using JuMP, GLPK, LinearAlgebra, DataFrames

# Define some input data about the test system
# Maximum power output of generators
g_max = [1000, 1000];
# Minimum power output of generators
g_min = [0, 300];
# Incremental cost of generators
c_g = [50, 100];
# Fixed cost of generators
c_g0 = [1000, 0]
# Incremental cost of wind generators
c_w = 50;
# Total demand
d = 1500;
# Wind forecast
w_f = 200;

# In this cell we create function solve_ed, which solves the economic dispatch problem for a given set of input parameters.
function solve_ed(g_max, g_min, c_g, c_w, d, w_f)
    #Define the economic dispatch (ED) model
    ed = Model(GLPK.Optimizer)

    # Define decision variables
    @variable(ed, 0 <= g[i = 1:2] <= g_max[i]) # power output of generators
    @variable(ed, 0 <= w <= w_f) # wind power injection

    # Define the objective function
    @objective(ed, Min, dot(c_g, g) + c_w * w)

    # Define the constraint on the maximum and minimum power output of each generator
    @constraint(ed, [i = 1:2], g[i] <= g_max[i]) #maximum
    @constraint(ed, [i = 1:2], g[i] >= g_min[i]) #minimum

    # Define the constraint on the wind power injection
    @constraint(ed, w <= w_f)

    # Define the power balance constraint
    @constraint(ed, sum(g) + w == d)

    # Solve statement
    optimize!(ed)

    # return the optimal value of the objective function and its minimizers
    return value.(g), value(w), w_f - value(w), objective_value(ed)
end

# Solve the economic dispatch problem
(g_opt, w_opt, ws_opt, obj) = solve_ed(g_max, g_min, c_g, c_w, d, w_f);

println("\n")
println("Dispatch of Generators: ", g_opt, " MW")
println("Dispatch of Wind: ", w_opt, " MW")
println("Wind spillage: ", w_f - w_opt, " MW")
println("\n")
println("Total cost: ", obj, "\$")

c_g_scale_df = DataFrame(Symbol("Dispatch of Generator 1(MW)") => Float64[],
               Symbol("Dispatch of Generator 2(MW)") => Float64[],
               Symbol("Dispatch of Wind(MW)") => Float64[],
               Symbol("Spillage of Wind(MW)") => Float64[],
               Symbol("Total cost(\$)") => Float64[])
for c_g1_scale = 0.5:0.1:3.0
    c_g_scale = [c_g[1] * c_g1_scale, c_g[2]] # update the incremental cost of the first generator at every iteration
    g_opt, w_opt, ws_opt, obj = solve_ed(g_max, g_min, c_g_scale, c_w, d, w_f) # solve the ed problem with the updated incremental cost
    push!(c_g_scale_df, (g_opt[1], g_opt[2], w_opt, ws_opt, obj))
end

ENV["COLUMNS"]=250 # Helps us display the complete table
c_g_scale_df

function solve_ed_inplace(c_w_scale)
    start = time()
    obj_out = Float64[]
    w_out = Float64[]
    g1_out = Float64[]
    g2_out = Float64[]

    ed = Model(GLPK.Optimizer)

    # Define decision variables
    @variable(ed, 0 <= g[i = 1:2] <= g_max[i]) # power output of generators
    @variable(ed, 0 <= w <= w_f ) # wind power injection

    # Define the objective function
    @objective(ed, Min, dot(c_g, g) + c_w * w)

    # Define the constraint on the maximum and minimum power output of each generator
    @constraint(ed, [i = 1:2], g[i] <= g_max[i]) #maximum
    @constraint(ed, [i = 1:2], g[i] >= g_min[i]) #minimum

    # Define the constraint on the wind power injection
    @constraint(ed, w <= w_f)

    # Define the power balance constraint
    @constraint(ed, sum(g) + w == d)

    optimize!(ed)

    for c_g1_scale = 0.5:0.01:3.0
        @objective(ed, Min, c_g1_scale*c_g[1]*g[1] + c_g[2]*g[2] + c_w_scale*c_w*w)
        optimize!(ed)
        push!(obj_out, objective_value(ed))
        push!(w_out, value(w))
        push!(g1_out, value(g[1]))
        push!(g2_out, value(g[2]))
    end
    elapsed = time() - start
    print(string("elapsed time: ", elapsed, " seconds"))
    return obj_out, w_out, g1_out, g2_out
end

solve_ed_inplace(2.0);

demandscale_df = DataFrame(Symbol("Dispatch of Generators(MW)") => Float64[],
               Symbol("Dispatch of Generator 2(MW)") => Float64[],
               Symbol("Dispatch of Wind(MW)") => Float64[],
               Symbol("Spillage of Wind(MW)") => Float64[],
               Symbol("Total cost(\$)") => Float64[])

for demandscale = 0.2:0.1:1.5
    g_opt,w_opt,ws_opt,obj = solve_ed(g_max, g_min, c_g, c_w, demandscale*d, w_f)

    push!(demandscale_df, (g_opt[1], g_opt[2], w_opt, ws_opt, obj))
end

demandscale_df

# In this cell we introduce binary decision u to the economic dispatch problem (function solve_ed)
function solve_uc(g_max, g_min, c_g, c_w, d, w_f)
    #Define the unit commitment (UC) model
    uc = Model(GLPK.Optimizer)

    # Define decision variables
    @variable(uc, 0 <= g[i=1:2] <= g_max[i]) # power output of generators
    @variable(uc, u[i = 1:2], Bin) # Binary status of generators
    @variable(uc, 0 <= w <= w_f ) # wind power injection

    # Define the objective function
    @objective(uc, Min, dot(c_g, g) + c_w * w)

    # Define the constraint on the maximum and minimum power output of each generator
    @constraint(uc, [i = 1:2], g[i] <= g_max[i]) #maximum
    @constraint(uc, [i = 1:2], g[i] >= g_min[i]) #minimum

    # Define the constraint on the wind power injection
    @constraint(uc, w <= w_f)

    # Define the power balance constraint
        @constraint(uc, sum(g) + w == d)

    # Solve statement
    optimize!(uc)

    status = termination_status(uc)
    if status != MOI.OPTIMAL
        return status, zeros(length(g)), 0.0, 0.0, zeros(length(u)), Inf
    end
    return status, value.(g), value(w), w_f - value(w), value.(u), objective_value(uc)
end

# Solve the economic dispatch problem
status, g_opt, w_opt, ws_opt, u_opt, obj = solve_uc(g_max, g_min, c_g, c_w, d, w_f);

println("\n")
println("Dispatch of Generators: ", g_opt[:], " MW")
println("Commitments of Generators: ", u_opt[:])
println("Dispatch of Wind: ", w_opt, " MW")
println("Wind spillage: ", w_f - w_opt, " MW")
println("\n")
println("Total cost: ", obj, "\$")

uc_df = DataFrame(Symbol("Commitment of Generator 1(MW)") => Float64[],
               Symbol("Commitment of Generator 2(MW)") => Float64[],
               Symbol("Dispatch of Generator 1(MW)") => Float64[],
               Symbol("Dispatch of Generator 2(MW)") => Float64[],
               Symbol("Dispatch of Wind(MW)") => Float64[],
               Symbol("Spillage of Wind(MW)") => Float64[],
               Symbol("Total cost(\$)") => Float64[])

for demandscale = 0.2:0.1:1.5
    status, g_opt, w_opt, ws_opt, u_opt, obj = solve_uc(g_max, g_min, c_g, c_w, demandscale*d, w_f)

    if status == MOI.OPTIMAL
    push!(uc_df, (u_opt[1], u_opt[2], g_opt[1], g_opt[2], w_opt, ws_opt, obj))
    else
        println("Status: $status for demandscale = $demandscale \n")
    end
end

uc_df


using DifferentialEquations
using Plots
plotlyjs()

#constants
l = 1.0                             # length [m]
m = 1.0                             # mass[Kg]
g = 9.81                            # gravitational acceleration [m/s²]

#function to compute ODE
function pendulum!(du,u,p,t)
    du[1] = u[2]                    # θ'(t) = ω(t)
    du[2] = -(g\l)*sin(u[1])  # ω'(t) = -(g/l) sin θ(t)
end

#initial conditions
θ₀ = π/6                         # initial angular deflection [rad]
ω₀ = 0.0                            # initial angular velocity [rad/s]
u₀ = [θ₀, ω₀]                       # initial state vector
tspan = (0.0,100.0)                  # time interval

#defining the problem and solving
prob = ODEProblem(pendulum!,u₀,tspan)
sol = solve(prob,saveat=0.1)

#plotting the solution
display(plot(sol,linewidth=2,xaxis="t (s)",
    color=["green" "blue"],label=["θ [rad]" "ω [rad/s]"],layout=(2,1)))
savefig("./plots/simple pendulum-plot.png")

# trajectory
t=sol.t
U=sol[1:end,:]
(θ,ω)=[U[x,:] for x in 1:size(U,1)]
x=l.*sin.(π.-θ)
y=zeros(size(t))
z=l.*cos.(π.-θ)
display(plot(x,y,z,linewidth=2,label="trajectory",
    xaxis="x (m)",yaxis="y (m)",zaxis="z (m)",grid=(:on,:black)))
savefig("./plots/simple pendulum-trajectory.png")

Modeling the Dragon Module as a Pendulum/Scripts/pendulum.jl


using Printf

agm1step(x, y) = (x + y) / 2, sqrt(x * y)

function approxπstep(x, y, z, n::Integer)
    a, g = agm1step(x, y)
    k = n + 1
    s = z + 2 ^ (k + 1) * (a ^ 2 - g ^ 2)
    return a, g, s, k
end

approxπ(a, g, s) = 4a ^ 2 / (1 - s)

function testmakepi()
	setprecision(512)
	a, g, s, k = BigFloat(1.0), 1 / √BigFloat(2.0), BigFloat(0.0), 0
	oldπ = BigFloat(0.0)
	println("Approximating π using ", precision(BigFloat), "-bit floats.")
	println("   k     Error  Result")
	for i in 1:100
		a, g, s, k = approxπstep(a, g, s, k)
		estπ = approxπ(a, g, s)
		if abs(estπ - oldπ) < 2eps(estπ) break end
		oldπ = estπ
		err = abs(π - estπ)
		@printf("%4d%10.1e%68.60e\n", i, err, estπ)
	end
end

testmakepi()


module MyCalculus

export EulerMethod

"""
    EulerMethod(f, x₀, y₀, step= 0.5, n=100.0)

Function to approximate solution to an ordinary differential equation using the Euler's Method.
dy/dx = f(x, y) where y(x₀) = y₀ and x₀ = x₀.

# Arguments
    f: function to approximate solution to, of the form f(x, y).
        This is the right hand side of the differential equation.
    x₀: initial x value condition
    y₀: initial y value condition
    step: step size for the Euler's Method
    n: number of steps to take

# Returns
    x: array of x values
    y: array of y values

"""
function EulerMethod(f::Function, x₀::Real, y₀::Real, step::Real, n::Real=100.0)

    y =[y₀] #initialize y array with intiial condition
    x = [x for x in range(x₀, step=step, length=n)] # array of x values starting with initial condition
    m = [] # this array will hold slope (right hand side of differential equation)

    push!(m, f(x[1], y[1])) # push initial slope onto array

    for k in 2:length(x) # iterate through x values starting at 2nd element

        push!(y, y[k-1] + m[k-1]*step) # push new y value to array using Euler's Method
        push!(m, f(x[k], y[k])) # push slope to array
    end

    return x, y

end

end


using DelimitedFiles
using LinearAlgebra

function convolve_linear(signal::Array{T, 2}, filter::Array{T, 2},
                         output_size) where {T <: Number}

    # convolutional output
    out = Array{Float64,2}(undef, output_size)
    sum = 0

    for i = 1:output_size[1]
        for j = 1:output_size[2]
            for k = max(1, i-size(filter)[1]):i
                for l = max(1, j-size(filter)[2]):j
                    if k <= size(signal)[1] && i-k+1 <= size(filter)[1] &&
                       l <= size(signal)[2] && j-l+1 <= size(filter)[2]
                        sum += signal[k,l] * filter[i-k+1, j-l+1]
                    end
                end
            end

            out[i,j] = sum
            sum = 0
        end
    end

    return out
end

function create_gaussian_kernel(kernel_size)

    kernel = zeros(kernel_size, kernel_size)

    # The center must be offset by 0.5 to find the correct index
    center = kernel_size * 0.5 + 0.5

    sigma = sqrt(0.1*kernel_size)

    for i = 1:kernel_size
        for j = 1:kernel_size
            kernel[i,j] = exp(-((i-center)^2 + (j-center)^2) / (2*sigma^2))
        end
    end

    return normalize(kernel)

end

function create_sobel_operators()
    Sx = [1.0, 2.0, 1.0]*[-1.0 0.0 1.0] / 9
    Sy = [-1.0, 0.0, 1.0]*[1.0 2.0 1.0] / 9

    return Sx, Sy
end

function compute_sobel(signal)
    Sx, Sy = create_sobel_operators()

    Gx = convolve_linear(signal, Sx, size(signal) .+ size(Sx))
    Gy = convolve_linear(signal, Sy, size(signal) .+ size(Sy))

    return sqrt.(Gx.^2 .+ Gy.^2)
end

# Simple function to create a square grid with a circle embedded inside of it
function create_circle(image_resolution, grid_extents, radius)
    out = zeros(image_resolution, image_resolution)

    for i = 1:image_resolution
        x_position = ((i-1)*grid_extents/image_resolution)-0.5*grid_extents
        for j = 1:image_resolution
            y_position = ((j-1)*grid_extents/image_resolution)-0.5*grid_extents
            if x_position^2 + y_position^2 <= radius^2
                out[i,j] = 1.0
            end
        end
    end

    return out
end

function main()

    # Random distribution in x
    x = rand(100, 100)

    # Gaussian signals
    y = [exp(-(((i-50)/100)^2 + ((j-50)/100)^2)/.01) for i = 1:100, j=1:100]

    # Normalization is not strictly necessary, but good practice
    normalize!(x)
    normalize!(y)

    # full convolution, output will be the size of x + y
    full_linear_output = convolve_linear(x, y, size(x) .+ size(y))

    # simple boundaries
    simple_linear_output = convolve_linear(x, y, size(x))

    # outputting convolutions to different files for plotting in external code
    writedlm("full_linear.dat", full_linear_output)
    writedlm("simple_linear.dat", simple_linear_output)

    # creating simple circle and 2 different Gaussian kernels
    circle = create_circle(50,2,0.5)

    normalize!(circle)

    small_kernel = create_gaussian_kernel(3)
    large_kernel = create_gaussian_kernel(25)

    small_kernel_output = convolve_linear(circle, small_kernel,
                                          size(circle).+size(small_kernel))
    large_kernel_output = convolve_linear(circle, large_kernel,
                                          size(circle).+size(large_kernel))

    writedlm("small_kernel.dat", small_kernel_output)
    writedlm("large_kernel.dat", large_kernel_output)

    # Using the circle for Sobel operations as well
    sobel_output = compute_sobel(circle)

    writedlm("sobel_output.dat", sobel_output)

end


# N is the number of dimensions of the prediction
abstract Loss{N}
typealias UnivariateLoss Loss{0}
typealias MultivariateLoss Loss{1}

## Abs loss (for regression)
#
#   loss(p, y) = |p - y|
#
immutable AbsLoss <: UnivariateLoss
end

value{T<:BlasReal}(::AbsLoss, p::T, y::T) = abs(p - y)
deriv{T<:BlasReal}(::AbsLoss, p::T, y::T) = sign(p - y)
value_and_deriv{T<:BlasReal}(::AbsLoss, p::T, y::T) = (r = p - y; (abs(r), sign(r)))

## Squared loss (for regression)
#
#   loss(p, y) := (1/2) * (p - y)^2
#
immutable SqrLoss <: UnivariateLoss
end

value{T<:BlasReal}(::SqrLoss, p::T, y::T) = half(abs2(p - y))
deriv{T<:BlasReal}(::SqrLoss, p::T, y::T) = p - y
value_and_deriv{T<:BlasReal}(::SqrLoss, p::T, y::T) = (r = p - y; v = half(abs2(r)); (v, r))

## Quantile loss (for quantile regression, asymmetric version of Abs loss)
#
#   loss(p, y) := t * (p - y)         ... (p >= y)
#               = (1 - t) * (y - p)   ... (p < y)
#
immutable QuantileLoss <: UnivariateLoss
    t::Float64

    function QuantileLoss(t::Real)
        zero(t) < t < one(t) || error("t must be a real value in (0, 1).")
        new(convert(Float64, t))
    end
end

function value{T<:BlasReal}(loss::QuantileLoss, p::T, y::T)
    t = convert(T, loss.t)
    p >= y ? t * (p - y) : (one(T) - t) * (y - p)
end

function deriv{T<:BlasReal}(loss::QuantileLoss, p::T, y::T)
    t = convert(T, loss.t)
    p > y ? t :
    p < y ? t - one(T) : zero(T)
end

function value_and_deriv{T<:BlasReal}(loss::QuantileLoss, p::T, y::T)
    t = convert(T, loss.t)
    if p > y
        (t * (p - y), t)
    elseif p < y
        c = t - one(T)
        (c * (p - y), c)
    else
        (zero(T), zero(T))
    end
end

## Epsilon Insensitive loss (for support vector regression)
#
#   loss(p, y) := 0                   ... abs(y - p) <= eps
#               = abs(y - p) - eps    ... otherwise
#
immutable EpsilonInsLoss <: UnivariateLoss
    epsilon::Float64

    function EpsilonInsLoss(epsilon::Real)
        new(convert(Float64, epsilon))
    end
end

function value{T<:BlasReal}(loss::EpsilonInsLoss, p::T, y::T)
    eps = convert(T, loss.epsilon)
    a = abs(p - y)
    a > eps ? a - eps : zero(T)
end

function deriv{T<:BlasReal}(loss::EpsilonInsLoss, p::T, y::T)
    eps = convert(T, loss.epsilon)
    r = p - y
    abs(r) > eps ? sign(r) : zero(T)
end

function value_and_deriv{T<:BlasReal}(loss::EpsilonInsLoss, p::T, y::T)
    eps = convert(T, loss.epsilon)
    r = p - y
    a = abs(r)
    a > eps ? (a - eps, sign(r)) : (zero(T), zero(T))
end

## Huber loss (for regression, smoothed version of Abs loss)
#
#   loss(p, y) := (1/2) * (p - y)^2      ... (|p - y| <= h)
#                 h * |p - y| - h^2/2    ... otherwise
#
immutable HuberLoss <: UnivariateLoss
    h::Float64

    function HuberLoss(h::Real)
        h > zero(h) || error("h must be a positive value.")
        new(convert(Float64, h))
    end
end

function value{T<:BlasReal}(loss::HuberLoss, p::T, y::T)
    h = convert(T, loss.h)
    a = abs(p - y)
    a <= h ? half(a * a) : h * a - half(h * h)
end

function deriv{T<:BlasReal}(loss::HuberLoss, p::T, y::T)
    h = convert(T, loss.h)
    r = p - y
    r > h ? h : r < -h ? -h : r
end

function value_and_deriv{T<:BlasReal}(loss::HuberLoss, p::T, y::T)
    h = convert(T, loss.h)
    r = p - y
    r > h ? (h * r - half(h * h), h) :
    r < -h ? (-h * r - half(h * h), -h) :
    (half(r * r), r)
end

## Hinge loss (for L1-SVM)
#
#   loss(p, y) := max(1 - y * p, 0)
#
immutable HingeLoss <: UnivariateLoss
end

value{T<:BlasReal}(::HingeLoss, p::T, y::T) = nonneg(one(T) - y * p)
deriv{T<:BlasReal}(::HingeLoss, p::T, y::T) = y * p < one(T) ? -y : zero(T)

function value_and_deriv{T<:BlasReal}(::HingeLoss, p::T, y::T)
    yp = y * p
    yp >= one(T) ? (zero(T), zero(T)) : (one(T) - yp, -y)
end

## Squared Hinge loss (for L2-SVM)
#
#   loss(p, y) := max(1 - y * p, 0)^2
#
immutable SqrHingeLoss <: UnivariateLoss
end

function value{T<:BlasReal}(::SqrHingeLoss, p::T, y::T)
    yp = y * p
    yp >= one(T) ? zero(T) : abs2(nonneg(one(T) - yp))
end

deriv{T<:BlasReal}(::SqrHingeLoss, p::T, y::T) = y * p < one(T) ? 2(p - y) : zero(T)

function value_and_deriv{T<:BlasReal}(::SqrHingeLoss, p::T, y::T)
    yp = y * p
    yp >= one(T) ? (zero(T), zero(T)) : (abs2(one(T) - yp), 2(p - y))
end

## Smoothed HingeLoss
#
#   loss(p, y) := 0              ... y * p > 1 + h
#                 1 - y * p      ... y * p < 1 - h
#                 (1 + h - y * p)^2 / 4h   ... otherwise
#
#  Reference
#
#   O. Chapelle, "Training a Support Vector Machine in the Primal", Neural Computation.
#
immutable SmoothedHingeLoss <: UnivariateLoss
    h::Float64

    function SmoothedHingeLoss(h::Real)
        h > zero(h) || error("h must be a positive value.")
        new(convert(Float64, h))
    end
end

function value{T<:BlasReal}(loss::SmoothedHingeLoss, p::T, y::T)
    h = convert(T, loss.h)
    yp = y * p
    yp >= one(T) + h ? zero(T) :
    yp <= one(T) - h ? one(T) - yp :
    abs2(one(T) + h - yp) / 4h
end

function deriv{T<:BlasReal}(loss::SmoothedHingeLoss, p::T, y::T)
    h = convert(T, loss.h)
    yp = y * p
    yp >= one(T) + h ? zero(T) :
    yp <= one(T) - h ? -y :
    y * (yp - one(T) - h) / 2h
end

function value_and_deriv{T<:BlasReal}(loss::SmoothedHingeLoss, p::T, y::T)
    h = convert(T, loss.h)
    yp = y * p
    if yp >= one(T) + h
        (zero(T), zero(T))
    elseif yp <= one(T) - h
        (one(T) - yp, -y)
    else
        z = yp - one(T) - h
        (abs2(z) / 4h, y * z / 2h)
    end
end

## Logistic loss (for logistic regression)
#
#   loss(p, y) := log(1 + exp(-y * p))
#
immutable LogisticLoss <: UnivariateLoss
end

value{T<:BlasReal}(::LogisticLoss, p::T, y::T) =
    (yp = y * p; yp >= zero(T) ? log1p(exp(-yp)) : log1p(exp(yp)) - yp)

function deriv{T<:BlasReal}(::LogisticLoss, p::T, y::T)
    yp = y * p
    if yp >= zero(T)
        e = exp(-yp)
        -y * e / (one(T) + e)
    else
        -y / (one(T) + exp(yp))
    end
end

function value_and_deriv{T<:BlasReal}(::LogisticLoss, p::T, y::T)
    yp = y * p
    if yp >= zero(T)
        e = exp(-yp)
        (log1p(e), -y * e / (one(T) + e))
    else
        e = exp(yp)
        (log1p(e) - yp, -y / (one(T) + e))
    end
end

## general functions for multivariate loss

grad{T<:BlasReal}(loss::MultivariateLoss, p::StridedVector{T}, y) =
    grad!(loss, zeros(T, length(p)), p, y)

value_and_grad{T<:BlasReal}(loss::MultivariateLoss, p::StridedVector{T}, y) =
    value_and_grad!(loss, zeros(T, length(p)), p, y)

## SumLoss
#
#   loss(p, y) := sum_k loss.intern(p[k], y[k])
#
immutable SumLoss{L<:UnivariateLoss} <: MultivariateLoss
    intern::L
end

SumLoss{L<:UnivariateLoss}(loss::L) = SumLoss{L}(loss)

typealias SumSqrLoss SumLoss{SqrLoss}
SumSqrLoss() = SumLoss{SqrLoss}(SqrLoss())

function value{T<:BlasReal}(s::SumLoss, p::StridedVector{T}, y::StridedVector{T})
    loss = s.intern
    k = length(p)
    @_checkdims k == length(y)
    s = value(loss, p[1], y[1])
    @inbounds for i = 2:k
        s += value(loss, p[i], y[i])
    end
    s
end

function grad!{T<:BlasReal}(s::SumLoss, g::StridedVector{T}, p::StridedVector{T}, y::StridedVector{T})
    loss = s.intern
    k = length(p)
    @_checkdims k == length(g) == length(y)
    @inbounds for i = 1:k
        g[i] = deriv(loss, p[i], y[i])
    end
    g
end

function value_and_grad!{T<:BlasReal}(s::SumLoss, g::StridedVector{T}, p::StridedVector{T}, y::StridedVector{T})
    loss = s.intern
    k = length(p)
    @_checkdims k == length(g) == length(y)
    s, dv = value_and_deriv(loss, p[1], y[1])
    g[1] = dv
    @inbounds for i = 2:k
        v, dv = value_and_deriv(loss, p[i], y[i])
        s += v
        g[i] = dv
    end
    (s, g)
end

## Multinomial logistic loss (for Multinomial logistic regression)
#
#   loss(p, y) := log(sum_k exp(p[k])) - p[y]
#
immutable MultiLogisticLoss <: MultivariateLoss
end

function value{T<:BlasReal}(::MultiLogisticLoss, p::StridedVector{T}, y::Integer)
    k = length(p)
    pmax = maximum(p)
    s = zero(T)
    @inbounds for i = 1:k
        s += exp(p[i] - pmax)
    end
    pmax + log(s) - p[y]
end

function grad!{T<:BlasReal}(::MultiLogisticLoss, g::StridedVector{T}, p::StridedVector{T}, y::Integer)
    k = length(p)
    @_checkdims length(g) == k
    pmax = maximum(p)
    s = zero(T)
    @inbounds for i = 1:k
        pi = exp(p[i] - pmax)
        g[i] = pi
        s += pi
    end
    @inbounds for i = 1:k
        g[i] /= s
    end
    g[y] -= one(T)
    g
end

function value_and_grad!{T<:BlasReal}(::MultiLogisticLoss, g::StridedVector{T}, p::StridedVector{T}, y::Integer)
    k = length(p)
    @_checkdims length(g) == k

    pmax = maximum(p)
    p_y = p[y]  # g and p can be the same array, so have to cache p[y] before p is overwritten

    s = zero(T)
    @inbounds for i = 1:k
        pi = exp(p[i] - pmax)
        g[i] = pi
        s += pi
    end

    @inbounds for i = 1:k
        g[i] /= s
    end
    g[y] -= one(T)
    return (pmax - p_y + log(s), g)
end


function RouwenNormalize!(A)
    A[2:end-1, :] ./= 2
    nothing
end

function rouwenupdate!(Πold, Πnew, p, q)
    Nold = size(Πold, 1)
    Πnew[1:Nold, 1:Nold] = p*Πold
    Πnew[1:Nold, 2:(Nold+1)] .+= (1-p)*Πold
    Πnew[2:(Nold+1), 1:Nold] .+= (1-q)*Πold
    Πnew[2:(Nold+1), 2:(Nold+1)] .+= q*Πold
    nothing
end

function rouwenmat(N, ρ, p, q)
    initmat = [p 1-p;1-q q]
    if N==2
        ret = initmat
    else
        currmat = initmat
        for n = 3:N
            nextmat = fill(zero(ρ), (n, n))
            rouwenupdate!(currmat, nextmat, p, q)
            currmat = nextmat
            RouwenNormalize!(currmat)
        end
        ret = currmat
    end
    return(ret)
end

"""
    rouwenhorst(npts, ρ, σ)

Discretizes AR(1) process `y′ = ρ y +  σ ϵ`.

# Example

npts = 10
rho = 0.9
sig = 0.01
points, Π = rouwenhorst(npts, rho, sig)

"""
function rouwenhorst(npts, ρ, σ)
    ω = σ/sqrt(1-ρ^2) # long run std dev
    # println("long run variance: $ω")
    q = (1+ρ)/2

    points = range(-ω*sqrt(npts - 1),
                   ω*sqrt(npts-1),
                   length=npts)

    # points = GenGrid(uniform(npts, bds))
    Π = (npts > 1) ? rouwenmat(npts, ρ, q, q) : fill(one(ρ), (1,1))

    return points, Π
end

"Returns a MarkovChain with discretized AR(1) from applying `rouwenhorst`"
rouwenhorst_mc(npts, ρ, σ)=MarkovChain(rouwenhorst(npts, ρ, σ)...)


function check_symplecticity(tab::Tableau{T}; atol=16eps(T), rtol=16eps(T)) where {T}
    a, b = tab.a, tab.b
    [isapprox(b[i] * a[i,j] + b[j] * a[j,i], b[i] * b[j]; atol=atol, rtol=rtol) for i in axes(a,1), j in axes(a,2)]
end

function issymplectic(tab::Tableau; kwargs...)
    all(check_symplecticity(tab; kwargs...))
end

function check_symplecticity(tab::PartitionedTableau{T}; atol=16eps(T), rtol=16eps(T)) where {T}
    a, b = tab.q.a, tab.q.b
    ā, b̄ = tab.p.a, tab.p.b
    ([isapprox(b[i] * ā[i,j] + b̄[j] * a[j,i], b[i] * b̄[j]; atol=atol, rtol=rtol) for i in axes(a,1), j in axes(a,2)],
     [isapprox(b[i], b̄[i]; atol=atol, rtol=rtol) for i in eachindex(b,b̄)])
end

function issymplectic(tab::PartitionedTableau; kwargs...)
    all(all.(check_symplecticity(tab; kwargs...)))
end

function compute_symplecticity_error(tab::Tableau)
    a, b = tab.a, tab.b
    [b[i] * a[i,j] + b[j] * a[j,i] - b[i] * b[j] for i in axes(a,1), j in axes(a,2)]
end

function get_symplectic_conjugate_coefficients!(a̅::AbstractMatrix{T}, a::AbstractMatrix{T}, b::AbstractVector{T}) where {T}
    @assert size(a) == size(a̅)
    @assert length(b) == size(a,1) == size(a,2)

    for i in axes(a̅, 1)
        for j in axes(a̅, 2)
            a̅[i,j] = b[j] / b[i] * ( b[i] - a[j,i] )
        end
    end

    return a̅
end

get_symplectic_conjugate_coefficients(a, b) = get_symplectic_conjugate_coefficients!(zero(a), a, b)
get_symplectic_conjugate_coefficients(tab::Tableau) = Tableau(tab.name, tab.s, get_symplectic_conjugate_coefficients(tab.a, tab.b), tab.b, tab.c)

function symplecticize(tab::Tableau; name=nothing, T=Float64)
    a̅ = get_symplectic_conjugate_coefficients(tab.a, tab.b)
    Tableau{T}(name === nothing ? Symbol(tab.name, "S") : name, tab.o, (tab.a .+ a̅) ./ 2, tab.b, tab.c)
end


export gauss_quad_linear,gauss_points
function gauss_quad_linear(N,a,b)
    N=N-1;
    N1=N+1;
    N2=N+2;
    xu=linspace(-1,1,N1)';
    # Initial guess
    y=cos.((2*(0:N)'+1)*pi/(2*N+2))+(0.27/N1)*sin.(pi*xu*N/N2);
    # Legendre-Gauss Vandermonde Matrix
    L=zeros(N1,N2);
    # Derivative of LGVM
    Lp=zeros(N1,N2);
    # Compute the zeros of the N+1 Legendre Polynomial
    # using the recursion relation and the Newton-Raphson method
    y0=2;
    # Iterate until new points are uniformly within epsilon of old points
    while (maximum(abs.(y-y0))>eps(1.0))
        L[:,1]=1;
        Lp[:,1]=0;
        L[:,2]=y;
        #Lp[:,2]=1;
        for k=2:N1
            L[:,k+1]=( (2*k-1)*y'.*L[:,k]-(k-1)*L[:,k-1] )/k;
        end
        Lp=(N2)*( L[:,N1]-y'.*L[:,N2] )./(1-y.^2)';
        y0=y;
        y=y0'-L[:,N2]./Lp;
        y=y'
    end
    y=y'
    # Linear map from[-1,1] to [a,b]
    x=(a*(1-y)+b*(1+y))/2;
    # Compute the weights
    w=(b-a)./((1-y.^2).*Lp.^2)*(N2/N1)^2;
    return x, w
end

function gauss_points(N)
    x,w = gauss_quad_linear(N,-1,1)
    gpt = zeros(N*N,2)
    gpw = zeros(N*N,1)
    for i = 1:N
        u = x[i]
        wu = w[i]
        for j = 1:N
            v = x[j]
            wv = w[j]
            gpt[(i-1)*N + j,1] = u
            gpt[(i-1)*N + j,2] = v
            gpw[(i-1)*N + j] = wu * wv
        end
    end
    return gpt,gpw
end


##
##  d e r i v a t i v e . j l  Numerical Derivatives
##

#-- Numerical gradient of multivariate function ------------------------------
#
function fd_gradient{T<:Real}(f::Function, x0::Array{T,1}; h::Real = 0.0)
    local heps::Real = h
    if heps == 0.0; heps = eps()^(1.0/3.0); end
    local n::Int = length(x0)
    if length(f(x0)) > 1
        error("Function 'f' must be a scalar function.")
    end

    local hh = zeros(n), gr = zeros(n)
    for i = 1:n
        hh[i] = heps
        gr[i] = ((f(x0 + hh) - f(x0 - hh)) / (2.0*heps))
        hh[i] = 0.0
    end
    return gr
end

function fd_gradient(f::Function, x0::Real; h::Real = 0.0)
    local heps::Real = h
    if heps == 0.0; heps = eps()^(1.0/3.0); end
    if length(f(x0)) > 1
        error("Function 'f' must be a scalar function.")
    end
    return (f(x0 + heps) - f(x0 - heps)) / (2.0*heps)
end

function fd_jacobian{T<:Real}(f::Function, x0::Array{T,1}; h::Real = 0.0)
    local heps::Real = h
    if heps == 0.0; heps = eps()^(1.0/3.0); end
	local n = length(x0), m = length(f(x0))
	local hh = zeros(n)
	jacob = zeros(m, n)
	for i = 1:n
		hh[i] = heps
		jacob[:, i] = (f(x0 + hh) - f(x0 - hh)) / (2.0*heps)
		hh[i] = 0.0
	end
	return jacob
end

function fd_hessian{T<:Real}(f::Function, x0::Array{T,1}; h::Real = 0.0)
    local heps::Real = h
    if heps == 0.0; heps = eps()^(1.0/4.0); end
    local n::Int = length(x0)
    if length(f(x0)) != 1
        error("Function 'f' must a scalar function of n variables.")
    end

    if n == 1
        H = (f(x0.-heps) - 2.0*f(x0) + f(x0.+heps)) / heps^2

    else
        H = zeros(n, n)
        hh = diagm(heps*ones(n))
        for i = 1:(n-1)
            hi = hh[:, i]
            H[i, i] = (f(x0-hi) - 2.0*f(x0) + f(x0+hi)) / heps^2
            for j = (i+1):n
                hj = hh[:, j]
                H[i, j] = (f(x0+hi+hj) - f(x0+hi-hj) - f(x0-hi+hj) + f(x0-hi-hj)) / (4.0*heps^2)
                H[j, i] = H[i, j]
            end
        end
        hi = hh[:, n]
        H[n, n] = (f(x0-hi) - 2.0*f(x0) + f(x0+hi)) / heps^2
    end

    return H
end

function fd_laplacian{T<:Real}(f::Function, x0::Array{T,1}; h::Real = 0.0)
    local heps::Real = h
    if heps == 0.0; heps = eps()^(1.0/4.0); end
    local n::Int = length(x0)
    local hh::Real = zeros(n)

    L = 0.0
    for i = 1:n
        hh[i] = heps
        L += (f(x0+hh) + f(x0-hh) - 2.0*f(x0)) / heps^2
        hh[i] = 0.0
    end

    return L
end

#-- Central difference combined with Richardson approximation ----------------
#
function numderiv(f::Function, x0::Real; n::Int = 16, h::Real = 0.1)

    local epsilon = eps(), err = Inf
    local hstep::Real = h

    j::Int = 1
    D = zeros(n, n)
    D[1, 1] = (f(x0+hstep) - f(x0-hstep))/(2*hstep)

    while j < n
        hstep = hstep / 2.0
        D[j+1, 1] = (f(x0+hstep) - f(x0-hstep)) / (2*hstep)
        for k = 1:j
            D[j+1,k+1] = D[j+1,k] + (D[j+1,k] - D[j,k]) / (4^k - 1)
        end

        err_new = 2 * abs(D[j+1,j+1] - D[j,j]) /
                  (abs(D[j+1,j+1]) + abs(D[j,j]) + epsilon)

        if err_new >= err; break; end
        err = err_new
        j += 1
    end

    return D[j,j]
end

#-- Complex-step derivative approach -----------------------------------------
#
function complex_step(f::Function, x0::Real; h::Real = 1e-20)
    # Check whether function accepts and returns complex values
    if !isa(f(x0), Real)
        error("Function 'f' must take on a real(!) value at 'x0'.")
    end

    fx0hi = try
        f(x0 + h*1im)
    catch err
        error("Function 'f' does not appear to accept complex arguments.")
    end

    # f(x0) must be real, fx0hi complex
    if !isa(fx0hi, Complex)
        error("'f(x0 + h*1im)' must be complex for 'complex-step' to work.")
    end

    return imag(fx0hi) / h
end


# # Implementations and proof of concept
#
# In this examples I want to show some of the implementations and how they are used
# as a proof of concept. This means that we will use the "low-level API", i.e. calling
# the methods directly instead of the `optimize` interface.
#
# Further, we wish to show that the implementations can at least solve some of the
# most common benchmark optimization problems.
#
# Before we start, I will define a seed and an RNG to enable reproducibility of the
# results presented here.

using Random

RANDOM_SEED = 458012;
rng = MersenneTwister(RANDOM_SEED);

#
# ## Nonlinear ``d``-dimensional global optimization problem
#
# Using `BioMetaheuristics.jl` is fairly straightforward, we will start by defining
# an d-dimensional nonlinear function to minimize,
# in this case we will use a popular function, the
# [Griewank function](http://mathworld.wolfram.com/GriewankFunction.html)
# defined as
#
# ```math
# f(\mathbf{x}) = \sum_{i=1}^d \frac{x_i^2}{4000} - \prod_{i=1}^d \cos{\left(
# \frac{x_i}{\sqrt{i}}\right)} + 1
# ```
#
# where ``d`` is the dimension of the problem. It's mostly evaluated within the
# boundaries ``-100 \leq x_i \leq 100``, and it has a **minimum** at ``\mathbf
# {x^*} = (0, \cdots, 0)``, and it evaluates to ``f(\mathbf{x^*}) = 0``.
#
# We define the function in `Julia` like this

function griewank(x)
    first_term = sum(x.^2) / 4000
    ## This variable will hold the result of the product,
    ## the second term in the function definition from above
    second_term = 1.0
    for (idx, val) in enumerate(x)
        second_term *= cos(val / sqrt(idx))
    end

    return first_term - second_term + 1.0
end

# Now, we wish to find the minimum of this function, and fortunately we know the
# true value so we can compare it later, we can use some of the implementations
# from `BioMetaheuristics.jl`, for example, [`PSO`](@ref).
# In this script we have chosen 30 particles within the population, `d` is equal
# to 20, next we define the boundaries and finally we declare that the algorithm
# will run for 20000 maximum iterations until it stops, having _converged_.

using BioMetaheuristics

val = PSO(
    griewank,
    Population(35, 10, -600.0, 600.0, rng),
    20_000,
    rng
)
println(val)

# Within a certain tolerance of about ``\epsilon = 1 \times 10^{-6}`` we have found
# the _global_ minimum of the function. We can actually check the value with the
# evaluation, notice that it actually returns `0`, as expected.

griewank(val.x)

# ## Nonlinear 2-dimensional global optimization problem
#
# Let us now tackle one of the most common optimization problems, which is
# finding the minimum of the [Rosenbrock function](https://en.wikipedia.org/wiki/Rosenbrock_function)
# which is a non-convex function, meaning that is does not have just one minimum
# or stationary point, it has several, so it is a difficult problem for classical
# optimization algorithms. In this example we will try to solve it using the
# [`SimulatedAnnealing`](@ref) implementation from `BioMetaheuristics.jl`.
#
# First, we define the Rosenbrock function in `Julia`

rosenbrock2d(x) =  (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2;

# We will apply the _Simulated Annealing_ algorithm to find the global optimum
val = SimulatedAnnealing(
    rosenbrock2d, -5.0, 5.0, 2, rng; low_temp=10_000
)
println(val)

# Again, within a certain tolerance we find the expected result which is
#
# ```math
# \mathbf{x}^{*} = (1, 1)
# ```
#
# and if we account for rounding errors and floating-point arithmetic, we
# can safely take this result as the best.


using Random: AbstractRNG
using Bijectors
import Bijectors: logabsdetjac, bijector
using Distributions: MultivariateDistribution, Continuous
import Distributions: _rand!, _logpdf
using StatsFuns

"""
    Kipping13()

A non-informative prior for two-parameter limb-darkening coefficients using *triangular sampling* ([Kipping 2013](https://ui.adsabs.harvard.edu/abs/2013MNRAS.435.2152K/)).

# Examples

```jldoctest
julia> using Random; rng = Random.seed!(10);

julia> rand(rng, Kipping13())
2-element Vector{Float64}:
 0.24716310305467298
 0.08836997249298882

julia> rand(rng, Kipping13(), 5)
2×5 Matrix{Float64}:
 0.0664907  0.124817   1.00732    0.806902  0.74165
 0.520411   0.222718  -0.389412  -0.314755  0.0768429
```

# References

> [Kipping (2013)](https://ui.adsabs.harvard.edu/abs/2013MNRAS.435.2152K/)
>
>   "Efficient, uninformative sampling of limb darkening coefficients for two-parameter laws"
"""
struct Kipping13 <: MultivariateDistribution{Continuous} end

Base.length(::Kipping13) = 2

function _rand!(rng::AbstractRNG, ::Kipping13, x::AbstractVector{T}) where T
    q1, q2 = rand(rng, T, 2)
    sqrtq1 = sqrt(q1)
    twoq2 = 2 * q2
    x[begin] = sqrtq1 * twoq2
    x[end] = sqrtq1 * (1 - twoq2)
    return x
end

_logpdf(::Kipping13, x::AbstractArray{T}) where {T} = zero(T)

struct Kipping13Transform <: Bijector{1} end

function (::Kipping13Transform)(x::AbstractVector)
    usum = sum(x)
    q = [usum^2, 0.5 * first(x) / usum]
    return @. log(q) - log(1 - q)
end

function (::Inverse{<:Kipping13Transform})(y::AbstractVector)
    tmp = map(logistic, y)
    sqrtq1 = sqrt(first(tmp))
    twoq2 = 2 * last(tmp)
    tmp[begin] = sqrtq1 * twoq2
    tmp[end] = sqrtq1 * (1 - twoq2)
    return tmp
end

logabsdetjac(::Kipping13Transform, y::Number) = -2 * softplus(-y) - y
logabsdetjac(k::Kipping13Transform, y) = sum((yi) -> logabsdetjac(k, yi), y)
bijector(::Kipping13) = Kipping13Transform()


# From algorithm 3.1

function bracket_minimum(f, x=0, s=1e-2, k=2.0)

    a, ya = x, f(x)
    b, yb = a + s, f(a + s)
    if yb > ya
        a, b = b, a
        ya, yb = yb, ya
        s = -s
    end
    while true
        c, yc = b + s, f(b + s)
        if yc > yb
            return (min(a,c), max(a,c))
        end
        a, ya, b, yb = b, yb, c, yc
        s *= k
    end
end

function test_f(x)
     return x^3 + 2 * x^2 + x + 3
end

@time a, b = bracket_minimum(test_f)


# # Munk Gyre analytical solution equations
# things between #### blocks are the code for checking exact solution in DGmodel test

using Pkg
Pkg.add("Latexify")
using Latexify
Pkg.add("Plots")
using Plots
Pkg.add("PyPlot")
pyplot()

# Problem parameters
# These will come from DG model/problem settings
###
τ₀=0.1      # Stress
ρ=1000      # density
f=1.e-4     # Coriolis
β=1.e-11    # Rate of change of Coriolis
H=1000      # Depth
L_x=1000.e3 # Zonal extent
L_y=1000.e3 # Meridional extent
Ah=1000.    # Viscosity
g=10.       # Gravity

δ_m=(Ah/β)^(1/3)
####

# Set up Munk formula per https://mitgcm.readthedocs.io/en/latest/examples/examples.html#model-solution
# Build up using expressions to catch typos
t1=:( cos((3^0.5*x)/(2*δ_m))+(1)/(3^0.5)*sin((3^0.5*x)/(2*δ_m)) )
display(latexify(t1))
t2=:( 1-exp((-x)/(2*δ_m) ) * $t1 )
display(latexify(t2))
t3=:( π*sin(π*y/L_y) )
display(latexify(t3))
t4=:( τ₀/(ρ*g*H)*(f/β)*(1-x/L_x) )
display(latexify(t4))
tfull=:( ($t4) * ($t3) * ($t2) )
display(latexify(tfull))
display(tfull)

# Actual function (a copy paste from display(tfull) line)
#### This is formula needed for checking numerical against exact
ηfun(x,y)=(((τ₀ / (ρ * g * H)) * (f / β) * (1 - x / L_x)) * (π * sin((π * y) / L_y)) * (1 - exp(-x / (2δ_m)) * (cos((3 ^ 0.5 * x) / (2δ_m)) + (1 / 3 ^ 0.5) * sin((3 ^ 0.5 * x) / (2δ_m)))))
####

# Now make a plot to check
xvals=collect(0:L_x/100:L_x)
yvals=collect(0:L_y/100:L_y)
nx=length(xvals)
ny=length(yvals)
η=zeros(nx,ny)
for j=1:ny
 for i=1:nx
  η[i,j]=ηfun(xvals[i],yvals[j])
 end
end

p2 = contour(xvals, yvals, η')
plot(p2)


function swiss_roll(n, σ = 0.1)
    # x = rand(n) .* (3.0 * π) .+ (1.5 * π)
    # y = rand(n) .* (30.0 * π)

    ### 1
    # f = permutedims([(x, y) -> x .* cos.(x),
    #                  (x, y) -> y,
    #                  (x, y) -> x .* sin.(x) ])
    # broadcast((f, x, y) -> f(x, y), f, x, y)

    ### 2
    # f = ((x, y) -> x .* cos.(x),
    #      (x, y) -> y,
    #      (x, y) -> x .* sin.(x) )
    # for i in axis(res, 1), j in axis(res, 2)
    #     res[i, j] = ff[j](x[i], y[i])
    # end
    # return res

    ### 3
    # res = Array{Float64, 2}(undef, n, 3)
    # f1(x, y) = x .* cos.(x)
    # f2(x, y) = y
    # f3(x, y) = x .* sin.(x)
    # res[:, 1] .= f1.(x, y) .+ randn(n) .* σ
    # res[:, 2] .= f2.(x, y) .+ randn(n) .* σ
    # res[:, 3] .= f3.(x, y) .+ randn(n) .* σ
    # return res

    ### 4
    # hcat(x .* cos.(x), y, x .* sin.(x));

    ### 5
    # f(x, y) = (x * cos(x), y, x * sin(x))
    # permutedims(reshape(reinterpret(Float64, f.(x, y)), (3, n)))

    xx() = rand() .* (3.0 * π) .+ (1.5 * π)
    yy() = rand() .* (30.0 * π)

    permutedims(reshape(reinterpret(Float64, [
        begin
            x = xx()
            y = yy()
            (x * cos(x), y, x * sin(x))
        end for i in 1:n
    ]), (3, n)))
end

function circles(n, σ = 0.1, factor = 0.8)
    res = Array{Float64, 2}(undef, n, 2)

    classes = rand(Bool, n)
    ff = classes .* factor
    res[:, 1] .= cos.(range(0, stop = 2 * π, length = n)) .* ff
    res[:, 2] .= sin.(range(0, stop = 2 * π, length = n)) .* ff

    res .+= randn(size(res)...) .* σ
    res, classes
end


function luhntest(x::Integer)
    d = reverse(digits(x))
    s = sum(d[1:2:end])
    s += sum(sum.(digits.(2d[2:2:end])))
    return s % 10 == 0
end

for card in [49927398716, 49927398717, 1234567812345678, 1234567812345670]
    println(luhntest(card) ? "PASS " : "FAIL ", card)
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Foundations of machine learning")

# ## What is learning?
#
# Computers read data, as we saw in notebooks 1 and 2. We can then build functions that model that data to make decisions, as we saw in notebooks 3 and 5.
#
# But how do you make sure that the model actually fits the data well? In the last notebook, we saw that we can fiddle with the parameters of our function defining the model to reduce the loss function. However, we don't want to have to pick the model parameters ourselves. Choosing parameters ourselves works *well enough* when we have a simple model and only a few data points, but can quickly become extremely complex for more detailed models and larger data sets.
#
# Instead, we want our machine to *learn* the parameters that fit the model to our data, without needing us to fiddle with the parameters ourselves. In this notebook, we'll talk about the "learning" in machine learning.

#-

# ### Motivation: Fitting parameters by hand
#
# Let's go back to our example of fitting parameters from notebook 3. Recall that we looked at whether the amount of green in the pictures could distinguish between an apple and a banana, and used a sigmoid function to model our choice of "apple or banana" using the amount of green in an image.

using Plots; gr()
using Images, Statistics

σ(x,w,b) = 1 / (1 + exp(-w*x+b))

apple =  load(datapath("data/10_100.jpg"))
banana = load(datapath("data/104_100.jpg"))
apple_green_amount =  mean(Float64.(green.(apple)))
banana_green_amount = mean(Float64.(green.(banana)));

w = 10.0 # Try manipulating w between 0 and 30 to see how the plot changes
b = 15.0 # Try manipulating b bewteen 0 and 30

plot(x->σ(x,w,b), 0, 1, label="Model", legend = :topleft, lw=3)
scatter!([apple_green_amount],  [0.0], label="Apple")
scatter!([banana_green_amount], [1.0], label="Banana")

# Intuitively, how did you tweak the sliders so that way the model sends apples to 0 and bananas to 1? Most likely, you did the following:
#
# #### Move the sliders a bit, see whether the curve moves in the right direction, and if it did, keep doing it.
#
# For a machine, "learning" is that same process, translated into math!

#-

# ## "Learning by nudging": The process of descent
#
# Let's start to formalize this idea. In order to push the curve in the "right direction", we need some measurement of "how right" and "how wrong" the model is. When we translate the idea of a "right direction" into math, we end up with a **loss function**, `L(w, b)`, as we saw in notebook 5. We say that the loss function is lowest when the model `σ(x, w, b)` performs the best.
#
# Now we want to create a loss function that is the lowest when the apple is at `0` and the banana is at `1`. If the data (the amount of green) for our apple is $x_1$, then our model will output $σ(x_1,w, b)$ for our apple. So, we want the difference $0 - σ(x_1, w, b)$ to be small. Similarly, if our data for our banana (the banana's amount of green) is $x_2$, we want the difference $1 - σ(x_2, w, b)$ to be small.
#
# To create our loss function, let's add together the squares of the difference of the model's output from the desired output for the apple and the banana. We get
#
# $$ L(w,b) = (0 - σ(x_1, w, b))^2 + (1 - σ(x_2, w, b))^2. $$
#
# $L(w, b)$ is lowest when it outputs `0` for the apple and `1` for the banana, and thus the cost is lowest when the model "is correct".
#
# We can visualize this function by plotting it in 3D with the `surface` function or in 2D with contour lines

## plotly() # The plotly backend is nice for 3d surface plots
gr() # The GR backend is good for faster interactive plots

#-

L(w, b) = (0 - σ(apple_green_amount,w,b))^2 + (1 - σ(banana_green_amount,w,b))^2

w_range = 10:0.1:13
b_range = 0:1:20

L_values = [L(w,b) for b in b_range, w in w_range]

w = 11.5 # Try manipulating w with values from w_range (between 10 and 13)
b = 10 # Try manipulating b with values from b_range (between 0 and 20)
##     p1 = surface(w_range, b_range, L_values, xlabel="w", ylabel="b", cam=(70,40), cbar=false, leg=false)
##     scatter!(p1, [w], [b], [L(w,b)+1e-2], markersize=5, color = :blue)
p1 = contour(w_range, b_range, L_values, levels=0.05:0.1:1, xlabel="w", ylabel="b", cam=(70,40), cbar=false, leg=false)
scatter!(p1, [w], [b], markersize=5, color = :blue)

p2 = plot(x->σ(x,w,b), 0, 1, label="Model", legend = :topleft, lw=3)
scatter!(p2, [apple_green_amount],  [0.0], label="Apple", markersize=10)
scatter!(p2, [banana_green_amount], [1.0], label="Banana", markersize=10, xlim=(0,1), ylim=(0,1))
plot(p1, p2, layout=(2,1))

# The blue ball on the 3D plot shows the current parameter choices, plotted as `(w,b)`. Shown below the 3D plot is a 2D plot of the corresponding model with those parameters. Notice that as the blue ball rolls down the hill, the model becomes a better fit. Our loss function gives us a mathematical notion of a "hill", and the process of "learning by nudging" is simply rolling the ball down that hill.

#-

# To do this mathematically, we need to know which direction is "downhill". Recall from calculus that the derivative of `L` with respect to `b` tells you how `L` changes when `b` changes. Thus to roll downhill, we should go in the direction where the derivative is negative (the function goes down) for each parameter. This direction is the negative of what's called the **gradient**, $\nabla L$. This means that the "learn by nudging method" can be rephrased in mathematical terms as:
#
# 1. Calculate the gradient
# 2. Move a little bit in the direction of the negative gradient
# 3. Repeat
#
# This process of rolling the ball in the direction of the negative gradient is called **gradient descent**; written mathematically, it is
#
# $$p_{n+1} = p_n - \eta \nabla L(p_n).$$
#
# Here, $p_n$ represents the vector of current parameters $(w, b)$; $\nabla L(p_n)$ is the gradient of the loss function, given those parameters. We start from $p_n$ and change it by $\eta \nabla L(p_n)$, where $\eta$ is a small step size that determines how far we move the parameters in the direction of the negative gradient; notice that if you step too far, you'll overshoot the minimum!. The result is $p_{n+1}$, the new vector of parameters.
#
# [Picture of Gradient Descent Vectors]
#
# If we repeat this process, then we will end up at parameters where the model correctly labels apples as `0` and bananas as `1`. When this happens, the model has learned from the data and can then read pictures and tell you whether they are apples or bananas!

#-

# #### Exercise 1
#
# Use the following terms to fill in the sentences below. Terms may be used more than once or not at all:
# > gradient, loss function, derivative, gradient descent, learning.
#
# * We can think of a _(A)_ as a 1D version of a _(B)_.
# * We can visualize a _(C)_ as a hill.
# * In the explanation above, rolling downhill is called _(D)_ and means traveling along the _(E)_.
# * To quantify the correctness of a model we use a _(F)_.
# * When our program can minimize a _(G)_ on its own, we say it is _(H)_.
#
#

#
# A)

# B)

# C)

# D)

# E)

# F)

# G)

# H)

#-

# #### Solution
#
# A) derivative

# B) gradient

# C) loss function

# D) gradient descent

# E) gradient

# F) loss function

# G) loss function

# H) learning

Courses/Foundations of machine learning/0900.What-is-learning.jl


function pagerank(
    g::AbstractGraph{U},
    α=0.85,
    n=100::Integer,
    ϵ=1.0e-6
    ) where U <: Integer

    # indegree(g, v) is estimated run-time to iterate over inneighbors(g, v)
    partitions = LightGraphs.optimal_contiguous_partition(indegree(g), nthreads(), nv(g))

    α_div_outdegree = Vector{Float64}(undef,nv(g))
    dangling_nodes = Vector{U}()
    @inbounds for v in vertices(g)
        if outdegree(g, v) == 0
            push!(dangling_nodes, v)
        end
        α_div_outdegree[v] = (α/outdegree(g, v))
    end

    nvg = Int(nv(g))
    # solution vector and temporary vector
    x = fill(1.0 / nvg, nvg)
    xlast = copy(x)
    @inbounds for _ in 1:n
        dangling_sum = 0.0
        for v in dangling_nodes
            dangling_sum += x[v]
        end
        # flow from teleprotation
        y = (1 - α + α * dangling_sum) * (1.0 / nvg)
        xlast .= y
        # flow from edges
        let x = x
            @threads for v_set in partitions
                for v in v_set
                    for u in inneighbors(g, v)
                        xlast[v] += (x[u] * α_div_outdegree[u])
                    end
                end
            end
        end

        # l1 change in solution convergence criterion
        err = 0.0
        for v in vertices(g)
            err += abs(xlast[v] - x[v])
            x[v] = xlast[v]
        end
        if (err < nvg * ϵ)
            return x
        end
    end
    error("Pagerank did not converge after $n iterations.") # TODO 0.7: change to InexactError with appropriate msg.
end


# Exploration of Boos Stanski Chapter 1 - consulting example

# You are trying to model maximum flooding level Q
# while A is floodplane
# and k and η are parameters to be estimated

# literature suggests over 100 years the maximum level is:
Q(k, A, η) = k * A^(η - 1)

# Take the log to linearlize
logQ(k, A, η) = log(k) + (η - 1)*log(A)

# Verify
round(logQ(5, 100, .5), digits=5) == round(log(Q(5, 100, .5)), digits=5)

# The problem is that we don't actually see any observation which spans 100 years

# Example is of 140 gauging stations - 140 areas
# Stations have between 6 and 83 years of observations.

# The expected value E for subset of observations X₁ which is smaller than subset
# X₂ which is smaller than the full set X₁₀₀ is going to have the following property:
# E(max(X₁)) < E(max(X₂)) < E(max(X₁₀₀))

# What we would like to know is max(X₁₀₀)

# But the maximum of any of our observed values will be biased downwards since we
# never observe 100 years.

# Authors suggest using an estimate of the the joint likelihood that any observation
# will be below the median using the following identity allowing unobserved and observed
# X_i through X_100 to be ordered
# P(X_100 ≤ t) = P(X_1 ≤, ... X_100 ≤ t) = Π P(X_i < t) = [F(t)]^100
# by iid P(X_100 ≤ t) => [F(t)]^100

# P(X_100 ≤ t) = [F(t)]^100 = 1/2
# → P(X_100 ≤ t) = F(t₀) = (1/2)^.01 → t₀ = .993

# Using Extreme value function
# P(X_i <= t) = exp(-exp(-(t-μ)/σ))

Q993(μ, σ) = σ*(-log(-log(.993))) + μ

# The Authors suggest that we can estimate μ and σ then plug them into the function
# to get the solution.

# lets try it

using Random, Distributions

waterlevel = rand(Poisson(50), 100)
observe = rand(Binomial(1, .25), 100)

μhat = mean(waterlevel[observe .== 1])
σhat = var(waterlevel[observe .== 1])^.5

Q993(μhat, σhat)
maximum(waterlevel)


# D(u)_function.jl

"""
Implementation of FHIP's susceptibility formula D(u) for the Optical Absorption of Polarons.

See FHIP 1962, equation (35c):
https://link.aps.org/doi/10.1103/PhysRev.127.1004.
"""

"""
ℜD(x::Float64, y::Float64, v::Float64, w::Float64, β::Float64)

    Calculate the real part of D(u) (equation (35c) in FHIP) for a complex argument u = x + iy. v and w are the variational Polaron parameters that minimise the free energy, for the supplied α Frohlich coupling. β is the reduced thermodynamical beta = ħω/kT with k being Boltzman's constant.
"""
function ℜD(x, y, v, w, β)
    x = BigFloat(x)
    y = BigFloat(y)
    β = BigFloat(β)
    v = BigFloat(v)
    w = BigFloat(w)

    R = (v^2 - w^2) / (w^2 * v)
    a_squared = β^2 / 4 + R * β * coth(β * v / 2)
    b = R * β / sinh(β * v / 2)

    w^2 * (a_squared - β^2 / 4 - b * cos(v * x) * cosh(v * (y - β / 2)) + x^2 + y * (β - y)) / (β * v^2)
end

"""
ℑD(x::Float64, y::Float64, v::Float64, w::Float64, β::Float64)

    Calculate the imaginary part of D(u) (equation (35c) in FHIP) for a complex argument u = x + iy. v and w are the variational Polaron parameters that minimise the free energy, for the supplied α Frohlich coupling. β is the reduced thermodynamical beta = ħω/kT with k being Boltzman's constant.
"""
function ℑD(x, y, v, w, β)
    x = BigFloat(x)
    y = BigFloat(y)
    β = BigFloat(β)
    v = BigFloat(v)
    w = BigFloat(w)

    R = (v^2 - w^2) / (w^2 * v)
    a_squared = β^2 / 4 + R * β * coth(β * v / 2)
    b = R * β / sinh(β * v / 2)

    w^2 * (b * sin(v * x) * sinh(v * (y - β / 2)) + 2 * x * (y - β / 2)) / (β * v^2)
end

# Create a complex number from the real and imaginary parts of D(u).

D(x, y, v, w, β) = ℜD(x, y, v, w, β) + 1im * ℑD(x, y, v, w, β)


abstract type actFun end

export actFun

export probToValue

### sigmoid

abstract type σ <: actFun end

"""
    return the Sigmoid output
    inputs must be matices
"""
σ(X, W, B) = 1 ./ (1 .+ exp.(.-(W * X .+ B)))
σ(Z) = 1 ./ (1 .+ exp.(.-Z))

export σ

"""
    return the derivative of Sigmoid function
"""
dσ(Z) = σ(Z) .* (1 .- σ(Z))

export dσ

@doc raw"""
    function probToValue(
        actFun::Type{σ},
        probs::AbstractArray{T,N},
        labels::Aa = nothing;
        evalConst = 0.5,
    ) where {Aa<:Union{<:AbstractArray,Nothing},T,N}

Convert the probabilities return out of sigmoid function to Bool value (i.e. 0,1) values based on comparing on a threshold value `evalConst`

# Return

- `Ŷ_bool` := Boolean valuse of the probabilites

- `acc` := Accuracy when `labels` provided
"""
function probToValue(
    actFun::Type{σ},
    probs::AbstractArray{T,N},
    labels::Aa = nothing;
    evalConst = 0.5,
) where {Aa<:Union{<:AbstractArray,Nothing},T,N}

    s = size{probs}
    Ŷ_bool = probs .> T(evalConst)
    acc = nothing
    if labels isa AbstractArray
        acc = sum(Ŷ_bool .== labels) / (s[end-1] * s[end])
        # println("Accuracy = $acc")
    end
    return Ŷ_bool, acc
end #predictpredict(probs::AbstractArray{T, 2},

### relu

abstract type relu <: actFun end

"""
    return the ReLU output
"""
function relu(Z::AbstractArray{T,N}) where {T,N}
    max.(zero(T), Z)
end #function relu(Z::AbstractArray{T,N}) where {T,N}

export relu

"""
    return the derivative of ReLU function
"""
function drelu(Z::AbstractArray{T,N}) where {T,N}
    return T.(Z .> zero(T))
end #function drelu(z::AbstractArray{T,N}) where {T,N}

export drelu

### softmax

abstract type softmaxFamily <: actFun end

abstract type softmax <: softmaxFamily end

"""
    compute the softmax function

"""
function softmax(Ŷ::AbstractArray{T,N}) where {T,N}
    Ŷ_exp = exp.(Ŷ)
    sumofexp = sum(Ŷ_exp, dims = N - 1)
    return Ŷ_exp ./ sumofexp
end #softmax

function dsoftmax(Ŷ, dim = 1)
    sŶ = softmax(Ŷ)
    T = eltype(Ŷ)
    softMat = AbstractArray{T,3}(undef, 0, 0, 0)
    sSize = size(Ŷ)[dim]
    for c in eachcol(sŶ)
        tmpMat = zeros(T, sSize, sSize)
        for i = 1:length(c)
            for j = 1:length(c)
                if i == j
                    tmpMat[i, j] = c[i] * (1 - c[j])
                else
                    tmpMat[i, j] = -c[i] * c[j]
                end
            end
        end
        softMat = cat(softMat, tmpMat, dims = 3)
    end

    return softMat
end #dsoftmax

export softmax, dsoftmax

@doc raw"""
    function probToValue(
        actFun::Type{S},
        probs::AbstractArray{T,N};
        labels = nothing,
    ) where {T,N,S<:softmaxFamily}

convert the probabilites out of `softmax` or softmax-like functions into `Bool` values, where the max value gets 1 and the other get zeros

# Return

- `Ŷ_bool` := Boolean valuse of the probabilites

- `acc` := Accuracy when `labels` provided
"""
function probToValue(
    actFun::Type{S},
    probs::AbstractArray{T,N};
    labels = nothing,
) where {T,N,S<:softmaxFamily}

    maximums = maximum(probs, dims = N - 1)
    Ŷ_bool = probs .== maximums
    acc = nothing
    if labels isa AbstractArray
        acc = 0
        bool_labels = Bool.(labels)
        ax = axes(bool_labels)[1:end-1]
        endax = axes(bool_labels)[end]
        trueFalse = Array{Bool,1}(undef, length(endax))
        @simd for i in endax
            lab = view(bool_labels, ax..., i)
            pred = view(Ŷ_bool, ax..., i)
            trueFalse[i] = (lab == pred)
        end
        acc = mean(trueFalse)
        # println("Accuracy = $acc")
    end

    return Ŷ_bool, acc
end #predictpredict(probs::AbstractArray{T, 2}, :softmax)

### tanh

abstract type tanh <: actFun end

Base.tanh(Z::AbstractArray{T,N}) where {T,N} = Base.tanh.(Z)

tanh(Z::AbstractArray{T,N}) where {T,N} = Base.tanh.(Z)

dtanh(Z::AbstractArray{T,N}) where {T,N} = 1 .- (Base.tanh.(Z)) .^ 2

export dtanh, tanh

### noAct

abstract type noAct <: actFun end

function noAct(Z)
    return Z
end

function dnoAct(Z)
    return ones(eltype(Z), size(Z)...)
end

export noAct, dnoAct


"""
    horner(coefficients, x)

Evaluate the given polynomial at position x using the Horner scheme.

```math
p(x) = \\sum_{n=0}^N c_n x^n
```
"""
function horner(coefficients::Vector{T}, x::Number) where T<:Number
    bn = coefficients[end]
    for n=length(coefficients)-1:-1:1
        bn = coefficients[n] + bn*x
    end
    bn
end

module hermite

"""
    a_nk(N)

Calculate the all coefficients for all Hermite polynomials up to order N.

```math
H_n(x) = \\sum_{k=0}^n a_{n,k} x^k
```

Returns a vector of length N+1 where the n-th entry contains all coefficients
for the n-th Hermite polynomial.
"""
function a(N::Int)
    a = Vector{Vector{Int}}(undef, N+1)
    a[1] = [1]
    a[2] = [0,2]
    am = a[2]
    for n=2:N
        an = zeros(Int, n+1)
        a[n+1] = an
        if iseven(n)
            an[1] = -am[2]
        end
        an[n+1] = 2*am[n]
        if iseven(n)
            for k=3:2:n-1
                an[k] = 2*am[k-1] - am[k+1]*k
            end
        else
            for k=2:2:n-1
                an[k] = 2*am[k-1] - am[k+1]*k
            end
        end
        am = an
    end
    a
end

"""
    A_nk(N)

Calculate the all scaled coefficients for all Hermite polynomials up to order N.

The scaled coefficients `A` are connected to the unscaled coefficients `a` by
the relation ``A_{n,k} = \\frac{a_{n,k}}{\\sqrt{2^n n!}}``

Returns a vector of length N+1 where the n-th entry contains all scaled
coefficients for the n-th Hermite polynomial.
"""
function A(N)
    A = Vector{Vector{Float64}}(undef, N+1)
    A[1] = [1.]
    A[2] = [0., sqrt(2)]
    Am = A[2]
    for n=2:N
        An = zeros(Float64, n+1)
        A[n+1] = An
        if iseven(n)
            An[1] = -Am[2]/sqrt(2*n)
        end
        An[n+1] = Am[n]*sqrt(2/n)
        if iseven(n)
            for k=3:2:n-1
                An[k] = Am[k-1]*sqrt(2/n) - Am[k+1]*k/sqrt(2*n)
            end
        else
            for k=2:2:n-1
                An[k] = Am[k-1]*sqrt(2/n) - Am[k+1]*k/sqrt(2*n)
            end
        end
        Am = An
    end
    A
end

end # module hermite


"""
    metnum.jl

Modul ini berisi metode untuk menyelesaikan

1. Akar pertaksamaan tak-linear: `bisection`, `regulaFalsi`, `fixpoint`, `newtonRaphson`, `secant`
2. SPL langsung: `backsub`, `elimGaussNonPivoting`, `elimGaussWithPivoting`, `LUtanpaP`, `LUdenganP`
3. SPL iteratif: `jacobi`, `gaussSeidel`, `rekons` ,`conGrad`
4. Interpolas: `lagrange`, `newton`
5. Regresi: `reglin`, `regpower`, `regpoly`
6. Turunan: `bedaPusat`, `richardson`
7. Integral: `komptrap`, `kompsimp`, `rekursif`, `romberg`, `adaptif`
8. PDB dengan MNA: `euler`, `heun`, `taylor`, `rungekutta`, `rungekuttasistem`, `rkf45`
9. PDB dengan MNB: `linearshooting`, `findiff`
10. PDP: `gelombang`, `panas`, `dirichlet`
"""
module metnum

using LinearAlgebra
using Statistics
using Dates
using IJulia

export regpoly, regpower, regulaFalsi, rekons, rekursif, richardson, rkf45
export romberg, rungekutta, rungekuttasistem, secant
export taylor, adaptif, backsub, bedaPusat, bisection
export conGrad, dirichlet, elimGaussNonPivoting, elimGaussWithPivoting
export euler, findiff, fixpoint, gaussSeidel, gelombang, heun, jacobi
export kompsimp, komptrap, lagrange, linearshooting, LUdenganP, LUtanpaP
export newton, newtonRaphson, panas, reglin, mulaipraktikum

include("regpoly.jl")
include("regpower.jl")
include("regulaFalsi.jl")
include("rekons.jl")
include("rekursif.jl")
include("richardson.jl")
include("rkf45.jl")
include("romberg.jl")
include("rungekutta.jl")
include("rungekuttasistem.jl")
include("secant.jl")
include("taylor.jl")
include("adaptif.jl")
include("backsub.jl")
include("bedaPusat.jl")
include("bisection.jl")
include("conGrad.jl")
include("dirichlet.jl")
include("elimGaussNonPivoting.jl")
include("elimGaussWithPivoting.jl")
include("euler.jl")
include("findiff.jl")
include("fixpoint.jl")
include("gaussSeidel.jl")
include("gelombang.jl")
include("heun.jl")
include("jacobi.jl")
include("kompsimp.jl")
include("komptrap.jl")
include("lagrange.jl")
include("linearshooting.jl")
include("LUdenganP.jl")
include("LUtanpaP.jl")
include("newton.jl")
include("newtonRaphson.jl")
include("panas.jl")
include("reglin.jl")
include("mulaipraktikum.jl")

end