#=
   From Allen Downey
   https://twitter.com/AllenDowney/status/1063460117029535746
   """
   Today's Bayesian problem of the week: Suppose we visit a wild animal preserve where we
   know that the only animals are lions and tigers and bears, but we don't know how
   many of each there are.

   During the tour, we see 3 lions, 2 tigers, and 1 bear. Assuming that every animal had an equal
   chance to appear in our sample, estimate the prevalence of each species.

   What is the probability that the next animal we see is a bear?
   """

   Also see: https://towardsdatascience.com/estimating-probabilities-with-bayesian-modeling-in-python-7144be007815
    From https://towardsdatascience.com/estimating-probabilities-with-bayesian-modeling-in-python-7144be007815
    """
    Species: lions    Prevalence: 44.44%.
    Species: tigers   Prevalence: 33.33%.
    Species: bears    Prevalence: 22.22%.
    """

   Distributions of variable animal7
   lion       =>    4526  (0.452600)
   tiger      =>    3251  (0.325100)
   bear       =>    2223  (0.222300)

   Cf ~/blog/lions_tigers_and_bears.blog
      ~/webppl/lions_tigers_and_bears.wppl

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function lions_tigers_and_bears()
    lion = 1
    tiger = 2
    bear = 3

    # Prior
    # probLion  ~ Beta(1,1)
    # probTiger ~ Beta(1,1)
    # probBear  ~ Beta(1,1)

    # The Dirichlet distribution ensures that the sum of probabilities is 1
    # i.e. we don't have to ensure this via some specific constraint.
    v = [3, 2, 1] # We saw three lions, two tigers, and one bear
    # v = [2,2,2]
    x ~ Dirichlet(v)

    # The probabilities to calculate ("aliased" for simplicity)
    probLion  ~ Dirac(x[1])
    probTiger ~ Dirac(x[2])
    probBear  ~ Dirac(x[3])

    n = 7
    o = tzeros(n)
    for i in 1:n
        o[i] ~ Categorical([probLion,probTiger,probBear])
    end

    # It shouldn't matter in what order we see the different animals.
    true ~ Dirac(o[1] == lion)
    true ~ Dirac(o[2] == lion)
    true ~ Dirac(o[3] == lion)
    true ~ Dirac(o[4] == tiger)
    true ~ Dirac(o[5] == tiger)
    true ~ Dirac(o[6] == bear)

    animal7 ~ Dirac(o[7])

    # probbear7 ~ Dirac(o[7] == bear)
    # problion7 ~ Dirac(o[7] == lion)
    # probtiger7 ~ Dirac(o[7] == tiger)

end

model = lions_tigers_and_bears()
num_chns = 4

# chns = sample(model, Prior(), 10_000)

# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(1000), 10_000)
# chns = sample(model, IS(), 10_000)

#
display(chns)

show_var_dist_pct(chns,:animal7,["lion","tiger","bear"])


123 == 123

123 == 100 + 23

123 == -123

typeof(true)

typeof(false)

typeof(45 == 45)

23 ≤ -5

4 ≠ 44

1.0 == 1

1.000000000001 == 1

3 < 4 && -15 > -10

3 < 4 || -15 > -10

!(-4 ≥ 4) && (1 > 2 || 1 < 2)

1 < 2 <= 2 < 3 == 3 > 2 >= 1 == 1 < 3 != 5

x = -2.3
-3 ≤ x ≤ 4

# Examples of if-else statements

function check_x(x)
    if 0 ≤ x ≤ 10
        println("x is between 0 and 10")
    elseif !(x < 0)
        println("x is not between 0 and 10, and x is not negative (meaning x > 10)")
    else
        println("x is not between 0 and 10, and x is not not negative (meaning x is negative)")
    end
end

check_x(π)

check_x(10.000001)

check_x(2^63)        # trick question - causes Int64 overflow

x = -3
if x ≥ 0 && sqrt(x) < 10
    # do something
else
    println("Sorry, need a non-negative number with square root strictly less than 10")
end

function fibonacci(n)
    # First special cases
    if n == 0
        return 0
    elseif n == 1
        return 1
    else
        # General case n ≥ 2 - initialize and iterate
        Fkold = 0
        Fk = 1
        for k = 2:n
            Fknew = Fk + Fkold
            # Swap previous values
            Fkold = Fk
            Fk = Fknew
        end
        return Fk
    end
end

fibonacci(10)

fibonacci(1000)   # Again be careful with overflow of Int64, but this one is OK

function PE1(n)
    s = 0
    for i = 1:n-1
        if i % 3 == 0 || i % 5 == 0
            s += i
        end
    end
    s
end

PE1(10)

PE1(1000)


using Plots
using LinearAlgebra

"
This code shows how errors for different finite difference methods converge as h->0.
"

# define forward, centered differences, D3 = some other difference approx
D1(u,x,h) = (u(x+h) - u(x))/h
D2(u,x,h) = (u(x+h) - u(x-h))/(2*h)
D3(u,x,h) = (u(x-2*h) - 6*u(x-h) + 3*u(x) + 2*u(x+h))/(6*h)

include("fdcoeffV.jl")
x = -2:2
a = fdcoeffV(1,x[3],x) # 1 = order of derivative, x̄, stencil points x
D4(u,x,h,c) = dot(c,(u(x-2*h),u(x-h),u(x),u(x+h),u(x+2*h)))/h

# function, exact derivative
u(x) = x^2 + sin(1+pi*x)
dudx(x) = 2*x + pi*cos(1+pi*x)
x = 0.0 # point at which we approximate the derivative

# measure how error changes as h decreases
hvec = 2.0 .^(-(0:8))
err = (zeros(size(hvec)),zeros(size(hvec)),zeros(size(hvec)),zeros(size(hvec)))
for (i,h) in enumerate(hvec)
    err[1][i] = abs(dudx(x) - D1(u,x,h))
    err[2][i] = abs(dudx(x) - D2(u,x,h))
    err[3][i] = abs(dudx(x) - D3(u,x,h))
    err[4][i] = abs(dudx(x) - D4(u,x,h,a))
end

plot(xaxis=:log,yaxis=:log,legend=:bottomright)
labels = ("Forward diff","Centered diff","Some other 4-point diff","5-point centered diff")
for i = 1:4
    plot!(hvec,err[i],marker=:dot,label=labels[i])
    plot!(hvec,hvec.^i,linestyle=:dash,label="Order $(i)")
end
display(plot!())


using Plots
gr(titlefontsize=12,size=(500,300),transpose=true,colorbar=false)

using FourierGPE
# Units
# this example works in oscillator units

## Initialize simulation

L = (40.0,40.0)
N = (256,256)
sim = Sim(L,N)
@unpack_Sim sim

## simulation parameters
μ = 15.0

# potential
import FourierGPE.V
V(x,y,t) = 0.5*(x^2 + y^2)

# TF state
ψtf(x,y,μ,g) = sqrt(μ/g)*sqrt(max(1.0-V(x,y,0.0)/μ,0.0)+im*0.0)

# make initial state
x,y = X
ψi = ψtf.(x,y',μ,g)
ϕi = kspace(ψi,sim)

@pack_Sim! sim

## evolve
sol = runsim(sim)

## pull out the ground state
ϕg = sol[end]
ψg = xspace(ϕg,sim)
showpsi(x,y,ψg)

R(w) = sqrt(2*μ/w^2)
R(1)
plot(x, abs2.(ψtf.(x,0.,μ,g)))
plot!(x, abs2.(ψg[:,128]))

## free expansion
V(x,y,t) = 0.0

# sim parameters
γ = 0.0
tf = 10.0
t = LinRange(ti,tf,Nt)
ϕi = ϕg

sime = Sim(L,N)
@pack! sime = tf,t,γ,ϕi

## evolve
sole = runsim(sime)

ϕf = sole[end]
ψf = xspace(ϕf,sime)
showpsi(x,y,ψf)

anim = @animate for i=1:Nt-6
    ψ = xspace(sole[i],sime)
    showpsi(x,y,ψ)
end

gif(anim,"./examples/expand.gif",fps=30)


using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("colorscheme.jl"))

using GLMakie, InteractiveDynamics, DynamicalSystems, OrdinaryDiffEq, LinearAlgebra
diffeq = (alg = Tsit5(), dtmax = 0.02, abstol = 1e-6, reltol = 1e-6)

# Modified version of Rozenweig-MacArthur model for predator-prey
# See https://arxiv.org/abs/2101.12107
function hassan_alkhayuon(u, p, t) # From talk of Hassan Alkhayuon
    r, c, μ, ν, α, β, χ, δ = p
    N, P = u
    common = α*N*P/(β+N)
    dN = r*N*(1 - (c/r)*N)*((N-μ)/(N+ν)) - common
    dP = χ*common - δ*P
    return SVector(dN, dP)
end

u0 = SVector(8.0, 0.01)
r1, r2 = 1.7, 2.5
# r, c, μ, ν, α, β, χ, δ = p
p = [r1, 0.19, 0.03, 0.003, 800, 1.5, 0.004, 2.2]
ds = ContinuousDynamicalSystem(hassan_alkhayuon, u0, p)

# I will re-use code from InteractiveDynamics.jl, so I'll be defining some necessary
# quantities that may not make perfect sense without looking at the source code.
# Nevertheless the source code is quite clear and is just used to initialize some
# interactive observables and stuff.
tail = 500                  # length of plotted trajectory (in units of `dt`)
lims = ((0, 15), (0, 0.04)) # limits of state space
color = to_color("#1e1528") # trajectory color

figure = Figure(resolution = (1000, 800))
pinteg = DynamicalSystems.parallel_integrator(ds, [u0]; diffeq...)
obs, finalpoints = InteractiveDynamics.init_trajectory_observables(1, pinteg, tail, SVector(1,2), identity)

main = figure[1,1] = InteractiveDynamics.init_main_trajectory_plot(
    ds, figure, SVector(1,2), lims, pinteg, [color], obs, NamedTuple(), finalpoints, 1.0
)

# Plot the stability basin and attractor
active_color = to_color("#d7800a")
passive_color = RGBAf0(0.3, 0.4, 0.4, 0.5)
r1_color = Observable(active_color)
r2_color = Observable(passive_color)

function plot_limitcycle_basin!(main, r, color)
    set_parameter!(ds, 1, r)
    # plot unstable manifold of "e3" fixed point
    zz = SVector(p[3], 0.0)
    J = ds.jacobian(zz, p, 0)
    ev = eigvecs(J)
    # system in reverse time: just negative `f`.
    pprev = (u, p, t) -> -hassan_alkhayuon(u, p, t)
    dsrev = ContinuousDynamicalSystem(pprev, SVector(1.0, 1.0), p)
    e = ev[:, 1] # eigenvector correpsonding to positive eigenvalue
    tr = trajectory(dsrev, 3.5, zz .+ 1e-3 .* e; dtmax = 0.01)
    lines!(main, columns(tr)...; linewidth = 2.0, color = color, linestyle = :dash)
    # plot limit cycle
    tr = trajectory(ds, 20.0; dt = 0.01, Ttr = 100.0)
    lines!(main, columns(tr)...; linewidth = 4.0, color = color)
    return
end

plot_limitcycle_basin!(main, r2, r2_color)
plot_limitcycle_basin!(main, r1, r1_color)

# add controls for start/stop and toggle the r value below the figure
figure[2, 1] = controllayout = GridLayout(tellwidth = false)
runbutton = controllayout[1, 1] = Button(figure; label = "run")
rbutton = controllayout[1, 2] = Button(figure; label = "toggle r")
run, rtoggle = runbutton.clicks, rbutton.clicks

# Set up interactive part of the application: start/stop time evolution
isrunning = Observable(false)
on(run) do clicks; isrunning[] = !isrunning[]; end
on(run) do clicks
    @async while isrunning[]
        DynamicalSystems.step!(pinteg)
        ob = obs[1]
        last_state = DynamicalSystems.get_state(pinteg, 1)
        ob[] = push!(ob[], last_state) # push and trigger update with `=`
        finalpoints[] = [x[][end] for x in obs]
        sleep(0.00001) # use sleep to slow down animation
        isopen(figure.scene) || break # crucial, ensures computations stop if closed window
    end
end

# Set up interactive part of the application: toggle the r value
on(rtoggle) do clicks
    r = (r1, r2)[mod(clicks, 2)+1]
    set_parameter!(ds, 1, r)
    u_modified!(pinteg, true)
    x, y = r1_color[], r2_color[]
    r1_color[], r2_color[] = y, x
end

display(figure)


module MultivariateCalculusTest

using VarianceComponentModels, LinearAlgebra, Random, Test

Random.seed!(123)

# test vech
@testset "vech" begin
  a = reshape(1:9, 3, 3)
  vecha = vech(a)
  @test vecha[1] == a[1, 1]
  @test vecha[2] == a[2, 1]
  @test vecha[3] == a[3, 1]
  @test vecha[4] == a[2, 2]
  @test vecha[5] == a[3, 2]
  @test vecha[6] == a[3, 3]
  a = 1.0
  @test vech(a) == a
end

# test trilind
@testset "trilind" begin
  n = 3
  A = randn(n, n)
  @test norm(vech(A) - A[trilind(A)]) ≈ 0.0
  @test norm([2, 3, 6] - trilind(A, -1)) ≈ 0.0
end

# test triuind
@testset "triuind" begin
  n = 2
  idx = triuind(n, n)
  @test length(idx) == binomial(n + 1, 2)
  @test idx[1] == 1
  @test idx[2] == 3
  @test idx[3] == 4
  idx = triuind(n)
  @test length(idx) == binomial(n + 1, 2)
  @test idx[1] == 1
  @test idx[2] == 3
  @test idx[3] == 4
  idx = triuind(randn(n, n))
  @test length(idx) == binomial(n + 1, 2)
  @test idx[1] == 1
  @test idx[2] == 3
  @test idx[3] == 4
  idx = triuind(randn(n, n), 1)
  @test length(idx) == binomial(n, 2)
  @test idx[1] == 3
end

# test commutation
@testset "commutation" begin
  m, n = 3, 2
  A = randn(m, n)
  @test norm(commutation(m, n) * vec(A) - vec(A')) ≈ 0.0
  @test norm(commutation(A) * vec(A) - vec(A')) ≈ 0.0
  @test norm(spcommutation(m, n) * vec(A) - vec(A')) ≈ 0.0
  @test norm(spcommutation(A) * vec(A) - vec(A')) ≈ 0.0
  B = randn(m, m)
  @test norm(commutation(m) * vec(B) - vec(B')) ≈ 0.0
  @test norm(commutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0
  @test norm(spcommutation(eltype(B), m) * vec(B) - vec(B')) ≈ 0.0
end

# test duplication
@testset "duplication" begin
  n = 3
  A = randn(n, n)
  A = 0.5(A + A') # symmetrize
  @test norm(duplication(n) * vech(A) - vec(A)) ≈ 0.0
  @test norm(duplication(A) * vech(A) - vec(A)) ≈ 0.0
  @test norm(spduplication(n) * vech(A) - vec(A)) ≈ 0.0
  @test norm(spduplication(A) * vech(A) - vec(A)) ≈ 0.0
end

# test chol_gradient
@testset "chol_gradient" begin
  n = 3
  A = randn(n, n)
  A = 0.5(A + A') # symmetrize
  L = tril(randn(n, n))
  # calculate gradient wrt L using chol_gradient
  dL1 = chol_gradient(vec(A), L)
  # alternative way to calculate gradient wrt L
  dL2 = 2.0vech((A * L) + L' * A' - Matrix(Diagonal(A * L)))
  @test norm(dL1 - dL2) ≈ 0.0
end

# test kron_gradient
@testset "kron_gradient" begin
  n, q, p, r = 2, 3, 4, 5
  X = randn(n, q)
  Y = randn(p, r)
  M = kron(X, Y)
  # calculate gradient wrt X using kron_gradient
  dX1 = kron_gradient(vec(M), Y, n, q)
  # alternative way to calculate gradient wrt X
  dX2 = zero(X)
  for j = 1:q
    for i = 1:n
      dX2[i, j] = dot(Y, M[(i-1)*p+1:i*p, (j-1)*r+1:j*r])
    end
  end
  @test norm(dX1 - vec(dX2)) < 1e-8
end

# test duplication
@testset "kronaxpy" begin
  m, n, p, q = 3, 4, 5, 6
  A = randn(m, n)
  X = randn(p, q)
  Y = zeros(m * p, n * q)
  kronaxpy!(A, X, Y)
  @test norm(Y - kron(A, X)) ≈ 0.0
end

# test bump_diagonal
@testset "bump_diag" begin
  A = randn(3, 3)
  B = copy(A)
  bump_diagonal!(B, 1e-3)
  @test B[1, 1] - A[1, 1] ≈ 1e-3
  @test B[2, 2] - A[2, 2] ≈ 1e-3
  @test B[3, 3] - A[3, 3] ≈ 1e-3
end

# test clamp_diagonal
@testset "clamp_diag" begin
  n = 3
  A = randn(n, n)
  B = copy(A)
  clamp_diagonal!(B, 0.0, 1.0)
  @test all(0.0 .≤ diag(B) .≤ 1.0)
end

end # module MultivariateCalculusTest


"""
    first_po(n::Int64, k::Int64) -> po_idx::Vector{Int64}

Notation-Generator: Calculate the first generator matrix in PO notation (padding ones indices) by n and k.
"""
function first_po(n::Int64, k::Int64)::Vector{Int64}
    return collect(1:(n-k))
end

"""
    next_po(current_po::Vector{Int64}, n::Int64, i=0) -> next::Vector{Int64}

Notation-Generator: Calculates the next generator matrix in PO notation (padding ones indices) in relation to a given generator matrix in PO notation.
"""
function next_po(current_po::Vector{Int64}, n::Int64, i=0)::Vector{Int64}
    if i > size(current_po,1)
        println("Error: i=$i")
        return nothing
    end
    next = copy(current_po)
    if current_po[end-i] + 1 > n - i
        return next_po(current_po, n, i+1)
    else
        next[end-i] += 1
        for j in (size(next,1) - i + 1):size(next,1)
           next[j] = next[j-1] +1
        end
        return next
    end
end

"""
    construct_generators(n::Int64, k::Int64, prnt = false) -> Ω::Int64, all_pos, all_generating_matrices

Matrix- and Notation-Generator: Calculate all generating matrices (including PO notations) and the number of generating matrices Ω.
"""
function construct_generators(n::Int64, k::Int64, prnt = false)
    # calculate number of possible generating matrices
    Ω = binomial(n,k)
    println("There are Ω = ($n choose $k) = $Ω  possible basic permutations.")

    # Generate all possible padding ones positions first
    # number of padding ones
    num_pos = n - k
    # save first PO
    all_pos = []
    push!(all_pos, first_po(n,k))
    # save first PO as matrix
    all_generating_matrices = SparseMatrixCSC{ℤ₂, Int64}[]
    push!(all_generating_matrices, po2matrix(all_pos[1],n))

    if prnt
        println(
            "1:\t ", all_pos[1],
            "\t↔ AI:\t ", po2ai(all_pos[1],n),
            "\tA² Test: ", multiplyZ2(all_generating_matrices[1],all_generating_matrices[1]) == id(n),
            ": ", all_generating_matrices[1], "\n\n")
    end

    # construct all other generating matrices
    for i in 2:Ω
        next = next_po(all_pos[end],n)
        push!(all_pos, next)
        push!(all_generating_matrices, po2matrix(all_pos[end],n))
        if prnt
            println(
                "$i:\t", next,
                "\t↔ AI:\t ", po2ai(all_pos[end],n),
                "\tA² Test: ", multiplyZ2(all_generating_matrices[end],all_generating_matrices[end]) == id(n),
                ": ", all_generating_matrices[end], "\n\n")
        end
    end

    # return all necessary variables
    return Ω, all_pos, all_generating_matrices
end


using Graphs
using StatsBase: Weights, sample!

function create_random_topology(n::Int, m::Int = ceil(Int, n / 2) + 1)
    @assert m > n / 2
    g = SimpleGraph(n)
    while ne(g) < m
        v1, v2 = rand(vertices(g), 2)
        add_edge!(g, v1, v2)
    end
    return g
end

function create_small_world_topology(n, k::Int = 2, p::Float64 = 0.05)
    @assert 1 ≤ k ≤ n
    @assert 0 ≤ p ≤ 1
    g = SimpleGraph(n)
    for v1 in vertices(g)
        for i = 1:k
            v2 = mod1(v1 + i, length(vertices(g)))
            add_edge!(g, v1, v2)
        end
    end
    if p > 0
        for e in edges(g)
            if rand() ≤ p
                v1 = src(e)
                rem_edge!(g, e)
                v2 = rand(vertices(g))
                add_edge!(g, v1, v2)
            end
        end
    end
    return g
end

function create_scale_free_topology(n::Int, n0::Int = 3, n1::Int = 2)
    @assert 2 ≤ n1 ≤ n0 ≤ n
    g = create_partial_ring(n, n0)
    _add_scale_free_topopolgy!(g, n0, n1)
    return g

end

function create_scale_free_topology(base_graph::SimpleGraph, n0::Int = 3, n1::Int = 2)
    @assert 2 ≤ n1 ≤ n0 ≤ length(vertices(base_graph))
    g = deepcopy(base_graph)
    _add_scale_free_topopolgy!(g, n0, n1)
    return g
end

function create_partial_ring(total_nodes::Int, ring_nodes::Int)
    g = SimpleGraph(total_nodes)
    for v1 = 1:ring_nodes
        v2 = mod1(v1 + 1, length(vertices(g)))
        add_edge!(g, v1, v2)
    end
    return g
end

function _add_scale_free_topopolgy!(g::SimpleGraph, n0::Int, n1::Int)
    n = length(vertices(g))
    neighbors = Vector{Int64}(undef, n1)
    for v1 = n0+1:n
        wv = Weights([degree(g, v) for v in vertices(g)])
        sample!(vertices(g), wv, neighbors, replace = false)
        for v2 in neighbors
            add_edge!(g, v1, v2)
        end
    end
    return g
end

is_tree(g::AbstractGraph) = is_connected(g) && ne(g) === nv(g) - 1

function flip_edge!(g::AbstractGraph, e::Edge)
    has_edge(g, e) || throw(error("Edge e is not in g!"))
    rem_edge!(g, e)
    add_edge!(g, reverse(e))
    return g

end


# # Lasso, Ridge and Elastic Net Regressions
#
# This notebook presents a simple implementation of Lasso and elastic net regressions.

#-

# ## Load Packages and Extra Functions

using DelimitedFiles, LinearAlgebra, Statistics, Plots, Convex, SCS

import MathOptInterface
const MOI = MathOptInterface

# # Loading Data
#
# We use the diabetes data from Efron et al, downloaded from https://web.stanford.edu/~hastie/StatLearnSparsity_files/DATA/diabetes.html and then converted from a tab to a comma delimited file.
#
# All data series are standardised (see below) to have zero means and unit standard deviation, which improves the numerical stability. (Efron et al do not standardise the scale of the response variable.)

(x,header) = readdlm("aux_files/diabetes.csv",',',header=true)
#display(header)
#display(x)

x = (x .- mean(x,dims=1))./std(x,dims=1)          #standardise

(Y,X) = (x[:,end],x[:,1:end-1]);                  #to get traditional names
xNames = header[1:end-1];

# # Lasso, Ridge and Elastic Net Regressions
#
# (a)  The regression is $Y = Xb + u$,
# where $Y$ and $u$ are $T \times 1$, $X$ is $T \times K$, and $b$ is the $K$-vector of regression coefficients.
#
# (b) We want to minimize $(Y-Xb)'(Y-Xb) + \gamma \sum |b_i| + \lambda \sum b_i^2$.
#
# (c) We can equally well minimise $b'Qb - 2c'b + \gamma \sum |b_i| + \lambda \sum b_i^2$,
# where $Q = X'X$ and $c=X'Y$
#
# (d) Lasso: $\gamma>0,\lambda=0$; Ridge: $\gamma=0,\lambda>0$; elastic net: $\gamma>0,\lambda>0$.

"""
    LassoEN(Y,X,γ,λ)

Do Lasso (set γ>0,λ=0), ridge (set γ=0,λ>0) or elastic net regression (set γ>0,λ>0).

## Input
- `Y::Vector`:     T-vector with the response (dependent) variable
- `Y::VecOrMat`:   TxK matrix of covariates (regressors)
- `γ::Number`:     penalty on sum(abs.(b))
- `λ::Number`:     penalty on sum(b.^2)

"""
function LassoEN(Y,X,γ,λ=0.0)

    K = size(X,2)

    b_ls = X\Y                    #LS estimate of weights, no restrictions

    Q  = X'X
    c  = X'Y                      #c'b = Y'X*b

    b  = Variable(K)              #define variables to optimize over
    L1 = quadform(b,Q)            #b'Q*b
    L2 = dot(c,b)                 #c'b
    L3 = norm(b,1)                #sum(|b|)
    L4 = sumsquares(b)            #sum(b^2)

    Sol = minimize(L1-2*L2+γ*L3+λ*L4)      #u'u + γ*sum(|b|) + λsum(b^2), where u = Y-Xb
    solve!(Sol,()->SCS.Optimizer(verbose = false))
    Sol.status == MOI.OPTIMAL ? b_i = vec(evaluate(b)) : b_i = NaN

    return b_i, b_ls

end

# The next cell makes a Lasso regression for a single value of γ.

K = size(X,2)
γ = 100

(b,b_ls) = LassoEN(Y,X,γ)

println("OLS and Lasso coeffs (with γ=$γ)")
display([["" "OLS" "Lasso"];xNames b_ls b])

# # Redo the Lasso Regression with Different Gamma Values
#
#
# We now loop over $\gamma$ values.
#
# Remark: it would be quicker to put this loop inside the `LassoEN()` function so as to not recreate `L1`-`L4`.

nγ = 101
γM = range(0; stop=600, length=nγ)             #different γ values

bLasso   = fill(NaN,size(X,2),nγ)       #results for γM[i] are in bLasso[:,i]
for i = 1:nγ
    bLasso[:,i], = LassoEN(Y,X,γM[i])
end

#-

plot(log.(γM),bLasso',
     title = "Lasso regression coefficients",
     xlabel = "log(γ)",
     label = permutedims(xNames),
     size = (600,400))

# # Ridge Regression
#
# We use the same function to do a ridge regression. Alternatively, do `b = inv(X'X + λ*I)*X'Y`.

nλ = 101
λM = range(0; stop=3000, length=nλ)

bRidge  = fill(NaN,size(X,2),nλ)
for i = 1:nλ
    bRidge[:,i], = LassoEN(Y,X,0,λM[i])
end

#-

plot(log.(λM),bRidge',
     title = "Ridge regression coefficients",
     xlabel = "log(λ)",
     label = permutedims(xNames),
     size = (600,400))

# # Elastic Net Regression

λ = 200
println("redo the Lasso regression, but with λ=$λ: an elastic net regression")

bEN   = fill(NaN,size(X,2),nγ)
for i = 1:nγ
    bEN[:,i], = LassoEN(Y,X,γM[i],λ)
end

#-

plot(log.(γM),bEN',
     title = "Elastic Net regression coefficients",
     xlabel = "log(γ)",
     label = permutedims(xNames),
     size = (600,400))
#-


#=
  Port of WebPPL code from https://github.com/probmods/ppaml2016/blob/gh-pages/chapters/5-election.md
  """
  Basic model
  Learning a state-wide preference from poll data
  """

  Clinton: 1 Trump: 2
  Distributions of variable (num:0)
  2.00000 =>    8750  (0.875000)
  1.00000 =>    1250  (0.125000)

  According to the WebPPL model, the probability that Clinton wins (won) is 0.12207
=#

using Turing
include("jl_utils.jl")

@model function sampleElection()
    function trueStatePref()
        pref ~ Beta(1,1)
        counts = Dict(:trump=>304, :clinton=>276)
        total = sum(values(counts))
        c ~ Binomial(total, pref)
        true ~ Dirac(c == counts[:clinton])
        return pref
    end
    clinton = 0
    trump = 1
    pref = trueStatePref()
    turnout = 2400000
    clintonVotes ~ Binomial(turnout,pref)
    trumpVotes ~ Dirac(turnout - clintonVotes)
    winner ~ Dirac(clintonVotes > trumpVotes ? clinton : trump)

    clintonWins ~ Dirac(winner == clinton)
    trumpWins ~ Dirac(winner == trump)
    return winner
end

model = sampleElection()

# chns = sample(model, MH(), 10_000)
# chns = sample(model, PG(15), 10_000)
# chns = sample(model, IS(), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, SMC(), MCMCThreads(), 10_000, 4)

display(chns)

chns_params = Turing.MCMCChains.get_sections(chns, :parameters)
genq = generated_quantities(model, chns_params)
println("Clinton: 0 Trump: 1")
show_var_dist_pct(genq)


# # Hovercraft Path Planning
# In this case study, we seek to determine an optimal control policy for the
# trajectory of a hovercraft that travels to a set of dynamic waypoints while
# trying to minimize the thrust input.

# ## Formulation

# The corresponding dynamic optimization problem is expressed:
# ```math
# \begin{aligned}
# 	&&\underset{x(t), v(t), u(t)}{\text{min}} &&& \int_{t \in T} |u(t)|_2^2 dt  \\
# 	&&\text{s.t.} &&& v(0) = v0\\
# 	&&&&& \frac{dx}{dt} = v(t), && t \in T\\
#     &&&&& \frac{dv}{dt} = u(t), && t \in T\\
#     &&&&& x(t_i) = xw_i, && i \in I
# \end{aligned}
# ```
# where ``x(t)`` is the Cartesian position, ``v(t)`` is the velocity, ``u(t)`` is
# the thrust input, ``xw_i, \ i \in I,`` are the waypoints, and ``T`` is the time
# horizon.

# ## Model Definition

# Let's implement this in `InfiniteOpt` and first import the packages we need:
using InfiniteOpt, Ipopt

# Next we'll specify our waypoint data:
xw = [1 4 6 1; 1 3 0 1] # positions
tw = [0, 25, 50, 60];    # times

# We initialize the infinite model and opt to use the Ipopt solver:
m = InfiniteModel(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0));

# Let's specify our infinite parameter which is time ``t \in [0, 60]``:
@infinite_parameter(m, t in [0, 60], num_supports = 61)

# Now let's specify the decision variables:
@variables(m, begin
    ## state variables
    x[1:2], Infinite(t)
    v[1:2], Infinite(t)
    ## control variables
    u[1:2], Infinite(t), (start = 0)
end)

# Specify the objective:
@objective(m, Min, ∫(u[1]^2 + u[2]^2, t))

# Set the initial conditions with respect to the velocity:
@constraint(m, [i = 1:2], v[i] == 0, DomainRestrictions(t => 0))

# Define the point physics ODEs which serve as our system model:
@constraint(m, [i = 1:2], ∂(x[i], t) == v[i])
@constraint(m, [i = 1:2], ∂(v[i], t) == u[i])

# Ensure we hit all the waypoints:
@constraint(m, [i = 1:2, j = eachindex(tw)], x[i] == xw[i, j], DomainRestrictions(t => tw[j]))

# ## Problem Solution

# Optimize the model:
optimize!(m)

# Extract the results:
x_opt = value.(x);

# Plot the results:
using Plots
scatter(xw[1,:], xw[2,:], label = "Waypoints", background_color = :transparent)
plot!(x_opt[1], x_opt[2], label = "Trajectory")
xlabel!("x_1")
ylabel!("x_2")

# That's it, now we have our optimal trajectory!

# ### Maintenance Tests
# These are here to ensure this example stays up to date.
using Test
@test termination_status(m) == MOI.LOCALLY_SOLVED
@test has_values(m)
@test x_opt isa Vector{<:Vector{<:Real}}


# Functions

# Point
#rand(rng::Random.AbstractRNG, ::Random.SamplerType{Point{T, N}}) where N where T = Point{T, N}(NTuple{N, T}(rand(rng, T, N)))
distance(p1::Point{T, N}, p2::Point{T, N}) where N where T = sqrt(sum((p1.coordinates .- p2.coordinates).^2))

function area(arr::Array{Point{T, 2}, 1}) where T
    length(arr) <= 2 && error("Area for two points cannot be computed!")
    area = 0.0
    x = 2
    y = 3
    flag = false
    while(x <= length(arr) && y <= length(arr))
        a = distance(arr[1], arr[x])
        b = distance(arr[x], arr[y])
        c = distance(arr[1], arr[y])
        s = (a + b + c)/2.0
        area += sqrt(s*(s - a)*(s - b)*(s - c))
        if flag == false
            flag = true
            x = y + 1
        else
            flag = false
            y = x + 1
        end
    end
    return area
end

function convexHull(points::Array{Point{T,2},1}) where T
    if length(points) < 3
        error("Minimum three points required")
    end
    n = length(points)
    function orientation(p::Point{T, 2}, q::Point{T, 2}, r::Point{T, 2}) where T <: Real
        v = (q.coordinates[2] - p.coordinates[2])*(r.coordinates[1] - q.coordinates[1])- (q.coordinates[1] - p.coordinates[1])*(r.coordinates[2] - q.coordinates[2])
        if v == 0 return 0 end
        return v > 0 ? 1 : 2
    end
    hull = Array{Point{T, 2}, 1}(undef, 0)
    l = 1
    for i in 2:n
        if points[i].coordinates[1] < points[l].coordinates[1]
            l = i
        end
    end
    p = l - 1
    q = -1
    while true
        push!(hull, points[p + 1])
        q = (p + 1) % n
        for i in 0:(n - 1)
            if(orientation(points[p + 1], points[i + 1], points[q+1]) == 2)
                q = i
            end
        end
        p = q
        if p == l - 1
            break
        end
    end
    return hull
end
# Line
function distance(p::Point{T, N}, l::Line{T, N}) where N where T
    t = dot(p.coordinates .- l.p.coordinates, l.v)/dot(l.v, l.v)
    return norm(p.coordinates .- l.p.coordinates .- t*l.v)
end
distance(l::Line{T, N}, p::Point{T, N}) where N where T = distance(p::Point{T, N}, l::Line{T, N})

function distance(l1::Line{T, N}, l2::Line{T, N}; err = 1e-10) where N where T
    if norm(l1.v .- l2.v) ≦ err
        return distance(l1.p, l2)
    else
        return zero(T)
    end
end

# Plane

function distance(p::Point{T, N}, q::Plane{T, N}) where N where T
    x = distance(p, q.p)
    y = distance(p, q.n)
    return sqrt(x^2 - y^2)
end

function boundingbox(a::Array{Point{T, N}, 1}) where N where T
    mn = Array{T,1}(undef, N)
    mn .= typemax(T)
    mx = Array{T,1}(undef, N)
    mx .= typemin(T)
    for i in 1:N, j in 1:length(a)
        mn[i] = min(mn[i], a[j].coordinates[i])
        mx[i] = max(mx[i], a[j].coordinates[i])
    end
    return (Point{T, N}(mn), Point{T, N}(mx))
end


ProjDir = dirname(@__FILE__)
cd(ProjDir) do

A = Float64[
  1/2 1/3 1/4 1/5 1/6;
  1/3 1/4 1/5 1/6 1/7;
  1/4 1/5 1/6 1/7 1/8;
  1/5 1/6 1/7 1/8 1/9;
  1/6 1/7 1/8 1/9 1/10;
]
b = ones(5)

(Al, Au, Ap) = lu(A)

@show y = Al \ b[Ap]
println()

@show x = Au \ y
println()

@show As = sparse(A)
println()

@show As\b
println()

@show F = lu(As)
println()

@show F.L * F.U == F.Rs .* A[F.p, :]
println()

@show F.L*F.U
println()

@show sparse(F.Rs.* A[F.p, F.q])
println()

@show lu(A)
println()

end


"""
Polynomial smoothing with the Savitzky Golay filters. Adapted for julia >= 1.0.
Sources: https://github.com/BBN-Q/Qlab.jl/blob/master/src/SavitskyGolay.jl
         https://gist.github.com/lnacquaroli/c97fbc9a15488607e236b3472bcdf097
Requires LinearAlgebra and DSP modules loaded.
"""

# TODO: could probably improve performance with some @inbounds sprinkling
function savitzkyGolay(x::AbstractVector, windowSize::Int, polyOrder::Int; deriv::Int=0)

  isodd(windowSize) || throw("Window size must be an odd integer.")
  polyOrder < windowSize || throw("Polynomial order must me less than window size.")

  halfWindow = Int( ceil((windowSize-1)/2) )

  # Setup the S matrix of basis vectors
  S = zeros.(windowSize, polyOrder+1)
  for ct = 0:polyOrder
    S[:,ct+1] = (-halfWindow:halfWindow).^(ct)
  end

  ## Compute the filter coefficients for all orders

  # From the scipy code it seems pinv(S) and taking rows should be enough
  G = S * pinv(S' * S)

  # Slice out the derivative order we want
  filterCoeffs = G[:, deriv+1] * factorial(deriv)

  # Pad the signal with the endpoints and convolve with filter
  paddedX = vcat(x[1]*ones(halfWindow), x, x[end]*ones(halfWindow))
  y = conv(filterCoeffs[end:-1:1], paddedX)

  # Return the valid midsection
  return y[2*halfWindow+1:end-2*halfWindow]

end


using Test, FinancialMonteCarlo,ArrayFire;
@show "HestonModel"

S0=100.0;
K=100.0;
r=0.02;
T=1.0;
d=0.01;
D=90.0;

Nsim=1000;
Nstep=30;
sigma=0.2;
sigma_zero=0.2;
kappa=0.01;
theta=0.03;
lambda=0.01;
rho=0.0;
mc=MonteCarloConfiguration(Nsim,Nstep,FinancialMonteCarlo.AFMode());
mc1=MonteCarloConfiguration(Nsim,Nstep,FinancialMonteCarlo.AntitheticMC(),FinancialMonteCarlo.AFMode());
toll=0.8;

rfCurve=ZeroRate(r);

FwdData=Forward(T)
EUData=EuropeanOption(T,K)
AMData=AmericanOption(T,K)
BarrierData=BarrierOptionDownOut(T,K,D)
AsianData1=AsianFloatingStrikeOption(T)
AsianData2=AsianFixedStrikeOption(T,K)
Model=HestonProcess(sigma,sigma_zero,lambda,kappa,rho,theta,Underlying(S0,d));

@show FwdPrice=pricer(Model,rfCurve,mc,FwdData);
@show EuPrice=pricer(Model,rfCurve,mc,EUData);
@show AmPrice=pricer(Model,rfCurve,mc,AMData);
@show BarrierPrice=pricer(Model,rfCurve,mc,BarrierData);
@show AsianPrice1=pricer(Model,rfCurve,mc,AsianData1);
@show AsianPrice2=pricer(Model,rfCurve,mc,AsianData2);

@test abs(FwdPrice-98.72567723404445)<toll
@test abs(EuPrice-17.62536090688433)<toll
@test abs(BarrierPrice-11.38748933756886)<toll
@test abs(AsianPrice1-9.762160560168732)<toll

@show FwdPrice=pricer(Model,rfCurve,mc,FwdData);
@show EuPrice=pricer(Model,rfCurve,mc,EUData);
@show AmPrice=pricer(Model,rfCurve,mc,AMData);
@show BarrierPrice=pricer(Model,rfCurve,mc,BarrierData);
@show AsianPrice1=pricer(Model,rfCurve,mc,AsianData1);
@show AsianPrice2=pricer(Model,rfCurve,mc,AsianData2);
tollanti=0.8
@test abs(FwdPrice-98.72567723404445)<tollanti
@test abs(EuPrice-17.62536090688433)<tollanti
@test abs(BarrierPrice-11.38748933756886)<tollanti
@test abs(AsianPrice1-9.762160560168732)<tollanti


using Random, Plots, StatsPlots, Distributions, DataFrames, Optim
# Bayesian Inference for Bernoulli parameter (inference success probability)
# Define HPD (highest posterior density) interval
# HPD are difficult to calculate and not be used nowadays.
# Maybe this? https://github.com/tpapp/HighestDensityRegions.jl
"""
HPD interval for beta dist
Optim.jl
http://julianlsolvers.github.io/Optim.jl/v0.9.3/user/tipsandtricks/#dealing-with-constant-parameters

    Inputs
    ------
    ci0:  initial value for calculating HPD interval numerically
    α:    parameter 1 of Beta dist
    β:    parameter 2 of Beta dist
    prob: probabilty of HPD interval

    Outputs
    -------
    HPD interval

"""
function beta_hpdi(ci0, α, β, prob)
    """
    Continuous equation for HPD interval

        Inputs
        ------
        v: HPD interval (2-element Vector)
        a: parameter 1 of Beta dist
        b: parameter 2 of Beta dist
        p: probabilty of HPD interval

        Outputs
        -------
        HPD interval condition equations (2-element Vector)

    """
    function hpdi_conditions(v, a, b, p)
        # HPD interval probability = p
        eq1 = cdf(Beta(a, b), v[2]) - cdf(Beta(a, b), v[1]) - p
        # p(lower|D) = p(upper|D)
        eq2 = pdf(Beta(a, b), v[2]) - pdf(Beta(a, b), v[1])
        return [eq1, eq2]
    end
    return nothing
end

function hpdi_conditions(v, a, b, p)
    # HPD interval probability = p
    eq1 = cdf(Beta(a, b), v[2]) - cdf(Beta(a, b), v[1]) - p
    # p(lower|D) = p(upper|D)
    eq2 = pdf(Beta(a, b), v[2]) - pdf(Beta(a, b), v[1])
    return [eq1, eq2]
end

function hpdi_conditions(v)
    # HPD interval probability = p
    eq1 = cdf(Beta(14, 38), v[2]) - cdf(Beta(14, 38), v[1]) - 0.95
    # p(lower|D) = p(upper|D)
    eq2 = pdf(Beta(14, 38), v[2]) - pdf(Beta(14, 38), v[1])
    return [eq1, eq2]
end

# Optimization
hpdi_conditions([0.17, 0.41])
Optim.optimize(hpdi_conditions, [0.17, 0.41])
hpdi_conditions([0, 0])
Optim.optimize(hpdi_conditions, [0, 0])
NLsolve.nlsolve(hpdi_conditions)

# Define calculation of posterior for Bernoulli parameter
"""
Posterior Distribution (Beta dist) for Bernoulli parameter π

    Inputs
    ------
    data: data (0 or 1)
    a0: parameter 1 of prior (Beta dist)
    b0: parameter 2 of prior (Beta dist)
    prob: credible interval probability (0 < prob < 1)

    Outputs
    -------
    results: posterior summary stats table (DataFrames)
    a: parameter 1 of posterior (Beta dist)
    b: parameter 2 of posterior (Beta dist)

"""
function bernoulli_stats(data, a0, b0, prob)
    # posterior
    n = size(data)[1]
    sum_data = sum(data)
    a = sum_data + a0
    b = n - sum_data + b0
    d_posterior = Beta(a, b)
    # mean
    mean_π = mean(d_posterior)
    # median
    median_π = median(d_posterior)
    # mode
    mode_π = mode(d_posterior)
    # Analytic Mode for Beta Dist
    # (a - 1.0) / (a + b - 2.0)
    # std
    sd_π = std(d_posterior)
    # credible interval
    # ci upper boud
    ci_π_upper = quantile(d_posterior, prob - (1-prob)/2)
    # ci lower boud
    ci_π_lower = quantile(d_posterior, (1-prob)/2)
    ci_π = [ci_π_lower, ci_π_upper]

    posterior_stats = reshape([mean_π, median_π, mode_π, sd_π, ci_π], 1, 5)
    stats_string = ["Mean", "Median", "Mode", "SD", "CI"]
    results = DataFrame(posterior_stats, stats_string)
    return results, a, b
end

# Generate moc data from Bernoulli
p = 0.25
n = 50
Random.seed!(99)
data = rand(Bernoulli(p), n)

# Set prior
a0 = 1.0
b0 = 1.0

# Calculate posterior statistics
prob = 0.95
results, a, b = bernoulli_stats(data, a0, b0, prob)
print(results)

# Visualize Posterior
plot(Beta(a0, b0), xlims=(0, 1), ylims=(0, 7), xlabel="Success Probability q", ylabel="Probability Density", label="prior: Beta(1, 1)", linestyle=:dash)
plot!(Beta(a, b), label="posterior: Beta($a, $b)", linestyle=:solid)

# Plot posterior by different prior
Random.seed!(99)
data = rand(Bernoulli(p), 250) # more data
value_size = [10, 50, 250]
value_a0 = [1.0, 6.0]
value_b0 = [1.0, 4.0]
styles = [:dot, :dashdot, :dash, :solid]

l = @layout [grid(1, 2)]
plots = []
for index = 1:2
    style_index = 1
    a0_i = value_a0[index]
    b0_i = value_b0[index]
    pl = plot(Beta(a0_i, b0_i), linestyle=styles[style_index], label="prior: Beta($a0_i, $b0_i)", color="cornflower blue",
             xlims=(0, 1), ylims=(0, 15.5), xlabel="Success Probability q", ylabel="Probability Density")
    for n_j in value_size
        style_index += 1
        sum_data = sum(data[1:n_j])
        a_j = sum_data + a0_i
        b_j = n_j - sum_data + b0_i
        plot!(Beta(a_j, b_j), linestyle=styles[style_index], label="posterior: Beta($a_j, $b_j)", color="cornflower blue")
    end
    push!(plots, pl)
end
plot(plots..., layout=l)

# Animation
# https://stackoverflow.com/questions/55794068/animating-subplots-using-plots-jl-efficiently
Random.seed!(99)
data = rand(Bernoulli(p), 250) # more data
value_size = Vector(1:1:100)
value_a0 = [1.0, 6.0]
value_b0 = [1.0, 4.0]

gr(fmt = :png)
anim = @animate for n_j in value_size
    sum_data = sum(data[1:n_j])
    a_j_1 = sum_data + value_a0[1]
    b_j_1 = n_j - sum_data + value_b0[1]
    # all_p[1][1][:z] = Beta(a_j, b_j)
    p1 = plot(Beta(a_j_1, b_j_1), color="cornflower blue", label="posterior: Beta($a_j_1, $b_j_1)", xlims=(0, 1), ylims=(0, 15.5), xlabel="Success Probability q", ylabel="Probability Density", title="n=$n_j")
    a0_1 = value_a0[1]
    b0_1 = value_b0[1]
    plot!(Beta(a0_1, b0_1), linestyle=:dot, label="prior: Beta($a0_1, $b0_1)")

    a_j_2 = sum_data + value_a0[2]
    b_j_2 = n_j - sum_data + value_b0[2]
    # all_p[2][1][:z] = Beta(a_j, b_j)
    p2 = plot(Beta(a_j_2, b_j_2), color="cornflower blue", label="posterior: Beta($a_j_2, $b_j_2)", xlims=(0, 1), ylims=(0, 15.5), xlabel="Success Probability q", ylabel="Probability Density", title="n=$n_j")
    a0_2 = value_a0[2]
    b0_2 = value_b0[2]
    plot!(Beta(a0_2, b0_2), linestyle=:dot, label="prior: Beta($a0_2, $b0_2)")

    layout = @layout [grid(1, 2)]
    plot(p1, p2; layout)
end
gif(anim, "posterior.gif", fps=10)


"""
    AMHCopula{d,T}

Fields:
  - θ::Real - parameter

Constructor

    AMHCopula(d, θ)

The [AMH](https://en.wikipedia.org/wiki/Copula_(probability_theory)#Most_important_Archimedean_copulas) copula in dimension ``d`` is parameterized by ``\\theta \\in [0,1)``. It is an Archimedean copula with generator :

```math
\\phi(t) = 1 - \\frac{1-\\theta}{e^{-t}-\\theta}
```
"""
struct AMHCopula{d,T} <: ArchimedeanCopula{d}
    θ::T
end
AMHCopula(d,θ) = (0 <= θ <= 1) ? AMHCopula{d,typeof(θ)}(θ) : @error "Theta must be between 0 and 1"
ϕ(  C::AMHCopula,t) = (1-C.θ)/(exp(t)-C.θ)
ϕ⁻¹(  C::AMHCopula,t) = log(C.θ + (1-C.θ)/t)

τ(C::AMHCopula) = 1 - 2(C.θ+(1-C.θ)^2*log(1-C.θ))/(3C.θ^2) # no closed form inverse...
function τ⁻¹(::Type{AMHCopula},τ)
    if τ == zero(τ)
        return τ
    end
    if τ > 1/3
        @warn "AMHCopula cannot handle kendall tau's greater than 1/3. We capped it to 1/3."
        return 1
    end
    return Roots.fzero(θ -> 1 - 2(θ+(1-θ)^2*log(1-θ))/(3θ^2) - τ,0.5)
end

radial_dist(C::AMHCopula) = 1 + Distributions.Geometric(1-C.θ)


"""
    CrossEntropy(NeuralNetwork)

Cross entropy loss function.

    Usage:
```julia-repl
CrossEntropy(NeuralNetwork)
```
Arguments:
- `NeuralNetwork` : The NeuralNetwork structure.
"""
function CrossEntropy(NeuralNetwork)
    ΣScore = 0.0

	for i in 1:size(NeuralNetwork.y, 1)
        ΣScore += NeuralNetwork.y[i] * log(1e-15 + NeuralNetwork.output[i])
    end

    MeanΣScore = 1.0 / size(NeuralNetwork.y, 1) * ΣScore
    loss = -MeanΣScore
	return loss
end


# the idea of this is to find a basis of the
# kernel of matrix. This is done using the eigenvalues.
function kernel_basis(A, tol = 1e-5)
	B = Array{Float64,1}[]
	kernel = nullspace(A);
	kernel_size = size(kernel)[2]
	for n = 1:kernel_size
		push!(B,kernel[:,n]);
	end
	return B
end


(@__DIR__) ∉ LOAD_PATH && push!(LOAD_PATH, (@__DIR__))

module SequencesDemo

using IntegerSequences

println("\nProduct 1*2*3")
p = ∏([1, 2, 3])
println(p)

println("\nList of first 10000 Kolakoski numbers (timing):")
@time KolakoskiList(10000)

println("\nList of first 10000 partition numbers (timing):")
@time PartitionNumberList(10000)

println("\nList of first 10000 Clausen numbers (timing):")
@time ClausenNumberList(10000)

println("\nList of first 10000 Ramanujan tau numbers (timing):")
@time RamanujanTauList(10000)

println("\nRamanujan tau of n = 10000:")
r = RamanujanTau(10000)
println(r)

println("\nAll partitions of 1, 2, 3, 4 and 5.")
for n in 1:5 Partition(n); println() end

println("\nNumber of points in square lattice on the
circle of radius √n.")
SeqShow(L004018(12))

println("\nThe product of the prime numbers dividing n.")
println(typeof([Radical(n) for n in 0:9]))
SeqShow([Radical(n) for n in 0:9])

println("\nOr:")
A = ZArray(9, Radical)
SeqShow(A)

println("\n=============")

println("\nThe exponential transform of Pascal's triangle.")
T = T055883(8)
ShowAsΔ(T)

println("\nThe row sums of the triangle above.")
Println(RowSums(T))

println("\nThe factorial of 100.")
f = F!(100)
println(f)

println("\nThe rising factorial ↑(20, 80).") # (a.k.a. Pochhammer)
rf = 20 ↑ 80
println(rf)

println("\nThe falling factorial ↓(80, 60).")
ff = 80 ↓ 60
println(ff)

println("\nThe number of divisors of n = 25920.")
t = τ(25920)
println(t)

println("\nThe sum of divisors of n = 25920.")
s = σ(25920)
println(s)

println("\nNumber of acyclic orientations of the
Turán graph T(2n, n) for n = 20.")
g = V033815(20)
println(g)

println("\nThe binomial function defined for all fmpz.
The same way as Maple and Mathematica compute the binomial.
First the familiar Pascal case:")
for n in 0:8
    for k in 0:n print(lpad(Binomial(n, k), 4)) end
    println()
end

println("\nAn extended region on the 2-dim lattice:")
for n in -5:5
    for k in -5:5
        print(lpad(Binomial(n, k), 5))
    end
    println()
end

println("\nKnuth, Graham and Patashnik write in
Concrete Mathematics: Hear us, O mathematicians
of the world! Let us not wait any longer! We can
make many formulas clearer by defining a new notation
now! Let us agree to write m ⊥ n, and to say m is
prime to n, if m and n are relatively prime.\n")
for n in 1:6
    for m in 1:6 print(⊥(m, n), " " ) end
    println()
end

println("\nUsing conversion to integer this can also be written as:\n")
for n in 1:6
    for m in 1:6 print(Int(⊥(m, n)), " " ) end
    println()
end

# Generating the OLMS logo.
# ⍊(n, k) means k is strong prime to n.
# n is strong prime to k iff n is prime to k and n does not divide k-1.
# Insert the line sum(L) == n - 5 && println(n) if you want to see why this
# triangle is called the Save-Prime-Triangle.

println("\nThe Save-Prime-Triangle, the logo of the OLMS
(open library of mathematical sequences).\n")
for n in 5:23
    T = [k for k in 1:n if ⍊(n, k)]
    L = [Int(k ∈ T) for k in 3:(n - 2)]
    println(L)
end

println("\n... have fun with Sequences!")

end # module


function HPDE_∂F∂x(func_F,u,Δx,dim)
    Uₓᴸ, Uₓᴿ = GeneralFluxLimiter(u,Δx)
    n = size(u,dim)
    ∂f∂x = copy(u)

    for i=1:n
        A⁺ , A⁻ = LRJacobian(func_F,u[:,i])
        ∂f∂x[:,i] = (A⁻ * Uₓᴸ[:,i] .+ A⁺ * Uₓᴿ[:,i])
    end
    return ∂f∂x
end

function HPDE_∂u∂t(prob::HPDEProblem,u)

    ∂F∂x = HPDE_∂F∂x(prob.func_F,u,prob.Grid.Δx,2)

    # future extentions
    #∂F∂x = HPDE_∂F∂x(prob.func_F,u,prob.Grid.Δx,1)
    #∂G∂y = HPDE_∂F∂x(prob.func_G,u,prob.Grid.Δy,2)
    #∂H∂z = HPDE_∂F∂x(prob.func_H,u,prob.Grid.Δz,3)
    #S = prob.func_S(u)

    ∂G∂y = 0
    ∂H∂z = 0
    #S = func_S(u)
    S = 0

    ∂u∂t = S .- (∂F∂x .+ ∂G∂y .+ ∂H∂z)

    return ∂u∂t
end

function LRJacobian(func_F,u)
    A = ForwardDiff.jacobian(func_F, u)
    #A = (abs.(A) .> 1e-10) .* A
    #println(A)
    Λ = diagm(eigvals(A))
    Λ⁺ = 0.5 .* (Λ .+ abs.(Λ))
    Λ⁻ = 0.5 .* (Λ .- abs.(Λ))
    P = eigvecs(A)
    P⁻¹ = inv(P)
    A⁺ = P * Λ⁺ * P⁻¹
    A⁻ = P * Λ⁻ * P⁻¹
    return A⁺, A⁻
end

function GeneralFluxLimiter(u,Δx)
    ϵ = 1.0e-8
    ∂u∂x(U⁺,U⁻) = (U⁺ .- U⁻) ./ Δx
    func_Ψ(r) = 0
    #func_Ψ(r) = κ_Scheme_new(r,1)
    func_Ψ(r) = SuperBee(r)
    #func_Ψ(r) = Koren(r)
    #func_Ψ(r) = 0
    func_r(Uₓ₁,Uₓ₂) = (Uₓ₁ .+ ϵ) ./ (Uₓ₂ .+ ϵ)
    func_U(Δxᵢ,r,uₓ) = (Δxᵢ / 2) .* func_Ψ(r) .* (uₓ .+ ϵ)
    Uᵢ₋₂ = [u[:,1] u[:,1] u[:,1:end-2]]
    Uᵢ₋₁ = [u[:,1] u[:,1:end-1]]
    Uᵢ   = copy(u)
    Uᵢ₊₁ = [u[:,2:end] u[:,end]]
    Uᵢ₊₂ = [u[:,3:end] u[:,end] u[:,end]]

    Uₓi₊¾ = ∂u∂x(Uᵢ₊₂,Uᵢ₊₁)
    Uₓi₊½ = ∂u∂x(Uᵢ₊₁,Uᵢ)
    Uₓi₋½ = ∂u∂x(Uᵢ,Uᵢ₋₁)
    Uₓi₋¾ = ∂u∂x(Uᵢ₋₁,Uᵢ₋₂)

    Uᴸi₊½ = Uᵢ₊₁ .- func_U(Δx,func_r(Uₓi₊½,Uₓi₊¾),Uₓi₊¾)
    Uᴸi₋½ = Uᵢ   .- func_U(Δx,func_r(Uₓi₋½,Uₓi₊½),Uₓi₊½)
    Uᴿi₊½ = Uᵢ   .+ func_U(Δx,func_r(Uₓi₊½,Uₓi₋½),Uₓi₋½)
    Uᴿi₋½ = Uᵢ₋₁ .+ func_U(Δx,func_r(Uₓi₋½,Uₓi₋¾),Uₓi₋¾)

    Uₓᴸ = (Uᴸi₊½ .- Uᴸi₋½) ./ Δx
    Uₓᴿ = (Uᴿi₊½ .- Uᴿi₋½) ./ Δx
    #return Uᴿi₊½, Uᴿi₋½, Uᴸi₊½ , Uᴸi₋½
    #return Uₓi₊¾, Uₓi₊½, Uₓi₋½, Uₓi₋¾
    return Uₓᴸ, Uₓᴿ
end

################## METHODS ####################

function Koren(r)
    return max.(0,min.(2 .* r,min.(1/3 .+ 2 /3 .* r,2)))
end

function UpWind_FO(r)
    return 0
end

function UpWind_SO(r)
    return κ_Scheme_new(r,-1)
end

function SCD(r)
    return κ_Scheme_new(r,1)
end

function UpWind_Q(r)
    return κ_Scheme_new(r,0.5)
end

function UpWind_C(r)
    return κ_Scheme_new(r,1/3.0)
end

function SuperBee(r)
    return max.(0,min.(r,1))
end


using NumericalMethodsforEngineers
using Test

a = [10. 1. -5;-20. 3. 20.;5. 3. 5.]
b = [1., 2., 6.]

c = a\b
f = lu(a);
(lower, upper) = lufac(a)
subfor!(lower, b)
subbac!(upper, b)

@test round.(b, digits=10) == round.(c, digits=10)


# Converts an FFT into a gammatonegram. Based on the gammatonegram/fft2gammatonemx MATLAB
# functions by Dan Ellis.
# D. P. W. Ellis (2009). "Gammatone-like spectrograms", web resource.
# http://www.ee.columbia.edu/~dpwe/resources/matlab/gammatonegram/

type Gammatonegram{T, F<:Real} <: DSP.Periodograms.TFR{T}
	amplitude::Matrix{T}
	frequencies::Vector{F}
	time::FloatRange{Float64}
end

function gammatonegram(x,sr::Integer,twin::Real,thop::Real,N::Integer,fmin,fmax,width)
	nfft = int(2^(ceil(log(2*twin*sr)/log(2))))
    nhop = iround(thop*sr)
    nwin = iround(twin*sr)
    (W,F) = fft2gammatonemx(nfft, sr, N, width, fmin, fmax)
    # perform FFT and weighting in amplitude domain
	S = stft(x, nwin, nwin-nhop; nfft=nfft, fs=sr, window=hanning)
	Y = 1/nfft*W*abs(S)
	Gammatonegram(Y, vec(F), ((0:size(Y,2)-1)*(nhop)+nwin/2)/sr)
end

function fft2gammatonemx(nfft::Integer, sr::Integer, N::Integer, width, fmin, fmax)
	# Allocating matrix to store the filterbank weights
	W = zeros(N, int(nfft/2+1));
	# Gammatone filterbank constants
	EarQ = 9.26449
	minBW = 24.7
	order = 1

	cfreqs = -(EarQ*minBW) + exp((1:N)'*(-log(fmax + EarQ*minBW) + log(fmin + EarQ*minBW))/N) * (fmax + EarQ*minBW);
	cfreqs = fliplr(cfreqs)
	GTord = 4
	ucirc = exp(im*2*pi*[0:(nfft/2)]/nfft)
	for k=1:N
	    cf = cfreqs[k];
	    ERB = width*((cf/EarQ).^order + minBW^order).^(1/order);
	    B = 1.019*2*pi*ERB;
	    r = exp(-B/sr);
	    theta = 2*pi*cf/sr;
	    pole = r*exp(im*theta);
	    T = 1/sr;

		A11 = -(2*T*cos(2*cf*pi*T)./exp(B*T) + 2*sqrt(3+2^1.5)*T*sin(2*
	                                                      cf*pi*T)./exp(B*T))/2;
	    A12 = -(2*T*cos(2*cf*pi*T)./exp(B*T) - 2*sqrt(3+2^1.5)*T*sin(2*
	                                                      cf*pi*T)./exp(B*T))/2;
	    A13 = -(2*T*cos(2*cf*pi*T)./exp(B*T) + 2*sqrt(3-2^1.5)*T*sin(2*
	                                                      cf*pi*T)./exp(B*T))/2;
	    A14 = -(2*T*cos(2*cf*pi*T)./exp(B*T) - 2*sqrt(3-2^1.5)*T*sin(2*
	                                                      cf*pi*T)./exp(B*T))/2;
	    zros = -[A11 A12 A13 A14]/T;

	    gain =  abs((-2*exp(4*im*cf*pi*T)*T +
	                2*exp(-(B*T) + 2*im*cf*pi*T).*T.*
	                (cos(2*cf*pi*T) - sqrt(3 - 2^(3/2))*
	                 sin(2*cf*pi*T))) .*
	               (-2*exp(4*im*cf*pi*T)*T +
	                2*exp(-(B*T) + 2*im*cf*pi*T).*T.*
	                (cos(2*cf*pi*T) + sqrt(3 - 2^(3/2)) *
	                 sin(2*cf*pi*T))).*
	               (-2*exp(4*im*cf*pi*T)*T +
	                2*exp(-(B*T) + 2*im*cf*pi*T).*T.*
	                (cos(2*cf*pi*T) -
	                 sqrt(3 + 2^(3/2))*sin(2*cf*pi*T))) .*
	               (-2*exp(4*im*cf*pi*T)*T + 2*exp(-(B*T) + 2*im*cf*pi*T).*T.*
	                (cos(2*cf*pi*T) + sqrt(3 + 2^(3/2))*sin(2*cf*pi*T))) ./
	               (-2 ./ exp(2*B*T) - 2*exp(4*im*cf*pi*T) +
	                2*(1 + exp(4*im*cf*pi*T))./exp(B*T)).^4);
	    W[k,:] = ((T^4)/gain) *
	        abs(ucirc-zros[1]).*abs(ucirc-zros[2]) .*
	        abs(ucirc-zros[3]).*abs(ucirc-zros[4]) .*
	        (abs((pole-ucirc).*(pole'-ucirc)).^-GTord);
	end
	W, cfreqs
end


#!/usr/bin/julia

# Trizen
# 28 August 2020
# https://github.com/trizen

# Generate all the positive divisors of n.

using Primes

function divisors(n)

    d = Int64[1]

    for (p,e) in factor(n)
        t = Int64[]
        r = 1

        for i in 1:e
            r *= p
            for u in d
                push!(t, u*r)
            end
        end

        append!(d, t)
    end

    return sort(d)
end

println(divisors(5040))


function kernel(x::REALMAT, x₀::REALMAT, kernel_struct::Kernel)
    @assert size(x, 2) == size(x₀, 2) "The number of columns of x and x₀ do not match."
    # Dispatch to kernel
    K = kernel_calculation(x, x₀, kernel_struct::Kernel)
    return K
end
kernel(x::REALMAT, kernel_struct::Kernel) = kernel(x, x, kernel_struct)

# Linear
function kernel_calculation(x, x₀, k::Linear)
    K = [dot(x[i1, :], x[i2, :]) for i1 in 1:size(x, 1), i2 in 1:size(x₀, 1)]
    return K * k.σ
end

# Radial Basis Function (Squared Exponential)
function kernel_calculation(x, x₀, k::RBF)
    K = zeros(size(x, 1), size(x₀, 1))
    for i1 in 1:size(x, 1)
        for i2 in 1:size(x₀, 1)
            K[i1, i2] = k.σ * exp(-sum((x[i1, :] .- x₀[i2, :]).^2)/(2*k.l^2))
        end
    end
    return K
end

# Rational Quadratic
function kernel_calculation(x, x₀, k::RationalQuadratic)
    K = zeros(size(x, 1), size(x₀, 1))
    for i1 in 1:size(x, 1)
        for i2 in 1:size(x₀, 1)
            K[i1, i2] = k.σ * (1.0 + sum((x[i1, :] .- x₀[i2, :])^2)/(2 * k.a * k.l^2))^-k.a
        end
    end
    return K
end

# Periodic
function kernel_calculation(x, x₀, k::Periodic)
    K = zeros(size(x, 1), size(x₀, 1))
    for i1 in 1:size(x, 1)
        for i2 in 1:size(x₀, 1)
            K[i1, i2] = k.σ * exp(-2 * sin(sum(abs.(x[i1, :] .- x₀[i2, :])))^2/k.l^2)
        end
    end
    return K
end

# Locally Periodic
function kernel_calculation(x, x₀, k::LocallyPeriodic)
    K = zeros(size(x, 1), size(x₀, 1))
    for i1 in 1:size(x, 1)
        for i2 in 1:size(x₀, 1)
            K[i1, i2] = k.σ * exp(-2 * sin(sum(abs.(x[i1, :] .- x₀[i2, :])))^2/k.l^2) * exp(-sum((x[i1, :] .- x[i2, :].^2)/(2*k.l^2)))
        end
    end
    return K
end


using AbstractAlgebra
using Nemo
using SymbolicIntegration
SI = SymbolicIntegration

using Test

@testset "Complex fields" begin
    QQx, x = PolynomialRing(Nemo.QQ, :x)
    k = FractionField(QQx)
    D0 = BasicDerivation(k)
    kI, I, D = SI.Complexify(k, D0)
    kI1, x1, I1, D1 = SI.switch_t_i(kI, D)

    f = (x^2+3x-1)//(x-7)*I + x^2//(1+x^3)
    f1 = SI.transform(f, x1, I1)
    f2 = SI.backtransform(f1, x, I)
    @test f2==f

    fd = D(f)
    fd1 = SI.transform(fd, x1, I1)
    f1d = D1(f1)
    @test fd1==f1d
end

@testset "ParametricLogarithmicDerivative complexified" begin
    QQx, x = PolynomialRing(Nemo.QQ, :x)
    k = FractionField(QQx)
    D0 = SI.BasicDerivation(k)
    kI, I, D0I = SI.Complexify(k, D0)

    kIt, t = PolynomialRing(kI, :t)

    H = 1//x+0*t
    D = SI.ExtensionDerivation(kIt, D0I, H)

    w = -I*1//(x*t^2)
    u = x^5*t+3*x*I
    n = 2
    m = 6

    f = (D(u)//u+m*w)//n
    n1, m1, u1, ρ = SI.ParametricLogarithmicDerivative(f, w, D)

    @test n1*f == D(u1)//u1 + m1*w
end


function evalPsiFromRectangleCoil(r0,z0,dr,dz,Ic,quadRule_r,quadRule_z, xr,xz; derivative=0)

	surf=dr*dz
	jc=Ic/surf

	#define quadrature points and weigths in the rectangle
	xgr=r0 .+ quadRule_r.x *0.5 *dr
	xgz=z0 .+ quadRule_z.x *0.5 *dz

	xgr_rep=repeat(xgr,size(xgz,1))
	xgz_rep=reshape(repeat(xgz,1,size(xgr,1))',size(xgr,1)*size(xgz,1))

	wgr=quadRule_r.w *0.5 *dr
	wgz=quadRule_z.w *0.5 *dz

	wgr_rep=repeat(wgr,size(wgz,1))
	wgz_rep=reshape(repeat(wgz,1,size(wgr,1))',size(wgr,1)*size(wgz,1))

	#
	G = @.Green(xr,xz,xgr_rep,xgz_rep, derivative=derivative)
	G_PXPG = getindex.(G,:G_xy)

	val=wgr_rep.*wgz_rep.*G_PXPG*jc
	val=sum(val)

	if(derivative==0)
		return (psi=val,)
	end
	if(derivative>0)
		dxrG = getindex.(G,:dxrG_xy)
		dxzG = getindex.(G,:dxzG_xy)

		dxrval=wgr_rep.*wgz_rep.*dxrG*jc
		dxzval=wgr_rep.*wgz_rep.*dxzG*jc

		dxrval=sum(dxrval)
		dxzval=sum(dxzval)

		return (psi=val, dxrpsi=dxrval, dxzpsi=dxzval)

	end

end

function assemblyRhsRectangleCoil(r0,z0,dr,dz,Ic,quadRule_r, quadRule_z, BEMesh, quadRule)

	# dimension
	n=size(BEMesh.edges,1)

	#
	rhs=zeros(n)

	#quadrature rule on BEMesh
	xGauss=quadRule.x
	wGauss=quadRule.w
	nQuad=size(quadRule.w,1)

	#in order to remap the quadrature we need the midpoints and length of edges
	e=collect(1:n)
	edgeToNode=BEMesh.edges
	h=BEMesh.length

    P=0.5*(BEMesh.coordinates[edgeToNode[:,1],:]+BEMesh.coordinates[edgeToNode[:,2],:])
	#@show P

	#quadrature points on all edges
	PG=zeros(size(P,1),2,nQuad)
	for i in 1:nQuad
		PG[:,:,i]=P + xGauss[i]*0.5*(BEMesh.coordinates[edgeToNode[:,2],:]-BEMesh.coordinates[edgeToNode[:,1],:])

		evalC=@.evalPsiFromRectangleCoil((r0,),(z0,),(dr,),(dz,),(Ic,),(quadRule_r,),(quadRule_z,), PG[:,1,i], PG[:,2,i])

		psiC = getindex.(evalC,:psi)

		rhs=rhs+0.5*wGauss[i]*h.*psiC
	end

	return rhs

end


"""
    randBall(nDims, nPts)

Draw nPts random points uniformely distributed in an nDims-dimensional ball.

# Implementation
Samples uniformely in a unit cube, then throws away unnecessary points.
Efficient only for low number of dimensions (nDims=1,2,3).

# Examples
```julia-repl
julia> using FPRS; using Plots;
julia> x = randBall(2,50);
julia> figure(); scatter(x[1,:], x[2,:]);
```
"""
function randBall(nDims::Integer, nPts::Integer)

    res = zeros(nDims, nPts)
    numInBall = 0

    while numInBall < nPts
        newPoint = 2.0 * rand(nDims) .- 1.0
        if norm(newPoint) <= 1
            numInBall += 1
            res[1:end, numInBall] = newPoint
        end
    end

    return res
end

"""
    randRotationMatrix(deflection=1.0; randSeed)

Return a random rotation matrix in 3d.

Sources:
-http://blog.lostinmyterminal.com/python/2015/05/12/random-rotation-matrix.html
-http://www.realtimerendering.com/resources/GraphicsGems/gemsiii/rand_rotation.c

# Arguments
- `deflection`: number between 0 and 1 controls the size of the perturbation.
                For 0, no rotation; for 1, competely random rotation.
                Small deflection => small perturbation.

# Examples
```julia-repl
julia> mx = randRotationMatrix();
```
"""
function randRotationMatrix(deflection=1.0)
    # Random.seed!(seedVal) # do not reseed rng

    theta = 2.0*deflection* pi * rand() # Rotation about the pole (Z)
    phi = 2.0* pi * rand() # For direction of pole deflection
    z = 2.0*deflection * rand() # For magnitude of pole deflection

    # Compute a vector V used for distributing points over the sphere
    # via the reflection I - V Transpose(V).  This formulation of V
    # will guarantee that if x[1] and x[2] are uniformly distributed,
    # the reflected points will be uniform on the sphere.  Note that V
    # has length sqrt(2) to eliminate the 2 in the Householder matrix.

    r =  sqrt(z)
    Vx = sin(phi) * r
    Vy = cos(phi) * r
    Vz = sqrt(2.0 - z)

    # Compute the row vector S = Transpose(V) * R, where R is a simple
    # rotation by theta about the z-axis.  No need to compute Sz since
    # it's just Vz.

    st = sin(theta)
    ct = cos(theta)
    Sx = Vx * ct - Vy * st
    Sy = Vx * st + Vy * ct

    # Construct the rotation matrix  ( V Transpose(V) - I ) R, which
    # is equivalent to V S - R.

    M = zeros(3,3)
    M[1,1] = Vx * Sx - ct;
    M[1,2] = Vx * Sy - st;
    M[1,3] = Vx * Vz;

    M[2,1] = Vy * Sx + st;
    M[2,2] = Vy * Sy - ct;
    M[2,3] = Vy * Vz;

    M[3,1] = Vz * Sx;
    M[3,2] = Vz * Sy;
    M[3,3] = 1.0 - z;   # This equals Vz * Vz - 1.0

    return M
end


correct = ( pi/4 + atan(3.0) ) / 2
for endpt in [neither, both, left, right]
    x, w = legendre(40, endpt)
    Q = sum(w .* 1 ./ ( 1 .+ (2x .- 1).^2))
    @test Q ≈ correct
end


# ffr, 19 April 2016

# for the moment, I decided to not include L, because it can
# be calculated from A and B
struct LF1dGrid
    N::Int64
    A::Float64
    B::Float64
    h::Float64
    grid::Array{Float64,1}
    D1jl::Array{Float64,2}
    D2jl::Array{Float64,2}
end

# Periodic LF
function init_LF1d_p( N::Int64, A::Float64, B::Float64, verbose=false )
    # Check argument
    if N % 2 == 0
        error("N must be an odd number\n")
    end
    L = (B - A)
    h = L/N
    grid = zeros(Float64,N)
    for i = 1:N
        grid[i] = A + 0.5*(B-A)*(2*i-1)/N
    end
    D1jl = zeros(Float64,N,N)
    D2jl = zeros(Float64,N,N)
    #
    # Diagonal elements
    #
    Nprimed = (N-1)/2
    for j = 1:N
        # the diagonal elements of D1jl are already zero
        D2jl[j,j] = -(2.0*pi/L)^2 * Nprimed * (Nprimed+1)/3.
    end
    #
    # Off diagonal elements
    #
    for j = 1 : N
        for l = j+1 : N
            #
            nn = j - l
            #
            tt1 = pi/L * (-1.0)^nn
            tt2 = sin(pi*nn/N)
            #
            tt3 = (2.0*pi/L)^2 * (-1.0)^nn * cos(pi*nn/N)
            tt4 = 2.0*sin(pi*nn/N)^2
            #
            D1jl[j,l] =  tt1/tt2
            D1jl[l,j] = -tt1/tt2
            #
            D2jl[j,l] = -tt3/tt4
            D2jl[l,j] = -tt3/tt4
        end
    end
    #
    LF = LF1dGrid( N, A, B, h, grid, D1jl, D2jl )
    if verbose
        @printf("Allocated: periodic 1d LBF grid: N, h = %d , %f\n", N, h)
    end
    #
    return LF
end # function init_LF1d_p

# cluster Lagrange function
function init_LF1d_c( N::Int64, A::Float64, B::Float64, verbose=false )
    L = B - A
    h = L/(N + 1.)
    #
    grid = zeros(Float64,N)
    for i = 1:N
        grid[i] = A + i*(B-A)/(N+1)
    end
    #
    D1jl = zeros(Float64,N,N)  # XXX this is not yet used
    D2jl = zeros(Float64,N,N)
    #
    # Diagonal part
    #
    pre = -pi^2/(2.0*L^2)
    for i = 1:N
        t1 = ( 2.0*(N+1)^2 + 1 )/3.0
        t2 = sin( i*pi/(N+1) )^2
        D2jl[i,i] = pre*( t1 - 1.0/t2 )
    end
    #
    # Off-diagonal
    #
    for l = 1 : N
        for j = l+1 : N
            nnm = l - j
            nnp = l + j
            pre = -pi^2 / (2*L^2) * (-1.0)^nnm
            t1 = sin( pi*nnm/2.0/(N+1) )^2
            t2 = sin( pi*nnp/2.0/(N+1) )^2
            #
            D2jl[l,j] = pre*( 1.0/t1 - 1.0/t2 )
            D2jl[j,l] = pre*( 1.0/t1 - 1.0/t2 )  # XXX is it faster to just recalculate?
        end
    end
    LF = LF1dGrid( N, A, B, h, grid, D1jl, D2jl )
    if verbose
        @printf("Allocated: cluster 1d LBF grid: N, h = %5d , %10.5f\n", N, h)
    end
    return LF
end

# Lagrange-sinc function
function init_LF1d_sinc( N::Int64, h::Float64, verbose=false )
    # Choice for A and B
    A = -(N-1)*h/2.0
    B =  (N-1)*h/2.0
    #
    grid = zeros(Float64,N)
    for i = 1:N
        grid[i] = A + (i-1)*h
    end
    #
    D1jl = zeros(Float64,N,N)  # XXX this is not yet used
    D2jl = zeros(Float64,N,N)
    #
    # Diagonal part
    #
    for i = 1:N
        D2jl[i,i] = -pi^2 / 3.0 / h^2
    end
    #
    # Off-diagonal
    #
    for j = 1 : N
        for i = j+1 : N
            D2jl[i,j] = -2.0*(-1.0)^(i-j)/( grid[i] - grid[j] )^2
            D2jl[j,i] = D2jl[i,j]  # XXX is it faster to just recalculate?
        end
    end
    LF = LF1dGrid( N, A, B, h, grid, D1jl, D2jl )
    if verbose
        @printf("Allocated: 1d Lagrange-sinc grid: N, h = %5d , %10.5f\n", N,  h)
    end
    return LF
end


"""
    Step{T, S<:Integer}(start, decay, step_sizes)
    Step(;λ, γ, step_sizes)

A step schedule decays exponentially by `γ` every step
in `step_sizes`.
The output conforms to
```text
λ * γ^{i - 1}
```
where `sum(step_sizes[1:(i - 1)]) < t <= sum(step_sizes[1:i])`

# Arguments:
- `start`/`λ`: the starting value
- `decay`/`γ`: the decay rate
- `step_sizes::Union{<:Integer, <:Vector}`: the step sizes
"""
struct Step{T, S<:Integer}
    start::T
    decay::T
    step_sizes::Vector{S}
end
Step(λ, γ, step_size::Integer) = Step(λ, γ, [step_size])
Step(;λ, γ, step_sizes) = Step(λ, γ, step_sizes)

function (schedule::Step)(t)
    i = findlast(x -> t > x, cumsum(schedule.step_sizes))
    i = isnothing(i) ? 0 :
            (i >= length(schedule.step_sizes)) ? length(schedule.step_sizes) - 1 : i

    return schedule.start * schedule.decay^i
end

Base.eltype(::Type{<:Step{T}}) where T = T
Base.IteratorSize(::Type{<:Step}) = Base.IsInfinite()

function Base.iterate(schedule::Step, state = (1, 1, 1))
    t, i, t0 = state
    if (i < length(schedule.step_sizes)) && (t >= t0 + schedule.step_sizes[i])
        # move onto next step range
        i += 1
        t0 = t
    end

    return schedule.start * schedule.decay^(i - 1), (t + 1, i, t0)
end

Base.axes(::Step) = (OneToInf(),)

"""
    Exp{T}(start, decay)
    Exp(;λ, γ)

A exponential decay schedule at rate `γ`.
The output conforms to
```text
λ * γ^{t - 1}
```

# Arguments:
- `start`/`λ`: the base value
- `decay`/`γ`: the decay rate
"""
struct Exp{T}
    start::T
    decay::T
end
Exp(;λ, γ) = Exp(λ, γ)

(schedule::Exp)(t) = schedule.start * schedule.decay^(t - 1)

Base.eltype(::Type{<:Exp{T}}) where T = T
Base.IteratorSize(::Type{<:Exp}) = Base.IsInfinite()

Base.iterate(schedule::Exp, t = 1) = schedule(t), t + 1

Base.axes(::Exp) = (OneToInf(),)

"""
    Poly{T, S<:Integer}(start, degree, max_iter)
    Poly(;λ, p, max_iter)

A polynomial schedule decays with degree `p`.
The output conforms to
```text
λ / (1 - (t - 1) / max_iter)^p
```

# Arguments
- `start`/`λ`: the base value
- `degree`/`p::Integer`: the degree of the polynomial
- `max_iter::Integer`: the total number of iterations
"""
struct Poly{T, S<:Integer}
    start::T
    degree::S
    max_iter::S
end
Poly(;λ, p, max_iter) = Poly(λ, p, max_iter)

function (schedule::Poly)(t)
    (t <= length(schedule)) || throw(BoundsError("Cannot index Poly for t > max_iter"))
    return schedule.start * (1 - (t - 1) / schedule.max_iter)^schedule.degree
end

Base.eltype(::Type{<:Poly{T}}) where T = T
Base.IteratorSize(::Type{<:Poly}) = Base.HasLength()
Base.length(schedule::Poly) = schedule.max_iter

Base.iterate(schedule::Poly, t = 1) = schedule(t), t + 1

Base.axes(schedule::Poly) = 1:length(schedule)

"""
    Inv{T, S<:Integer}(start, decay, degree)
    Inv(;λ, γ, p)

A decay schedule that inversely decays with rate `γ`.
The output conforms to
```text
λ / (1 + (t - 1) * γ)^p
```

# Arguments
- `start`/`λ`: the base value
- `decay`/`γ`: the decay rate
- `degree`/`p::Integer`: the degree of decay
"""
struct Inv{T<:Number, S<:Integer}
    start::T
    decay::T
    degree::S
end
Inv(;λ, γ, p) = Inv(λ, γ, p)

(schedule::Inv)(t) = schedule.start / (1 + (t - 1) * schedule.decay)^schedule.degree

Base.eltype(::Type{<:Inv{T}}) where T = T
Base.IteratorSize(::Type{<:Inv}) = Base.IsInfinite()

Base.iterate(schedule::Inv, t = 1) = schedule(t), t + 1

Base.axes(::Inv) = (OneToInf(),)


using Test, DiscreteExteriorCalculus
const DEC = DiscreteExteriorCalculus

@testset "parametrize_subspace" begin
    M, v = [[0,1] [-1,1] [0,1]], [-.5,1]
    a, b = DEC.parametrize_subspace(M, v)
    @test M * a ≈ v
    @test M * (a + b * [1]) ≈ v
    @test M * (a + b * [-2]) ≈ v

    M, v = [[.5, 1] [-.5, 1] [-.5, 1]], [0.0, 1]
    a, b = DEC.parametrize_subspace(M, v)
    @test M * a ≈ v
    @test M * (a + b * [1]) ≈ v
    @test M * (a + b * [-2]) ≈ v

    M, v = [[.5, 1] [-.5, 1]], [0.0, 1]
    a, b = DEC.parametrize_subspace(M, v)
    @test size(b) == (2,0)
    @test M * a ≈ v
end

@testset "sign_of_permutation" begin
    @test DEC.sign_of_permutation([1],[1]) == 1
    @test DEC.sign_of_permutation([1,2],[2,1]) == -1
    @test DEC.sign_of_permutation([1,2],[1,2]) == 1
end


using DifferentialEquations
function f(du,u,p,t)
    du[1] = -u[1]
end
u0 = [10.0]
const V = 1
prob = ODEProblem(f,u0,(0.0,10.0))

sol = solve(prob,Tsit5())
using Plots; gr()
plot(sol)

condition(u,t,integrator) = t==4 && u[1]/V<4
affect!(integrator) = integrator.u[1] += 10
cb = DiscreteCallback(condition,affect!)

sol = solve(prob,Tsit5(),tstops=[4.0],callback=cb)
using Plots; gr()
plot(sol)

println(sol(4.00000))
println(sol(4.000000000001))

function f(du,u,p,t)
    du[1] = -u[1]/6
end
u0 = [10.0]
const V = 1
prob = ODEProblem(f,u0,(0.0,10.0))

sol = solve(prob,Tsit5())
using Plots; gr()
plot(sol)

sol = solve(prob,Tsit5(),tstops=[4.0],callback=cb)
using Plots; gr()
plot(sol)

using SciMLTutorials
SciMLTutorials.tutorial_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


module Operators

export grad,lapq,bclap,bcdiv,adv,cgsolv

using LinearAlgebra
using SparseArrays

function grad(velovec,p,u,v,pvec,nx,ny,dx,dy)

    G = spzeros(length(velovec),length(pvec))

    for i = 1:nx
        for j = 1:ny

            if i != nx

                G[u[j,i],p[j,i+1]] =    1/dx
                G[u[j,i],p[j,i]]   = -1/dx

            end

            if j != ny

                G[v[j,i],p[j+1,i]] =    1/dy
                G[v[j,i],p[j,i]]   = -1/dy

            end
        end
    end

    return G
end

function lapq(velovec,p,u,v,nx,ny,dx,dy)

    cuti = (nx-1)*ny

L = spzeros(length(velovec),length(velovec))

    for i = 1:nx-1
        for j = 1:ny

        if i != 1 && j != 1 && i != nx-1 && j != ny

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -2/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

        # left
        if i == 1 && j != 1 && j != ny

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
#             L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -2/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

        if i == nx-1 && j != 1 && j != ny

#             L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -2/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

        if i != 1 && j == 1 && i != nx-1

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

        if i != 1 && j == ny && i != nx-1

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

            # four corners
        if (j == 1 && i == 1)

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
#             L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy

        end

        if (j == 1 && i == nx-1)

#             L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy

        end

        if (j == ny && i == 1)

            L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
#             L[u[j,i],u[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy

        end

        if (j == ny && i == nx-1)

#             L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[u[j,i],u[j,i]] =     -2/dx/dx
            L[u[j,i],u[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[u[j,i],u[j,i]] +=     -3/dy/dy
            L[u[j,i],u[j-1,i]] =    1/dy/dy

        end
    end
end

    for i = 1:nx
        for j = 1:ny-1

        if i != 1 && j != 1 && i != nx && j != ny-1

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -2/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy
        end

        if i == 1 && j != 1 && j != ny-1

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
#             L[v[j,i],v[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy
        end

        if i == nx && j != 1 && j != ny-1

#             L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy
        end

        if i != 1 && j == 1 && i != nx

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -2/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy
        end

        if i != 1 && j == ny-1 && i != nx

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -2/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy
        end

            # four corners
        if (j == 1 && i == 1)

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
#             L[u[j,i],u[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy

        end

        if (j == 1 && i == nx)

#             L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

            L[v[j,i],v[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
#             L[u[j,i],u[j-1,i]] =    1/dy/dy

        end

        if (j == ny-1 && i == 1)

            L[v[j,i],v[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
#             L[u[j,i],u[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy

        end

        if (j == ny-1 && i == nx)

#             L[u[j,i],u[j,i+1]] =    1/dx/dx
            L[v[j,i],v[j,i]] =     -3/dx/dx
            L[v[j,i],v[j,i-1]] =    1/dx/dx

#             L[u[j,i],u[j+1,i]] =    1/dy/dy
            L[v[j,i],v[j,i]] +=     -2/dy/dy
            L[v[j,i],v[j-1,i]] =    1/dy/dy

        end
    end
end

    return L
end

function bclap(velovec,u_up,u_bottom,u_left,u_right,v_up,v_bottom,v_left,v_right,p,u,v,nx,ny,dx,dy)

    cuti = (nx-1)*ny

    bcL = spzeros(length(velovec))

    bcL_u_left = deepcopy(u_left)
    bcL_u_right = deepcopy(u_right)

    bcL_v_up = deepcopy(v_up)
    bcL_v_bottom = deepcopy(v_bottom)

    bcL_u_up = deepcopy(u_up)
    bcL_u_bottom = deepcopy(u_bottom)
    bcL_v_left = deepcopy(v_left)
    bcL_v_right = deepcopy(v_right)

#     # extrapolation
#     for i = 1:length(bcL_u_up)
#         bcL_u_up[i] = 2*u_up[i] - velovec[u[end,i]]
#         bcL_u_bottom[i] = 2*u_bottom[i] - velovec[u[1,i]]
#     end

#     for j = 1:length(bcL_v_left)
#         bcL_v_left[j] = 2*v_left[j] - velovec[v[j,1]]
#         bcL_v_right[j] = 2*v_right[j] - velovec[v[j,end]]
#     end

    # u B.C.
    for i = 1:length(bcL_u_up)

        bcL[u[end,i]] += bcL_u_up[i]*2   /dy/dy
        bcL[u[1,i]]   += bcL_u_bottom[i]    *(2)/dy/dy

    end

    for j = 1:length(bcL_u_left)

        bcL[u[j,end]] += bcL_u_right[j]*1   /dx/dx
        bcL[u[j,1]]   += bcL_u_left[j]    *(1)/dx/dx

    end

    # v B.C.
    for i = 1:length(bcL_v_up)

        bcL[v[end,i]] += bcL_v_up[i]*1   /dy/dy
        bcL[v[1,i]]   += bcL_v_bottom[i]    *(1)/dy/dy

    end

    for j = 1:length(bcL_v_left)

        bcL[v[j,end]] += bcL_v_right[j]*2   /dx/dx
        bcL[v[j,1]]   += bcL_v_left[j]    *(2)/dx/dx

    end

    return bcL
end

function bcdiv(Dbc_up,Dbc_bottom,Dbc_left,Dbc_right,pvec,p,uy,vx,dx,dy)

    Dbc = spzeros(length(pvec))

    for j = 1:length(uy)

        Dbc[p[j,end]] += Dbc_right[j]*1   /dx
        Dbc[p[j,1]]   += Dbc_left[j]    *(-1)/dx

    end

    for i = 1:length(vx)

        Dbc[p[end,i]] += Dbc_up[i]*1   /dy
        Dbc[p[1,i]]   += Dbc_bottom[i]    *(-1)/dy

    end

    return deepcopy(Dbc)
end

function adv(velovec,bcu_up,bcu_bottom,bcu_left,bcu_right,bcv_up,bcv_bottom,bcv_left,bcv_right,p,u,v,nx,ny,dx,dy)
    q = deepcopy(0*velovec)

    for i = 1:nx-1
        for j = 1:ny

            if i != 1 && j != 1 && i != nx-1 && j != ny

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

            end

    if i == 1 && j != 1 && j != ny

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(bcu_left[j] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

    end

    if i == nx-1 && j != 1 && j != ny

            u_right = 0.5*(velovec[u[j,i]] + bcu_right[j])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

    end

        if j == 1 && i != 1 && i != nx-1

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = bcu_bottom[i]
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(bcv_bottom[i] + bcv_bottom[i+1])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

        end

            if j == ny && i != 1 && i != nx-1

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = bcu_up[i]
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(bcv_up[i] + bcv_up[i+1])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

        end

            # four corners
            if j == 1 && i == 1

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(bcu_left[j] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = bcu_bottom[i]
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(bcv_bottom[i] + bcv_bottom[i+1])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

            end

            if j == 1 && i == nx-1

            u_right = 0.5*(velovec[u[j,i]] + bcu_right[j])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
            u_bottom = bcu_bottom[i]
            v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
            v_bottom = 0.5*(bcv_bottom[i] + bcv_bottom[i+1])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

            end

            if j == ny && i == 1

            u_right = 0.5*(velovec[u[j,i]] + velovec[u[j,i+1]])
            u_left  = 0.5*(bcu_left[j] + velovec[u[j,i]])

            u_up     = bcu_up[i]
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(bcv_up[i] + bcv_up[i+1])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

            end

            if j == ny && i == nx-1

            u_right = 0.5*(velovec[u[j,i]] + bcu_right[j])
            u_left  = 0.5*(velovec[u[j,i-1]] + velovec[u[j,i]])

            u_up     = bcu_up[i]
            u_bottom = 0.5*(velovec[u[j-1,i]] + velovec[u[j,i]])
            v_up     = 0.5*(bcv_up[i] + bcv_up[i+1])
            v_bottom = 0.5*(velovec[v[j-1,i]] + velovec[v[j-1,i+1]])

            q[u[j,i]] = (u_right*u_right - u_left*u_left)/dx + (u_up*v_up - u_bottom*v_bottom)/dy

            end

    end
end

    for i = 1:nx
        for j = 1:ny-1

            if i != 1 && j != 1 && i != nx && j != ny-1

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

            if i == 1 && j != 1 && j != ny-1

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = bcv_left[j]
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(bcu_left[j] + bcu_left[j+1])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

    if i == nx && j != 1 && j != ny-1

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = bcv_right[j]
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(bcu_right[j] + bcu_right[j+1])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

    end

        if j == 1 && i != 1 && i != nx

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + bcv_bottom[i])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

        end

            if j == ny-1 && i != 1 && i != nx

                v_up     =  0.5*(velovec[v[j,i]] + bcv_up[i])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

        end

        # four corners

            if i == 1 && j == 1

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + bcv_bottom[i])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = bcv_left[j]
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(bcu_left[j] + bcu_left[j+1])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

            if i == 1 && j == ny-1

                v_up     =  0.5*(velovec[v[j,i]] + bcv_up[i])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = 0.5*(velovec[v[j,i]] + velovec[v[j,i+1]])
                v_left      = bcv_left[j]
                u_right     = 0.5*(velovec[u[j,i]] + velovec[u[j+1,i]])
                u_left      = 0.5*(bcu_left[j] + bcu_left[j+1])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

            if i == nx && j == 1

                v_up     = 0.5*(velovec[v[j,i]] + velovec[v[j+1,i]])
                v_bottom = 0.5*(velovec[v[j,i]] + bcv_bottom[i])

                v_right     = bcv_right[j]
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(bcu_right[j] + bcu_right[j+1])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

            if i == nx && j == ny-1

                v_up     =  0.5*(velovec[v[j,i]] + bcv_up[i])
                v_bottom = 0.5*(velovec[v[j,i]] + velovec[v[j-1,i]])

                v_right     = bcv_right[j]
                v_left      = 0.5*(velovec[v[j,i]] + velovec[v[j,i-1]])
                u_right     = 0.5*(bcu_right[j] + bcu_right[j+1])
                u_left      = 0.5*(velovec[u[j,i-1]] + velovec[u[j+1,i-1]])

            q[v[j,i]] = (u_right*v_right - u_left*v_left)/dx + (v_up*v_up - v_bottom*v_bottom)/dy

            end

        end
    end

    return -q
end

function cgsolv(A,b,x,imax,ϵ)
    i=0
    r=b-A*x
    d=deepcopy(r)
    δnew = r'*r

    # added by myself
    r0 = b
    δ0 = r0'*r0

    while i<imax && δnew > ϵ^2 * δ0
        q = A*d
        α = δnew/(d'*q)
        x = x + α*d
        if mod(i,50) == 0
            r=b-A*x
        else
            r=r-α*q
        end

        δold = deepcopy(δnew)
        δnew = r'*r
        β = δnew/δold
        d = r + β*d

        i+=1
    end

    x = x .- x[1]

    return i,x
end

end


export R_coeff, Q_coeff, impedanceM1
include("constant.jl")
#using .constant

# plan-wave reflection coefficient
"""

    R_coeff(Zc)

Calculate plan-wave reflection coefficient

# Arguments
- `Zc`: complex number, ground characteristic impedance

# Output
- `𝛹`: deg, grazing angle
- `R_mag`: coefficient magnitude
- `R_phase`: deg, phase

# Example
```julia-repl
julia> Zc = 10 + 10im
julia> 𝛹, R_mag, R_phase = R_coeff(Zc)
```

"""
function R_coeff(Zc;len::Int64=1000)
    𝛹 = range(0.0,π/2, length = len)
    R_coeff = ( sin.(𝛹) .- 1/Zc ) ./ ( sin.(𝛹) .+ 1/Zc )
    THETA = Float32.(𝛹*180/π)
    RMAG = Float32.(abs.(R_coeff))
    RPHASE = Float32.(angle.(R_coeff)*180/π)
    return THETA, RMAG, RPHASE
end

#@btime R_coeff(Zc)
#𝛹, R_mag, R_phase =   R_coeff(Zc)
#plot(𝛹,R_phase)

# spherical-wave reflection coefficient
"""

    Q_coeff(Zc,R2,f)

Calculate spherical-wave reflection coefficient

# Arguments:

- `Zc`: complex number, characteristic impedance
- `R2`: m, length of reflection ray path
- `f`: Hz, frequency

# Outputs:
- `𝛹`: deg, grazing angle
- `Q_mag`: coefficient magnitude
- `Q_phase`: deg, phase

# Example:
```julia-repl
julia> Zc = 10 + 10im
julia> 𝛹, Q_mag, Q_phase = R_coeff(Zc)
```

"""
function Q_coeff(Zc,R2,f,len=100)
    λ = c0/f
    k = 2*π/λ
    𝛹 = range(0,π/2, length = len)

    Q_coeff = randn(ComplexF64, (length(𝛹), 1))
    for i=1:length(𝛹)
        d = sqrt(1im*k*R2/2) * (1/Zc + sin(𝛹[i]))
        Fd = 1 + 1im*sqrt(π)*d*erfcx(-1im*(d))
        Q_coeff[i] = (Zc*sin(𝛹[i]) - 1 + 2*Fd) / (Zc*sin(𝛹[i]) + 1)
    end

    return Float32.(𝛹*180/π), Float32.(abs.(Q_coeff[:,1])), Float32.(angle.(Q_coeff[:,1])*180/π)

end


"""
    detrend(signal::Vector{<:Real}; p::Int = 1, coefs::Union{Vector{<:Real}, Nothing, Real} = nothing, return_coefs::Bool = false)

Detrend a signal by removing polynomial trend of order p using build in least squares.
Choose p = 0 to remove only mean or input coefficients from previous detrending to detrend by those.

# Args:

*   signal::Vector: Data Vector containing te signal
*   p::Int: order of polynomial
*   coefs::Union{Vector{<:Real}, Nothing, Real}: Coefficients to do the same detrending on different signal
*   return_coefs::Bool: if true returns coefficients detrended by

# Return:

* newSignal: The detrended sigal

    Or if return_coefs = true:

* (newSignal, coefs): The detrended signal and the estimated coefficients from order 0 to  p

# Examples

```julia
julia> signal = sin.([1:100;]) + 0.03 .* [1:100;]
julia> detrend(signal)
Vector{Float}
```

"""
function detrend(signal::Vector{<:Real}; p::Int = 1, coefs::Union{Vector{<:Real}, Nothing, Real} = nothing, return_coefs::Bool = false)
    p >= 0 || throw(DomainError("Order p hast to be positive or zero, is: $p"))
    # Build up regression matrix x
    x = ones(length(signal))
    xt = cumsum(x)
    for i in 1:p
      x = hcat(x, xt .^ i)
    end
    # If no coefs given estimate them
    if coefs === nothing
        coefs = x \ signal
    else
        length(coefs) == p+1 || throw(DomainError("p is $p, but length of coefs is $(length(coefs))"))
    end
    trend = x * coefs

    if return_coefs
        return (signal .- trend), coefs
    else
        return signal .- trend
    end
end

"""
    theilSenEstimator(x, y)

Calculate the Teil Sen Estimator (median of all slopes m = (yⱼ - yᵢ)/(xⱼ - xᵢ )). Stable up to ~27% outliers

# Args:

* 'x::Vector': Data Vector containing x values
* 'y::Vector': Data Vector containing y values

# Return:

* '(m, b)::Tuple': m represents slope, b the intersect

# Examples

```julia
julia> theilSenEstimator(1:10, 1:10)
(1.0,0.0)
```

"""
function theilSenEstimator(x::Vector{<:Real}, y::Vector{<:Real})
    len = length(y)
    len == length(x) || throw(DomainError("Input Vectors have to be of same length"))
    indices = 1:len
    m = Vector{Float64}(undef, len - 1)
    for ii = 1:len-1
        indices = mod.(indices, len) .+ 1
        m[ii] = median((y - y[indices]) ./ (x - x[indices]))
    end
    m = median(m)
    b = median(y .- m*x)
    return (m, b)
end

"""
    arDecomposition(x::AbstractVector, p::Union{Int, UnitRange{Int}}; nfreq::Int = 124, sf::Real = 1, verbose::Bool = false)

Perform a decomosition of the AR spectrum of signal `x` with order `p`. Returns the complex spectra associated to each AR-pole (nfreq x p) and the respectie center frequencies (p) and a frequency vector (nfreq)

# Arguments

* x: Signal to be decomposed
* p: order of the AR process, if a UnitRange is given, the optimal order will be chosen through AIC

# Keywords

* nfreq: resolution of the analysed spectra, defaults to 256
* sf: sampling frequency in Hz, defaults to 1
* verbose: print some information oder selection, defaults to false
"""
function arDecomposition(x::AbstractVector, p::Union{Int, UnitRange{Int}}; nfreq::Int = 124, sf::Real = 1, verbose::Bool = false)
    minimum(p) > 0 || throw(DomainError("Order p has to be positive"))
    if typeof(p) != Int
        aicValues = map(order-> length(x) * log(lpc(x, order)[2]) + 2order, p)
        p = p[findmin(aicValues)[2]]
        verbose && @info "Order was choosen as p: $p by AIC"
    end

    # estimate AR coefs
    arcoefs, γ = lpc(x, p)
    arPoly = Polynomial(reverse([1; arcoefs]))
    poles = Polynomials.roots(arPoly)

    z2f(z) = abs(real((2π*im)^-1 * log(z)))
    centerFrequencies = z2f.(poles) .* sf

    f = range(0, 0.5, length = nfreq)
    f2z(f) = exp(2π*im * f)
    z = f2z.(f)

    S = zeros(Complex, length(f), p)
    γk(k) = 2γ / (poles[k] * prod(poles[k] .- poles[1:end .!= k]) * prod(poles[k]^-1 .- poles))
    for k in 1:p
        Sk(z) = ((γk(k) * poles[k]) / (z^-1 - poles[k]) + γk(k) + (γk(k) * poles[k]) / (z - poles[k]))
        S[:, k] = Sk.(z)
    end

    return S, centerFrequencies, f .* sf
end

"""
    getSpectralComponent(S::AbstractMatrix, centerFrequencies::AbstractVector, frange::Tuple{<:Real, <:Real} = (0.0, Inf))

Extract the spectral components of a decomposed AR spectrum `S`, where the center frequencies fall in `frange`. See also `arDecomposition`.

"""
function getSpectralComponent(S::AbstractMatrix, centerFrequencies::AbstractVector, frange::Tuple{<:Real, <:Real} = (0.0, Inf))
    frange[1] < frange[2] || throw(DomainError("fmin needs to be < fmax"))
    return vec(sum((real.(S[:, findall(f -> frange[1] <= f <= frange[2], centerFrequencies)])), dims = 2))
end


using CSV, TimeSeries, Dates, Statistics, StatsBase, Measures, Plots; pyplot()

df = CSV.read("../data/oneOnEpsilonBlogs.csv",copycols = true)
tsA = TimeArray(Date.(df.Day,Dates.DateFormat("m/d/y")),df.Users)
tsB = moving(mean,tsA,7,padding = true)
tsC = TimeArray(timestamp(tsA), values(tsA) - values(tsB))

dow = dayofweek.(timestamp(tsA))
dayDiv = [filter((x)->!isnan(x), values(tsC)[dow .== d]) for d in 1:7]
dayMeans = mean.(dayDiv)
dailyCorrection = dayMeans[dow]
errs = filter((x)->!isnan(x),values(tsC) - dailyCorrection)
diffs = diff(errs)

lags = 1:50
acc = autocor(diffs,lags)

default(legend = false)
p1 = plot(diffs, c=:blue,
        xlabel="Day",ylabel="Difference Between Corrected Days")
p2 = plot(lags,acc,line=:stem, c=:blue,
        xlabel="Lag",ylabel="Autocorrelation")
plot(p1,p2,size=(900,400),margin = 5mm)


using QuasiGeostrophy, Plots, FFTW, BenchmarkTools
using LinearAlgebra
import Plots: plot
import QuasiGeostrophy: compute

##
Ωxy = S¹(0,2π) × S¹(0,2π)
Nx = 2^8; Ny = 2^8;
fourier_grid = create_grid((Nx, Ny), Ωxy)
x, y = fourier_grid.grid
fourier_transform = Transform(fourier_grid)

names = ("ϕ1", "ϕ2", "ϕ3", "ϕ4")
create_fields(names = names,
              grid = fourier_grid,
              transform = fourier_transform)
f1 = @. sin(x) + 0im * y
f2 = @. sin(y) + 0im * x
f3 = @. sin(x) * sin(y) # product
f4 = @. sin(x) + sin(y) # sum
f1 = fourier_transform.forward * f1
f2 = fourier_transform.forward * f2
f3 = fourier_transform.forward * f3
f4 = fourier_transform.forward * f4
ϕ1.data .= f1
ϕ2.data .= f2
ϕ3.data .= f3
ϕ4.data .= f4
## Check Algebra
norm((ϕ1 * ϕ2 - ϕ3).data)/norm((ϕ3).data)

f_ϕ1 = Field(ϕ1, BasicMetaData("ϕ1"))
f_ϕ2 = Field(ϕ2, BasicMetaData("ϕ2"))
tt =  2 * f_ϕ1 + f_ϕ2 * f_ϕ1 + 2 + tanh(f_ϕ1) + 2*2
compute(a::FourierField) = a
compute(tt)
evaluate(tt)

## Check Calculus
kx, ky = fourier_grid.wavenumbers
∂x = FourierOperator(im .* kx, FourierOperatorMetaData("∂x"))
∂y = FourierOperator(im .* ky, FourierOperatorMetaData("∂y"))

∂x(ϕ)

∂x(ϕ) + 1 + (∂x^2 + ∂y^2)(ϕ)
##
dmd = DerivativeMetaData(FourierOperator(im .* kx, FourierOperatorMetaData("∂x")), "x")
∂x = Operator(nothing, dmd)
∂x(f_ϕ1)
dmd = DerivativeMetaData(FourierOperator(im .* ky, FourierOperatorMetaData("∂y")), "y")
∂y = Operator(nothing, dmd)
∂ʸϕ1 = ∂y(f_ϕ2) * ∂x(f_ϕ1)
evaluate(∂ʸϕ1)
tmp = compute(∂ʸϕ1)
p1 = plot(tmp)

a1 = compute(∂x)^2
a2 = compute(∂x^2)
norm(a1.op - a2.op)

Δ1 = compute(∂x)^2 + compute(∂y)^2
Δ2 = compute(∂x^2+∂y^2)
norm(Δ1.op - Δ2.op)


"""
    Hypergeometric(s, f, n)

A *Hypergeometric distribution* describes the number of successes in `n` draws without replacement from a finite population containing `s` successes and `f` failures.

```math
P(X = k) = {{{s \\choose k} {f \\choose {n-k}}}\\over {s+f \\choose n}}, \\quad \\text{for } k = \\max(0, n - f), \\ldots, \\min(n, s).
```

```julia
Hypergeometric(s, f, n)  # Hypergeometric distribution for a population with
                         # s successes and f failures, and a sequence of n trials.

params(d)       # Get the parameters, i.e. (s, f, n)
```

External links

* [Hypergeometric distribution on Wikipedia](http://en.wikipedia.org/wiki/Hypergeometric_distribution)

"""
struct Hypergeometric <: DiscreteUnivariateDistribution
    ns::Int     # number of successes in population
    nf::Int     # number of failures in population
    n::Int      # sample size

    function Hypergeometric(ns::Real, nf::Real, n::Real)
        @check_args(Hypergeometric, ns >= zero(ns) && nf >= zero(nf))
        @check_args(Hypergeometric, zero(n) <= n <= ns + nf)
        new(ns, nf, n)
    end
end

@distr_support Hypergeometric max(d.n - d.nf, 0) min(d.ns, d.n)

### Parameters

params(d::Hypergeometric) = (d.ns, d.nf, d.n)

### Statistics

mean(d::Hypergeometric) = d.n * d.ns / (d.ns + d.nf)

function var(d::Hypergeometric)
    N = d.ns + d.nf
    p = d.ns / N
    d.n * p * (1.0 - p) * (N - d.n) / (N - 1.0)
end
mode(d::Hypergeometric) = floor(Int, (d.n + 1) * (d.ns + 1) / (d.ns + d.nf + 2))

function modes(d::Hypergeometric)
    if (d.ns == d.nf) && mod(d.n, 2) == 1
        [(d.n-1)/2, (d.n+1)/2]
    else
        [mode(d)]
    end
end

skewness(d::Hypergeometric) = (d.nf-d.ns)*sqrt(d.ns+d.nf-1)*(d.ns+d.nf-2*d.n)/sqrt(d.n*d.ns*d.nf*(d.ns+d.nf-d.n))/(d.ns+d.nf-2)
function kurtosis(d::Hypergeometric)
    N = d.ns + d.nf
    a = (N-1) * N^2 * (N * (N+1) - 6*d.ns * (N-d.ns) - 6*d.n*(N-d.n)) + 6*d.n*d.ns*(d.nf)*(N-d.n)*(5*N-6)
    b = (d.n*d.ns*(N-d.ns) * (N-d.n)*(N-2)*(N-3))
    a/b
end

### Evaluation & Sampling

@_delegate_statsfuns Hypergeometric hyper ns nf n

rand(d::Hypergeometric) = convert(Int, StatsFuns.RFunctions.hyperrand(d.ns, d.nf, d.n))

struct RecursiveHypergeomProbEvaluator <: RecursiveProbabilityEvaluator
    ns::Float64
    nf::Float64
    n::Float64
end

RecursiveHypergeomProbEvaluator(d::Hypergeometric) = RecursiveHypergeomProbEvaluator(d.ns, d.nf, d.n)

nextpdf(s::RecursiveHypergeomProbEvaluator, p::Float64, x::Integer) =
    ((s.ns - x + 1) / x) * ((s.n - x + 1) / (s.nf - s.n + x)) * p

Base.broadcast!(::typeof(pdf), r::AbstractArray, d::Hypergeometric, rgn::UnitRange) =
    _pdf!(r, d, rgn, RecursiveHypergeomProbEvaluator(d))

function Base.broadcast(::typeof(pdf), d::Hypergeometric, X::UnitRange)
    r = similar(Array{promote_type(partype(d), eltype(X))}, indices(X))
    r .= pdf.(d,X)
end


using LinearAlgebra
using Plots

# generate the data for the example
function generateSamples(n, k, mu::Float64)
  # generate theta
  theta = rand(Float64, k)
  @. theta = (theta -0.5) * 10

  # generate y with noise
  y = Vector{Float64}(undef, n+k -1)
  for j in 1:n
    y[j] = 1 * (rand(Float64) -0.5)
  end
  for j in n+1:n+k-1
    y[j] = 0
  end

  # generate X, update y
  supportPoints = rand(Float64, n)
  @. supportPoints = (supportPoints - 0.5)
  supportPoints = sort(supportPoints)
  X = Matrix{Float64}(undef, k, n+k-1)

  for j in 1:n
    X[1,j] = 1
    y[j] = theta[1]
    for i in 2:k
      X[i,j] = X[i-1, j] * supportPoints[j]
      y[j] += X[i,j] * theta[i]
    end
  end

  # identidy matrix multiplied with vector of weights, first row
  for j in n+1:n+k-1
    X[1,j] = 0
    for i in 1:k
      if (j - n + 1 == i)
        X[i,j] = sqrt(mu * (j-n))
      end
    end
  end

  return X, y, theta
end

function evalPolynomial(grid, coeff)
  k = length(coeff)
  len = length(grid)
  f = zeros(Float64, len)
  for i in 1:k
    for j in 1:len
      f[j] *= grid[j]
      f[j] += coeff[k + 1 - i]
    end
  end
  return f
end

function plotAll(X::Matrix{Float64}, y::Vector{Float64}, theta::Vector{Float64}, n, k)
  x = Vector{Float64}(undef, n)
  val = Vector{Float64}(undef, n)
  for j in 1:n
    x[j] = X[2,j]
    val[j] = y[j]
  end
  plot(x, val ,label="", seriestype=:scatter) #xlims=[x[1], x[n]]

  grid = collect(range(x[1], step=0.01, stop=x[n]))
  f = evalPolynomial(grid, theta)
  plot!(grid, f, label="theta")

  theta2 = X' \ y # solve in least squares sense
  f2 = evalPolynomial(grid, theta2)
  plot!(grid, f2, label="regression theta")
end

function wrap(n, k, mu::Float64)
  X, y, theta = generateSamples(n, k, mu)
  plotAll(X, y, theta, n, k)
end


using SymEngine
using HCubature
using LinearAlgebra

function L₂(X;x=symbols("x"),y=symbols("y"))
    res=Matrix{SymEngine.Basic}(undef,3,size(X,2))
    for i in 1:size(X,2)
        res[1,i]=diff(X[1,i],x)
        res[2,i]=diff(X[2,i],y)
        res[3,i]=diff(X[1,i],y)+diff(X[2,i],x)
    end
    return res
end

x,y=symbols("x,y")
xᵢ=symbols("x₁ x₂ x₃")
yᵢ=symbols("y₁ y₂ y₃")
#T9G9
A=symbols("A")

# aᵢ=[y[j]-y[m] for (j,m) in zip([2,3,1],[3,1,2])]
# bᵢ=[y[j]-y[m] for (j,m) in zip([2,3,1],[3,1,2])]
# cᵢ=[-x[j]+x[m] for (j,m) in zip([2,3,1],[3,1,2])]
aᵢ=symbols("a₁ a₂ a₃")
bᵢ=symbols("b₁ b₂ b₃")
cᵢ=symbols("c₁ c₂ c₃")
Lᵢ=1/2/A .*(aᵢ .+bᵢ .*x .+cᵢ .*y)
Nᵤᵨᵢ=[0.5*Lᵢ[i]*(bᵢ[m]*Lᵢ[j]-bᵢ[j]*Lᵢ[m]) for (i,j,m) in zip([1,2,3],[2,3,1],[3,1,2])]
Nᵥᵨᵢ=[0.5*Lᵢ[i]*(cᵢ[m]*Lᵢ[j]-cᵢ[j]*Lᵢ[m]) for (i,j,m) in zip([1,2,3],[2,3,1],[3,1,2])]

Nᵢ=Array{Basic}(undef,2,9)
for i in 1:3
    Nᵢ[:,3i-2:3i]=[Lᵢ[i] 0 Nᵤᵨᵢ[i];
                   0 Lᵢ[i] Nᵥᵨᵢ[i]]
end
B=L₂(Nᵢ)
D₀=E₀/(1-ν₀^2)
D=D₀*[1 ν₀ 0;
      ν₀ 1 0;
      0  0 (1-ν₀)/2]

K=transpose(B)*D*B
open("./k_GT9.jl","w+") do f
    for i in 1:size(K,1)
        for j in 1:size(K,2)
            write(f,"K["*string(i)*","*string(j)*"]="*string(K[i,j])*"\n")
        end
    end
end