function fornberg_weights!(c::AbstractMatrix{T}, x, ξ) where T
    nd,m = size(c) .- 1
    c = OffsetArray(c, 0:nd, 0:m)
    x = OffsetArray(x, 0:nd)
    length(x) == nd+1 ||
        throw(DimensionMismatch("Number of grid points does not match requested number of derivative coefficients"))

    c₁ = one(T)
    c₄ = x[0] - ξ

    c .= zero(T)
    c[0,0] = 1

    for i = 1:nd
        mn = min(i,m)
        c₂ = one(T)
        c₅ = c₄
        c₄ = x[i] - ξ
        for j = 0:i-1
            c₃ = x[i] - x[j]
            c₂ *= c₃
            for k = mn:-1:1
                c[i,k] = c₁*(k*c[i-1,k-1] - c₅*c[i-1,k])/c₂
            end
            c[i,0] = -c₁*c₅*c[i-1,0]/c₂
            for k = mn:-1:1
                c[j,k] = (c₄*c[j,k] - k*c[j,k-1])/c₃
            end
            c[j,0] = c₄*c[j,0]/c₃
        end
        c₁ = c₂
    end
    c
end

function fornberg_weights(x, ξ::T, m::Integer) where T
    c = zeros(promote_type(eltype(x),T), length(x), m+1)
    fornberg_weights!(c, x, ξ)
    c
end

function fornberg_all_weights!(c::AbstractArray{T,3}, x, ξ) where T
    nd,m = size(c,1) .- 1,size(c,3) .- 1
    c = OffsetArray(c, 0:nd, 0:nd, 0:m)
    x = OffsetArray(x, 0:nd)
    length(x) == nd+1 ||
        throw(DimensionMismatch("Number of grid points does not match requested number of derivative coefficients"))

    c .= zero(T)
    c[0,0,0] = 1

    c₁ = one(T)
    c₄ = x[0] - ξ

    for i = 1:nd
        mn = min(i,m)
        c₂ = one(T)
        c₅ = c₄
        c₄ = x[i] - ξ
        for j = 0:i-1
            c₃ = x[i] - x[j]
            c₂ *= c₃
            c[j,i,0] = c₄*c[j,i-1,0]/c₃
            for k = 1:mn
                c[j,i,k] = (c₄*c[j,i-1,k] - k*c[j,i-1,k-1])/c₃
            end
        end
        c[i,i,0] = -c₁*c₅*c[i-1,i-1,0]/c₂

        for k = 1:mn
            c[i,i,k] = c₁*(k*c[i-1,i-1,k-1] - c₅*c[i-1,i-1,k])/c₂
        end
        c₁ = c₂
    end
    c
end

function fornberg_all_weights(x, ξ::T, m::Integer) where T
    c = zeros(promote_type(eltype(x),T), length(x), length(x), m+1)
    fornberg_all_weights!(c, x, ξ)
    c
end

function fornberg_test_driver(x, ξ)
    pnum(n::AbstractFloat) = printfmt("{1:12.4f}", n)
    pnum(n) = print(format(n, width=12))
    pnum(n::Rational) =
        print(format(isinteger(n) ? Int(n) : n, width=12))

    M = min(4,length(x)-1)
    C = fornberg_all_weights(x, ξ, M)
    N = size(C,2)-1
    print("j ")
    for j = 0:N
        pnum(j)
    end
    println()
    print("x ")
    foreach(pnum, x)
    println()
    println()
    for (ki,k) = enumerate(0:M)
        printfmtln("k = {1:2d}", k)
        for i = k:N
            ii = i+1
            printfmt("{1:2d}", i)
            foreach(pnum, view(C,1:ii,ii,ki))
            println()
        end
        println()
    end
    println()
    C = fornberg_weights(x, ξ, M)
    for (ki,k) = enumerate(0:M)
        printfmt("{1:2d}", k)
        foreach(pnum, view(C,:,ki))
        println(stdout)
    end
end


function ldlfor!(a::Matrix{Float64}, b::Vector{Float64})
  n = size(a, 1)
  for i in 1:n
    total = b[i]
    if i > 1
      for j in 1:(i-1)
        total -= a[j, i] * b[j]
      end
      #(i=i, total=total, ad=a[i, i], newb=total / a[i, i]) |> display
    end
    b[i] = total / a[i, i]
    #i == 1 && (i=i, total=total, ad=a[i, i], newb=total / a[i, i]) |> display
  end
end

function ldlfor(a::Matrix{Float64}, b::Vector{Float64})
  bt = deepcopy(b)
  n = size(a, 1)
  for i in 1:n
    total = bt[i]
    if i > 1
      for j in 1:(i-1)
        total -= a[j, i] * bt[j]
      end
      #(i=i, total=total, ad=a[i, i], newb=total / a[i, i]) |> display
    end
    bt[i] = total / a[i, i]
    #i == 1 && (i=i, total=total, ad=a[i, i], newb=total / a[i, i]) |> display
  end
  bt
end

export
  ldlfor!,
  ldlfor


# ------------------------------------------------------------------------------------------
# ## Neural networks
#
# Now that we know what neurons are, we are ready for the final step: the neural network!. A
# neural network is literally made out of a network of neurons that are connected together.
#
# So far, we have just looked at single neurons, that only have a single output.
# What if we want multiple outputs?
#
#
# ### Multiple output models
#
# What if we wanted to distinguish between apples, bananas, *and* grapes? We could use
# *vectors* of `0` or `1` values to symbolize each output.
#
#
#
# The idea of using vectors is that different directions in the space of outputs encode
# information about different types of inputs.
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# Now we extend our previous model to give multiple outputs by repeating it with different
# weights. For the first element of the array we'd use:
#
# $$\sigma(x;w^{(1)},b^{(1)}) := \frac{1}{1 + \exp(-w^{(1)} \cdot x + b^{(1)})};$$
#
# then for the second we'd use
#
# $$\sigma(x;w^{(2)},b^{(2)}) := \frac{1}{1 + \exp(-w^{(2)} \cdot x + b^{(2)})};$$
#
# and if you wanted $n$ outputs, you'd have for each one
#
# $$\sigma(x;w^{(i)},b^{(i)}) := \frac{1}{1 + \exp(-w^{(i)} \cdot x + b^{(i)})}.$$
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# Notice that these equations are all the same, except for the parameters, so we can write
# this model more succinctly, as follows. Let's write $b$ in an array:
#
# $$b=\left[\begin{array}{c}
# b_{1}\\
# b_{2}\\
# \vdots\\
# b_{n}
# \end{array}\right]$$
#
# and put our array of weights as a matrix:
#
# $$ \mathsf{W}=\left[\begin{array}{c}
# \\
# \\
# \\
# \\
# \end{array}\begin{array}{cccc}
# w_{1}^{(1)} & w_{2}^{(1)} & \ldots & w_{n}^{(1)}\\
# w_{1}^{(2)} & w_{2}^{(2)} & \ldots & w_{n}^{(2)}\\
# \vdots & \vdots &  & \vdots\\
# w_{1}^{(n)} & w_{2}^{(n)} & \ldots & w_{n}^{(n)}
# \end{array}\right]
# $$
#
# We can write this all in one line as:
#
# $$\sigma(x;w,b)= \left[\begin{array}{c}
# \sigma^{(1)}\\
# \sigma^{(2)}\\
# \vdots\\
# \sigma^{(n)}
# \end{array}\right] = \frac{1}{1 + \exp(-\mathsf{W} x + b)}$$
#
# $\mathsf{W} x$ is the operation called "matrix multiplication"
#
# [Show small matrix multiplication]
#
# It takes each column of weights and does the dot product against $x$ (remember, that's how
# $\sigma^{(i)}$ was defined) and spits out a vector from doing that with each column. The
# result is a vector, which makes this version of the function give a vector of outputs
# which we can use to encode larger set of choices.
#
# Matrix multiplication is also interesting since **GPUs (Graphics Processing Units, i.e.
# graphics cards) are basically just matrix multiplication machines**, which means that by
# writing the equation this way, the result can be calculated really fast.
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# This "multiple input and multiple output" version of the sigmoid function is known as a
# *layer of neurons*.
#
# Previously we worked with a single neuron, which we visualized as
#
#
#
# where we have two pieces of data (green) coming into a single neuron (pink) that returned
# a single output. We could use this single output to do binary classification - to identify
# an image of a fruit as `1`, meaning banana or as `0`, meaning not a banana (or an apple).
#
# To do non-binary classification, we can use a layer of neurons, which we can visualize as
#
#
#
# We now have stacked a bunch of neurons on top of each other to hopefully work together and
# train to output results of more complicated features.
#
# We still have two input pieces of data, but now have several neurons, each of which
# produces an output for a given binary classification:
# * neuron 1: "is it an apple?"
# * neuron 2: "is it a banana?"
# * neuron 3: "is it a grape?"
# ------------------------------------------------------------------------------------------

.nbexports/introductory-tutorials/broader-topics-and-ecosystem/intro-to-ml/15. ML - Intro to neural networks.jl


# # An equation of second order in two dimmensions
#
# Let \\(\Omega = (0, 1)^2\\) the unit square and denote \\( \Gamma = \partial \Omega\\) its boundary.
# Consider the following problem. Given \\(f \in C(\Omega)\\), find a function \\(u\\) satisfying
# \Delta^2 u = f in \Omega
# u = \Delta u = 0 on \Gamma

using Revise
using SymFEL
using LinearAlgebra
using SparseArrays
using WriteVTK
using JLD2

## discretization parameters
# we use the mesh square.msh (in gmsh format)
# obtained from square.geo using gmsh
# this mesh is formed by quad elements
# we use gmsh module for read the mesh
# there are some problems using Threads and gmsh (in linux)
# the file is square.jld2 is prepared separetely
@load "square-simple.jld2"

# elementary matrices - P2 x P2
elem_M = SymFEL.get_square_hermite_em((3, 3), (0, 0), (0, 0))
elem_K = SymFEL.get_square_hermite_em((3, 3), (2, 0), (2, 0)) +
    SymFEL.get_square_hermite_em((3, 3), (2, 0), (0, 2)) +
    SymFEL.get_square_hermite_em((3, 3), (0, 2), (2, 0)) +
    SymFEL.get_square_hermite_em((3, 3), (0, 2), (0, 2))

dx = norm(nodes_coordinate[:, elements_bound[1,1]] - nodes_coordinate[:, elements_bound[2,1]])
elem_K_dx = convert(Matrix{Float64}, elem_K.subs(h, dx))
elem_M_dx = convert(Matrix{Float64}, elem_M.subs(h, dx));

# global matrices
K = SymFEL.assemble_squaremesh_FE_matrix(elem_K_dx, elements_int,
                                           order1=1, order2=1,
                                           dof1=4, dof2=4)
M = SymFEL.assemble_squaremesh_FE_matrix(elem_M_dx, elements_int,
                                           order1=1, order2 = 1,
                                           dof1=4, dof2=4)

f = zeros(Float64, 4*nodes_N)
f[4*((1:nodes_N) .- 1) .+ 1] = 4*pi^4*sin.(pi * nodes_coordinate[1,:]) .* sin.(pi * nodes_coordinate[2,:])
f[4*((1:nodes_N) .- 1) .+ 2] = 4*pi^5*cos.(pi * nodes_coordinate[1,:]) .* sin.(pi * nodes_coordinate[2,:])
f[4*((1:nodes_N) .- 1) .+ 3] = 4*pi^5*sin.(pi * nodes_coordinate[1,:]) .* cos.(pi * nodes_coordinate[2,:])
f[4*((1:nodes_N) .- 1) .+ 4] = 4*pi^6*cos.(pi * nodes_coordinate[1,:]) .* cos.(pi * nodes_coordinate[2,:])

F = M * f

F[4*(nodes_boundary.-1).+1] = zeros(nodes_boundary_N)
A = copy(K)

# boundary condition
tgv = 1e30
A[4*(nodes_boundary.-1).+1, 4*(nodes_boundary.-1).+1] += tgv * sparse(Matrix{Float64}(I, nodes_boundary_N, nodes_boundary_N))

u = A \ F
u_exact = zeros(Float64, 4*nodes_N)
u_exact[4*((1:nodes_N) .- 1) .+ 1] = sin.(pi * nodes_coordinate[1,:]) .* sin.(pi * nodes_coordinate[2,:])
u_exact[4*((1:nodes_N) .- 1) .+ 2] = pi * cos.(pi * nodes_coordinate[1,:]) .* sin.(pi * nodes_coordinate[2,:])
u_exact[4*((1:nodes_N) .- 1) .+ 3] = pi * sin.(pi * nodes_coordinate[1,:]) .* cos.(pi * nodes_coordinate[2,:])
u_exact[4*((1:nodes_N) .- 1) .+ 4] = pi^2 * cos.(pi * nodes_coordinate[1,:]) .* cos.(pi * nodes_coordinate[2,:])

err = u - u_exact

println("L2 error : ", sqrt(err' * M * err))
println("H2 error : ", sqrt(err' * K * err))

# export to vtk
cells = [MeshCell(VTKCellTypes.VTK_QUAD, elements_int[1:4, i]) for i = 1:elements_int_N]

points_x = nodes_coordinate[1, :]
points_y = nodes_coordinate[2, :]
vtkfile = vtk_grid("ex4-output", points_x, points_y, cells)

vtkfile["u", VTKPointData()] = u[4*((1:nodes_N) .- 1) .+ 1]
vtkfile["ux", VTKPointData()] = u[4*((1:nodes_N) .- 1) .+ 2]
vtkfile["uy", VTKPointData()] = u[4*((1:nodes_N) .- 1) .+ 3]
vtkfile["uxy", VTKPointData()] = u[4*((1:nodes_N) .- 1) .+ 4]
outfiles = vtk_save(vtkfile)


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    MaternVariogram(range=r, sill=s, nugget=n, order=ν)
    MaternVariogram(ball; sill=s, nugget=n, order=ν)

A Matérn variogram with range `r`, sill `s` and nugget `n`.
The parameter `ν` is the order of the Bessel function.
Optionally, use a custom metric `ball`.
"""
struct MaternVariogram{T,B} <: Variogram
  sill::T
  nugget::T
  order::T
  ball::B
end

MaternVariogram(ball; sill=1.0, nugget=0.0, order=1.0) =
  MaternVariogram(sill, nugget, order, ball)

MaternVariogram(; range=1.0, sill=1.0, nugget=0.0, order=1.0) =
  MaternVariogram(sill, nugget, order, MetricBall(range))

function (γ::MaternVariogram)(h::T) where {T}
  r = radius(γ.ball)
  s = γ.sill
  n = γ.nugget
  ν = γ.order

  # shift lag by machine precision to
  # avoid explosion at the origin
  h′ = sqrt(2ν)*(h + eps(T)) / r
  Β  = besselk(ν, h′)
  Γ  = gamma(ν)

  (s - n) * (1 - 2^(1 - ν) / Γ * h′^ν * Β) + (h′ > 0) * n
end

isstationary(::Type{<:MaternVariogram}) = true


include("08_two_site_operator_custom.jl")

println("
#######################################################
# Tutorial 9                                          #
#                                                     #
# 2-site energy minimization via gradient             #
# optimization                                        #
#######################################################
")

# Used for computing gradients "automatically".
# Julia's source-to-source reverse mode AD library.
using Zygote

# Energy function to minimize with gradient descent.
# Depends implicitly on the Hamiltonian.
function E(ψ)
  ψHψ = inner(ψ', H, ψ)

  # Alternative:
  # Hψ = apply(H, ψ)
  # ψHψ = inner(ψ, Hψ)

  ψψ = inner(ψ, ψ)
  return ψHψ / ψψ
end

# Extremely simple gradient descent.
# Better to use a library like `OptimKit.jl` in general.
function minimize(f, ∂f, x; nsteps, γ)
  for n in 1:nsteps
    println("n = ", n, ", f_x = ", f(x))

    # Gradient descent step with step size `γ`
    x = x - γ * ∂f(x)
  end
  return x
end

# Use the cat state as a starting state
ψ₀ = (Zp1 * Zp2 + Zm1 * Zm2) / √2

# Minimize the energy!
∂E(ψ) = gradient(E, ψ)[1]
ψ = minimize(E, ∂E, ψ₀; nsteps=10, γ=0.1)

@show E(ψ₀), norm(∂E(ψ₀))
@show E(ψ), norm(∂E(ψ))


# The SteinParzenARKernel is a kernel derived by Parzen in 1961,
# which arises from a second order autoregressive process. Its
# inner product is for the Sobolev space H^2([a, b]) for any
# a < b.
#
# k(x, y) = (4*alpha*gamma^2)^-1 * exp{-alpha*|x - y|} cos(w*|x-y|)
#   + (alpha / w) * sin(w*|x-y|)
#
# where w^2 = gamma^2 - alpha^2 > 0. The associated norm is
#
# |u|_H = 4*alpha*gamma^2*u(a)^2 + 4*alpha*u'(a)^2 +
#  \int_[a,b] (u''(t) + 2*alpha^2*u'(t) + gamma^2*u(t))^2

type SteinParzenARKernel <: SteinTensorizedKernel
    # the alpha parameter
    alpha::Float64
    # the gamma parameter
    gamma::Float64
end

# have default parameters of alpha = 1/sqrt(2) and gamma = 1.0
SteinParzenARKernel() = SteinParzenARKernel(1.0/sqrt(2), 1.0)

# utility to get w
function getw(ker::SteinParzenARKernel)
    sqrt(ker.gamma^2 - ker.alpha^2)
end

# utility method for wacky constant
function getc1(ker::SteinParzenARKernel)
    1.0 / (4.0 * ker.alpha * ker.gamma^2)
end

function ki(ker::SteinParzenARKernel, x::Float64, y::Float64)
    w = getw(ker)
    c = getc1(ker)
    alpha = ker.alpha

    c * exp(-alpha * abs(x-y)) * cos(w * abs(x-y)) + (alpha/w) * sin(w * abs(x-y))
end

function gradxki(ker::SteinParzenARKernel, x::Float64, y::Float64)
    w = getw(ker)
    c = getc1(ker)
    alpha = ker.alpha
    s = sign(x - y)

    -c * s * exp(-alpha * abs(x-y)) * (
        alpha * cos(w * abs(x-y)) +
        w * sin(w * abs(x-y))
    ) + alpha * s * cos(w * abs(x-y))
end

function gradxyki(ker::SteinParzenARKernel, x::Float64, y::Float64)
    w = getw(ker)
    c = getc1(ker)
    alpha = ker.alpha
    s = sign(x - y)

    -c * exp(-alpha * abs(x-y)) * (
        (alpha^2 - w^2) * cos(w * abs(x-y)) +
        2 * alpha * w * sin(w * abs(x-y))
    ) + alpha * w * sin(w * abs(x-y))
end


function Qmat(N, m, lam)
    # Integration matrix
    Q = 1
    round(m) !== m ? (return Q = Qmat05(N, lam)) : nothing
    Q = Q*Qmat1(N, lam)
    return Q
end

function Qmat05(N, lam)
    # Half integral matrix
    if lam == 0.5
        v = (2/sqrt(pi)) ./ (2*collect(0:N-1).+1)
        Q = myspdiagm(spdiagm(0=>v, 1=>-v[2:end]), N)
    elseif lam == 1
        v = (sqrt(pi)/2) .* ones(N)
        Q = myspdiagm(spdiagm(-1=>v[2:end], 0=>v), N)
    end
    return Q
end

function Qmat1(N, lam)
    # Full integral matrix
    if lam == 0.5
        v = 1 ./ (2*collect(0:N-1).+1)
        Q = myspdiagm(spdiagm(-1 => v, 1 => -v[2:end]), N)
        Q[1, 1] = 1
    elseif lam == 1
        nn = collect(0:N)
        v = 1 ./ (2*nn.+1)
        v2 = 2 ./ (4*nn.^2 .-1)
        Q = myspdiagm(spdiagm(-1=>v[2:end], 0=>v2[2:end], 1=>-v[2:end-1]), N)
    end
    return Q
end

myspdiagm(M, N) = size(M, 1) !== N ? (return M[1:N, 1:N]) : (return M)


# L2,1 norm/Sum of norms of columns or rows (times a constant)

export NormL21

"""
**Sum-of-``L_2`` norms**

    NormL21(λ=1, dim=1)

Returns the function
```math
f(X) = λ⋅∑_i\\|x_i\\|
```
for a nonnegative `λ`, where ``x_i`` is the ``i``-th column of ``X`` if `dim == 1`, and the ``i``-th row of ``X`` if `dim == 2`.
In words, it is the sum of the Euclidean norms of the columns or rows.
"""
struct NormL21{R <: Real, I <: Integer} <: ProximableFunction
    lambda::R
    dim::I
    function NormL21{R,I}(lambda::R, dim::I) where {R <: Real, I <: Integer}
        if lambda < 0
            error("parameter λ must be nonnegative")
        else
            new(lambda, dim)
        end
    end
end

is_convex(f::NormL21) = true

NormL21(lambda::R=1, dim::I=1) where {R <: Real, I <: Integer} = NormL21{R, I}(lambda, dim)

function (f::NormL21)(X::AbstractArray{T, 2}) where {R, T <: RealOrComplex{R}}
    nslice = R(0)
    n21X = R(0)
    if f.dim == 1
        for j = 1:size(X, 2)
            nslice = R(0)
            for i = 1:size(X, 1)
                nslice += abs(X[i, j])^2
            end
            n21X += sqrt(nslice)
        end
    elseif f.dim == 2
        for i = 1:size(X, 1)
            nslice = R(0)
            for j = 1:size(X, 2)
                nslice += abs(X[i, j])^2
            end
            n21X += sqrt(nslice)
        end
    end
    return f.lambda * n21X
end

function prox!(Y::AbstractArray{T, 2}, f::NormL21, X::AbstractArray{T, 2}, gamma::Real=1) where {R, T <: RealOrComplex{R}}
    gl = gamma * f.lambda
    nslice = R(0)
    n21X = R(0)
    if f.dim == 1
        for j = 1:size(X, 2)
            nslice = R(0)
            for i = 1:size(X, 1)
                nslice += abs(X[i, j])^2
            end
            nslice = sqrt(nslice)
            scal = 1 - gl / nslice
            scal = scal <= 0 ? R(0) : scal
            for i = 1:size(X, 1)
                Y[i, j] = scal * X[i, j]
            end
            n21X += scal * nslice
        end
    elseif f.dim == 2
        for i = 1:size(X, 1)
            nslice = R(0)
            for j = 1:size(X, 2)
                nslice += abs(X[i, j])^2
            end
            nslice = sqrt(nslice)
            scal = 1-gl/nslice
            scal = scal <= 0 ? R(0) : scal
            for j = 1:size(X, 2)
                Y[i, j] = scal * X[i, j]
            end
            n21X += scal * nslice
        end
    end
    return f.lambda * n21X
end

fun_name(f::NormL21) = "sum of Euclidean norms"
fun_dom(f::NormL21) = "AbstractArray{Real,2}, AbstractArray{Complex, 2}"
fun_expr(f::NormL21) = "x ↦ λsum(||x_i||)"
fun_params(f::NormL21) = "λ = $(f.lambda), dim = $(f.dim)"

function prox_naive(f::NormL21, X::AbstractArray{T,2}, gamma::Real=1.0) where T <: RealOrComplex
    Y = max.(0, 1 .- f.lambda * gamma ./ sqrt.(sum(abs.(X).^2, dims=f.dim))) .* X
    return Y, f.lambda * sum(sqrt.(sum(abs.(Y).^2, dims=f.dim)))
end


using .PDMats: PDMat

export kernelpdmat

"""
    Compute a positive-definite matrix in the form of a `PDMat` matrix see [PDMats.jl]() with the cholesky decomposition precomputed
    The algorithm recursively tries to add recursively a diagonal nugget until positive definiteness is achieved or that the noise is too big
"""
function kernelpdmat(
        κ::Kernel,
        X::AbstractMatrix;
        obsdim::Int = defaultobs
        )
    K = kernelmatrix(κ,X,obsdim=obsdim)
    Kmax =maximum(K)
    α = eps(eltype(K))
    while !isposdef(K+α*I) && α < 0.01*Kmax
        α *= 2.0
    end
    if α >= 0.01*Kmax
        throw(ErrorException("Adding noise on the diagonal was not sufficient to build a positive-definite matrix:\n\t- Check that your kernel parameters are not extreme\n\t- Check that your data is sufficiently sparse\n\t- Maybe use a different kernel"))
    end
    return PDMat(K+α*I)
end

kernelpdmat(κ::Kernel,X::AbstractVector{<:Real};obsdim=defaultobs) = kernelpdmat(κ,reshape(X,1,:),obsdim=2)


"""
    tnmean(a, b)

Mean of the truncated standard normal distribution in the interval [a, b].
"""
function tnmean(a::Real, b::Real)
    if !(a ≤ b)
        return oftype(middle(a, b), NaN)
    elseif a == b
        return middle(a, b)
    elseif abs(a) > abs(b)
        return -tnmean(-b, -a)
    elseif isinf(a) && isinf(b)
        return zero(middle(a, b))
    end

    @assert a < b && abs(a) ≤ abs(b)
    @assert a ≤ 0 ≤ b || 0 < a < b

    Δ = (b - a) * middle(a, b)
    #Δ = one(Δm1) + Δm1

    if a ≤ 0 ≤ b
        m = √(2/π) * expm1(-Δ) * exp(-a^2 / 2) / erf(b/√2, a/√2)
    elseif 0 < a < b
        z = exp(-Δ) * erfcx(b/√2) - erfcx(a/√2)
        iszero(z) && return middle(a, b)
        m = √(2/π) * expm1(-Δ) / z
    end
    return clamp(m, a, b)
end

"""
    tnmean(a, b, μ, σ)

Mean of the truncated normal distribution, where μ, σ are the mean and standard
deviation of the untruncated distribution.
"""
function tnmean(a, b, μ, σ)
    α = (a - μ) / σ
    β = (b - μ) / σ
    return μ + tnmean(α, β) * σ
end

"""
    tnmean(a, b)

Mean of the truncated standard normal distribution.

    tnmean(a, b, μ, σ)

Mean of the truncated normal distribution, where μ, σ are the mean and standard
deviation of the untruncated distribution.
"""
tnmean(a, b) = tnmean(a, b)

"""
    tnmom1i(a, b)

Mean of the normal distribution with variance -1 and mean 0, truncated to [a,b].
"""
function tnmom1i(a::Real, b::Real)
    if !(-Inf < a ≤ b < Inf)
        return oftype(middle(a, b), NaN)
    elseif a == b
        return middle(a, b)
    elseif abs(a) > abs(b)
        return -tnmom1i(-b, -a)
    end

    @assert -Inf < a < b < Inf && abs(a) ≤ abs(b)
    @assert a ≤ 0 ≤ b || 0 < a < b

    Δ = (b - a) * middle(a, b)
    m = (1/√2) * expm1(-Δ) / (dawson(a/√2) * exp(-Δ) - dawson(b/√2))
    return clamp(m, a, b)
end

"""
    tnmom1i(a, b, μ, σ)

Mean of the normal distribution with variance -σ^2 and mean μ,
truncated to [a,b].
"""
tnmom1i(a, b, μ, σ) = μ + σ * tnmom1i((a - μ)/σ, (b - μ)/σ)


#=
The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.

Find the sum of all the primes below two million.
=#
using Primes
@time println(sum(primes(2_000_000)))


"""
Random graph model exhibiting low GCC and high ALCC. Creates a fixed number
of cliques each with a random number of nodes. Nodes can partake in multiple
cliques.

Based on "Random clique covers for graphs with local density and global sparsity"
by Sinead A. Williamson and Mauricio Tec. The original paper may be found
at https://arxiv.org/pdf/1704.03913.pdf.
"""

using MatrixNetworks
using SpecialFunctions # Used for gamma
using LinearAlgebra # Matrix transpose
using SparseArrays
using Distributions: Poisson

"""
`rcc`
=====================

Generate a random graph by creating random clique covers.

Arguments
---------
    - `a::Float64 (> 0)`: The expected number of vertices per clique
    - `s::Float64 (∈ [0,1))`: Controls how close the degree distribution is to a power law
    - `c::Float64 (> -s)`: Controls (but not equal to) the expected number of cliques each vertex is a part of.
                           Larger c = less overlap
    - `N::Int64 (=10)`: The total number of cliques

Preconditions
-------------
    - `a > 0`
    - `s ∈ [0,1)`
    - `c > -s`
    - `N >= 0`

Examples
--------
~~~~
random_clique_cover(10.,0.9,0.1;N=100) # 100-clique graph with ~10 nodes per clique with high degree skew and medium clique overlap
random_clique_cover(50.,0.1,5.) # 10-clique graph with ~50 nodes per clique with more uniform degree distribution and high clique overlap
~~~~

The random cliques are generated using the following process. Imagine a fixed
number of restaurant customers coming up to an (infinitely long) buffet one by one.
The first customer takes the first n dishes, where n is random. Each customer
thereafter
    1) samples each already-tried dish at random, with the probability
    of taking a dish decreasing with the number of times it has been tried
    2) selects the next k (also random) dishes that have not yet been tried.

This graph model works by substituting customers for cliques and dishes for nodes. Bon appétit!
"""
function rcc(a::Float64, s::Float64, c::Float64; N::Int64=10)
    @assert a > 0
    @assert s >= 0 && s < 1
    @assert c > -s
    @assert N >= 0

    P(x) = rand(Poisson(x)) # Sample from Poisson distribution with λ = x

    ### Approximate Γ(n+c+s−1)/Γ(n+c) to avoid overflow with large n ###
    function gamma_ratio_approx(n) # Permits large N, since it avoids using large numbers
        radical = ((ℯ^(1-s)) * ((n+c+s)^(s+0.5)) / ((n+c+1)^1.5)) ^ (1/(n+c))
        return (radical * (n+c+s)/(n+c+1)) ^ (n+c) # Derived from Stirling's approximation for gamma
    end

    agammr = a*gamma(c+1) / gamma(c+s) # Constant term for determining the number of new nodes to add to a clique

    cliques = [[i for i = 1:P(a)]] # Stores cliques
    clique_count = [1 for i = 1:size(cliques[1],1)] # Stores how many cliques each node is part of

    for n = 1:N-1
        push!(cliques, []) # New empty clique
        for k = 1:size(clique_count,1) # Decide which of existing nodes to place in new clique
            if rand(Float64) <= (clique_count[k]-s) / (n+c)
                push!(cliques[n+1], k)
                clique_count[k] += 1
            end
        end

        for j = 1:P(agammr*gamma_ratio_approx(n)) # Add new nodes to graph and clique
            push!(cliques[n+1], size(clique_count,1)+1)
            push!(clique_count, 1)
        end
    end

    Z = zeros(Int64, N, size(clique_count,1)) # Convert clique list to edge clique cover.
                                              # Z[i,j] = 1 iff clique i contains node j.
    for i = 1:N
        for j in cliques[i]
            Z[i,j] = 1
        end
    end

    mat = Z' * Z # Convert edge clique cover to corresponding adjacency matrix
    mat -= Diagonal(mat) # Simplify matrix
    mat = min.(mat, 1)

    return MatrixNetwork(sparse(mat))
end


using Distributions, Random
Random.seed!(1)

N = 10^6
sig = 1.7

data1 = sqrt.(-(2* sig^2)*log.(rand(N)))

distG = Normal(0,sig)
data2 = sqrt.(rand(distG,N).^2 + rand(distG,N).^2)

distR = Rayleigh(sig)
data3 = rand(distR,N)

mean.([data1, data2, data3])


eye(A) = A^0
isnormal(A) = size(A,1) == size(A,2) && A'*A == A*A'
isunitary(A) = size(A,1) == size(A,2) && A'*A == eye(A)


using LinearAlgebra
A = [5 -2 -2; -2 4 -1; -2 -1 3]
L = cholesky(A)
##
println("A = ")
display(A)
println("L = ")
display(L.L)
##
@show norm(A - L.L*L.L')


function qnwbeta(n::Int, a::T, b::S) where {T <: Real, S <: Real}
    a -= 1
    b -= 1

    maxit = 25

    x = zeros(n)
    w = zeros(n)

    for i=1:n
        if i == 1
            an = a / n
            bn = b / n
            r1 = (1 + a) * (2.78 / (4 + n * n) + 0.768an / n)
            r2 = 1 + 1.48 * an + 0.96bn + 0.452an*an + 0.83an*bn
            z = 1 - r1 / r2

        elseif i == 2
            r1 = (4.1 + a) / ((1 + a) * (1 + 0.156a))
            r2 = 1 + 0.06 * (n - 8) * (1 + 0.12a) / n
            r3 = 1 + 0.012b * (1 + 0.25 * abs(a)) / n
            z = z - (1 - z) * r1 * r2 * r3

        elseif i == 3
            r1 = (1.67 + 0.28a) / (1 + 0.37a)
            r2 = 1 + 0.22 * (n - 8) / n
            r3 = 1 + 8 * b / ((6.28 + b) * n * n)
            z = z - (x[1] - z) * r1 * r2 * r3

        elseif i == n - 1
            r1 = (1 + 0.235b) / (0.766 + 0.119b)
            r2 = 1 / (1 + 0.639 * (n - 4) / (1 + 0.71 * (n - 4)))
            r3 = 1 / (1 + 20a / ((7.5+ a ) * n * n))
            z = z + (z - x[n-3]) * r1 * r2 * r3

        elseif i == n
            r1 = (1 + 0.37b) / (1.67 + 0.28b)
            r2 = 1 / (1 + 0.22 * (n - 8) / n)
            r3 = 1 / (1 + 8 * a / ((6.28+ a ) * n * n))
            z = z + (z - x[n-2]) * r1 * r2 * r3

        else
            z = 3 * x[i-1] - 3 * x[i-2] + x[i-3]
        end

        ab = a + b

        for its = 1:maxit
            temp = 2 + ab
            p1 = (a - b + temp * z) / 2
            p2 = 1
            for j=2:n
              p3 = p2
              p2 = p1
              temp = 2 * j + ab
              aa = 2 * j * (j + ab) * (temp - 2)
              bb = (temp - 1) * (a * a - b * b + temp * (temp - 2) * z)
              c = 2 * (j - 1 + a) * (j - 1 + b) * temp
              p1 = (bb * p2 - c * p3) / aa
            end
            pp = (n * (a - b - temp * z) * p1 +
                  2 * (n + a) * (n + b) * p2) / (temp * (1 - z * z))
            z1 = z
            z = z1 - p1 ./ pp
            if abs(z - z1) < 3e-14 break end
        end

        if its >= maxit
            error("Failure to converge in qnwbeta1")
        end

        x[i] = z
        w[i] = temp / (pp * p2)
    end

    x = (1 - x) ./ 2
    w = w * exp(gammaln(a + n) +
                gammaln(b + n) -
                gammaln(n + 1) -
                gammaln(n + ab + 1))
    w = w / (2 * exp(gammaln(a + 1) +
                     gammaln(b + 1) -
                     gammaln(ab + 2)))

    return x, w
end


module Haversine

using Parameters

R = 6.371e6 # earth's volumetric mean radius in meters

@with_kw struct GeoLocation
    λ::Real
    ϕ::Real
end

function HaversineDistance(p1::GeoLocation, p2::GeoLocation)::Float64
    λ1, ϕ1 = p1.λ, p1.ϕ
    λ2, ϕ2 = p2.λ, p2.ϕ
    Δϕ = ϕ2 - ϕ1
    Δλ = λ2 - λ1
    a = sind(Δϕ / 2)^2 + cosd(ϕ1) * cosd(ϕ2) * sind(Δλ / 2)^2
    c = 2 * atan(sqrt(a), sqrt(1 - a))
    return c * R
end

function HaversineBearing(p1::GeoLocation, p2::GeoLocation)::Float64
    λ1, ϕ1 = p1.λ, p1.ϕ
    λ2, ϕ2 = p2.λ, p2.ϕ
    Δλ = λ2 - λ1
    θ = atand(sind(Δλ) * cosd(ϕ2), cosd(ϕ1) * sind(ϕ2) - sind(ϕ1) * cosd(ϕ2) * cosd(Δλ))
    return θ
end

function HaversineDestination(geopoint::GeoLocation, θ::Real, d::Real)::GeoLocation
    λ1, ϕ1 = geopoint.λ, geopoint.ϕ
    δ = d / R
    ϕ2 = asind(sind(ϕ1) * cos(δ) + cosd(ϕ1) * sin(δ) * cosd(θ))
    λ2 = λ1 + atand(sind(θ) * sin(δ) * cosd(ϕ1), cos(δ) - sind(ϕ1) * sind(ϕ2))
    return GeoLocation(λ=λ2, ϕ=ϕ2)
end

function HaversineDistance(p1::AbstractArray{GeoLocation}, p2::AbstractArray{GeoLocation})::Vector{Float64}
    return map(HaversineDistance, p1, p2)
end

function HaversineBearing(p1::AbstractArray{GeoLocation}, p2::AbstractArray{GeoLocation})::Vector{Float64}
    return map(HaversineBearing, p1, p2)
end

function HaversineDestination(
        p::Union{AbstractArray, GeoLocation},
        θ::Union{AbstractArray, Real},
        d::Union{AbstractArray, Real}
    )::Vector{GeoLocation}
    if isa(p, GeoLocation)
        return broadcast((y, z) -> HaversineDestination(p, y, z), θ, d)
    end
    return broadcast((x, y, z) -> HaversineDestination(x, y, z), p, θ, d)
end

export GeoLocation, HaversineDistance, HaversineBearing, HaversineDestination

end


# Linear Interpolation
function (A::LinearInterpolation{<:AbstractVector{<:Number}})(t::Number)
  idx = findfirst(x->x>=t,A.t)-1
  idx == 0 ? idx += 1 : nothing
  θ = (t - A.t[idx])/ (A.t[idx+1] - A.t[idx])
  (1-θ)*A.u[idx] + θ*A.u[idx+1]
end

function (A::LinearInterpolation{<:AbstractMatrix{<:Number}})(t::Number)
  idx = findfirst(x->x>=t,A.t)-1
  idx == 0 ? idx += 1 : nothing
  θ = (t - A.t[idx])/ (A.t[idx+1] - A.t[idx])
  (1-θ)*A.u[:,idx] + θ*A.u[:,idx+1]
end

# Quadratic Interpolation
function (A::QuadraticInterpolation{<:AbstractVector{<:Number}})(t::Number)
  idx = findfirst(x->x>=t,A.t)-1
  idx == 0 ? idx += 1 : nothing
  if idx == length(A.t) - 1
    i₀ = idx - 1; i₁ = idx; i₂ = i₁ + 1;
  else
    i₀ = idx; i₁ = i₀ + 1; i₂ = i₁ + 1;
  end
  l₀ = ((t-A.t[i₁])*(t-A.t[i₂]))/((A.t[i₀]-A.t[i₁])*(A.t[i₀]-A.t[i₂]))
  l₁ = ((t-A.t[i₀])*(t-A.t[i₂]))/((A.t[i₁]-A.t[i₀])*(A.t[i₁]-A.t[i₂]))
  l₂ = ((t-A.t[i₀])*(t-A.t[i₁]))/((A.t[i₂]-A.t[i₀])*(A.t[i₂]-A.t[i₁]))
  A.u[i₀]*l₀ + A.u[i₁]*l₁ + A.u[i₂]*l₂
end

function (A::QuadraticInterpolation{<:AbstractMatrix{<:Number}})(t::Number)
  idx = findfirst(x->x>=t,A.t)-1
  idx == 0 ? idx += 1 : nothing
  if idx == length(A.t) - 1
    i₀ = idx - 1; i₁ = idx; i₂ = i₁ + 1;
  else
    i₀ = idx; i₁ = i₀ + 1; i₂ = i₁ + 1;
  end
  l₀ = ((t-A.t[i₁])*(t-A.t[i₂]))/((A.t[i₀]-A.t[i₁])*(A.t[i₀]-A.t[i₂]))
  l₁ = ((t-A.t[i₀])*(t-A.t[i₂]))/((A.t[i₁]-A.t[i₀])*(A.t[i₁]-A.t[i₂]))
  l₂ = ((t-A.t[i₀])*(t-A.t[i₁]))/((A.t[i₂]-A.t[i₀])*(A.t[i₂]-A.t[i₁]))
  A.u[:,i₀]*l₀ + A.u[:,i₁]*l₁ + A.u[:,i₂]*l₂
end

# Lagrange Interpolation
function (A::LagrangeInterpolation{<:AbstractVector{<:Number}})(t::Number)
  idxs = findRequiredIdxs(A,t)
  if A.t[idxs[1]] == t
    return A.u[idxs[1]]
  end
  N = zero(A.u[1]); D = zero(A.t[1]); tmp = N
  for i = 1:length(idxs)
    mult = one(A.t[1])
    for j = 1:(i-1)
      mult *= (A.t[idxs[i]] - A.t[idxs[j]])
    end
    for j = (i+1):length(idxs)
      mult *= (A.t[idxs[i]] - A.t[idxs[j]])
    end
    tmp = inv((t - A.t[idxs[i]]) * mult)
    D += tmp
    N += (tmp * A.u[idxs[i]])
  end
  N/D
end

function (A::LagrangeInterpolation{<:AbstractMatrix{<:Number}})(t::Number)
  idxs = findRequiredIdxs(A,t)
  if A.t[idxs[1]] == t
    return A.u[:,idxs[1]]
  end
  N = zero(A.u[:,1]); D = zero(A.t[1]); tmp = D
  for i = 1:length(idxs)
    mult = one(A.t[1])
    for j = 1:(i-1)
      mult *= (A.t[idxs[i]] - A.t[idxs[j]])
    end
    for j = (i+1):length(idxs)
      mult *= (A.t[idxs[i]] - A.t[idxs[j]])
    end
    tmp = inv((t - A.t[idxs[i]]) * mult)
    D += tmp
    @. N += (tmp * A.u[:,idxs[i]])
  end
  N/D
end

function (A::AkimaInterpolation{<:AbstractVector{<:Number}})(t::Number)
  i = searchsortedlast(A.t, t)
  i == 0 && return A.u[1]
  i == length(A.t) && return A.u[end]
  wj = t - A.t[i]
  @evalpoly wj A.u[i] A.b[i] A.c[i] A.d[i]
end

# ZeroSpline Interpolation
function (A::ZeroSpline{<:AbstractVector{<:Number}})(t::Number)
  if A.dir === :left
    # :left means that value to the left is used for interpolation
    i = searchsortedlast(A.t, t)
    return A.u[max(1, i)]
  else
    # :right means that value to the right is used for interpolation
    i = searchsortedfirst(A.t, t)
    return A.u[min(length(A.t), i)]
  end
end
 function (A::ZeroSpline{<:AbstractMatrix{<:Number}})(t::Number)
  if A.dir === :left
    # :left means that value to the left is used for interpolation
    i = searchsortedlast(A.t, t)
    return A.u[:, max(1, i)]
  else
    # :right means that value to the right is used for interpolation
    i = searchsortedfirst(A.t, t)
    return A.u[:, min(length(A.t), i)]
  end
end

# QuadraticSpline Interpolation
function (A::QuadraticSpline{<:AbstractVector{<:Number}})(t::Number)
  i = findfirst(x->x>=t,A.t)
  i == 1 ? i += 1 : nothing
  Cᵢ = A.u[i-1]
  σ = 1//2 * (A.z[i] - A.z[i-1])/(A.t[i] - A.t[i-1])
  A.z[i-1] * (t - A.t[i-1]) + σ * (t - A.t[i-1])^2 + Cᵢ
end

# CubicSpline Interpolation
function (A::CubicSpline{<:AbstractVector{<:Number}})(t::Number)
  i = findfirst(x->x>=t,A.t)
  i == nothing ? i = length(A.t) - 1 : i -= 1
  i == 0 ? i += 1 : nothing
  I = A.z[i] * (A.t[i+1] - t)^3 / (6A.h[i+1]) + A.z[i+1] * (t - A.t[i])^3 / (6A.h[i+1])
  C = (A.u[i+1]/A.h[i+1] - A.z[i+1]*A.h[i+1]/6)*(t - A.t[i])
  D = (A.u[i]/A.h[i+1] - A.z[i]*A.h[i+1]/6)*(A.t[i+1] - t)
  I + C + D
end

# BSpline Curve Interpolation
function (A::BSplineInterpolation{<:AbstractVector{<:Number}})(t::Number)
  # change t into param [0 1]
  idx = findfirst(x->x>=t,A.t) - 1
  idx == 0 ? idx += 1 : nothing
  t = A.p[idx] + (t - A.t[idx])/(A.t[idx+1] - A.t[idx]) * (A.p[idx+1] - A.p[idx])
  n = length(A.t)
  N = spline_coefficients(n,A.d,A.k,t)
  ucum = zero(eltype(A.u))
  for i = 1:n
    ucum += N[i] * A.c[i]
  end
  ucum
end

# BSpline Curve Approx
function (A::BSplineApprox{<:AbstractVector{<:Number}})(t::Number)
  # change t into param [0 1]
  idx = findfirst(x->x>=t,A.t) - 1
  idx == 0 ? idx += 1 : nothing
  t = A.p[idx] + (t - A.t[idx])/(A.t[idx+1] - A.t[idx]) * (A.p[idx+1] - A.p[idx])
  n = length(A.t)
  N = spline_coefficients(A.h,A.d,A.k,t)
  ucum = zero(eltype(A.u))
  for i = 1:A.h
    ucum += N[i] * A.c[i]
  end
  ucum
end

# Loess
function (A::Loess{<:AbstractVector{<:Number}})(t::Number)
  tmp = sort(abs.(A.t .- t))
  w = abs.(A.t .- t) ./ tmp[A.q]
  for i = 1:length(A.t)
    if w[i] <= one(A.t[1])
      w[i] = (1 - (w[i] ^ 3)) ^ 3
    else
      w[i] = zero(A.t[1])
    end
  end
  w = Diagonal(w)
  b = inv(transpose(A.x) * w * A.x) * transpose(A.x) * w * A.u
  u = zero(t[1])
  for (idx,v) in enumerate(b)
    u += v*(t^(idx-1))
  end
  u
end

# GaussianProcess
function (A::GPInterpolation{<:AbstractVector{<:Number}})(t::AbstractVector{<:Number})
  rand(A.gp,t,1)
end

function (A::GPInterpolation{<:AbstractVector{<:Number}})(t::Number)
  rand(A.gp,[t],1)[1]
end

# Curvefit
function (A::CurvefitCache{<:AbstractVector{<:Number}})(t::Union{AbstractVector{<:Number},Number})
  A.m(t,A.pmin)
end


using SpecialFunctions
using Statistics
using FFTW

function fMultiSinGen(N::Integer,
                      P::Integer,
                      M::Integer;
                      fMin::Float64=0.99,
                      fMax::Float64=1.00,
                      fs::Float64=1.0,
                      type_signal::String="full",
                      nGroup::Integer=3,
                      uStd::Float64=1.)
    """
    generates a zero-mean random phase multisine with std = 1
    INPUT
    options.N: number of points per period
    options.P: number of periods
    options.M: number of realizations
    options.fMin: minimum excited frequency
    options.fMax: maximum escited frequency
    options.fs: sample frequency
    options.type: "full", "odd", "oddrandom"

    OPTIONAL
    options.nGroup: in case of oddrandom, 1 out of nGroup odd lines is
                     discarded. Default = 3
    options.std: std of the generated signals. Default = 1

    OUTPUT
    u: NPxM record of the generated signals
    lines: excited frequency lines -> 1 = dc, 2 = fs/N

    copyright:
    Maarten Schoukens
    Vrije Universiteit Brussel, Brussels Belgium
    10/05/2017

    translated from Matlab to Julia by
    Wouter Kouw
    TU Eindhoven, Eindhoven, Netherlands
    22/01/2021

    This work is licensed under a
    Creative Commons Attribution-NonCommercial 4.0 International License
    (CC BY-NC 4.0)
    https://creativecommons.org/licenses/by-nc/4.0/
    """

    # Lines selection - select which frequencies to excite
    f0 = fs/N
    linesMin = Int64(ceil(fMin / f0) + 1)
    linesMax = Int64(floor(fMax / f0) + 1)
    lines = linesMin:linesMax

    # Remove DC component
    if lines[1] == 1; lines = lines[2:end]; end

    if type_signal == "full"
        # do nothing
    elseif type_signal == "odd"

        # remove even lines - odd indices
        if Bool(mod(lines[1],2)) # lines(1) is odd
            lines = lines[2:2:end]
        else
            lines = lines[1:2:end]
        end

    elseif type_signal == "oddrandom"

        # remove even lines - odd indices
        if Bool(mod(lines[1],2)) # lines(1) is odd
            lines = lines[2:2:end]
        else
            lines = lines[1:2:end]
        end

        # remove 1 out of nGroup lines
        nLines = length(lines)
        nRemove = floor(nLines / nGroup)
        removeInd = rand(1:nGroup, [1 nRemove])
        removeInd = removeInd + nGroup*[0:nRemove-1]
        lines = lines(!removeInd)
    end
    nLines = length(lines)

    # multisine generation - frequency domain implementation
    U = zeros(ComplexF64, N,M)

    # excite the selected frequencies
    U[lines,:] = exp.(2im*pi*rand(nLines,M))

    # go to time domain
    u = real(ifft(U))

    # rescale to obtain desired rms std
    u = uStd * u ./ std(u[:,1])

    # generate P periods
    u = repeat(u, outer=(P,1))

    return u, lines
end


"""
    AbstractNumbers

An abstract type to represent the number system on which a manifold is built.

This provides concrete number types for dispatch. The two most common number types are
the fields [`RealNumbers`](@ref) (`ℝ` for short) and [`ComplexNumbers`](@ref) (`ℂ`).
"""
abstract type AbstractNumbers end

"""
    ℝ = RealNumbers()

The field of real numbers.
"""
struct RealNumbers <: AbstractNumbers end

"""
    ℂ = ComplexNumbers()

The field of complex numbers.
"""
struct ComplexNumbers <: AbstractNumbers end

"""
    ℍ = QuaternionNumbers()

The division algebra of quaternions.
"""
struct QuaternionNumbers <: AbstractNumbers end

const ℝ = RealNumbers()
const ℂ = ComplexNumbers()
const ℍ = QuaternionNumbers()

Base.show(io::IO, ::RealNumbers) = print(io, "ℝ")
Base.show(io::IO, ::ComplexNumbers) = print(io, "ℂ")
Base.show(io::IO, ::QuaternionNumbers) = print(io, "ℍ")

^(𝔽::AbstractNumbers, n) = Euclidean(n...; field = 𝔽)

@doc doc"""
    real_dimension(𝔽::AbstractNumbers)

Return the real dimension $\dim_ℝ 𝔽$ of the [`AbstractNumbers`] system `𝔽`.
The real dimension is the dimension of a real vector space with which a number in `𝔽` can be
identified.
For example, [`ComplexNumbers`](@ref) have a real dimension of 2, and
[`QuaternionNumbers`](@ref) have a real dimension of 4.
"""
function real_dimension(𝔽::AbstractNumbers)
    error("real_dimension not defined for number system $(𝔽)")
end
real_dimension(::RealNumbers) = 1
real_dimension(::ComplexNumbers) = 2
real_dimension(::QuaternionNumbers) = 4


using DataDrivenDiffEq
using ModelingToolkit
using OrdinaryDiffEq

using Dierckx
using Statistics
using LinearAlgebra
using Plots
gr()

# Create a test problem
function lorenz(u,p,t)
    x, y, z = u

    ẋ = 10.0*(y - x)
    ẏ = x*(28.0-z) - y
    ż = x*y - (8/3)*z
    return [ẋ, ẏ, ż]
end

u0 = [-8.;8.;27.]
tspan = (0.0,200.0)
dt = 0.001
prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob, Tsit5(), reltol = 1e-12, abstol = 1e-12,  saveat = dt)

plot(sol,vars=(1,2,3))

# Differential data from equations
X = Array(sol)

# Time delay coordinates
stackmax = 100

H = zeros(eltype(x), stackmax, size(X,2)-stackmax)
for i in 1:stackmax
    H[i, :] = X[1, i:end-stackmax+i-1]
end

m,n = minimum(size(H)), maximum(size(H))
U, S, V = svd(H, full = false)
τ = DataDrivenDiffEq.optimal_svht(m,n)
r = length(S[S .> τ*median(S)])
r = minimum([15, r])

z = Array(V[:, 1:r]')
dz = similar(z)

for (i, vi) in enumerate(eachrow(z))
    x_int = Spline1D(sol.t[1:length(vi)], vi)
    dz[i, :] = derivative(x_int, sol.t[1:length(vi)])
end

plot(plot(z[1, :]),plot(z[end, :].^2),  layout = (2, 1))

@variables u[1:r]

basis = Basis(u, u)
opt = SR3(1e-1)
b = SInDy(z[:, 1:end], dz[1:end-1, 1:end], basis, maxiter = 1000, opt = opt, normalize = true)

# Coincides with the paper results
println(b)


# Hellwarth 1999 PRB - Part IV; T-dep of the Feynman variation parameter
# A Friday afternoon of hacking to try and implement the T-dep electron-phonon coupling from the above PRB
# Which was unusually successful! And more or less reproduced Table III

# one-dimensional numerical integration in Julia using adaptive Gauss-Kronrod quadrature
using QuadGK

# Equation numbers follow above Hellwarth 1999 PRB
# 62b
A(v,w,β)=3/β*( log(v/w) - 1/2*log(2*π*β) - log(sinh(v*β/2)/sinh(w*β/2)))

# 62d
Y(x,v,β)=1/(1-exp(-v*β))*(1+exp(-v*β)-exp(-v*x)-exp(v*(x-β)))
# 62c integrand
f(x,v,w,β)=(exp(β-x)+exp(x))/(w^2*x*(1-x/β)+Y(x,v,β)*(v^2-w^2)/v)^(1/2)
# 62c
B(v,w,β,α) = α*v/(sqrt(π)*(exp(β)-1)) * quadgk(x->f(x,v,w,β),0,β/2)[1]
#62e
C(v,w,β)=3/4*(v^2-w^2)/v * (coth(v*β/2)-2/(v*β))

F(v,w,β,α)=-(A(v,w,β)+B(v,w,β,α)+C(v,w,β)) #(62a)

# Can now evaluate, e.g.
# F(v,w,β,α)=F(7.2,6.5,1.0,1.0)
# BUT - this is just the objective function! Not the optimised parameters.
# Also there's a scary numeric integration (quadgk) buried within...

"Print out F(alpha,beta) for a specific v,w; as a test"
function test_fns()
    @printf("\t\t")
    for α in 1:5
        @printf("α=%d\t\t",α)
    end
    @printf("\n")

    for β in 1:0.25:3.0
        v=w=4
        print("β: $β  \t||")
        for α in 1:5
            @printf("%f\t",F(v,w,β,α))
        end
        println()
    end
end

test_fns()  # OK - very primitive!

"
These are 1D traces along the solution for Alpha=Beta=1 in Helwarth PRB TABLE III,
this was used to correct a transcription error in the above typed-in equations
It was also good to see what F(v,w) looked like as a function of v and w near an optimal solution"
function test_trace()
	v=7.20
	w=6.5
	α=1.0
	β=1.0

	for v=6:0.1:8
    	@printf("%f %f\n",v,F(v,w,β,α))
	end

	@printf("\n")
	v=7.20
	for w=6:0.1:7
    	@printf("%f %f\n",w,F(v,w,β,α))
	end
end
test_trace()

# Angle for the ringside seats, when the fall, don't blame me, Bring on the Major Leagues
using Optim
# Julia package stuffed full of magic, does auto-differentation & etc. etc.

Fopt(x) = F(x[1],x[2],1,1)

function test_Fopt()
    show(Fopt([7.2,6.5]))
# OK! It looks like I can bury the alpha, beta parameters (which we don't optimise), by wrapping our function in a function definition.
    initial=[7.2,6.5]

    show(optimize(Fopt,  initial, LBFGS()))

    show(optimize(Fopt, initial, BFGS(), Optim.Options(autodiff=true)))
end

test_Fopt()

function test_Optim()
	# After a bit of fiddling, I figured out how to add bounds, to stop that 'DomainError',
	# which occurs where the you are evaluating log(-ve Real), i.e. w<0.0 or v<0.0

	initial=[7.2,6.5]

	lower=[0.0,0.0]
	upper=[10.0,10.0]

	@printf("\t\t")
	for α in 1:5
		@printf("α=%d\t\t",α)
	end
	@printf("\n")

	for β in 1:0.25:3.0
		print("β: $β  \t||")
		for α in 1:5
			myf(x) = F(x[1],x[2],β,α)
			solution=optimize(DifferentiableFunction(myf), initial, lower, upper, Fminbox(); optimizer = ConjugateGradient, optimizer_o=Optim.Options(autodiff=true))
			minimum=Optim.minimizer(solution)

			v=minimum[1]
			w=minimum[2]
			#print(solution,"\t")
			@printf("%.2f %.2f\t",v,w)

		end
		println()
	end
end

test_Optim()

function test_Optimisers()
	# So that looks really good! I was super stoked to see how close these values are to TABLE III in Hellwarth
	# However, the solutions all start on (7.20,6.50) so that top-left data point is cheating, whereas the
	# others have some disagreement / noise associated with them
	# I was wondering whether it might be a function of the optimiser, so thought I'd try them all

	initial=[7.1,6.5]
	# Main use of these bounds is stopping v or w going negative, at which you get a NaN error as you are evaluating log(-ve Real)
	lower=[1.0,1.0]
	upper=[10.0,10.0]

	for optimizer in [BFGS, LBFGS, ConjugateGradient] # Newton, GradientDescent, NelderMead - steps outside box & log(-ve)->NaN error
		@printf("\n\t\t##### NOW TRIALING: %s #####\n\n",optimizer)

		@printf("\t\t")
		for α in 1:5
			@printf("α=%d\t\t",α)
		end
		@printf("\n")

		for β in 1:0.25:3.0
			print("β: $β  \t||")
			for α in 1:5
				myf(x) = F(x[1],x[2],β,α)
				res=optimize(DifferentiableFunction(myf), initial, lower, upper, Fminbox(); optimizer = optimizer, optimizer_o=Optim.Options(autodiff=true))
				minimum=Optim.minimizer(res)
				#show(Optim.converged(res)) # All came out as 'true'

				#print(solution,"\t")
				@printf("%.2f %.2f\t",minimum[1],minimum[2])
			end
			println()
		end
	end
end

test_Optimisers()


module IsingMCMC

function genFunctor( Jd, Bd)
    Jd = Jd
    Bd = Bd

    # E(S) = - 1/2\sum_{i,j, i not j} J_{i,j} si sj - \sum_i B_i si
    function E(S, N)
        local Et = 0
        for i in 1:N
            for (j,Jv) in Jd[i]
                @inbounds Et += -S[i]*Jv*S[j]/2
            end
            @inbounds Et += -S[i]*Bd[i]
        end
        Et
    end

    # differential energy
    function dE(S, k, N)
        local dEt = 0
        for (j,Jv) in Jd[k]
            @inbounds dEt += S[k]*Jv*S[j]*2
        end
        @inbounds dEt += S[k]*Bd[k]*2
        dEt
    end

    # probability for dE
    PrE(dE, T) = exp(-dE/T)

    # flip spin at x
    function flipx!(list, x)
        list[x] *= -1
    end

    # MCMC
    function MCMC( T, N, trial)
        # initialize
        simE = zeros(Int, div(trial,1000)+1)
        S = ones(Int8, N)
        for i in 1:N
            if rand() < 0.5
                flipx!(S,i) # random flip at first
            end
        end

        # MCMC trial
        Ec = E(S,N) # current energy
        simE[1] = Ec
        local k, de
        @inbounds for t in 1:trial
            k  = rand(1:N) # Gibbs sampling position
            de = dE(S, k, N)
            if rand() < PrE(de, T) # MH criteria
                flipx!(S, k) # change k
                Ec += de     # change E
            end
            (t % 1000 == 0) && (simE[div(t,1000)+1] = Ec)
        end

        (simE, S)
    end

    return (E, MCMC)
end # of functor

end # of module


""" Julia program to Check if a number is a Mystery Number or Not.

A mystery number is that number which can be expressed as a sum of two
numbers and those two numbers must be the reverse of each other
"""

function reverse(n)
    rev = 0
    while n > 0
        rev = rev * 10
        rev = rev + (n % 10);
        n = n ÷ 10
    end
    return rev
end

function check_mystery_num(n)
    # Let us iterate from 1 upto n/2, to check if there is any pair, such that thier
    # sum equals the given number and check if the pair are reverse of each other
    temp = n ÷ 2
    for i in 1:(temp)
        if (reverse(i) == (n - i))
            return true
        end
    end
    return false
end

print("Enter the number: ")
num = readline()
num = parse(Int, num)
res = check_mystery_num(num)
if res
    println("The given number $num is a Mystery Number.")
else
    println("The given number $num is not a Mystery Number.")
end

"""
Time Complexity: O(n*log(n)), where 'n' is the given number
Space Complexity: O(1)

SAMPLE INPUT AND OUTPUT

SAMPLE 1
Enter the number: 43234
The given number 43234 is a Mystery Number.

SAMPLE 2
Enter the number: 1234
The given number 1234 is not a Mystery Number.

"""


function rbc()
    @parameters α β ρ δ σ Ω_1
    @variables k z c q i
    @make_markov k z c q i
    x = [k, z]
    y = [c, q, i]
    p = [α, β, ρ]
    p_f = [δ, σ, Ω_1]

    H = [1 / c - (β / c_p) * (α * exp(z_p) * k_p^(α - 1) + (1 - δ)),
         c + k_p - (1 - δ) * k - q,
         q - exp(z) * k^α,
         z_p - ρ * z,
         i - (k_p - (1 - δ) * k)]

    x̄ = [k_ss ~ (((1 / β) - 1 + δ) / α)^(1 / (α - 1)), z_ss ~ 0]
    ȳ = [c_ss ~ (((1 / β) - 1 + δ) / α)^(α / (α - 1)) -
                δ * (((1 / β) - 1 + δ) / α)^(1 / (α - 1)),
         q_ss ~ (((1 / β) - 1 + δ) / α)^(α / (α - 1)),
         i_ss ~ δ * (((1 / β) - 1 + δ) / α)^(1 / (α - 1))]

    x̄_iv = [z_ss ~ 0, k_ss ~ (((1 / β) - 1 + δ) / α)^(1 / (α - 1))]
    ȳ_iv = [q_ss ~ (((1 / β) - 1 + δ) / α)^(α / (α - 1)),
            c_ss ~ (((1 / β) - 1 + δ) / α)^(α / (α - 1)) -
                    δ * (((1 / β) - 1 + δ) / α)^(1 / (α - 1)),
            i_ss ~ δ * (((1 / β) - 1 + δ) / α)^(1 / (α - 1))]

    n_ϵ = 1
    n_z = 2
    n_x = length(x)
    n_y = length(y)
    n_p = length(p)
    Γ = reshape([σ], n_ϵ, n_ϵ)
    η = reshape([0; -1], n_x, n_ϵ) # η is n_x * n_ϵ matrix

    Q = zeros(n_z, n_y + n_x) # The order is [y, x]
    Q[1, 1] = 1.0 # c
    Q[2, 3] = 1.0 # i

    Ω = [Ω_1, Ω_1]

    return H, (; x, y, x̄, ȳ, Γ, η, p_f, p, x̄_iv, ȳ_iv, Ω, Q), "rbc"
end


using DelimitedFiles

function sigmoid(x)
        return 1/(1+exp.(-x));
end

# Set manually so that scaling is equivalent to training set.
resistanceMean = 49999.784590407784;
resistanceStdDev = 28617.43200535356;
capacitanceMean = 4.894379617495234e-6;
capacitanceStdDev = 2.9095702774112495e-6;
timeMean = 0.6064870129571632;
timeStdDev = 0.7246955112804028;

layerOneWeights = readdlm("layer1.csv", ',', Float64, '\n');
layerTwoWeights = readdlm("layer2.csv", ',', Float64, '\n');

data = readdlm(ARGS[1], ',', Float64, '\n');
numOfTestPoints = size(data, 2);

inputs = vcat(data[1:3,:], ones(numOfTestPoints)'); # Adds row of ones for bias term in weights
expectedOutput = data[4,:];
print(size(expectedOutput))

# Normalize inputs
inputs[1,:] .-= resistanceMean;
inputs[1,:] ./= resistanceStdDev;
inputs[2,:] .-= capacitanceMean;
inputs[2,:] ./= capacitanceStdDev;
inputs[3,:] .-= timeMean;
inputs[3,:] ./= timeStdDev;

global cost = 0.0;

for i = 1:numOfTestPoints
        netOne = layerOneWeights * inputs[:,i];
        outputOne = vcat(sigmoid.(netOne), 1);

        netTwo = layerTwoWeights * outputOne;
        outputTwo = netTwo[1];

        absoluteError = outputTwo - expectedOutput[i];
        global cost += absoluteError .^ 2;
        percentError = abs(absoluteError / expectedOutput[i]) * 100;

        print("Percent error at test point ", i, ": ", percentError, "% \n")
end

global cost /= numOfTestPoints;

print("Total MSE: ", cost, '\n')


module SolarModel

using ..vSmartMOM               # For locating default solar T
using DocStringExtensions       # For simplifying docstring
using DelimitedFiles            # For easily reading in solar spectrum
using Interpolations            # For interpolating solar spectrum
using Pkg.Artifacts             # For default solar spectrum

"""
    $(FUNCTIONNAME)(T::Real, ν_grid::Vector)

Produce the black-body planck spectrum (mW/m²-sr-cm⁻¹), given the temperature (K)
and calculation grid (ν in cm⁻¹)

"""
function planck_spectrum_wn(T::Real, ν_grid::Vector)

    c1 = 1.1910427 * 10^(-5)    # mW/m²-sr-cm⁻¹
    c2 = 1.4387752              # K⋅cm

    # L(ν, T) = c1⋅ν³/(exp(c2⋅ν/T) - 1)
    radiance = c1 .* (ν_grid.^3) ./ (exp.(c2 * ν_grid / T) .- 1)

    return radiance
end

"""
    $(FUNCTIONNAME)(T::Real, λ_grid::Vector)

Produce the black-body planck spectrum (W/m²-sr-μm), given the temperature (K)
and calculation grid (λ in μm)

"""
function planck_spectrum_wl(T::Real, λ_grid::Vector)

    c1 = 1.1910427 * 10^8    # W/m²-sr-μm
    c2 = 1.4387752 * 10^4    # K⋅μm

    # L(ν, T) = c1⋅ν³/(exp(c2⋅ν/T) - 1)
    radiance = c1 ./ (λ_grid.^5 .* (exp.(c2 ./ (λ_grid * T)) .- 1))

    return radiance
end

# W/m²-sr-μm to Ph/s-m²-sr-um
# λ_grid in micron
function watts_to_photons(λ_grid::Vector, radiance::Vector)

    h = 6.62607015e-34 # J⋅Hz−1
    c = 299792458 # m/s

    E_per_λ = h * c ./ (λ_grid / 1e6)
    photons = radiance ./ E_per_λ

    return photons
end

"""
    $(FUNCTIONNAME)(T::Real; stride_length::Integer = 100)

Produce the black-body planck spectrum (mW/m²-sr-cm⁻¹), given the temperature (K).
Use a unit calculation grid and check for convergence every `stride_length` cm⁻¹ until the
spectrum dies off.

"""
function planck_spectrum_wn(T::Real; stride_length::Integer = 100)

    # νs, starting with ν0 = 1.0 cm⁻¹
    νs = [1.0]

    # radiances corresponding with νs
    radiances = planck_spectrum_wn(T, νs)

    # Loop until convergence
    while true

        # Add the next ν
        νs = vcat(νs, collect(νs[end] + 1 : νs[end] + stride_length))

        # Compute the next radiance
        radiances = vcat(radiances, planck_spectrum_wn(T, νs[(end - stride_length + 1) : end]))

        # Exit if spectrum has died off
        (radiances[end] < radiances[1]) && break

    end

    return [νs[1:(end-1)] radiances[1:(end-1)]]
end

"""
    $(FUNCTIONNAME)(solar, ν_grid)

Interpolate a solar linelist to the ν_grid
"""
function itp_solar_to_ν_grid(solar, ν_grid)

    solar_idx_start = maximum((argmin(abs.(solar[:, 1] .- minimum(ν_grid))) - 10, 1))
    solar_idx_end   = minimum((argmin(abs.(solar[:, 1] .- maximum(ν_grid))) + 10, length(solar[:,1])))

    solar_subset = solar[solar_idx_start:solar_idx_end, :]

    itp = LinearInterpolation(solar_subset[:, 1],
                              solar_subset[:, 2])

    return itp.(ν_grid)
end

"""
    $(FUNCTIONNAME)(file_name::String)

Get the solar transmission from the specified file
"""
solar_transmission_from_file(file_name::String) = readdlm(file_name)

"""
    $(FUNCTIONNAME)(file_name::String, ν_grid::Union{AbstractRange{<:Real}, AbstractArray})

Get the solar transmission from the specified file, and interpolate to wavenumber grid
"""
function solar_transmission_from_file(file_name::String,
                                      ν_grid::Union{AbstractRange{<:Real}, AbstractArray})

    solar = solar_transmission_from_file(file_name)
    return itp_solar_to_ν_grid(solar, ν_grid)
end

"""
    $(FUNCTIONNAME)(ν_grid::Union{AbstractRange{<:Real}, AbstractArray} = 600.0:0.01:26316.0)

Get the default solar transmission and interpolate to wavenumber grid (entire grid if not specified)
"""
function default_solar_transmission(ν_grid::Union{AbstractRange{<:Real}, AbstractArray} = 600.0:0.01:26316.0)

    @info "Using line-list from:\nToon, G. C., Solar line list for GGG2014, TCCON data archive, hosted by the Carbon Dioxide Information Analysis Center, Oak Ridge National Laboratory, Oak Ridge, Tennessee, U.S.A., doi:10. 14291/tccon.ggg2014.solar.R0/1221658, 2014."
    @info "Found at: https://mark4sun.jpl.nasa.gov/toon/solar/solar_spectrum.html"

    filename = joinpath(dirname(pathof(RadiativeTransfer)), "SolarModel", "solar.out")
    !isfile(filename) && download("http://web.gps.caltech.edu/~cfranken/hitran_2016/solar_merged_20160127_600_26316_100.out", filename)

    return hcat(ν_grid, solar_transmission_from_file(filename, ν_grid))
end

"""
    $(FUNCTIONNAME)(ν_grid::Union{AbstractRange{<:Real}, AbstractArray} = 600.0:0.01:26316.0)

Get the default solar spectrum and interpolate to wavenumber grid (entire grid if not specified)
"""
function default_solar_spectrum_at_earth(ν_grid::Union{AbstractRange{<:Real}, AbstractArray} = 600.0:0.01:26316.0)

    T = 5777 # K
    λ_grid = reverse(1e4 ./ ν_grid) # Wavenumber to micron
    black_body = reverse(SolarModel.watts_to_photons(λ_grid, planck_spectrum_wl(T, λ_grid) * 2.1629e-05 * pi))
    solar_transmission = default_solar_transmission(ν_grid)[:,2]

    return hcat(ν_grid, black_body .* solar_transmission)
end

export planck_spectrum_wn, planck_spectrum_wl, solar_transmission_from_file, default_solar_transmission

end


abstract type SpectralEstimate{D,T} <: AbstractVector{T} end

@doc raw"""
    Periodogram(timeseries, Δ)

Compute the periodogram for the provided timeseries with sampling rate Δ.

# Arguments
- `timeseries`: A `Vector` if univariate and an `n` by `d` `Matrix` if multivariate, where `n` is the number of observations and `d` is the dimension of the timeseries.
- `Δ`: A positive real number.

The periodogram is defined as
```math
\boldsymbol I(ω)&=\boldsymbol J(ω) \boldsymbol J(ω)^H \quad \text{where} \quad \boldsymbol J(ω) = \sqrt{\frac{Δ}{2π n}}\sum_{t=0}^{n-1} \boldsymbol{P}_{tΔ}e^{-itΔ ω}
```

Note the periodogram is in terms of angular frequency here, and uses the normalisation ``Δ/2π``.
The choice of normalisation is essentially arbitrary; however, this matches our definition for the spectral density function.
"""
struct Periodogram{D,T,V} <: SpectralEstimate{D,T}
    Ω::V
    ordinates::Vector{T}
    function Periodogram(timeseries::Matrix{T}, Δ::Real) where {T}
        Δ > 0 || throw(ArgumentError("Δ should be a positive."))
        J = fftshift(fft(timeseries, 1),1)
        n = size(J,1)
        D = size(J,2)
        ordinates = [(J[ii, :]*J[ii, :]'.*(Δ / (2π * n))) for ii ∈ 1:size(J,1)]
        Ω = fftshift(fftfreq(n, 2π/Δ))
        new{D, eltype(ordinates), typeof(Ω)}(Ω, ordinates)
    end
    function Periodogram(timeseries::Vector{T}, Δ::Real) where {T}
        Δ > 0 || throw(ArgumentError("Δ should be a positive."))
        J = fftshift(fft(timeseries, 1),1)
        n = size(J,1)
        ordinates = abs.(J.^2).*(Δ / (2π * n))
        Ω = fftshift(fftfreq(n, 2π/Δ))
        new{1, eltype(ordinates), typeof(Ω)}(Ω, ordinates)
    end
    function Periodogram(Ω, ordinates::Vector{T}) where {T}
        new{size(ordinates[1],1), T, typeof(Ω)}(Ω, ordinates)
    end
end

@doc raw"""
    BartlettPeriodogram(timeseries, Δ, segmentlength)

Compute the Bartlett periodogram for the provided timeseries with sampling rate Δ.

# Arguments
- `timeseries`: A `Vector` if univariate and an `n` by `d` `Matrix` if multivariate, where `n` is the number of observations and `d` is the dimension of the timeseries.
- `Δ`: A positive real number.
- `segmentlength`: the length of series used in each segment.

Computes an estimate of the spectral density function using Bartlett's method. Using the same normalisation as `Periodogram`.

# External links

* [Bartlett's method on Wikipedia](https://en.wikipedia.org/wiki/Bartlett%27s_method)

"""
struct BartlettPeriodogram{D,T,V} <: SpectralEstimate{D,T}
    Ω::V
    ordinates::Vector{T}
    function BartlettPeriodogram(timeseries::Matrix{T}, Δ::Real, segmentlength::Int = Int(100÷Δ)) where {T}
        nsegments = size(timeseries, 1) ÷ segmentlength
        P = Periodogram(timeseries[1:segmentlength, :], Δ)

        for ii = 1:nsegments-1
            P += Periodogram(timeseries[segmentlength*ii.+(1:segmentlength), :], Δ)
        end

        BartlettPeriodogram(P/nsegments)
    end
    function BartlettPeriodogram(timeseries::Vector{T}, Δ::Real, segmentlength::Int = Int(100÷Δ)) where {T}
        nsegments = size(timeseries, 1) ÷ segmentlength
        P = Periodogram(timeseries[1:segmentlength], Δ)

        for ii = 1:nsegments-1
            P += Periodogram(timeseries[segmentlength*ii.+(1:segmentlength)], Δ)
        end

        BartlettPeriodogram(P/nsegments)
    end
    function BartlettPeriodogram(p::Periodogram{D,T}) where {D,T}
        new{D, T, typeof(p.Ω)}(p.Ω, p.ordinates)
    end
    function BartlettPeriodogram(Ω, ordinates::Vector{T}) where {T}
        new{size(ordinates[1],1), T, typeof(Ω)}(Ω, ordinates)
    end
end

struct CoherancyEstimate{D,T,V} <: SpectralEstimate{D,T}
    Ω::V
    ordinates::Vector{T}

    function CoherancyEstimate(timeseries::Matrix{T}, Δ::Real, segmentlength::Int = Int(100÷Δ)) where {T}
        S = BartlettPeriodogram(timeseries, Δ, segmentlength)
        CoherancyEstimate(S)
    end

    function CoherancyEstimate(S::SpectralEstimate{D,T}) where {D,T}
        newordinates = [[s[i,j] / sqrt(s[i,i]*s[j,j]) for i ∈ 1:D, j ∈ 1:D] for s ∈ S.ordinates]
        new{D,eltype(newordinates),typeof(S.Ω)}(S.Ω, newordinates)
    end

    function CoherancyEstimate(Ω, ordinates::Vector{T}) where {T}
        new{size(ordinates[1],1), T, typeof(Ω)}(Ω, ordinates)
    end
end

ndims(::SpectralEstimate{D,T}) where {D,T} = D
size(ŝ::SpectralEstimate) = (length(getfreq(ŝ)),)
getindex(ŝ::T, inds) where {T<:SpectralEstimate} = getconstructor(T)(getfreq(ŝ)[inds], getordinate(ŝ)[inds])
getindex(ŝ::SpectralEstimate, ind::Int) = (getfreq(ŝ)[ind],getordinate(ŝ)[ind])
log10(ŝ::T) where {T<:SpectralEstimate} = getconstructor(T)(getfreq(ŝ), log10.(getordinate(ŝ)))
getfreq(p::Periodogram) = p.Ω
getfreq(b::BartlettPeriodogram) = b.Ω
getfreq(c::CoherancyEstimate) = c.Ω
getordinate(p::Periodogram) = p.ordinates
getordinate(b::BartlettPeriodogram) = b.ordinates
getordinate(c::CoherancyEstimate) = c.ordinates
getconstructor(::Type{<:Periodogram}) = Periodogram
getconstructor(::Type{<:BartlettPeriodogram}) = BartlettPeriodogram
getconstructor(::Type{<:CoherancyEstimate}) = CoherancyEstimate

@recipe function f(ŝ::SpectralEstimate)
    HermitianPlot(getfreq(ŝ), getordinate(ŝ))
end

@recipe function f(ŝ::SpectralEstimate{1,T}) where {T} # univariate plotting
    getfreq(ŝ), getordinate(ŝ)
end

function Base.:+(ŝ₁::T, ŝ₂::T) where {T<:SpectralEstimate}
    getfreq(ŝ₁) == getfreq(ŝ₂) || error("Frequencies must be the same to add spectral estimates.")
    getconstructor(T)(getfreq(ŝ₁), getordinate(ŝ₁) .+ getordinate(ŝ₂))
end

function Base.:/(ŝ::T, a::Real) where {T<:SpectralEstimate}
    getconstructor(T)(getfreq(ŝ), getordinate(ŝ)./a)
end


using Compat

if VERSION.minor == 7
  @eval using LinearAlgebra, SparseMatrices
end

ProjDir = dirname(@__FILE__)
cd(ProjDir) do

A = Float64[
  1/2 1/3 1/4 1/5 1/6;
  1/3 1/4 1/5 1/6 1/7;
  1/4 1/5 1/6 1/7 1/8;
  1/5 1/6 1/7 1/8 1/9;
  1/6 1/7 1/8 1/9 1/10;
]
b = ones(5)

(Al, Au, Ap) = lu(A)

@show y = Al \ b[Ap]
println()

@show x = Au \ y
println()

@show As = sparse(A)
println()

@show As\b
println()

@show F = lufact(As)
println()

if VERSION.minor == 7
  @show F.L * F.U == F.Rs .* A[F.p, :]
  println()

  @show F.L*F.U
  println()

  @show sparse(F.Rs.* A[F.p, F.q])
  println()
else
  @show ys = F[:L] \ (F[:Rs] .* b)
  println()

  @show x = F[:U] \ ys
  println()

  @show F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]
  println()

  @show F[:L]*F[:U]
  println()

  @show sparse((F[:Rs] .* A)[F[:p], F[:q]])
  println()
end

@show lufact(A)
println()

end


export
    LinearRegressionModel,
    fit!,
    predict

mutable struct LinearRegressionModel
    df::DataFrame
    label::Symbol
    features::Vector{Symbol}
    argv::Vector{Float64}
end

function LinearRegressionModel(df::DataFrame, label::Symbol, features::Vector{Symbol})
    return LinearRegressionModel(df, label, features, rand(length(features)+1))
end

function LinearRegressionModel(df::DataFrame, label::Symbol, features::Symbol)
    return LinearRegressionModel(df, label, [features], rand(2))
end

function g(model::LinearRegressionModel, row_n::Int64)
    xs = collect(model.df[row_n, model.features])
    pushfirst!(xs, 1)
    y = sum(model.argv .* xs)

    return y
end

function loss(model::LinearRegressionModel)
    n = nrow(model.df)
    l = 0
    for i in 1:n
        l += (g(model, i) - model.df[i, model.label])^2
    end

    return l/n
end

function fit!(model::LinearRegressionModel; lr=1e-4, atol::Float64=1e-6, show=false)
    while (l = loss(model)) > atol
        show && println("Loss: $l")
        dl_da = 0
        dl_db = zeros(length(model.features))
        for i in 1:nrow(model.df)
            # intersection
            dl_da += g(model, i) - model.df[i, model.label]

            # sloaps
            for (j, f) in enumerate(model.features)
                dl_db[j] += (g(model, i) - model.df[i, model.label]) * model.df[i, f]
            end
        end

        # intersection
        model.argv[1] -= lr * 2 * dl_da
        # slopes
        for j in 2:length(model.argv)
            model.argv[j] -= lr * 2 * dl_db[j-1]
        end
    end
end

function predict(model::LinearRegressionModel, xs::Vector{<:Real})
    pushfirst!(xs, 1)
    y = sum(model.argv .* xs)

    return y
end

function predict(model::LinearRegressionModel, x::Real)
    return predict(model, [x])
end


include("11_two_site_state_preparation.jl")

println("
#######################################################
# Tutorial 12                                         #
#                                                     #
# n-site circuit optimization with MPS                #
#######################################################
")

# Beyond full state, use approximations
n = 30
i = [Index(2, "S=1/2") for j in 1:n]

# Product state |Z+Z+Z+…Z+⟩
Zp = MPS(i, "Z+")

# Product state |Z-Z-Z-…Z-⟩
Zm = MPS(i, "Z-")

inner(Zp, Zp)
inner(Zm, Zp)

# Do some arithmatic with them, like tensors:
# TODO: fix this when normalize PR is merged!
#Cat = (Zp + Zm) / √2
#
# Performs a truncated MPS addition,
# returns an MPS approximate up to some tolerance.
Cat = (Zp + Zm) * (1 / √2)
@show maxlinkdim(Zp)
@show maxlinkdim(Zm)
@show maxlinkdim(Cat)
@show norm(Cat)
@show inner(Cat, Zp)
@show inner(Cat, Zm)

# Apply operators:
j = n ÷ 2
Xⱼ = op("X", i[j])
XⱼZp = apply(Xⱼ, Zp)

XⱼZp = MPS(i, [k == j ? "Z-" : "Z+" for k in 1:n])
@show inner(XⱼZp, apply(Xⱼ, Zp))

function ising(n; h)
  # Symbolic representation
  # of the Hamiltonian ("operator sum")
  H = OpSum()
  for j in 1:(n - 1)
    H -= "Z", j, "Z", j + 1
  end
  for j in 1:n
    H += h, "X", j
  end
  return H
end

h = 0.5
H = MPO(ising(n; h=h), i)

# Performs a truncated MPO*MPS contraction,
# returns an MPS approximate up to some tolerance.
HZp = H * Zp
@show inner(Zp', HZp)
@show inner(Zp', H, Zp)

# Gradient energy minimization
# Already written above! New Hamiltonian.
function E(ψ)
  ψHψ = inner(ψ', H, ψ)
  ψψ = inner(ψ, ψ)
  return ψHψ / ψψ
end

function minimize(f, ∂f, x; nsteps, γ, kwargs...)
  for n in 1:nsteps
    println("n = ", n, ", f_x = ", f(x))
    x = -(x, γ * ∂f(x); kwargs...)
  end
  return x
end

ψ⁰ = MPS(i, "Z+")
∂E(x) = gradient(E, x)[1]
ψ = minimize(E, ∂E, ψ⁰; nsteps=50, γ=0.1, maxdim=10, cutoff=1e-5)

# TODO: change to:
# ψ /= norm(ψ)
ψ *= inv(norm(ψ))

Eᵈᵐʳᵍ, ψᵈᵐʳᵍ = dmrg(H, ψ⁰; nsweeps=10, maxdim=10, cutoff=1e-5)

@show maxlinkdim(ψ⁰)
@show maxlinkdim(ψ)
@show maxlinkdim(ψᵈᵐʳᵍ)
@show E(ψ⁰), norm(∂E(ψ⁰))
@show E(ψ), norm(∂E(ψ))
@show E(ψᵈᵐʳᵍ), norm(∂E(ψᵈᵐʳᵍ))

#
# Circuit optimization
#

# Form the circuit from parameters
Ry_layer(θ, i) = [op("Ry", i[j]; θ=θ[j]) for j in 1:n]
CX_layer(i) = [op("CX", i[j], i[j+1]) for j in 1:2:(n-1)]

function U(θ, i; nlayers)
  n = length(i)
  Uᶿ = Ry_layer(θ[1:n], i)
  for l in 1:(nlayers - 1)
    Uᶿ = [Uᶿ; CX_layer(i)]
    Uᶿ = [Uᶿ; Ry_layer(θ[(1:n) .+ l * n], i)]
  end
  return Uᶿ
end

nlayers = 6
maxdim = 10
cutoff = 1e-5

# Find the circuit `U(θ)` that minimizes:
# E(θ) = ⟨0|U(θ)† H U(θ)|0⟩ = ⟨θ|H|θ⟩
function E(θ)
  # Apply the circuit:
  # |θ⟩ = U(θ)|0⟩
  ψᶿ = apply(U(θ, i; nlayers=nlayers), ψ⁰; maxdim=maxdim, cutoff=cutoff)

  # Compute the expecation value: ⟨θ|H|θ⟩
  # No need to normalize!
  return inner(ψᶿ', H, ψᶿ)
end

println()
println("Circuit optimization")
θ⁰ = zeros(nlayers * n)
∂E(ψ) = gradient(E, ψ)[1]
θ = minimize(E, ∂E, θ⁰; nsteps=20, γ=0.1)

ψᶿ = apply(U(θ, i; nlayers=nlayers), ψ⁰; maxdim=maxdim, cutoff=cutoff)

@show maxlinkdim(ψ⁰)
@show maxlinkdim(ψᶿ)
@show E(θ⁰), norm(∂E(θ⁰))
@show E(θ), norm(∂E(θ))

# TODO: Finish this implementation
# Example of preparing the ground state
# Find `U(θ)` that minimizes
#
# F(θ) = -|⟨ψ|U(θ)|0⟩|²
function F(θ)
  # Apply the circuit:
  # |θ⟩ = U(θ)|0⟩
  ψᶿ = apply(U(θ, i; nlayers=nlayers), ψ⁰; maxdim=maxdim, cutoff=cutoff)

  # -|⟨ψ|θ⟩|²
  return -abs(inner(ψ, ψᶿ))^2
end

println()
println("State preparation")
θ⁰ = zeros(nlayers * n)
∂F(ψ) = gradient(F, ψ)[1]
θ = minimize(F, ∂F, θ⁰; nsteps=20, γ=0.1)

ψᶿ = apply(U(θ, i; nlayers=nlayers), ψ⁰; maxdim=maxdim, cutoff=cutoff)

@show maxlinkdim(ψ⁰)
@show maxlinkdim(ψᶿ)
@show F(θ⁰), norm(∂F(θ⁰))
@show F(θ), norm(∂F(θ))


## R code 5.3
m5.1 <- quap(
    alist(
        D ~ dnorm( mu , sigma ) ,
        mu <- a + bA * A ,
        a ~ dnorm( 0 , 0.2 ) ,
        bA ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data = d )

## R code 5.6
d$M <- scale( d$Marriage )
m5.2 <- quap(
    alist(
        D ~ dnorm( mu , sigma ) ,
        mu <- a + bM * M ,
        a ~ dnorm( 0 , 0.2 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data = d )

## R code 5.10
m5.3 <- quap(
    alist(
        D ~ dnorm( mu , sigma ) ,
        mu <- a + bM*M + bA*A ,
        a ~ dnorm( 0 , 0.2 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        bA ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data = d )

## R code 5.13
m5.4 <- quap(
    alist(
        M ~ dnorm( mu , sigma ) ,
        mu <- a + bAM * A ,
        a ~ dnorm( 0 , 0.2 ) ,
        bAM ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data = d )

## R code 5.19
data(WaffleDivorce)
d <- list()
d$A <- standardize( WaffleDivorce$MedianAgeMarriage )
d$D <- standardize( WaffleDivorce$Divorce )
d$M <- standardize( WaffleDivorce$Marriage )

m5.3_A <- quap(
    alist(
      ## A -> D <- M
        D ~ dnorm( mu , sigma ) ,
        mu <- a + bM*M + bA*A ,
        a ~ dnorm( 0 , 0.2 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        bA ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 ),
      ## A -> M
        M ~ dnorm( mu_M , sigma_M ),
        mu_M <- aM + bAM*A,
        aM ~ dnorm( 0 , 0.2 ),
        bAM ~ dnorm( 0 , 0.5 ),
        sigma_M ~ dexp( 1 )
    ) , data = d )

## R code 5.29
m5.5_draft <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bN*N ,
        a ~ dnorm( 0 , 1 ) ,
        bN ~ dnorm( 0 , 1 ) ,
        sigma ~ dexp( 1 )
    ) , data=d )

## R code 5.32
m5.5_draft <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bN*N ,
        a ~ dnorm( 0 , 1 ) ,
        bN ~ dnorm( 0 , 1 ) ,
        sigma ~ dexp( 1 )
    ) , data=dcc )

## R code 5.34
m5.5 <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bN*N ,
        a ~ dnorm( 0 , 0.2 ) ,
        bN ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data=dcc )

## R code 5.37
m5.6 <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bM*M ,
        a ~ dnorm( 0 , 0.2 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data=dcc )

## R code 5.38
m5.7 <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bN*N + bM*M ,
        a ~ dnorm( 0 , 0.2 ) ,
        bN ~ dnorm( 0 , 0.5 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data=dcc )

## R code 5.41
# M -> K <- N
# M -> N
n <- 100
M <- rnorm( n )
N <- rnorm( n , M )
K <- rnorm( n , N - M )
d_sim <- data.frame(K=K,N=N,M=M)

m5.7.A <- quap(
    alist(
        K ~ dnorm( mu , sigma ) ,
        mu <- a + bN*N + bM*M ,
        a ~ dnorm( 0 , 0.2 ) ,
        bN ~ dnorm( 0 , 0.5 ) ,
        bM ~ dnorm( 0 , 0.5 ) ,
        sigma ~ dexp( 1 )
    ) , data=d_sim )

## R code 5.47
m5.8 <- quap(
    alist(
        height ~ dnorm( mu , sigma ) ,
        mu <- a[sex] ,
        a[sex] ~ dnorm( 178 , 20 ) ,
        sigma ~ dunif( 0 , 50 )
    ) , data=d )
precis( m5.8 , depth=2 )

## R code 5.51
d$K <- scale( d$kcal.per.g )
m5.9 <- quap(
    alist(
        K ~ dnorm( mu , sigma ),
        mu <- a[clade_id],
        a[clade_id] ~ dnorm( 0 , 0.5 ),
        sigma ~ dexp( 1 )
    ) , data=d )

## R code 5.53
m5.10 <- quap(
    alist(
        K ~ dnorm( mu , sigma ),
        mu <- a[clade_id] + h[house],
        a[clade_id] ~ dnorm( 0 , 0.5 ),
        h[house] ~ dnorm( 0 , 0.5 ),
        sigma ~ dexp( 1 )
    ) , data=d )


# This file investigates integration error with respect to free variables in hidden 1D integration

using FractalTools
using Makie

# Construct interpolation data
f(x) = sin(2π * x)  + 1
g(x) = cos(2π * x)  + 1
dx   = 0.1
xi   = 0.
xf   = 1.
x    = collect(xi : dx : xf)
y    = f.(x)
z    = g.(x)
pts  = collect.(zip(x, y, z))

# Compute errors
fvals = (cos(2π * xi) - cos(2π * xf)) / (2π) + (xf - xi)
freevars = 0.001 : 0.01 : 0.999
mse = map(freevars) do freevar
    ival = integrate(pts, HInterp1D(fill(freevar, 2, 2)))[1]
    abs(fval - ival)
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="MSE", title="1D Hidden Interpolation MSE")
stem!(ax, freevars, mse, color=:black)
save(joinpath(@__DIR__, "hinteg1d_error.png"), fig)
display(fig)


# ------------------------------------------------
# basic math operators like dot mul,dot add,etc ..
# ------------------------------------------------
import Base.+
import Base.-
import Base.*
import Base.^

export dotAdd
export dotMul
export matAddVec
export matMulVec

function Base.:+(x::Variable{T}, constant) where T
    # a matrix add a constant element by element
    C = eltype(ᵛ(x))(constant)
    y = Variable{T}(ᵛ(x) .+ C, x.backprop)
    if x.backprop
        function matAddScalarBackward()
            if need2computeδ!(x)
                δ(x) .+= δ(y)
            end
            ifNotKeepδThenFreeδ!(y);
        end
        push!(graph.backward, matAddScalarBackward)
    end
    return y
end

function Base.:+(constant, var::Variable{T}) where T
    return var + constant;
end

function Base.:-(x::Variable{T}, constant) where T
    # a matrix minus a constant element by element
    C = eltype(ᵛ(x))(constant)
    y = Variable{T}(ᵛ(x) .- C, x.backprop)
    if x.backprop
        function matMinusScalarBackward()
            if need2computeδ!(x)
                δ(x) .+= δ(y)
            end
            ifNotKeepδThenFreeδ!(y);
        end
        push!(graph.backward, matMinusScalarBackward)
    end
    return y
end

function Base.:-(constant, x::Variable{T}) where T
    # a matrix minus a constant element by element
    C = eltype(ᵛ(x))(constant)
    y = Variable{T}(C .- ᵛ(x), x.backprop)
    if x.backprop
        function scalarMinusMatBackward()
            if need2computeδ!(x)
                δ(x) .-= δ(y)
            end
            ifNotKeepδThenFreeδ!(y);
        end
        push!(graph.backward, scalarMinusMatBackward)
    end
    return y
end

function Base.:*(x::Variable{T}, constant) where T
    # a matrix multiplies a constant element by element
    C = eltype(ᵛ(x))(constant)
    y = Variable{T}(ᵛ(x) .* C, x.backprop)
    if x.backprop
        function matMulScalarBackward()
            if need2computeδ!(x)
                δ(x) .+= δ(y) .* constant
            end
            ifNotKeepδThenFreeδ!(y);
        end
        push!(graph.backward, matMulScalarBackward)
    end
    return y
end

function Base.:*(constant, var::Variable{T}) where T
    return var * constant
end

function Base.:^(x::Variable{T}, n::Int) where T
    # 矩阵、列向量与常数按元素做幂指数运算
    n = eltype(ᵛ(x))(n)
    y = Variable{T}(ᵛ(x) .^ n, x.backprop)
    if x.backprop
        function powerBackward()
            if need2computeδ!(x)
                δ(x) .+= n .* ᵛ(y) ./ ᵛ(x) .* δ(y);
            end
            ifNotKeepδThenFreeδ!(y);
        end
        push!(graph.backward, powerBackward)
    end
    return y
end

function Base.:+(x::Variable{T1}, y::Variable{T2}) where {T1,T2}
    # a matrix add a matrix element by element: z = x + y
   @assert T1 <: T2 || T1 >: T2
   T = T1 <: T2 ? T1 : T2
   @assert (x.shape == y.shape) "2 inputs shall be the same size"
   backprop = (x.backprop || y.backprop)
   z = Variable{T}(ᵛ(x) + ᵛ(y), backprop)
   if backprop
       function add2varBackward()
           if need2computeδ!(x) δ(x) .+= δ(z) end
           if need2computeδ!(y) δ(y) .+= δ(z) end
           ifNotKeepδThenFreeδ!(z);
       end
       push!(graph.backward, add2varBackward)
   end
   return z
end

function Base.:-(x::Variable{T1}, y::Variable{T2}) where {T1,T2}
    # a matrix minus a matrix element by element : z = x - y
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    @assert (x.shape == y.shape) "2 inputs shall be the same size"
    backprop = (x.backprop || y.backprop)
    z = Variable{T}(ᵛ(x) - ᵛ(y), backprop)
    if backprop
        function minus2varBackward()
            if need2computeδ!(x) δ(x) .+= δ(z) end
            if need2computeδ!(y) δ(y) .-= δ(z) end
            ifNotKeepδThenFreeδ!(z);
        end
        push!(graph.backward, minus2varBackward)
    end
    return z
end

"""
    dotAdd(var1::Variable{T1}, var2::Variable{T2}) where {T1,T2}
a tensor add a tensor element by element
"""
function dotAdd(x::Variable{T1}, y::Variable{T2}) where {T1,T2}
    # a tensor add a tensor element by element: z = x .+ y
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    @assert (x.shape == y.shape) "2 inputs shall be the same size"
    backprop = (x.backprop || y.backprop)
    z = Variable{T}(ᵛ(x) .+ ᵛ(y), backprop)
    if backprop
        function dotAddBackward()
            if need2computeδ!(x) δ(x) .+= δ(z) end
            if need2computeδ!(y) δ(y) .+= δ(z) end
            ifNotKeepδThenFreeδ!(z);
        end
        push!(graph.backward, dotAddBackward)
    end
    return z
end

"""
    dotMul(var1::Variable{T1}, var2::Variable{T2}) where {T1,T2}
a tensor multiplies a tensor element by element
"""
function dotMul(x::Variable{T1}, y::Variable{T2}) where {T1,T2}
    # a tensor multiplies a tensor element by element: z = x .* y
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    @assert (x.shape == y.shape) "2 inputs shall be the same size"
    backprop = (x.backprop || y.backprop)
    z = Variable{T}(ᵛ(x) .* ᵛ(y), backprop)
    if backprop
        function dotMulBackward()
            if need2computeδ!(x) δ(x) .+= δ(z) .* ᵛ(y) end
            if need2computeδ!(y) δ(y) .+= δ(z) .* ᵛ(x) end
            ifNotKeepδThenFreeδ!(z);
        end
        push!(graph.backward, dotMulBackward)
    end
    return z
end

function Base.:*(W::Variable{T1}, X::Variable{T2}) where {T1,T2}
    # matrix W multiplies matrix X
    # 矩阵相乘 Y[i,j] = sum(W[i,k]*X[k,j],k=...)
    # W -- 权重矩阵
    # X -- n个输入列向量组成的矩阵
    # Y -- n个输出列向量组成的矩阵
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    backprop = (W.backprop || X.backprop)
    Y = Variable{T}(ᵛ(W) * ᵛ(X), backprop)
    if backprop
        function matMulBackward()
            if need2computeδ!(W) δ(W) .+= δ(Y)  * ᵛ(X)' end
            if need2computeδ!(X) δ(X) .+= ᵛ(W)' * δ(Y)  end
            ifNotKeepδThenFreeδ!(Y);
        end
        push!(graph.backward, matMulBackward)
    end
    return Y
end

"""
    matAddVec(var1::Variable{T1}, var2::Variable{T2}) where {T1,T2}
a matrix tensor `var1` adds a vector tensor `var2`
"""
function matAddVec(M::Variable{T1}, V::Variable{T2}) where {T1,T2}
    # M -- 充当和节点，非学习的参数
    # V -- 偏置列向量，要学习的参数
    # Z = M .+ V
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    @assert (M.shape[1]==V.shape[1] && V.shape[2]==1)
    backprop = (M.backprop || V.backprop)
    Z = Variable{T}(ᵛ(M) .+ ᵛ(V), backprop)
    if backprop
        function matAddVecBackward()
            if need2computeδ!(M)
                δ(M) .+= δ(Z)
            end
            if need2computeδ!(V)
                δ(V) .+= sum(δ(Z), dims=2)
            end
            ifNotKeepδThenFreeδ!(Z);
        end
        push!(graph.backward, matAddVecBackward)
    end
    return Z
end

"""
    matAddVec(var1::Variable{T1}, var2::Variable{T2}) where {T1,T2}
a matrix tensor `var1` multiplies a vector tensor `var2`
"""
function matMulVec(M::Variable{T1}, V::Variable{T2}) where {T1,T2}
    # M -- 一般充当激活节点，非网络需要学习的参数
    # V -- 列向量，循环权重，是网络需要学习的参数
    # Z = M .* V
    @assert T1 <: T2 || T1 >: T2
    T = T1 <: T2 ? T1 : T2
    @assert (M.shape[1]==V.shape[1] && V.shape[2]==1)
    backprop = (M.backprop || V.backprop)
    Z = Variable{T}(ᵛ(M) .* ᵛ(V), backprop)
    if backprop
        function matMulVecBackward()
            if need2computeδ!(M) δ(M) .+=     δ(Z) .* ᵛ(V)          end
            if need2computeδ!(V) δ(V) .+= sum(δ(Z) .* ᵛ(M), dims=2) end
            ifNotKeepδThenFreeδ!(Z);
        end
        push!(graph.backward, matMulVecBackward)
    end
    return Z
end


using ADCME
using PyPlot

function residual_and_jacobian(θ, u)
    X = ae(u, config, θ) + 1.0     # (1)
    Xp = tf.gradients(X, u)[1]
    Xpp = tf.gradients(Xp, u)[1]
    up = [u[2:end];constant(zeros(1))]
    un = [constant(zeros(1)); u[1:end-1]]
    R = Xp .* ((up-un)/2h)^2 + X .* (up+un-2u)/h^2 - φ
    dRdu = Xpp .* ((up-un)/2h)^2 + Xp.*(up+un-2u)/h^2 - 2/h^2*X
    dRdun = -Xp[2:end]/h .* (up-un)[2:end]/2h + X[2:end]/h^2
    dRdup = Xp[1:end-1]/h .* (up-un)[1:end-1]/2h + X[1:end-1]/h^2
    J = spdiag(n-1,
        -1=>dRdun,
        0=>dRdu,
        1=>dRdup)     # (2)
    return R, J
end

config = [20,20,20,1]
n = 100
h = 1/n
x = collect(LinRange(0, 1.0, n+1))

φ = @. (1 - 2*x)*(-100*x^2*(2*x - 2) - 200*x*(1 - x)^2)/(100*x^2*(1 - x)^2 + 1)^2 - 2 - 2/(100*x^2*(1 - x)^2 + 1)
φ = φ[2:end-1]
θ = Variable(ae_init([1,config...]))
u0 = constant(zeros(n-1))
function L(u)    # (3)
  u_obs = (@. x * (1-x))[2:end-1]
  loss = mean((u - u_obs)^2)
end
loss, solution, grad = NonlinearConstrainedProblem(residual_and_jacobian, L, θ, u0)
X_pred = ae(collect(LinRange(0.0,0.25,100)), config, θ) + 1.0

sess = Session(); init(sess)
BFGS!(sess, loss, grad, θ)
x_pred, sol = run(sess, [X_pred, solution])

figure(figsize=(10,4))
subplot(121)
s = LinRange(0.0,0.25,100)
x_exact = @. 1/(1+100*s^2) + 1
plot(s, x_exact, "-", linewidth=3, label="Exact")
plot(s, x_pred, "o", markersize=2, label="Estimated")
legend()
xlabel("u")
ylabel("X(u)")

subplot(122)
s = LinRange(0.0,1.0,101)[2:end-1]
plot(s, (@. s * (1-s)), "-", linewidth=3, label="Exact")
plot(s, sol, "o", markersize=2, label="Estimated")
legend()
xlabel("x")
ylabel("u")
savefig("nn.png")


## Packages
using NLsolve
using Parameters
using BenchmarkTools
using ForwardDiff

## Files
include("NewtonRoot.jl")
include("Riccati.jl")

######################## Preliminary ###################################
# Construct gradient and hessian at SS
function getM(md,kss,hss)
    @unpack θ,ψ,δ,γ_n,γ_z,ρ,β_hat = md

    f(x::Vector) = log((x[1]^θ)*((exp(x[2])*x[5])^(1-θ))-(1+γ_n)*(1+γ_z)*x[4]+(1-δ)*x[1])+ψ*log(1-x[5]);
    z_bar = [kss,0.0,1.0,kss,hss];
    grad = ForwardDiff.gradient(f,z_bar);
    hess = ForwardDiff.hessian(f,z_bar);

    # Apply Kydland and Prescott's method
    e = [0;0;1;0;0];

    M = e.*(f(z_bar)-grad'*z_bar.+(0.5.*z_bar'*hess*z_bar))*e' +
        0.5*(grad*e'-e*z_bar'*hess-hess*z_bar*e'+e*grad') +
        0.5*hess;

    # Translating M into the matrices we need:
    Q = M[1:3,1:3];
    W = M[1:3,4:5];
    R = M[4:5,4:5];

    A = [0 0 0; 0 ρ 0; 0 0 1];
    B = [1 0; 0 0; 0 0];
    C = [0; 1; 0];

    #Mapping to the problem without discounting (1 VARIABLES ARE ~ IN LECTURE NOTES)
    A_tld = sqrt(β_hat)*(A-B*(R\W'));
    B_tld = sqrt(β_hat)*B;
    Q_tld = Q-W*(R\W');

    return A_tld,B_tld,Q_tld,R,W
end

################### LQ Approximation ##################################
function LQ(A_tld,B_tld,R,Q_tld;tol=1.0e-10)

    Pn, Fn = Riccati(A_tld, B_tld, R, Q_tld, tol);
    F = Fn + R\W';
    P = Pn;

    return P,F
end

################### Vaughan ##################################
function Vaughan(A_tld,B_tld,Q_tld,R,W)

    L = size(A_tld)[1]
    H = [inv(A_tld)  (A_tld\B_tld)*(R\B_tld');
        Q_tld/A_tld Q_tld*(A_tld\B_tld)*(R\B_tld')+A_tld'];

    V = eigen(H).vectors
    #Note that Julia puts the eigenvalues out of the unit circle in the bottom of the matrix,
    #while in the lecture notes they are at the top
    P = V[L+1:end,L+1:end]/(V[1:L,L+1:end]);
    F1 = (R+B_tld'*P*B_tld)\B_tld'*P*A_tld;
    F = F1+R\W';

    return P,F
end


"""
    PGeneralizedGaussian(α, μ, p)

The *p-Generalized Gaussian distribution*, more commonly known as the exponential
power or the generalized normal distribution, with scale `α`, location `μ`, and
shape `p` has the probability density function

```math
f(x, \\mu, \\alpha, p) = \\frac{p}{2\\alpha\\Gamma(1/p)} e^{-(\\frac{|x-\\mu|}{\\alpha})^p} \\quad x \\in (-\\infty, +\\infty) , \\alpha > 0, p > 0
```

The p-Generalized Gaussian (GGD) is a parametric distribution that incorporates the
Normal and Laplacian distributions as special cases where `p = 1` and `p = 2`. As `p → ∞`,
the distribution approaches the Uniform distribution on `[μ-α, μ+α]`.

```julia
PGeneralizedGaussian()           # GGD with shape 2, scale 1, location 0, (the Normal distribution)
PGeneralizedGaussian(α, μ, p)    # GGD with location α, scale μ, and shape p

params(d)                       # Get the parameters, i.e. (α, μ, p)
shape(d)                        # Get the shape parameter, p
scale(d)                        # Get the scale parameter, α
location(d)                     # Get the location parameter, μ
```

External Links
 * [Generalized Gaussian on Wikipedia](http://en.wikipedia.org/wiki/Generalized_normal_distribution)
 * [Reference implementation paper](https://www.researchgate.net/publication/254282790_Simulation_of_the_p-generalized_Gaussian_distribution)
 """
struct PGeneralizedGaussian{T1<:Real, T2<:Real, T3<:Real} <: ContinuousUnivariateDistribution
    μ::T1
    α::T2
    p::T3
    PGeneralizedGaussian{T1,T2,T3}(μ::T1,α::T2,p::T3) where {T1<:Real, T2<:Real, T3<:Real} = new{T1,T2,T3}(µ, α, p)
end

function PGeneralizedGaussian(μ::T1,α::T2,p::T3; check_args::Bool=true) where {T1<:Real, T2<:Real, T3<:Real}
    check_args && @check_args(PGeneralizedGaussian, α > zero(α) && p > zero(p))
    return PGeneralizedGaussian{T1,T2,T3}(μ,α,p)
end

"""
    PGeneralizedGaussian(p)

Builds a p-generalized Gaussian with `μ=0.0, α=1.0`
"""
PGeneralizedGaussian(p::Real; check_args::Bool=true) = PGeneralizedGaussian(zero(p), one(p), p; check_args=check_args)

"""
    PGeneralizedGaussian()

Builds a default p-generalized Gaussian with `μ=0.0, α=√2, p=2.0`, corresponding
to the normal distribution with `μ=0.0, σ=1.0`.
"""
PGeneralizedGaussian() = PGeneralizedGaussian(0.0, √2, 2.0, check_args=false) # approximate scale with unity std deviation and shape 2

#### Conversions

convert(::Type{PGeneralizedGaussian{T1,T2,T3}}, μ::S1, α::S2, p::S3) where {T1 <: Real, T2 <: Real, T3 <:Real, S1 <: Real, S2 <: Real, S3 <: Real} = PGeneralizedGaussian(T1(μ),T2(α),T3(p))
function convert(::Type{PGeneralizedGaussian{T1,T2,T3}}, d::PGeneralizedGaussian{S1,S2,S3}) where {T1 <: Real, T2 <: Real, T3 <: Real, S1 <: Real, S2 <: Real, S3 <: Real}
    return PGeneralizedGaussian(T1(d.μ), T2(d.α), T3(d.p), check_args=false)
end

@distr_support PGeneralizedGaussian -Inf Inf

#### Parameters
partype(::PGeneralizedGaussian{T1,T2,T3}) where {T1,T2,T3} = promote_type(T1,T2,T3)

params(d::PGeneralizedGaussian) = (d.μ, d.α, d.p)
location(d::PGeneralizedGaussian) = d.μ
shape(d::PGeneralizedGaussian) = d.p
scale(d::PGeneralizedGaussian) = d.α

#### Statistics

mean(d::PGeneralizedGaussian) = d.μ
median(d::PGeneralizedGaussian) = d.μ
mode(d::PGeneralizedGaussian) = d.μ

var(d::PGeneralizedGaussian) = (d.α^2) * (gamma(3.0 * inv(d.p)) / gamma(inv(d.p)))
std(d::PGeneralizedGaussian) = (d.α) * sqrt(gamma(3.0 * inv(d.p)) / gamma(inv(d.p)))

skewness(d::PGeneralizedGaussian{T1, T2, T3}) where {T1,T2,T3} = zero(T1)
kurtosis(d::PGeneralizedGaussian) = gamma(5.0 * inv(d.p)) * gamma(inv(d.p)) / (gamma(3.0 * inv(d.p))^2) - 3.0
entropy(d::PGeneralizedGaussian) = inv(d.p) - log( d.p / (2.0 * d.α * gamma(inv(d.p))))

#### Evaluation

function pdf(d::PGeneralizedGaussian, x::Real)
    (μ, α, p) = params(d)
    return ( p / ( 2.0 * α * gamma(1 / p) ) ) * exp( -( abs(x - μ) / α )^p )
end
logpdf(d::PGeneralizedGaussian, x::Real) = log(pdf(d, x))

"""
    cdf(d, x)

Calculates the CDF of the distribution. To determine the CDF, the incomplete
gamma function is required. The CDF  of the Gamma distribution provides this,
with the necessary 1/Γ(a) normalization.
"""
function cdf(d::PGeneralizedGaussian, x::Real)
    (μ, α, p) = params(d)
    v = cdf(Gamma(inv(p), 1), (abs(x - μ) / α)^p) * inv(2)
    return typeof(v)(1/2) + sign(x - μ) * v
end

#### Sampling

"""
    rand(rng, d)

Extract a sample from the p-Generalized Gaussian distribution 'd'. The sampling
procedure is implemented from from [1].
[1]  Gonzalez-Farias, G., Molina, J. A. D., & Rodríguez-Dagnino, R. M. (2009).
Efficiency of the approximated shape parameter estimator in the generalized
Gaussian distribution. IEEE Transactions on Vehicular Technology, 58(8),
4214-4223.
"""
function rand(rng::AbstractRNG, d::PGeneralizedGaussian)

    # utilizing the sampler from the Gamma distribution.
    g = Gamma(inv(d.p), 1)

    # random variable with value -1 or 1 with probability (1/2).
    b = 2.0 * rand(Bernoulli()) -1

    return d.μ + inv(sqrt(d.α)) * rand(rng, g)^inv(d.p) * b
end


export distance

"""
    distance(H1::AbstractHyperrectangle, H2::AbstractHyperrectangle;
             [p]::Real=2)

Compute the standard distance between two hyperrectangular sets, defined as

```math
    \\inf_{x \\in H_1, y \\in H_2} \\{ d(x, y) \\}.
```

### Input

- `H1` -- hyperrectangular set
- `H2` -- hyperrectangular set
- `p`  -- (optional; default: `2`) value of the ``p``-norm

### Output

The distance, which is zero if the sets intersect and otherwise the ``p``-norm
of the shortest line segment between any pair of points.

### Notes

See also [`hausdorff_distance`](@ref) for an alternative distance notion.
"""
function distance(H1::AbstractHyperrectangle, H2::AbstractHyperrectangle;
                  p::Real=2)
    n = dim(H1)
    @assert n == dim(H2) "incompatible set dimensions $n and $(dim(H2))"

    N = promote_type(eltype(H1), eltype(H2))
    d = Vector{N}(undef, n)
    @inbounds for i in 1:n
        # find relative position in dimension i
        # (if c1 == c2, the results are equivalent independent of the branch)
        if center(H1, i) >= center(H2, i)
            lhs = low(H1, i)
            rhs = high(H2, i)
        else
            lhs = low(H2, i)
            rhs = high(H1, i)
        end
        if _leq(lhs, rhs)
            d[i] = zero(N)
        else
            d[i] = rhs - lhs
        end
    end
    return norm(d, p)
end

"""
    distance(S::AbstractSingleton, X::LazySet; [p]::Real=2.0)

Compute the distance between the singleton `S` and the set `X` with respect to
the given `p`-norm.

### Input

- `S` -- singleton, i.e., a set with one element
- `X` -- set
- `p` -- (optional, default: `2.0`) the `p`-norm used; `p = 2.0` corresponds to
         the usual Euclidean norm

### Output

A scalar representing the distance between the element wrapped by `S` and the
set `X`.
"""
function distance(S::AbstractSingleton, X::LazySet; p::Real=2.0)
    return distance(element(S), X; p=p)
end

distance(X::LazySet, S::AbstractSingleton; p::Real=2.0) = distance(S, X; p=p)

distance(S1::AbstractSingleton, S2::AbstractSingleton; p::Real=2.0) =
    distance(element(S1), element(S2); p=p)

# disambiguation
distance(S::AbstractSingleton, H::AbstractHyperrectangle; p::Real=2.0) =
    distance(element(S), H; p=p)
distance(H::AbstractHyperrectangle, S::AbstractSingleton; p::Real=2.0) =
    distance(element(S), H; p=p)


# polar plots

#JL Plots
using Plots

r(θ) = 1 + cos(θ) * sin(θ)^2
Plots.plot(r, 0, 2π, proj=:polar, lims=(0,1.5))

r(θ) = 1 + cos(θ) * sin(θ)^2
θ = range(0, stop=2π, length=50)
Plots.plot(θ, r.(θ), proj=:polar, lims=(0,1.5), c=:steelblue,  linewidth=2)

# Polar with markers

Θ = range(0, stop = 2π, length = 100)
pr = abs.(0.1 * randn(100) + sin.(3Θ))
Plots.plot(Θ, pr, proj = :polar, m = 2, c=:steelblue,  linewidth=2)