# Rational polynomial interpolation

"""
Linear Rational LeastSquares

The following curvit is done:

`y = p(x) / q(x)`

where `p(x)` and `q(x)` are polynomials.

The linear case is solved by doing a least square fit on

`y*q(x) = p(x)`

where the zero order term o `q(x)` is assumed to be 1.
"""
function linear_rational_fit(x::AbstractVector{T}, y::AbstractVector{T}, p, q) where T<:Number
    n = size(x,1)
    A = zeros(T, n, q+p+1)
    for i = 1:n
        A[i,1] = one(T)
        for k = 1:p
            A[i,k+1] = x[i]^k
        end
        for k = 1:q
            A[i, p+1+k] = -y[i] * x[i]^k
        end
    end

    A \ y
#    coefs[1:p+1], [1.0; coefs[p+2:end]]

end

"""
# Type defining a rational polynomial

A rational polynomial is the ratio of two polynomials
and it is often useful in approximating functions.
"""
struct RationalPoly{T<:Number} <: LeastSquares
    num::Poly{T}
    den::Poly{T}
end
RationalPoly(a::AbstractVector{T}, b::AbstractVector{T}) where {T<:Number} = RationalPoly(Poly(a), Poly(b))
RationalPoly(p::Integer, q::Integer, ::Type{T}=Float64) where {T<:Number} = RationalPoly{T}(Poly(zeros(T,p+1)), Poly(zeros(T,q+1)))
RationalPoly(coefs::AbstractVector{T}, p, q) where {T<:Number} = RationalPoly(coefs[1:p+1],[1.0; coefs[p+2:end]])

"Evaluate a rational polynomial"
ratval(r::RationalPoly{T}, x) where {T<:Number} = polyval(r.num, x) ./ polyval(r.den, x)

"`call` overload for calling directly `ratval`"
(r::RationalPoly)(x) = ratval(r, x)

"Auxiliary function used in nonlinear least squares"
function make_rat_fun(p, q)
    r = RationalPoly(p, q, Float64)

    function(x, a)
        for i=0:p
            r.num[i] = a[i+1]
        end
        r.den[0] = 1
        for i = 1:q
            r.den[i] = a[p+1+i]
        end
        polyval(r.num, x[1]) / polyval(r.den, x[1]) - x[2]
    end

end

"""
# Carry out a nonlinear least squares of rational polynomials

Find the polynomial coefficients that best approximate
the points given by `x` and `y`.
"""
function rational_fit(x, y, p, q, eps=1e-8, maxiter=200)

    coefs0  = linear_rational_fit(x, y, p, q)

    fun = make_rat_fun(p, q)

    coefs, converged, niter = nonlinear_fit(hcat(x, y), fun, coefs0, eps, maxiter)

    coefs

end

function curve_fit(::Type{RationalPoly}, x, y, p, q, eps=1e-8, maxiter=200)
    RationalPoly(rational_fit(x, y, p, q, eps, maxiter), p, q)
end


#=
The series, 1^1 + 2^2 + 3^3 + ... + 10^10 = 10405071317.

Find the last ten digits of the series, 1^1 + 2^2 + 3^3 + ... + 1000^1000.
=#
function calc()
  a = [i^BigInt(i) for i in 1:1000]
  d = reverse(digits(sum(a))[1:10])
  result = ""
  for digit in d
    result *= "$digit"
  end
  result
end
@time println(calc())


using JuMP, Cbc, CSV, PrettyTables

#Importar datos de Excel
nodes_import = CSV.read("Node_Coordinates.csv", header = true)

#Definir que hay en cada columna del excel y cuantos nodos hay en total
num_nodes = size(nodes_import,1)
idcol = 1
Xcol = 2
Ycol = 3

#Generar la Matriz de distancias a partir del numero de nodos
distance_matrix = Array{Float64}(undef, (num_nodes, num_nodes))
#Llenar la matriz de distancias entre todos los puntos usando Pitagoras
for n in 1:num_nodes
    for s in 1:num_nodes
        distance_matrix[n,s] = sqrt((nodes_import[n,Ycol] - nodes_import[s,Ycol])^2 +
        (nodes_import[n,Xcol] - nodes_import[s,Xcol])^2)
    end
end

println(pretty_table(nodes_import))
println(distance_matrix)

mtzModel = Model(Cbc.Optimizer)
@variable(mtzModel, rutas[1:num_nodes,1:num_nodes], binary=true)
@variable(mtzModel, 2 <= u[2:num_nodes] <= num_nodes)
@objective(mtzModel, Min, sum(rutas[:,:].*distance_matrix[:,:]))
for i = 1:num_nodes
    @constraint(mtzModel, sum(rutas[i,:]) == 1)
    @constraint(mtzModel, sum(rutas[:,i]) == 1)
    @constraint(mtzModel, rutas[i,i] == 0)
end
for i = 2:num_nodes
    for j = 2:num_nodes
        @constraint(mtzModel, u[i]-u[j]+1 <= (num_nodes-1)*(1-rutas[i,j]))
    end
end
stats = JuMP.optimize!(mtzModel)
print(pretty_table(JuMP.value.(rutas)))


#' ---
#' title: Rocket Control
#' ---

#' **Originally Contributed by**: Iain Dunning

#' This tutorial shows how to solve a nonlinear rocketry control problem.
#' The problem was drawn from the [COPS3](http://www.mcs.anl.gov/~more/cops/cops3.pdf) benchmark.

#' Our goal is to maximize the final altitude of a vertically launched rocket.
#' We can control the thrust of the rocket, and must take account of
#' the rocket mass, fuel consumption rate, gravity, and aerodynamic drag.

#' Let us consider the basic description of the model (for the full description,
#' including parameters for the rocket, see the COPS3 PDF)

#' ### Overview
#' We will use a discretized model of time, with a fixed number of time steps, $n$.
#' We will make the time step size $\Delta t$, and thus the final time $t_f = n \cdot \Delta t$, a variable in the problem.
#' To approximate the derivatives in the problem we will use the [trapezoidal rule](http://en.wikipedia.org/wiki/Trapezoidal_rule).

#' ### State and Control
#' We will have three state variables:
#'
#' * Velocity, $v$
#' * Altitude, $h$
#' * Mass of rocket and remaining fuel, $m$
#'
#' and a single control variable, thrust $T$.
#' Our goal is thus to maximize $h(t_f)$.
#' Each of these corresponds to a JuMP variable indexed by the time step.

#' ### Dynamics
#' We have three equations that control the dynamics of the rocket:
#'
#' Rate of ascent: $$h^\prime = v$$
#' Acceleration: $$v^\prime = \frac{T - D(h,v)}{m} - g(h)$$
#' Rate of mass loss: $$m^\prime = -\frac{T}{c}$$
#'
#' where drag $D(h,v)$ is a function of altitude and velocity, and gravity $g(h)$ is a function of altitude.

#' These forces are defined as
#'
#' $$D(h,v) = D_c v^2 exp\left( -h_c \left( \frac{h-h(0)}{h(0)} \right) \right)$$
#' and
#' $$g(h) = g_0 \left( \frac{h(0)}{h} \right)^2$$
#'
#' The three rate equations correspond to JuMP constraints,
#' and for convenience we will represent the forces with nonlinear expressions.

using JuMP, Ipopt

# Create JuMP model, using Ipopt as the solver
rocket = Model(with_optimizer(Ipopt.Optimizer, print_level=0))

# Constants
# Note that all parameters in the model have been normalized
# to be dimensionless. See the COPS3 paper for more info.
h_0 = 1    # Initial height
v_0 = 0    # Initial velocity
m_0 = 1    # Initial mass
g_0 = 1    # Gravity at the surface

T_c = 3.5  # Used for thrust
h_c = 500  # Used for drag
v_c = 620  # Used for drag
m_c = 0.6  # Fraction of initial mass left at end

c     = 0.5 * sqrt(g_0 * h_0)  # Thrust-to-fuel mass
m_f   = m_c * m_0            # Final mass
D_c   = 0.5 * v_c * m_0 / g_0    # Drag scaling
T_max = T_c * g_0 * m_0        # Maximum thrust

n = 800   # Time steps

@variables(rocket, begin
    Δt ≥ 0, (start = 1/n) # Time step
    # State variables
    v[1:n] ≥ 0            # Velocity
    h[1:n] ≥ h_0          # Height
    m_f ≤ m[1:n] ≤ m_0    # Mass
    # Control
    0 ≤ T[1:n] ≤ T_max    # Thrust
end)

# Objective: maximize altitude at end of time of flight
@objective(rocket, Max, h[n])

# Initial conditions
@constraints(rocket, begin
    v[1] == v_0
    h[1] == h_0
    m[1] == m_0
    m[n] == m_f
end)

# Forces
# Drag(h,v) = Dc v^2 exp( -hc * (h - h0) / h0 )
@NLexpression(rocket, drag[j = 1:n], D_c * (v[j]^2) * exp(-h_c * (h[j] - h_0) / h_0))
# Grav(h)   = go * (h0 / h)^2
@NLexpression(rocket, grav[j = 1:n], g_0 * (h_0 / h[j])^2)
# Time of flight
@NLexpression(rocket, t_f, Δt * n)

# Dynamics
for j in 2:n
    # h' = v

    # Rectangular integration
    # @NLconstraint(rocket, h[j] == h[j - 1] + Δt * v[j - 1])

    # Trapezoidal integration
    @NLconstraint(rocket,
        h[j] == h[j - 1] + 0.5 * Δt * (v[j] + v[j - 1]))

    # v' = (T-D(h,v))/m - g(h)

    # Rectangular integration
    # @NLconstraint(rocket, v[j] == v[j - 1] + Δt *(
    #                 (T[j - 1] - drag[j - 1]) / m[j - 1] - grav[j - 1]))

    # Trapezoidal integration
    @NLconstraint(rocket,
        v[j] == v[j-1] + 0.5 * Δt * (
            (T[j] - drag[j] - m[j] * grav[j]) / m[j] +
            (T[j - 1] - drag[j - 1] - m[j - 1] * grav[j - 1]) / m[j - 1]))

    # m' = -T/c

    # Rectangular integration
    # @NLconstraint(rocket, m[j] == m[j - 1] - Δt * T[j - 1] / c)

    # Trapezoidal integration
    @NLconstraint(rocket,
        m[j] == m[j - 1] - 0.5 * Δt * (T[j] + T[j-1]) / c)
end

# Solve for the control and state
println("Solving...")
status = optimize!(rocket)

# Display results
# println("Solver status: ", status)
println("Max height: ", objective_value(rocket))

#+

# Can visualize the state and control variables
using Gadfly

#+

h_plot = plot(x = (1:n) * value.(Δt), y = value.(h)[:], Geom.line,
                Guide.xlabel("Time (s)"), Guide.ylabel("Altitude"))
m_plot = plot(x = (1:n) * value.(Δt), y = value.(m)[:], Geom.line,
                Guide.xlabel("Time (s)"), Guide.ylabel("Mass"))
v_plot = plot(x = (1:n) * value.(Δt), y = value.(v)[:], Geom.line,
                Guide.xlabel("Time (s)"), Guide.ylabel("Velocity"))
T_plot = plot(x = (1:n) * value.(Δt), y = value.(T)[:], Geom.line,
                Guide.xlabel("Time (s)"), Guide.ylabel("Thrust"))
draw(SVG(6inch, 6inch), vstack(hstack(h_plot, m_plot), hstack(v_plot, T_plot)))


"""
    ω(M, k)

Return the `M`th root of unity raised to the `k`th power.
"""
@inline ω(M, k) = exp(-2im*π*k/M)


using DifferentialEquations, Plots, ParameterizedFunctions
gr()
lorenz = @ode_def Lorenz begin
  dx = σ*(y-x)
  dy = ρ*x-y-x*z
  dz = x*y-β*z
end σ β ρ

p = [10.0,8/3,28]
u0 = [1., 5., 10.]
tspan = (0., 100.)
prob = ODEProblem(lorenz, u0, tspan, p)
sol = solve(prob)

plot(sol)

plot(sol,vars=(:x,:y,:z))

plot(sol,vars=[:x])

plot(sol,vars=(1,2,3))
plot(sol,vars=[1])

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
xaxis="Time (t)",yaxis="u(t) (in mm)",label=["X","Y","Z"])

scatter(sol,vars=[:x])

plot(sol,vars=(1,2,3),denseplot=false)

plot(sol,vars=(1,2,3),plotdensity=100)

plot(sol,vars=(1,2,3),plotdensity=10000)

plot(sol,vars=(1,2,3))
scatter!(sol,vars=(1,2,3),plotdensity=100)

p = plot(sol,vars=(1,2,3))
scatter!(p,sol,vars=(1,2,3),plotdensity=100)
title!("I added a title")

using DiffEqTutorials
DiffEqTutorials.tutorial_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


# William Kahan's method, accuracy is within 2 ulp

function csd_ad_minus_bc{T<:AbstractFloat}(a::T, b::T, c::T, d::T)
    bcHi  = b*c
    bcLo  = fma(-b, c,  bcHi)  # exact
    hi    = fma( a, d, -bcHi)  # close
    hi - bcLo                  # compensated
end

csd_ab_minus_cd{T<:AbstractFloat}(a::T, b::T, c::T, d::T) = ad_minus_bc(a,c,d,b)

function csdDet2x2{T<:AbstractFloat}(m2x2::Matrix{T})
   if size(k) != (2,2)
       throw(DomainError())
   end
   csd_ad_minus_bc(reshape(m2x2,4,1)...)
end

function csdCross3D{T<:AbstractFloat}(a::Vector{T}, b::Vector{T})
    a1,a2,a3 = a[1],a[2],a[3]
    b1,b2,b3 = b[1],b[2],b[3]

    x = ad_minus_bc(a2, a3, b2, b3)
    y = ad_minus_bc(a3, a1, b3, b1)
    z = ad_minus_bc(a1, a2, b1, b2)

    [x,y,z]
end


module GaussianMarkovRandomFields

using LDLFactorizations
import LDLFactorizations: LDLFactorization
using LinearAlgebra, SparseArrays
using Distributions
using Random
using Memoize # For recursive marginal variance implementation

export GMRF, cholesky_ldl, logdet_ldl, prec

function cholesky_ldl(A)
    F = ldl(A)
    L = cholesky(F)
    return L, F.P
end

function LinearAlgebra.cholesky(F::LDLFactorizations.LDLFactorization)
    # https://en.wikipedia.org/wiki/Cholesky_decomposition#LDL_decomposition
	# Need next two lines for type-stability, the `getproperty` method for LDLFactorization
	# objects is not type-stable
	d = getfield(F, :d)
	L = SparseMatrixCSC(F.n, F.n, F.Lp, F.Li, F.Lx)
    all(d .> 0) || PosDefException(0)
    L = ((L + I) * Diagonal(sqrt.(d)))#[invperm(F.P), invperm(F.P)]
    return L
end

function logdet_from_chol(L)
    return 2 * sum(log(L[i]) for i in diagind(L))
end

function logdet_ldl(M::AbstractMatrix)
    L, P = cholesky_ldl(M)
    return logdet_from_chol(L)
end

struct GMRF{Tv<:AbstractVector{<:Real},
        Tm<:AbstractMatrix{<:Real},
        Tl<:AbstractMatrix{<:Real}} <: AbstractMvNormal
    μ::Tv
    Q::Tm
    L::Tl
end

function GMRF(μ, Q)#::AbstractVector, Q::AbstractMatrix)
    n = LinearAlgebra.checksquare(Q)
    length(μ) == n || DimensionMismatch("The dimensions of μ and Q are inconsistent.")
    L, P = cholesky_ldl(Q)
    return GMRF(μ, Q, L)
end

function GMRF(Q::Tv) where {Tv <: AbstractMatrix{<:Real}}
    n = LinearAlgebra.checksquare(Q)
    return GMRF(zeros(eltype(Q), n), Q)
end

function GMRF(μ::AbstractVector, F::LDLFactorization)
	P = F.P
	L = cholesky(F)
	Q = (L * L')[invperm(P), invperm(P)]
    return GMRF(μ, Q, F.L)
end
GMRF(F::LDLFactorization) = GMRF(zeros(size(F.L, 1)), F)

function Distributions._logpdf(d::GMRF, x::AbstractVector{T}) where T
    k = length(d.μ)
    ld = logdet_from_chol(d.L)
    x0 = x .- d.μ
    return -0.5 * (-ld + dot(x0, d.Q, x0) + k*log(2pi))
end

Base.length(d::GMRF) = length(mean(d))
Base.eltype(d::GMRF) = eltype(mean(d))
prec(d::GMRF) = d.Q
Distributions.mean(d::GMRF) = d.μ

# Uses the recursive method of Rue (2005)
# Finds the "future" non-zero indices
function ℐ(L::AbstractSparseMatrix, i::Integer)
	Lcol = L[:, i]
	ind, val = findnz(Lcol)
	ind[ind .> i]
end
# Recursively calculate the marginal variances. Should be memoized to avoid
# duplicated calculations and (probably more importantly) stack overflows in
# large precision matrices.
@memoize function cov_element(L::AbstractSparseMatrix, i::Integer, j::Integer)
	# Reverse indices to calculate upper triangle entry of Σ, otherwise get zero below
	if (i > j)
		 (j, i) = (i, j)
	end
	Σij = (i == j) ? 1 / L[i, i]^2 : zero(L[i, i])
	for k in ℐ(L, i)
        # `cov_element` will be zero if k > j; indices are switched if
        # necessary; this is where recursion occurs.
		Σij -= 1 / L[i, i] * L[k, i] * cov_element(L, k, j)
	end
	Σij
end
function Distributions.var(d::GMRF)
    n = length(d)
    v = Vector{Float64}(undef, n)
    # Iterate *backwards*
    for idx in reverse(eachindex(v))
        v[idx] = cov_element(d.L, idx, idx)
    end
    return v
end

Distributions.cov(d::GMRF) = error( "ERROR: The covariance matrix of a GMRF `g` can often be " *
    "dense and can cause the computer to run out of memory. If you are sure you have "*
    "enough memory, you can invert the precision matrix with `inv(Matrix(prec(g)))`")

function Distributions._rand!(rng::Random.AbstractRNG, d::GMRF, x::AbstractArray)
    z = randn(rng, size(x))
    x .= mean(d) .+ d.L \ z
end

end # module


"""
Compute the inverse of the lattice. Takes special care of 1D or 2D cases.
"""
function compute_inverse_lattice(lattice::AbstractMatrix{T}) where {T}
    # Note: pinv pretty much does the same, but the implied SVD causes trouble
    #       with interval arithmetic and dual numbers, so we go for this version.
    n_dim = count(!iszero, eachcol(lattice))
    @assert 1 ≤ n_dim ≤ 3
    if n_dim == 3
        inv(lattice)
    else
        Mat3{T}([
            inv(lattice[1:n_dim, 1:n_dim])   zeros(T, n_dim, 3 - n_dim);
            zeros(T, 3 - n_dim, 3)
        ])
    end
end

"""
Compute the reciprocal lattice.
We use the convention that the reciprocal lattice is the set of G vectors such
that G ⋅ R ∈ 2π ℤ for all R in the lattice.
"""
function compute_recip_lattice(lattice::AbstractMatrix{T}) where {T}
    2T(π) * compute_inverse_lattice(lattice')
end

"""
Compute unit cell volume volume. In case of 1D or 2D case, the volume is the length/surface.
"""
function compute_unit_cell_volume(lattice)
    n_dim = count(!iszero, eachcol(lattice))
    abs(det(lattice[1:n_dim, 1:n_dim]))
end

"""Compute the diameter of the unit cell"""
function diameter(lattice::AbstractMatrix)
    # brute force search
    diam = zero(eltype(lattice))
    for vec in Vec3.(Iterators.product(-1:1, -1:1, -1:1))
        diam = max(diam, norm(lattice * vec))
    end
    diam
end


"""
    EGPpower(σ, ξ, κ)

*EGPpower* corresponds to the first extended GP model of Naveau et al. (2016), with the power law distribution G(v) = v^κ.

It is a three parameters family: κ controls the shape of the lower tail, σ is a scale parameter, and ξ controls the rate of upper tail decay.

```julia
EGPpower(σ, ξ, κ)   # EGP of Naveau et al. (2016) (type 1) with scale parameter σ, rate of upper tail decay ξ and shape of the lower tail κ.

params(d)           # Get the parameters, i.e. (σ, ξ, κ)
```

Reference :

* Naveau, P., Huser, R., Ribereau, P., and Hannart, A. (2016), Modeling jointly low, moderate, and heavy rainfall intensities without a threshold selection, Water Resour. Res., 52, 2753– 2769, doi:10.1002/2015WR018552.
"""
struct EGPpower{T<:Real} <: ContinuousUnivariateDistribution
    σ::T    # scale parameter
    ξ::T    # rate of upper tail decay
    κ::T    # shape of the lower tail

    function EGPpower{T}(σ::T, ξ::T, κ::T) where {T <: Real}
        new{T}(σ, ξ, κ)
    end
end

function EGPpower(σ::T, ξ::T, κ::T; check_args=true) where {T <: Real}
    check_args && @check_args(EGPpower, σ > zero(σ) && κ > zero(κ))
    return EGPpower{T}(σ, ξ, κ)
end

EGPpower(σ::Real, ξ::Real, κ::Real) = EGPpower(promote(σ, ξ, κ)...)
EGPpower(σ::Integer, ξ::Integer, κ::Integer) = EGPpower(float(σ), float(ξ), float(κ))

minimum(d::EGPpower) = 0.0
maximum(d::EGPpower) = Inf * (d.ξ >= 0) - ( d.σ / d.ξ ) * (d.ξ < 0)
insupport(d::EGPpower, x::Real) = minimum(d) <= x <= maximum(d)

#### Parameters

scale(d::EGPpower) = d.σ
decay(d::EGPpower) = d.ξ    # noms à vérifier...
shape(d::EGPpower) = d.κ

params(d::EGPpower) = (d.σ, d.ξ, d.κ)
partype(::EGPpower{T}) where {T} = T

#### Evaluation

function logpdf(d::EGPpower{T}, x::Real) where T<:Real
    μ = 0
    (σ, ξ, κ) = params(d)

    pd = GeneralizedPareto(μ, 1, ξ)

    lg(v::Real) = log(κ) + (κ-1)*log(v)

    p = -log(σ) + Distributions.logpdf(pd, x/σ) + lg(Distributions.cdf(pd, x/σ))

    return p
end

pdf(d::EGPpower, x::Real) = exp(logpdf(d, x))

function logcdf(d::EGPpower{T}, x::Real) where T<:Real
    # À vérifier
    μ = 0
    (σ, ξ, κ) = params(d)

    pd = GeneralizedPareto(μ, 1, ξ)

    #G(v::Real, κ::Real) = v^κ
    #lG(v::Real, κ::Real) = κ*log(v)

    p = κ*Distributions.logcdf(pd, x/σ)

    return p
end

cdf(d::EGPpower, x::Real) = exp(logcdf(d, x))

function quantile(d::EGPpower{T}, p::Real) where T<:Real
    @assert zero(p)<p<one(p) "the quantile level should be between 0 and 1."

    (σ, ξ, κ) = params(d)

    invG(p::Real, κ::Real) = p^(1/κ)

    x = (σ/ξ)*(((1 - invG(p,κ))^(-ξ)) - 1)

    return x
end

#### Sampling

function rand(rng::AbstractRNG, d::EGPpower)
    # Generate a Float64 random number uniformly in (0,1].
    u = 1 - rand(rng)

    invG(p::Real, κ::Real) = p^(1/κ)

    return (d.σ/d.ξ)*(((1 - invG(u,d.κ))^(-d.ξ)) - 1)
end

sampler(d::EGPpower) = d    # à vérifier


var2(x) = mean(x.^2) .- mean(x)^2

function waic( ll::AbstractArray; pointwise=false , log_lik="log_lik" , kwargs... )

    n_samples, n_obs = size(ll)
    pD = zeros(n_obs);

    lpd = reshape(logsumexp(ll .- log(n_samples); dims=1), n_obs);
    for i in 1:n_obs
        pD[i] = var2(ll[:,i])
    end

    waic_vec = (-2) .* ( lpd - pD );
    if pointwise == false
        waics = sum(waic_vec)
        lpd = sum(lpd)
        pD = sum(pD)
    else
        waics = waic_vec
    end

    local se
    try
        se = sqrt( n_obs*var2(waic_vec) )
    catch e
        println(e)
        se = nothing
    end

    (WAIC=waics, lppd=lpd, penalty=pD, std_err=se)
end

export
    var2,
    waic


using ContinuousTransformations
using FastGaussQuadrature
using ArgCheck

export Quadrature, quadrature_standard_normal, quadrature_standard_uniform

"""
Univariate numerical quadrature. A lightweight container for the
`nodes` and the `weights`, calculated from some quadrature rule. The
`domain` is provided for information purposes.
"""
immutable Quadrature{TD <: AbstractInterval, T}
    domain::TD
    nodes::Vector{T}
    weights::Vector{T}
    function Quadrature(domain, nodes, weights)
        @argcheck length(nodes) == length(weights)
        new(domain, nodes, weights)
    end
end

function Quadrature{TD, T}(domain::TD, nodes::Vector{T}, weights::Vector{T})
    Quadrature{TD, T}(domain, nodes, weights)
end

ContinuousTransformations.domain(q::Quadrature) = q.domain

Base.length(q::Quadrature) = length(q.nodes)

function Base.show(io::IO, q::Quadrature)
    print(io, "Quadrature of $(length(q)) nodes on $(q.domain)")
end

(q::Quadrature)(f) = dot(f.(q.nodes), q.weights)

"""
Return a quadrature that integrates a function under the standard
normal distribution using `N` nodes.
"""
function quadrature_standard_normal(N::Int)
    nodes, weights = gausshermite(N)
    Quadrature(ℝ, nodes*√2, weights/√π)
end

"""
Return a quadrature that integrates a function on the unit domain
[0,1] under the standard uniform distribution using `N` nodes.
"""
function quadrature_standard_uniform(N::Int)
    nodes, weights = gausslegendre(N)
    Quadrature(𝕀, fma.(nodes, 0.5, 0.5), weights .* 0.5)
end


# Local Tangent Space Alignment (LTSA)
# ---------------------------
# Principal Manifolds and Nonlinear Dimension Reduction via Local Tangent Space Alignment,
# Zhang, Zhenyue; Hongyuan Zha (2004),  SIAM Journal on Scientific Computing 26 (1): 313–338.
# doi:10.1137/s1064827502419154.

#### LTSA type
struct LTSA{T <: AbstractFloat} <: SpectralResult
    k::Int
    λ::AbstractVector{T}
    proj::Projection{T}

    LTSA{T}(k::Int, λ::AbstractVector{T}, proj::Projection{T}) where T = new(k, λ, proj)
end

## properties
outdim(M::LTSA) = size(M.proj, 1)
projection(M::LTSA) = M.proj

eigvals(M::LTSA) = M.λ
neighbors(M::LTSA) = M.k

## show & dump
function show(io::IO, M::LTSA)
    print(io, "LTSA(outdim = $(outdim(M)), neighbors = $(neighbors(M)))")
end

function dump(io::IO, M::LTSA)
    show(io, M)
    println(io, "eigenvalues: ")
    Base.showarray(io, transpose(M.λ), header=false, repr=false)
    println(io)
    println(io, "projection:")
    Base.showarray(io, M.proj, header=false, repr=false)
end

## interface functions
function transform(::Type{LTSA}, X::DenseMatrix{T}; d::Int=2, k::Int=12) where T<:AbstractFloat
    n = size(X, 2)

    # Construct NN graph
    D, I = find_nn(X, k)

    B = spzeros(n,n)
    for i=1:n
        # re-center points in neighborhood
        μ = mean(X[:,I[:,i]], dims=2)
        δ_x = X[:,I[:,i]] .- μ

        # Compute orthogonal basis H of θ'
        θ_t = svd(δ_x).V[:,1:d]

        # Construct alignment matrix
        G = hcat(ones(k)./sqrt(k), θ_t)
        B[I[:,i], I[:,i]] =  B[I[:,i], I[:,i]] + Matrix{Float64}(LinearAlgebra.I, k, k) - G*transpose(G)
    end

    # Align global coordinates
    λ, V = decompose(B, d)
    return LTSA{T}(k, λ, transpose(V))
end


# Doesn't work for colinear/parallel lines. (𝘂 × 𝘃 = 𝟬).
# For 𝗹₁(r) = 𝘅₁ + r𝘂 and 𝗹₂(s) = 𝘅₂ + s𝘃
# 1) 𝘅₁ + r𝘂 = 𝘅₂ + s𝘃                  subtracting 𝘅₁ from both sides
# 2) r𝘂 = (𝘅₂-𝘅₁) + s𝘃                  𝘄 = 𝘅₂-𝘅₁
# 3) r𝘂 = 𝘄 + s𝘃                        cross product with 𝘃 (distributive)
# 4) r(𝘂 × 𝘃) = 𝘄 × 𝘃 + s(𝘃 × 𝘃)        𝘃 × 𝘃 = 𝟬
# 5) r(𝘂 × 𝘃) = 𝘄 × 𝘃                   let 𝘄 × 𝘃 = 𝘅 and 𝘂 × 𝘃 = 𝘇
# 6) r𝘇 = 𝘅                             dot product 𝘇 to each side
# 7) r𝘇 ⋅ 𝘇 = 𝘅 ⋅ 𝘇                     divide by 𝘇 ⋅ 𝘇
# 8) r = (𝘅 ⋅ 𝘇)/(𝘇 ⋅ 𝘇)
# We need to ensure r, s ∈ [0, 1], hence we need to solve for s too.
# 1) 𝘅₂ + s𝘃 = 𝘅₁ + r𝘂                     subtracting 𝘅₂ from both sides
# 2) s𝘃 = -𝘄 + r𝘂                          cross product with 𝘄
# 3) s(𝘃 × 𝘄) = -𝘄 × 𝘄 + r(𝘂 × 𝘄)          𝘄 × 𝘄 = 𝟬
# 4) s(𝘃 × 𝘄) = r(𝘂 × 𝘄)                   using 𝘂 × 𝘄 = -(𝘄 × 𝘂), likewise for 𝘃 × 𝘄
# 5) s(𝘄 × 𝘃) = r(𝘄 × 𝘂)                   let 𝘄 × 𝘂 = 𝘆. use 𝘄 × 𝘃 = 𝘅
# 6) s𝘅 = r𝘆                               dot product 𝘅 to each side
# 7) s(𝘅 ⋅ 𝘅) = r(𝘆 ⋅ 𝘅)                   divide by (𝘅 ⋅ 𝘅)
# 9) s = r(𝘅 ⋅ 𝘆)/(𝘅 ⋅ 𝘅)
# The cross product of two vectors in the plane is a vector of the form (0, 0, k),
# hence, in 2D:
# r = (𝘅 ⋅ 𝘇)/(𝘇 ⋅ 𝘇) = x₃/z₃
# s = r(𝘅 ⋅ 𝘆)/(𝘅 ⋅ 𝘅) = y₃/z₃
function Base.intersect(l₁::LineSegment{Point{2,T}},
                        l₂::LineSegment{Point{2,T}}) where {T}
    𝘄 = l₂[1] - l₁[1]
    𝘂₁= l₁[2] - l₁[1]
    𝘂₂= l₂[2] - l₂[1]
    z = 𝘂₁ × 𝘂₂
    r = (𝘄 × 𝘂₂)/z
    s = (𝘄 × 𝘂₁)/z
    valid = 0 ≤ r && r ≤ 1 && 0 ≤ s && s ≤ 1
    return valid ? l₂(s) : Point(T(1e6),T(1e6))
end


### A Pluto.jl notebook ###
# v0.14.4

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 876e6928-abdc-11eb-0fea-d51de2d4b971
md""" ### Quick Julia tutorial...
"""

# ╔═╡ 9e95da34-fc21-4e7e-99a9-aea83edacd19
# global variable
example = 9.5

# ╔═╡ 34282b28-b73f-4aa7-96be-31dd6700084f
areaofcircle(r) = pi * r^2

# ╔═╡ 451499e9-76d9-4088-af2f-f35a9222afca
#= this updates
~automatically~
if you change the first cell
(and here's how you do multi-line commenting!) =#

bep = areaofcircle(example)

# ╔═╡ 1a3a323e-3f6b-4d65-bcbe-7c1c98aeb150
md"""
##### Some words on scope...
In Julia, like all other languages, there are globally and locally scoped variables. Locally scoped can be considered hard or soft; difference being that soft local variables unassigned locally and defined globally reassign the global variable, whereas hard scope remains local.

Julia uses lexical scoping, meaning that a function's scope refers to the one in which it was defined. For example, calling the module "Bleep" from a global scope will use the scope independent to Bleep, NOT the caller's global scope:
"""

# ╔═╡ 3f2ba373-77c6-4462-b30b-94fc8602298e
module Bleep
	example = 1
	foo() = example
end

# ╔═╡ 6315a551-f39b-40be-8c84-d3fbf34de4d0
Bleep.foo()

# ╔═╡ fa32bf73-e41a-4db5-ab2a-3e5a9e383aef
md"""
And this new module's independent scope is separate from the caller's global scope:
"""

# ╔═╡ 0cd019ae-3521-4323-9371-020190603204
module Deeee
	beh = example
end

# ╔═╡ 92cfd5a5-09ef-401d-a4a6-53d2576b314f
md"""Also, changing a global variable with a function will result in that variable remaining modified - called 'pass-by-sharing'. Conventionally, these functions will have '!' after their names.
"""

# ╔═╡ 92713261-8080-4797-8e48-0f0c2ce0e391
function changer!(bees)
    bees .+= 1
    return
end

# ╔═╡ 997f3fe0-d920-4055-b81e-b259fce128a3
ty = [1.0,2.0,3.0]

# ╔═╡ b6fa822c-7a55-46fe-b3a3-f191fe3ebd3d
begin
	changer!(ty)
	ty
end

# ╔═╡ 883334fb-f81b-4c24-9527-571a26ce7ce8
md"""
Additionally, if one variable is defined as another, any modification to the new variable will result in the same change in the original.
"""

# ╔═╡ dc423ed7-c9ee-416a-a0b2-a3c1408221cd
begin
	blip = [1,2,3]
	behhh = blip
end

# ╔═╡ b9aab8cc-c738-4f3c-a845-52f5078d1478
behhh[2] = 4

# ╔═╡ 0b04f67a-e413-4c9a-b298-610a294561b8
blip

# ╔═╡ 251313ac-b425-47c5-aae8-d78e5167b052
md"""
Other fun things include binding with a variety of inputs (copied from their sample notebook)... click the eyeball on the left to see the cell!
"""

# ╔═╡ 9f9fb7dd-b0f6-4bb8-b523-6053b5bc39dc
md"""
`zoot = ` $(@bind zoot html"<input type=range min=5 max=35>")

`beef = ` $(@bind beef html"<input type=text >")

`crr = ` $(@bind crr html"<input type=button value='Click'>")

`drrt = ` $(@bind drrt html"<input type=checkbox >")

`efff = ` $(@bind efff html"<select><option value='one'>First</option><option value='two'>Second</option></select>")

`fooop = ` $(@bind fooop html"<input type=color >")

"""

# ╔═╡ b03cc059-d281-4973-a22a-fabeb5de06e0
(zoot, beef, crr, drrt, efff, fooop)

# ╔═╡ c423ec0a-dbf7-4400-99f9-8357ac59edd1
md"""
#### Working with arrays, concatenation, and plotting...
"""

# ╔═╡ d5e187ee-c6b0-4490-80e0-12a47e809d27
begin
	years = collect(2001:2005)
	oranges = [3, 7, 12, 89, 20]
	pears = zeros(5,1)
	apples = rand(1:100, 5)
	bananas = rand(1:100, 5)
end

# ╔═╡ d50d1e34-9ef2-41de-8dac-bb763685fd5f
fruits = hcat(apples, oranges, pears, bananas)

# ╔═╡ 2f248615-aaca-44ce-a0d9-0ac31e12ba52
begin
	using Plots
	plot(years, fruits, layout=(4, 1), legend=false)
end

# ╔═╡ 6658d1dc-aefd-449b-8d81-f2b9f6a6fe3c
sort(fruits, dims=2)

# ╔═╡ 3bc93ea9-0718-47b9-ae4c-d1286641c9f2
all(i->(4<=i<=6), [4,5,6])

# ╔═╡ 28108af6-a838-417b-85a5-5e3fff0ae49b
md"""
#### Fun with structs!

Julia structs do not require inheritance like OOP languages do - of which Julia is not considered to be, due to the fact that you cannot attach a method (a behavior of a function) to a type. Dispatch, or the choice of method, depends on the first argument in OOP, while in Julia, it chooses which of a function's methods to call depending on the types of all the arguments. This is called multiple dispatch, and is the primary differentiator from other languages.

Types can be either system-defined or user-defined, and allow for structs to define a "supertype" which can be called by any number of functions or structs, as follows.

"""

# ╔═╡ cca3129b-8ccb-4747-b75d-cb34f3ff5b23
struct Limbs
	hands :: Float64
	leg :: Integer
end

# ╔═╡ b1ce5d0c-c3f6-411c-8f04-c44854fa89b1
struct boy
	body_parts :: Limbs #declaring body_parts as a Composite Type, defined by Limbs
	# only inner constructors have access to this "new" function
	boy(hands, leg) = new(Limbs(hands, leg))
end

# ╔═╡ 26730fa4-254c-4bef-81e5-39803ebcaa2e
begin
	joseph = boy(2.,1)
	joseph.body_parts.hands
end

# ╔═╡ 921051ca-54dc-4563-a116-0756cca43557
md"""
This will come into play with user-defined type "Cell" that can be acted on by any number of functions.
"""

# ╔═╡ 8a3ee669-bad2-4086-b90a-1744b85dcf76
md"""
##### Differences from other languages that may not be immediately obvious...

https://docs.julialang.org/en/v1/manual/noteworthy-differences/

https://erik-engheim.medium.com/defining-custom-units-in-julia-and-python-513c34a4c971
"""

# ╔═╡ 39d444ec-f84a-4d1b-9ba4-19af636bf3dd

# ╔═╡ Cell order:
# ╟─876e6928-abdc-11eb-0fea-d51de2d4b971
# ╠═9e95da34-fc21-4e7e-99a9-aea83edacd19
# ╠═34282b28-b73f-4aa7-96be-31dd6700084f
# ╠═451499e9-76d9-4088-af2f-f35a9222afca
# ╟─1a3a323e-3f6b-4d65-bcbe-7c1c98aeb150
# ╠═3f2ba373-77c6-4462-b30b-94fc8602298e
# ╠═6315a551-f39b-40be-8c84-d3fbf34de4d0
# ╟─fa32bf73-e41a-4db5-ab2a-3e5a9e383aef
# ╠═0cd019ae-3521-4323-9371-020190603204
# ╟─92cfd5a5-09ef-401d-a4a6-53d2576b314f
# ╠═92713261-8080-4797-8e48-0f0c2ce0e391
# ╠═997f3fe0-d920-4055-b81e-b259fce128a3
# ╠═b6fa822c-7a55-46fe-b3a3-f191fe3ebd3d
# ╟─883334fb-f81b-4c24-9527-571a26ce7ce8
# ╠═dc423ed7-c9ee-416a-a0b2-a3c1408221cd
# ╠═b9aab8cc-c738-4f3c-a845-52f5078d1478
# ╠═0b04f67a-e413-4c9a-b298-610a294561b8
# ╟─251313ac-b425-47c5-aae8-d78e5167b052
# ╟─9f9fb7dd-b0f6-4bb8-b523-6053b5bc39dc
# ╠═b03cc059-d281-4973-a22a-fabeb5de06e0
# ╟─c423ec0a-dbf7-4400-99f9-8357ac59edd1
# ╠═d5e187ee-c6b0-4490-80e0-12a47e809d27
# ╠═d50d1e34-9ef2-41de-8dac-bb763685fd5f
# ╠═6658d1dc-aefd-449b-8d81-f2b9f6a6fe3c
# ╠═2f248615-aaca-44ce-a0d9-0ac31e12ba52
# ╠═3bc93ea9-0718-47b9-ae4c-d1286641c9f2
# ╟─28108af6-a838-417b-85a5-5e3fff0ae49b
# ╠═cca3129b-8ccb-4747-b75d-cb34f3ff5b23
# ╠═b1ce5d0c-c3f6-411c-8f04-c44854fa89b1
# ╠═26730fa4-254c-4bef-81e5-39803ebcaa2e
# ╟─921051ca-54dc-4563-a116-0756cca43557
# ╟─8a3ee669-bad2-4086-b90a-1744b85dcf76
# ╠═39d444ec-f84a-4d1b-9ba4-19af636bf3dd


# Simple example of an eigenproblem:
#
#           y
#             |
#             |            u_y = 0
#          Ly ----------------------------------
#             |                                |
#             |                                |
#             |                                |
#       u = 0 |     u_xx + u_yy + λ u = 0      | u = 0
#             |                                |
#             |                                |
#             |                                |
#           -----------------------------------|-----  x
#                          u_y = 0              Lx
#
# Here, exact solutions are
#
#      u = sin( n π x / Lx ) cos( k π y / Ly )
#
#                      2               2
#      λ = ( n π / Lx )  + ( k π / Ly )
#
# for n = 1, 2, 3, ... and k = 0, 1, 2, ....

using FinElt
using FinElt.PlanarPoisson
using Printf
using Arpack

include("params.jl")
const nev = 4

λ = Float64[]
for n = 1:nev, k = 0:nev-1
    push!(λ, (n*pi/Lx)^2+(k*pi/Ly)^2)
end

sort!(λ)

essential_bc = [ "Left", "Right" ]

err = zeros(nev, refinements+1)
@printf("Eigenvalue errors\n\n")
@printf("%7s|", "N")
for j = 1:nev
    @printf("%16s|", "λ_$j    ")
end
@printf(" elapsed\n")
@printf("%84s\n", "-"^84)
for k = 0:refinements
    start = time()
    mesh = read_msh_file("rect$k.msh")
    ep = EigenProblem(mesh, essential_bc)
    add_bilin_form!(ep, "Omega", grad_dot_grad!,   :LHS)
    add_bilin_form!(ep, "Omega", func_times_func!, :RHS)
    A, B = assembled_eigenproblem_matrices(ep)
    d, nconv, niter, nmult, resid = eigs(A, B, nev=nev, which=:SM,
                                         ritzvec=false)
    finish = time()
    err[:,k+1] = abs.(d-λ[1:nev])
    N = size(A, 1)
    if k == 0
        elapsed = finish - start
        @printf("%7d|", N)
        for j = 1:nev
            @printf("%9.2e %6s|", err[j,k+1], "")
        end
        @printf(" %7.4f\n", elapsed)
    else
        elapsed = finish - start
        @printf("%7d|", N)
        for j = 1:nev
            rate = log2(err[j,k]/err[j,k+1])
            @printf("%9.2e %6.4f|", err[j,k+1], rate)
        end
        @printf(" %7.4f\n", elapsed)
    end
end


using Plots, LaTeXStrings; pyplot()

a, c, d = 2, 1, 5
next(x,y) = [a*x*(1-x) - x*y, -c*y + d*x*y]
equibPoint = [(1+c)/d ,(d*(a-1)-a*(1+c))/d]

initX = [0.8,0.05]
tEnd = 100

traj = [[] for _ in 1:tEnd]
traj[1] = initX

for t in 2:tEnd
    traj[t] = next(traj[t-1]...)
end

scatter([traj[1][1]], [traj[1][2]],
	c=:black, ms=10,
	label="Initial state")
plot!(first.(traj),last.(traj),
	c=:blue, ls=:dash, m=(:dot, 5, Plots.stroke(0)),
	label="Model trajectory")
scatter!([equibPoint[1]], [equibPoint[2]],
	c=:red, shape=:cross, ms=10, label="Equlibrium point",
	xlabel=L"X_1", ylabel=L"X_2")


#    Problem :
#    *********
#
#    The Boundary Value problem.
#    This is the nonlinear least-squares version without fixed variables.
#
#
#    Source:  problem 28 in
#    J.J. More', B.S. Garbow and K.E. Hillstrom,
#    "Testing Unconstrained Optimization Software",
#    ACM Transactions on Mathematical Software, vol. 7(1), pp. 17-41, 1981.
#
#    Implementation translated from:
#    See also Buckley#17 (p. 94):
#
#
#    SIF input: Ph. Toint, Dec 1989 and Nick Gould, Oct 1992.
#
#    classification SUR2-MN-V-0
#    The number of variables is N.
#
# Daniel Henderson, 08/2021

f = x -> begin
	n = lastindex(x)
	h = 1.0/(1 + n)^2
	f1 = (2x[1]-x[2]+0.5h^2*(x[1] + h + 1)^3)^2
	fn = (2x[n]-x[n-1]+0.5h^2*(x[n] + n*h + 1)^3)^2
    return sum((2x[i]-x[i-1]-x[i+1]+0.5h^2*(x[i] + i*h + 1)^3)^2 for i in 2:n-1) + f1 + fn
end

g! = (g, x) -> begin
	r = similar(g)
	n = lastindex(x)
	h = 1.0/(1 + n)^2
	for i in 2:n-1
		r[i] = 2x[i]-x[i-1]-x[i+1]+0.5h^2*(x[i] + i*h + 1)^3
	end
	r[1] = 2x[1]-x[2]+0.5h^2*(x[1] + h + 1)^3
	r[n] = 2x[n]-x[n-1]+0.5h^2*(x[n] + n*h + 1)^3
	for i in 2:n-1
		g[i] = -r[i-1] -r[i+1] + r[i]*(2+3/2.0*h^2*(x[i] + i*h + 1)^2)
	end
	g[1] = -r[2] + r[1] *(2+3/2.0*h^2*(x[1] + h + 1)^2)
	g[n] = -r[n-1] + r[n] *(2+3/2.0*h^2*(x[n] + n*h + 1)^2)
    return g
end

fg! = (g, x) -> begin
	r = similar(g)
	n = lastindex(x)
	h = 1.0/(1 + n)^2
	fx = (2x[1]-x[2]+0.5h^2*(x[1] + h + 1)^3)^2 + (2x[n]-x[n-1]+0.5h^2*(x[n] + n*h + 1)^3)^2
	for i in 2:n-1
		r[i] = 2x[i]-x[i-1]-x[i+1]+0.5h^2*(x[i] + i*h + 1)^3
		fx += r[i]^2
	end
	r[1] = 2x[1]-x[2]+0.5h^2*(x[1] + h + 1)^3
	r[n] = 2x[n]-x[n-1]+0.5h^2*(x[n] + n*h + 1)^3
	for i in 2:n-1
		g[i] = -r[i-1] -r[i+1] + r[i]*(2+3/2.0*h^2*(x[i] + i*h + 1)^2)
	end
	g[1] = -r[2] + r[1] *(2+3/2.0*h^2*(x[1] + h + 1)^2)
	g[n] = -r[n-1] + r[n] *(2+3/2.0*h^2*(x[n] + n*h + 1)^2)
	return fx, g
end

init = (n::Int=5000) -> begin
	n < 2 && @warn("MOREBV: number of variables must be ≥ 2")
	n = max(n, 2)
    return n, 0.5ones(n)
end
@warn "Debug MOREBV"
#TestSet["MOREBV"] = UncProgram("MOREBV",  f, g!, fg!, init)


export intersection_params, line_segment_point, each_intersect, find_juncts, find_connect, connections

"""
```julia
intersection_params(w₁::Line, w₂::Line)
```
Given two lines computes the parameters `[t,s]` so that
the intersection of the lines occurs at
`(w₁.p₂-w₁.p₁)t+w₁.p₁==(w₂.p₂-w₂.p₁)s+w₂.p₁`. If the
lines are parallel, one is a point and not a line, or
the lines are colinear the system of equations will
not be consitent and so will return some solution
like `[NaN,NaN]` or `[Inf,Inf]`. The lines intersect on
the segments given whenever both `0≤t≤1` and `0≤s≤1`.
"""
function intersection_params(w₁::Line, w₂::Line)
    hcat(w₁.p₂-w₁.p₁,w₂.p₁-w₂.p₂)\(w₂.p₁-w₁.p₁)
end

"""
```julia
line_segment_point(t::T, w::Line)
```
Computes the point on the line `w` at parameter `t` where
`t` is the parameter of the line so that `p₂*t+(1-t)p₁`
is the parametric equation of the line.
"""
function line_segment_point(t::T, w::Line) where T<:Real
    w.p₂*t+(1-t)*w.p₁
end

"""
```julia
each_intersect(f!,arr₁,arr₂,dims,arg)
```
    Executes `f!(i₁,i₂,arr₁,arr₂,dims,ps,p,arg)` for any intersection between pairs of
    lines in `arr₁` and `arr₂`. `i₁` and `i₂` are the indices of the lines in `arr₁`
    and `arr₂` respectively, `ps` is the vector of line intersection parameters as
    computed by `intersection_params(arr₁[i₁],arr₂[i₂])`, and `p` is the point at which
    the intersection occurs.
"""
function each_intersect(f!, arr₁::Array{Line{S,N}}, arr₂::Array{Line{S,N}}, dims::SVector{N,S}, arg) where {S,N}
    for i₁ ∈ eachindex(arr₁)
        for i₂ ∈ eachindex(arr₂)
            ps = intersection_params(arr₁[i₁],arr₂[i₂])
            if (0.0 ≤ ps[1] ≤ 1.0) && (0.0 ≤ ps[2] ≤ 1.0)
                p = line_segment_point(ps[1],arr₁[i₁])
                if sum( zero(SVector{N, S}) .≤ p .≤ dims ) == N
                    f!(i₁,i₂,arr₁,arr₂,dims,ps,p,arg)
                end
            end
        end
    end
    arg
end

"""
```julia
each_intersect(f!,arr,dims,arg)
```
    Executes `f!(i₁,i₂,arr,dims,ps,p,arg)` for any intersection between pairs of
    lines in `arr`. `i₁` and `i₂` are the indices of the lines in `arr`, `ps` is
    the vector of line intersection parameters as computed by
    `intersection_params(arr[i₁],arr[i₂])`, and `p` is the point at which the
    intersection occurs.
"""
function each_intersect(f!, arr::Array{Line{S,N}}, dims::SVector{N,S}, arg) where {S,N}
    n = length(arr)
    for i₁ ∈ 2:(n)
        for i₂ ∈ 1:(i₁-1)
            ps = intersection_params(arr[i₁],arr[i₂])
            if (0.0 ≤ ps[1] ≤ 1.0) && (0.0 ≤ ps[2] ≤ 1.0)
                p = line_segment_point(ps[1],arr[i₁])
                if sum( zero(SVector{N, S}) .≤ p .≤ dims ) == N
                    f!(i₁,i₂,arr,dims,ps,p,arg)
                end
            end
        end
    end
    arg
end

"""
```julia
find_juncts(net)
```
Given an `StickNetwork` object finds all points of intersection between lines.
"""
function find_juncts(net::StickNetwork{T,N}) where {T,N}
    each_intersect(net.lines, net.dims, Array{Tuple{SVector{N,T},Tuple{Int,Int}},1}()) do i₁,i₂,arr,dims,ps,p,juncts
        push!(juncts,(p,(i₁,i₂)))
    end
end

"""
```julia
find_connect(net)
```
Given an `StickNetwork` object finds all pairs `(i₁,i₂)` with an intersection between lines `i₁` and `i₂` in `net`.
"""
function find_connect(net::StickNetwork{T,N}) where {T,N}
    each_intersect(net.lines, net.dims, Array{Tuple{Int,Int},1}()) do i₁,i₂,arr,dims,ps,p,connects
        push!(connects,(i₁,i₂))
    end
end

"""
```julia
connections(arr₁,arr₂,dims)
```
Finds all connections between arrays of `Line`s `arr₁` and `arr₂`.
Returns `Tuple{Vector{Int},Vector{Int}}` with each vector representing the source and destination line indices for each connection.
"""
function connections(arr₁::Array{Line{S,N}}, arr₂::Array{Line{S,N}}, dims::SVector{N,S}) where {S,N}
    each_intersect(arr₁, arr₂, dims, (Int[],Int[])) do i₁,i₂,arr₁,arr₂,dims,ps,p,connects
        push!(connects[1],i₁)
        push!(connects[2],i₂)
    end
end

"""
```julia
connections(arr,dims)
```
Finds all connections between `Line`s in `arr`.
Returns `Tuple{Vector{Int},Vector{Int}}` with each vector representing the source and destination line indices for each connection.
"""
function connections(arr::Array{Line{S,N}}, dims::SVector{N,S}) where {S,N}
    each_intersect(arr, dims, (Int[],Int[])) do i₁,i₂,arr,dims,ps,p,connects
        push!(connects[1],i₁)
        push!(connects[2],i₂)
    end
end


function metric(J::AbstractArray{T,2}) where {T}
    g = J'*J
    return g
end

function covariant_basis(c::Coordinate{T}) where {T}
    return ForwardDiff.jacobian(c.R,c.u)
end

function covariant_metric(c::Coordinate)
    J = covariant_basis(c)
    return metric(J)
end

function contravariant_basis(c::Coordinate)
    Jco = covariant_basis(c)
    gco = metric(Jco)
    g = inv(gco)
    J = Jco*g
    return J
end

function contravariant_metric(c::Coordinate)
    J = contravariant_basis(c)
    g = metric(J)
end

abstract type CoordinateVector{T} <: AbstractArray{T,1} end

struct ContravariantVector{T,F} <: CoordinateVector{T}
    data::Vector{T}       #Contravariant Components of the Vector
    coord::Coordinate{T,F}  #Vector Coordinate
    J::Matrix{T}
    g::Matrix{T}
    h::Vector{T}
end

function ContravariantVector(data::AbstractVector{T}, coord::Coordinate{T,F}; unit_basis = false) where {T,F}
    J = covariant_basis(coord)
    g = metric(J)
    h = sqrt.(diag(g))
    if unit_basis
        data = data./h
    end
    ContravariantVector{T,F}(data, coord, J, g, h)
end

struct CovariantVector{T,F} <: CoordinateVector{T}
    data::Vector{T}       #Covariant Components of the Vector
    coord::Coordinate{T,F}  #Vector Coordinate
    J::Matrix{T}
    g::Matrix{T}
    h::Vector{T}
end

function CovariantVector(data::AbstractVector{T}, coord::Coordinate{T,F}; unit_basis = false) where {T,F}
    J = contravariant_basis(coord)
    g = metric(J)
    h = sqrt.(diag(g))
    if unit_basis
        data = data./h
    end
    CovariantVector{T,F}(data, coord, J, g, h)
end

# Array Interface for CoordinateVector
parent(A::T) where {T<:CoordinateVector} = A.data
size(A::T) where {T<:CoordinateVector} = size(A.data)
axes(A::T) where {T<:CoordinateVector} = axes(A.data)
parenttype(::Type{CoordinateVector{T}}) where {T} = Vector{T}
IndexStyle(::Type{T}) where {T<:CoordinateVector} = IndexStyle(parenttype(T))

@propagate_inbounds getindex(A::T, i::Int) where {T<:CoordinateVector} = A.data[i]
@propagate_inbounds setindex!(A::CoordinateVector{T}, v::T, i::Int) where {T} = A.data[i] = v

function convert(::Type{T}, x::S) where {T<:CoordinateVector,S<:CoordinateVector}
    data = x.g*x.data
    g = inv(x.g)
    J = x.J*g
    h = sqrt.(diag(g))
    T(data, x.coord, J, g, h)
end

convert(::Type{T}, x::T) where {T<:CoordinateVector} = x
convert(x::CovariantVector) = convert(ContravariantVector,x)
convert(x::ContravariantVector) = convert(CovariantVector,x)

function unit_basis_components(x::T) where {T<:CoordinateVector}
    return x .* x.h
end


# PS2  Dayuan Xie
# using Pkg
# Pkg.add("Optim")
# Pkg.add("HTTP")
# Pkg.add("GLM")

using Optim
using HTTP
using GLM
using LinearAlgebra
using Random
using Statistics
using DataFrames
using CSV
using FreqTables

function q6()

# Question 1
f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2
startval = rand(1)
result = optimize(minusf, startval, BFGS())
println(result)

# Question 2
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
df = CSV.read(HTTP.get(url).body)
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.married.==1

function ols(beta, X, y)
    ssr = (y.-X*beta)'*(y.-X*beta)
    return ssr
end

beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
println(beta_hat_ols.minimizer)

bols = inv(X'*X)*X'*y
df.white = df.race.==1
bols_lm = lm(@formula(married ~ age + white + collgrad), df)
println(bols_lm)

# Question 3
function logit(alpha, X, d)
    loglike=d'*X*alpha-sum(log.(ones(size(X,1),1)+exp.(X*alpha)))
    return loglike
end
alpha_hat_logit = optimize(a -> -(logit(a, X, y)), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
println(alpha_hat_logit.minimizer)

# question 4
alpha_glm= glm(@formula(married ~ age + white + collgrad), df, Binomial(), LogitLink())
println(alpha_glm)

# Question 5
freqtable(df, :occupation) # note small number of obs in some occupations
df = dropmissing(df, :occupation)
df[df.occupation.==8 ,:occupation] .= 7
df[df.occupation.==9 ,:occupation] .= 7
df[df.occupation.==10,:occupation] .= 7
df[df.occupation.==11,:occupation] .= 7
df[df.occupation.==12,:occupation] .= 7
df[df.occupation.==13,:occupation] .= 7
freqtable(df, :occupation) # problem solved

X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.occupation

y1=y.==1
y2=y.==2
y3=y.==3
y4=y.==4
y5=y.==5
y6=y.==6
y7=y.==7

Y = cat(y1,y2,y3,y4,y5,y6,dims=2)

function csum(A)
    Colsum = []
    for i=1:size(A,1)
        sum(A[i,:])
        push!(Colsum,sum(A[i,:]))
    end
        return Colsum
end

function mlogit(alpha, X, d)
    loglike=tr(d*(X*alpha)')-sum(log.(csum(exp.(X*alpha))))
    return loglike
end

beta_hat_mlogit1 = optimize(b -> -mlogit(b, X, Y), zeros(size(X,2),6), LBFGS(), Optim.Options(g_tol=10-5, iterations=100_000, show_trace=true))
println(beta_hat_mlogit1.minimizer)

beta_hat_mlogit2 = optimize(b -> -mlogit(b, X, Y), rand(size(X,2),6), LBFGS(), Optim.Options(g_tol=10-5, iterations=100_000, show_trace=true))
println(beta_hat_mlogit2.minimizer)

beta_hat_mlogit3 = optimize(b -> -mlogit(b, X, Y), 2*rand(size(X,2),6).-1, LBFGS(), Optim.Options(g_tol=10-5, iterations=100_000, show_trace=true))
println(beta_hat_mlogit3.minimizer)

end

# Question 6
q6()


"""
    calculateEnergy(r, r_j, r_dist, p_j, m, m_j, mu, mu_sum, ignore_H0)

Calculates the sum of kynetic and potential energy of the N-body hamiltonian system:

```math
\\mathcal{H}_{\\mathrm{N-body}}(\\textbf{q}, \\textbf{p}) = \\sum_{i=1}^{N} \\frac{\\textbf{p'}^{2}_{i}}{2 m_{i}'} - \\sum_{i=1}^{N} \\sum_{j=i+1}^{N} \\frac{Gm_{i}m_{j}}{\\mid \\textbf{q}_{i} - \\textbf{q}_{j} \\mid} \\\\
```

# Args

* `r`: NxD matrix where N is the number of bodies and D is the number of dimensions, each row contains the positions of the bodies in cartesian coordinates.
* `r_j`:  NxD matrix, it contains the positions of the bodies in jacobi coordinates.
* `r_dist`: array of N elements, in which distances from the bodies to the barycenter are saved.
* `p_j`: NxD matrix, it contains the momentum of the bodies in jacobi coordinates.
* `m`: array of N elements, where element number i contains the mass of i-th body.
* `m_j`: array of N elements, each containing the i-th jacobi mass (``m'_{i}``)
```math
m'_{1} = m_{sum_{N}} \\\\
m'_{i} = m_{i} \\frac{m_{sum_{i-1}}}{m_{sum_{i}}} \\\\
\\mathrm{,where } \\hspace{0.5cm} m_{sum_{i}} = \\sum_{j = 1}^{i} m_{j}
```
* `mu`: array of N elements, where element with index i contains the standard gravitational parameter of the i-th body
```math
mu_{i} = G \\cdot m_{j} \\\\
```
* `mu_{sum}`: array of N elements, where the standard gravitational parameter of the bodies from 1 to i is saved in the i-th element
```math
mu_{sum_{i}} = \\sum_{j = 1}^{i} G \\cdot m_{j} \\\\
```
* `ignore_H0`: boolean that determines if Hamiltonian ``\\mathcal{H_{0}}`` will be ignored (when calculating the energy).

# Returns

The energy of the ``\\mathcal{H}_{N-body}`` system.
"""
function calculateEnergy(r, r_j, r_dist, p_j, m, m_j, mu, mu_sum, ignore_H0)
    energy = 0.0

    # H_0
    if !ignore_H0
        energy += (dot(p_j[1,:],p_j[1,:]) / (2.0*m_j[1]))
    end

    # H_Kepler
    for i in 2:size(r)[1]
        energy += (dot(p_j[i,:],p_j[i,:]) / (2.0*m_j[i])) #- (mu_sum[i] * m_j[i] / r_dist[i])
    end
    #println(energy)

    # H_Interaction step 1
    #for i in 2:size(r)[1]
    #    energy += (mu_sum[i] * m_j[i] / r_dist[i])
    #end
    #println(energy)

    # H_Interaction step 2
    for i in 1:size(r)[1]
        for j in (i+1):size(r)[1]
            energy -= (mu[i] * m[j] / norm(r[i,:] - r[j,:]))
        end
    end

    return energy
end


using Plotly;
include("../interps.jl")
f(x) = 2016x^7 + 12x^5 +x^4 + x^3 + x^2 + x + 1

xs = linspace(-1,1, 6)
Newton = interpNewton(xs,map(f,xs))

xss = linspace(-1,1, 10000)
trace1 = scatter(;x=xss, y= map(Newton,xss), name="Newton")
trace2 = scatter(;x=xss, y=  map(f        ,xss), name="f")
plot([trace1, trace2])

#zera czebyszewa
k = 6
xs = []
for i = 0:(k-1)
push!(xs, cos(pi*(2*i+1)/(2*k) ))
end
xs = sort(xs)
@show xs

Newton = interpNewton(xs,map(f,xs))

xss = linspace(-1,1, 10000)
trace1 = scatter(;x=xss, y= map(Newton,xss), name="Newton")
trace2 = scatter(;x=xss, y=  map(f        ,xss), name="f")
plot([trace1, trace2])


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    EigenAnalysis(proj)

The eigenanalysis of the covariance with a given projection `proj`.

## Projections

* `:V` - Uncorrelated variables (PCA transform)
* `:VD` - Uncorrelated variables and variance one (DRS transform)
* `:VDV` - Uncorrelated variables and variance one (SDS transformation)

The `:V` projection used in the PCA transform projects the data on the eigenvectors
V of the covariance matrix.

The `:VD` projection used in the DRS transform. Similar to the `:V` projection,
but the eigenvectors are multiplied by the squared inverse of the eigenvalues D.

The `:VDV` projection used in the SDS transform. Similar to the `:VD` transform,
but the data is projected back to the basis of the original variables using the Vᵀ matrix.

See [https://geostatisticslessons.com/lessons/sphereingmaf](https://geostatisticslessons.com/lessons/sphereingmaf)
for more details about these three variants of eigenanalysis.

# Examples

```julia
EigenAnalysis(:V)
EigenAnalysis(:VD)
EigenAnalysis(:VDV)
```
"""
struct EigenAnalysis <: Transform
  proj::Symbol

  function EigenAnalysis(proj)
    @assert proj ∈ (:V, :VD, :VDV) "invalid projection"
    new(proj)
  end
end

assertions(::Type{EigenAnalysis}) = [assert_continuous]

isrevertible(::Type{EigenAnalysis}) = true

function apply(transform::EigenAnalysis, table)
  # basic checks
  for assertion in assertions(transform)
    assertion(table)
  end

  # original columns names
  names = Tables.columnnames(table)

  # table as matrix
  X = Tables.matrix(table)

  # center the data
  μ = mean(X, dims=1)
  Y = X .- μ

  # eigenanalysis of covariance
  S, S⁻¹ = eigenmatrices(transform, Y)

  # project the data
  Z = Y * S

  # table with transformed columns
  𝒯 = (; zip(names, eachcol(Z))...)
  newtable = 𝒯 |> Tables.materializer(table)

  newtable, (μ, S, S⁻¹)
end

function revert(::EigenAnalysis, newtable, cache)
  # transformed column names
  names = Tables.columnnames(newtable)

  # table as matrix
  Z = Tables.matrix(newtable)

  # retrieve cache
  μ, S, S⁻¹ = cache

  # undo projection
  Y = Z * S⁻¹

  # undo centering
  X = Y .+ μ

  # table with original columns
  𝒯 = (; zip(names, eachcol(X))...)
  𝒯 |> Tables.materializer(newtable)
end

function reapply(transform::EigenAnalysis, table, cache)
  # basic checks
  for assertion in assertions(transform)
    assertion(table)
  end

  # original columns names
  names = Tables.columnnames(table)

  # table as matrix
  X = Tables.matrix(table)

  # retrieve cache
  μ, S, S⁻¹ = cache

  # center the data
  Y = X .- μ

  # project the data
  Z = Y * S

  # table with transformed columns
  𝒯 = (; zip(names, eachcol(Z))...)
  𝒯 |> Tables.materializer(table)
end

function eigenmatrices(transform, Y)
  proj = transform.proj

  Σ = cov(Y)
  λ, V = eigen(Σ)

  if proj == :V
    S   = V
    S⁻¹ = transpose(V)
  elseif proj == :VD
    Λ   = Diagonal(sqrt.(λ))
    S   = V * inv(Λ)
    S⁻¹ = Λ * transpose(V)
  elseif proj == :VDV
    Λ   = Diagonal(sqrt.(λ))
    S   = V * inv(Λ) * transpose(V)
    S⁻¹ = V * Λ * transpose(V)
  end

  S, S⁻¹
end

"""
    PCA()

The PCA transform is a shortcut for
`ZScore() → EigenAnalysis(:V)`.

See also: [`ZScore`](@ref), [`EigenAnalysis`](@ref).
"""
PCA() = ZScore() → EigenAnalysis(:V)

"""
    DRS()

The DRS transform is a shortcut for
`ZScore() → EigenAnalysis(:VD)`.

See also: [`ZScore`](@ref), [`EigenAnalysis`](@ref).
"""
DRS() = ZScore() → EigenAnalysis(:VD)

"""
    SDS()

The SDS transform is a shortcut for
`ZScore() → EigenAnalysis(:VDV)`.

See also: [`ZScore`](@ref), [`EigenAnalysis`](@ref).
"""
SDS() = ZScore() → EigenAnalysis(:VDV)


# PROX_L1    The proximal operator of the l1 norm.
#
#   prox_l1(v,lambda) is the proximal operator of the l1 norm
#   with parameter lambda.
function x = prox_l1(v, lambda)
    x = max(0, v - lambda) - max(0, -v - lambda)
    return x
end


## Wetted-area method
#======================================================#

# Raymer form factor for wing, pylon, nacelle, fuselage? (Eq. 12.30)
form_factor_wing(x_c, t_c, Λ_m, M) = (1 + 0.6t_c / x_c + 100t_c^4) * (1.34M^0.18 * cos(Λ_m)^0.28)

function form_factor(wing :: HalfWing, M)
	# (x/c)_max, (t/c)_max, (t/c)_max sweep angles
	xcs, tcs, sweeps = max_tbyc_sweeps(wing, 60)
	Kf 				 = form_factor_wing.(xcs, tcs, sweeps, M)
end

# Schlichting averaged skin-friction coefficients
cf_lam(Re_c, k_lam = 1.) = 1.328 / √(Re_c * k_lam)
cf_turb(Re_c, M) = 0.455 / log10(Re_c)^2.58 / (1 + 0.144M^2)^0.65
cf_schlichting(Re, Re_xtr, M) = max(cf_lam(Re), cf_turb(Re, M) - (Re_xtr / 320 - 39) / Re)

function wetted_area_drag(mean_chords, S_wets, K_fs, x_tr, V, ρ, M, μ)
	# Skin-friction coefficients
	Re_c 		= reynolds_number.(ρ, V, mean_chords, μ)
	Re_xtr 		= reynolds_number.(ρ, V, mean_chords .* x_tr, μ)
	cfs 		= cf_schlichting.(Re_c, Re_xtr, M)

	# Profile drag
	Dp_by_q	 	= sum(@. cfs * S_wets * K_fs)
end

function wetted_area_drag(wing :: HalfWing, x_tr, V, ρ, a_ref = 330., μ = 1.5e-5)
	# Chord processing
	mean_chords = (fwdsum ∘ chords)(wing) / 2

	# Wetted areas
	S_wets = @. mean_chords * wing.spans / cos(wing.dihedrals)

	# Form factors
	M 	 = V / a_ref
	K_fs = form_factor(wing, M)

	wetted_area_drag(mean_chords, S_wets, K_fs, x_tr, V, ρ, M, μ)
end

profile_drag_coefficient(wing :: HalfWing, x_tr, V, rho_ref, a_ref, area_ref, μ) = wetted_area_drag(wing, x_tr, V, rho_ref, a_ref, μ) / area_ref
profile_drag_coefficient(wing :: Wing, x_tr, V, rho_ref, a_ref, area_ref, μ) = profile_drag_coefficient(left(wing), x_tr, V, rho_ref, a_ref, area_ref, μ) + profile_drag_coefficient(right(wing), x_tr, V, rho_ref, a_ref, area_ref, μ)

## Local-friction and local-dissipation method


# ---
# title: 1017. Convert to Base -2
# id: problem1017
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/convert-to-base-2/description/>
# hidden: true
# ---
#
# Given a number `N`, return a string consisting of `"0"`s and `"1"`s that
# represents its value in base `**-2**` (negative two).
#
# The returned string must have no leading zeroes, unless the string is `"0"`.
#
#
#
# **Example 1:**
#
#
#
#     Input: 2
#     Output: "110"
#     **Explantion:** (-2) ^ 2 + (-2) ^ 1 = 2
#
#
# **Example 2:**
#
#
#
#     Input: 3
#     Output: "111"
#     **Explantion:** (-2) ^ 2 + (-2) ^ 1 + (-2) ^ 0 = 3
#
#
# **Example 3:**
#
#
#
#     Input: 4
#     Output: "100"
#     **Explantion:** (-2) ^ 2 = 4
#
#
#
#
# **Note:**
#
#   1. `0 <= N <= 10^9`
#
#
## @lc code=start
using LeetCode

function base_neg2(N::Int)
    cnt = 0
    res = ""
    while N > 0
        bit = (N & 1)
        N -= bit * (-1) ^ cnt
        N >>= 1
        cnt += 1
        res = string(bit) * res
    end
    res
end
## @lc code=end


using Distributions, Plots, Colors

# Gaussian with known variance
# mu ~ Normal(mu0, sig0)
# x_i ~ Normal(mu, sig_like)
# Ref: https://en.wikipedia.org/wiki/Conjugate_prior#Continuous_distributions
function compute_posterior_parameters(xs, mu0, sig0, sig_like)
    n = length(xs)
    var = inv(1 / sig0^2 + n / sig_like^2)
    mu = var * (mu0 / sig0^2 + sum(xs) / sig_like^2)
    return mu, sqrt(var)
end

mu0 = 0.0
sig0 = 1.0
sig_like = 1.0
prior = Normal(mu0, sig0)

# Inference
obs = [5.0]
po1 = compute_posterior_parameters(obs, mu0, sig0, sig_like)
posterior_1 = Normal(compute_posterior_parameters(obs, mu0, sig0, sig_like)...)

# Inference with more data
push!(obs, 1.0)
po2 = compute_posterior_parameters(obs, mu0, sig0, sig_like)
posterior_2 = Normal(compute_posterior_parameters(obs, mu0, sig0, sig_like)...)

julia_purple = parse(Colorant, RGBA(0.702, 0.322, 0.8))
julia_brown = parse(Colorant, RGBA(0.8, 0.2, 0.2))
julia_green = parse(Colorant, RGBA(0.133, 0.541, 0.133))

xs = -3:0.01:5
lw = 15.0
plot(xs, pdf.(Ref(prior), xs), color=julia_purple, linewidth=lw, lab="prior", grid=false, legend=false, background_color = RGBA(1, 1, 1, 0))
plot!(xs, pdf.(Ref(posterior_1), xs), color=julia_brown, linewidth=lw, lab="posterior 1")
plot!(xs, pdf.(Ref(posterior_2), xs), color=julia_green, linewidth=lw, lab="posterior 2")
xaxis!(false)
yaxis!(false)

savefig("turing-logo.svg")


# Silverman's rule of thumb for KDE bandwidth selection
function bandwidth(data::Vector, alpha::Float64 = 0.9)
    # Determine length of data
    ndata = length(data)

    # Calculate width using variance and IQR
    var_width = std(data)
    q25, q75 = quantile(data, [0.25, 0.75])
    quantile_width = (q75 - q25) / 1.34

    # Deal with edge cases with 0 IQR or variance
    width = min(var_width, quantile_width)
    if width == 0.0
        if var_width == 0.0
            width = 1.0
        else
            width = var_width
        end
    end

    # Set bandwidth using Silverman's rule of thumb
    return alpha * width * ndata^(-0.2)
end

# Store both grid and density for KDE over the real line
immutable UnivariateKDE
    x::Vector
    density::Vector
end

# Algorithm AS 176 for calculating univariate KDE
function kde(data::Vector, npoints::Integer = 512)
    # Determine length of data
    ndata = length(data)

    # Set bandwidth
    window = bandwidth(data)

    # Check that the window is a positive constant
    if window <= 0.0
        error("Window must be positive")
    end

    # Find interval that will contain almost all mass
    dlo = min(data) - 3 * window
    dhi = max(data) + 3 * window

    # Check that interval for estimation is valid
    if dlo >= dhi
        error("Interval must be specified as lower bound, upper bound")
    end

    # Set up a grid for discretized data
    grid = zeros(Float64, npoints)

    # Define some more constants
    step = (dhi - dlo) / npoints
    ainc = 1.0 / (ndata * step)
    npoints2 = fld(npoints, 2)
    hw = window / step
    fac1 = 32.0 * (atan(1.0) * hw / npoints)^2

    # Discretize the data using a histogram
    dlo1 = dlo - step
    for i in 1:ndata
        j = fld(data[i] - dlo1, step)
        if j >= 1 && j <= npoints
            grid[j] = grid[j] + ainc
        end
    end

    # Transform to Fourier basis
    ft = rfft(grid)

    # Find transform of KDE by convolving grid with the
    # Fourier transform of a Gaussian kernel
    for j = 2:length(ft)
        ft[j] *= exp(-fac1 * (j-1)^2)
    end

    # Invert the Fourier transform to get the KDE
    density = irfft(ft, npoints)

    # Fix any noise that crept in
    for j in 1:npoints
        if density[j] < 0.0
            density[j] = 0.0
        end
    end

    # Expand the grid over which KDE was calculated
    x = [dlo:step:(dhi - step)]

    return UnivariateKDE(x, density)
end


"Algorithms for discovering sparse bases for the symmetry-allowed space of exchange couplings."

# ##### Math behind `basis_for_symmetry_allowed_couplings()` #####
#
# A crystal spacegroup consists of a set of symmetry operations. Each symop is
# defined by an orthogonal (rotation or reflection) matrix R and a translation
# vector T. The symop transforms every atom position `x` to a new one `x′ = Rx +
# T`, but leaves the crystal as a whole invariant. Consider some bond b=(i,j),
# defined as an ordered pair of atoms, not necessarily in the same crystal unit
# cell. Suppose this bond carries an exchange interaction of the form `S_iᵀ J
# S_j`. After applying a symop defined by (R, T), the bond b=(i,j) maps to a
# transformed bond b′=(i′,j′). This new bond must carry an exchange interaction
# of the form `S_{i′}ᵀ J′ S_{j′}`, where J′ is related to J, as we will now
# show. Besides mapping the atom j to a new position j′, the symop also
# transforms its spin vector from `S_j` to `S′_{j′} = R S_j`. Similarly, `S_iᵀ`
# maps to `S′_{i′}ᵀ = S_iᵀ Rᵀ`. The energy along a bond, however, is invariant
# under the symmetry transformation, so we require `S_iᵀ J S_j = S′_{i′}ᵀ J′
# S′_{j′}`, or
#
#   `S_iᵀ J S_j = S_iᵀ Rᵀ J′ R S_j`.
#
# This equation must be valid for arbitrary `S_i` and `S_j`, which implies `J =
# Rᵀ J′ R`, or equivalently,
#
#   J′ = R J Rᵀ
#
# because the matrix R is orthogonal. The conclusion is that specifying the
# exchange matrix J for one bond implicitly constrains the exchange matrices J′
# for all symmetry equivalent bonds.
#
# Often a symmetry operation will map a bond b into itself, or into the same
# bond but with atom positions reversed. The existence of such symops constrains
# the space of allowed exchange matrices J for the bond b. Specifically, we
# require
#
#   (1)  J = R J Rᵀ   or   (2)  Jᵀ = R J Rᵀ
#
# for every symop `s = (R, T)` that maps `b` into `b` (constraint 1), or into
# `reverse(b)` (constraint 2). The intersection of all such constraints defines
# the symmetry-allowed space of exchange matrices J. The allowed matrices can be
# expressed as a linear combination of basis matrices because all constraints
# are linear in J.
#
# It is convenient to express the constraints (1) or (2) in the form `F J = 0`,
# where F denotes the linear operator such that `F J = R J Rᵀ - J` or `F J = R J
# Rᵀ - Jᵀ`. Here, we are viewing the 3×3 matrix J as a flattened 9-dimensional
# vector, and F as a 9x9 matrix. In the first case, `F = R⊗R - I`. In the second
# case, we should replace `I` by the suitable transpose operation,
# `transpose_op_3x3`, defined explicitly below.
#
# Any linear combination of "vectors" (3x3 matrices) in the null space of F,
# i.e. `F J = 0`, satisfies the symmetry constraint (1) or (2) for the given
# symop. The singular value decomposition
#
#     F = U Σ Vᵀ
#
# can be used to produce an orthogonal basis for this null space. It is spanned
# by columns v of V corresponding to the zero singular values. The space spanned
# by v equivalently represented as a projection matrix,
#
#     P = v vᵀ
#
# When there are multiple constraints (F1, F2, ... Fn) we should take the
# intersection of the spanned spaces of P1, ... Pn. To calculate this
# intersection, form the product of the projectors:
#
#     P = P1 P2 ... Pn
#
# The allowable J values correspond to the eigenvectors of P with eigenvalue 1.
# An orthogonal basis for this space can again be calculated with an SVD.

# A 9x9 matrix that, when applied to a flattened 3x3 matrix (viewed as a
# 9-dimensional vector), generates the transposed 3x3 matrix in flattened form.
const transpose_op_3x3 = SMatrix{9, 9, Float64}([
    1 0 0  0 0 0  0 0 0
    0 0 0  1 0 0  0 0 0
    0 0 0  0 0 0  1 0 0

    0 1 0  0 0 0  0 0 0
    0 0 0  0 1 0  0 0 0
    0 0 0  0 0 0  0 1 0

    0 0 1  0 0 0  0 0 0
    0 0 0  0 0 1  0 0 0
    0 0 0  0 0 0  0 0 1
])

# Returns a projection operator P that maps to zero any symmetry-unallowed
# coupling matrix J. The space spanned by the eigenvectors of P with eigenvalue
# 1 represents the allowed coupling matrices J.
function projector_for_symop(cryst::Crystal, s::SymOp, parity::Bool)
    # Cartesian-space rotation operator corresponding to `s`
    R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)

    # Constraint is modeled as `F J = 0`
    F = kron(R, R) - (parity ? SMatrix{9, 9, Float64}(I) : transpose_op_3x3)

    # Orthogonal column vectors that span the null space of F
    v = nullspace(F; atol=1e-12)

    # Projector onto the null space of F
    P = SMatrix{9, 9, Float64}(v * v')
    return P
end

# Return an operator P that implicitly gives the space of symmetry allowed
# coupling matrices for bond b. Specifically, x is an allowed coupling if and
# only if it is an eigenvector of P with eigenvalue 1, i.e., `P x = x`.
function symmetry_allowed_couplings_operator(cryst::Crystal, b::BondRaw)
    P = SMatrix{9, 9, Float64}(I)
    for (s, parity) in symmetries_between_bonds(cryst, b, b)
        P = P * projector_for_symop(cryst, s, parity)
    end
    # Allowed coupling matrices J are simultaneously eigenvectors for all
    # projectors above, with eigenvalue 1.
    return P
end

# Check whether a coupling matrix J is consistent with symmetries of a bond
function is_coupling_valid(cryst::Crystal, b::BondRaw, J::Mat3)
    for (s, parity) in symmetries_between_bonds(cryst, b, b)
        R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)
        # TODO use symprec to handle case where matrix R is not precise
        if norm(R*J*R' - (parity ? J : J')) > 1e-12
            return false
        end
    end
    return true
end

function is_coupling_valid(cryst::Crystal, b::Bond, J::Mat3)
    return is_coupling_valid(cryst, BondRaw(cryst, b), J)
end

# Orthonormal basis of 3x3 symmetric matrices
const sym_basis = begin
    b = [diagm([1, 0, 0]),
         diagm([0, 1, 0]),
         diagm([0, 0, 1]),
         [0 1 0
          1 0 0
          0 0 0]/√2,
         [0 0 1
          0 0 0
          1 0 0]/√2,
         [0 0 0
          0 0 1
          0 1 0]/√2,]
    SMatrix{9, 6, Float64}(hcat(reshape.(b, 9)...))
end

# Orthonormal basis of 3x3 antisymmetric matrices
const asym_basis = begin
    b = [[ 0  1  0
          -1  0  0
           0  0  0]/√2,
         [ 0  0 -1
           0  0  0
           1  0  0]/√2,
         [ 0  0  0
           0  0  1
           0 -1  0]/√2]
    SMatrix{9, 3, Float64}(hcat(reshape.(b, 9)...))
end

@assert sym_basis * sym_basis' + asym_basis * asym_basis' ≈ I

# Given an m×n matrix A with empty nullspace, linearly combine the n columns to
# make them sparser. Solution was proposed here:
# https://math.stackexchange.com/q/4227648/660903 . Closely related to finding
# reduced row echolon form.
function sparsify_columns(A; atol)
    if size(A, 2) <= 1
        return A
    else
        # By assumption, the n columns of A are linearly independent
        @assert isempty(nullspace(A; atol))
        # Since row rank equals column rank, it should be possible to find n
        # linearly independent rows in A. Store these independent rows of A as
        # column vectors in indep_rows.
        indep_rows = zeros(Float64, size(A, 2), 0)
        for row in eachrow(A)
            # Add `row` to list if linearly independent with existing ones
            if isempty(nullspace(hcat(indep_rows, row); atol))
                indep_rows = hcat(indep_rows, row)
            end
        end
        return A * inv(indep_rows)'
    end
end

const _basis_elements_by_priority = [1, 5, 9, 8, 3, 4]

function _score_basis_matrix(J)
    return findfirst(i -> abs(J[i]) > 1e-12, _basis_elements_by_priority)
end

"""    basis_for_symmetry_allowed_couplings(cryst::Crystal, b::Bond)

Returns a list of ``3×3`` matrices that form a linear basis for the
symmetry-allowed coupling matrices associated with bond `b`.
"""
function basis_for_symmetry_allowed_couplings(cryst::Crystal, b::BondRaw)
    # Expected floating point precision for 9x9 matrix operations
    atol = 1e-12

    P = symmetry_allowed_couplings_operator(cryst, b)
    # Any solution to the original symmetry constraints `R J Rᵀ = J` or `R J Rᵀ
    # = Jᵀ` decomposes into purely symmetric/antisymmetric solutions. Therefore
    # we can pick a basis that separates into symmetric and antisymmetric parts.
    # We will do so by decomposing P. By construction, P = P_sym+P_asym.
    P_sym  = P *  sym_basis *  sym_basis'
    P_asym = P * asym_basis * asym_basis'

    acc_sym = SVector{9, Float64}[]
    acc_asym = SVector{9, Float64}[]

    # If any "reference" basis vectors are eigenvalues of P_sym with eigenvalue
    # 1, use them as outputs, and remove them from P_sym
    for x in eachcol(sym_basis)
        if isapprox(P_sym*x, x; atol)
            push!(acc_sym, x)
            P_sym = P_sym * (I - x*x')
        end
    end
    # Same for P_asym
    for x in eachcol(asym_basis)
        if isapprox(P_asym*x, x; atol)
            push!(acc_asym, x)
            P_asym = P_asym * (I - x*x')
        end
    end

    # Search for eigenvectors of P_sym with eigenvalue 1. These provide an
    # orthonormal basis for symmetric couplings.
    v = nullspace(P_sym-I; atol)
    v = sparsify_columns(v; atol)
    append!(acc_sym, eachcol(v))
    # Same for P_asym
    v = nullspace(P_asym-I; atol)
    v = sparsify_columns(v; atol)
    append!(acc_asym, eachcol(v))

    # Sort basis elements according to the indices where the nonzero elements
    # first appear
    sort!(acc_sym;  by=_score_basis_matrix)
    sort!(acc_asym; by=_score_basis_matrix)

    acc = [acc_sym; acc_asym]
    return map(acc) do x
        # Normalize each basis vector so that its maximum component is 1. The
        # shift by atol avoids unnecessary sign change in the case where the
        # maximum magnitude values of x appear symmetrically as ±c for some c.
        _, i = findmax(abs.(x.+atol))
        x = x / x[i]

        # Reinterpret as 3x3 matrix
        x = Mat3(reshape(x, 3, 3))

        # Double check that x indeed satifies the necessary symmetries
        @assert is_coupling_valid(cryst, b, x)

        return x
    end
end

function basis_for_symmetry_allowed_couplings(cryst::Crystal, b::Bond)
    return basis_for_symmetry_allowed_couplings(cryst, BondRaw(cryst, b))
end

"""
    all_symmetry_related_couplings_for_atom(cryst::Crystal, i::Int, b::Bond, J)

Given a reference bond `b` and coupling matrix `J` on that bond, return a list
of symmetry-equivalent bonds (constrained to start from atom `i`), and a
corresponding list of symmetry-transformed coupling matrices.
"""
function all_symmetry_related_couplings_for_atom(cryst::Crystal, i::Int, b_ref::Bond, J_ref)
    J_ref = Mat3(J_ref)
    @assert is_coupling_valid(cryst, b_ref, J_ref)

    bs = Bond[]
    Js = Mat3[]

    for b in all_symmetry_related_bonds_for_atom(cryst, i, b_ref)
        push!(bs, b)
        (s, parity) = first(symmetries_between_bonds(cryst, BondRaw(cryst, b), BondRaw(cryst, b_ref)))
        R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)
        push!(Js, R * (parity ? J_ref : J_ref') * R')
    end

    return (bs, Js)
end

"""
    all_symmetry_related_couplings(cryst::Crystal, b::Bond, J)

Given a reference bond `b` and coupling matrix `J` on that bond, return a list
of symmetry-equivalent bonds and a corresponding list of symmetry-transformed
coupling matrices.
"""
function all_symmetry_related_couplings(cryst::Crystal, b_ref::Bond, J_ref)
    J_ref = Mat3(J_ref)

    bs = Bond[]
    Js = Mat3[]

    for i in eachindex(cryst.positions)
        (bs_i, Js_i) = all_symmetry_related_couplings_for_atom(cryst, i, b_ref, J_ref)
        append!(bs, bs_i)
        append!(Js, Js_i)
    end

    return (bs, Js)
end


function noisy_circle_data(N, σ)
    r = 1 .+ σ*randn(1,N)
    θ = rand(Uniform(0, 2π), (1,N))

    rcosθ = r .* cos.(θ)
    rsinθ = r .* sin.(θ)

    data = [rcosθ; rsinθ]

    return data
end


module llh

using LinearAlgebra  #: cholesky, logdet, diag, inv, triu
using AxUtil
using NNlib: softmax
using ..Misc

export gmm_llh, gmm_llh_invLT

function log_gauss_llh(X, mu, sigma; bypass=false)
    if !bypass
        out = _log_gauss_llh(X, mu, sigma)
    else
        out = - ones(size(X, 2))*Inf
    end
    return out
end

function _log_gauss_llh(X, mu, sigma)
    d = size(X,1)
    invLT = Matrix(inv(cholesky(sigma).L))
    Z = invLT*(X .- mu)
    exponent = -0.5*sum(Z.^2, dims=1)  |> dropdim1
    lognormconst = -d*log(2*pi)/2 -0.5*logdet(sigma)  #.-0.5*(-2*sum(log.(diag(invLT))))
    return exponent .+ lognormconst
end

function _log_gauss_llh_invLT(X, mu, invLT)
    d = size(X,1)
    Z = invLT*(X .- mu)
    exponent = -0.5*sum(Z.^2, dims=1)  |> dropdim1
    lognormconst = -d*log(2*pi)/2 .-0.5*(-2*sum(log.(diag(invLT))))
    return exponent .+ lognormconst
end

function gmm_llh(X, pis, mus, sigmas; thrsh_comp=0.005)
    p, n = size(X)
    k = length(pis)
    inactive_ixs = pis[:] .< thrsh_comp

    P = zeros(k, n)
    for j = 1:k
        P[j,:] = log_gauss_llh(X, mus[j,:], sigmas[:,:,j],
            bypass=inactive_ixs[j]) .+ log(pis[j])
    end
    return AxUtil.Math.logsumexpcols(P)
end

function gmm_llh_invLT(X, pis, mus, invLTs::Array{T,1}; disp=false, thrsh_comp=0.005) where T <: AbstractMatrix
    p, n = size(X)
    k = length(pis)
    inactive_ixs = pis[:] .< thrsh_comp

    P = zeros(k, n)
    for j = 1:k
        if !inactive_ixs[j]
            P[j,:] = _log_gauss_llh_invLT(X, mus[j,:], invLTs[j]) .+ log(pis[j])
        else
            P[j,:] .= -Inf
        end
    end
    return AxUtil.Math.logsumexpcols(P)
end

function responsibilities(X, mus::AbstractArray{T,2}, sigmas::AbstractArray{T, 3}, pis::AbstractArray{T, 1}) where T <: AbstractFloat
    softmax(reduce(vcat, [log_gauss_llh(X, mus[j,:], sigmas[:,:,j]) .+ log(pis[j]) for j in 1:size(mus, 1)]'))
end

end


export Pow

"""
`Pow([domainType=Float64::Type,] dim_in::Tuple)`

Elementwise power `p` non-linear operator with input dimensions `dim_in`.

"""
struct Pow{T,N,I<:Real} <: NonLinearOperator
	dim::NTuple{N,Int}
    p::I
end

function Pow(DomainType::Type, DomainDim::NTuple{N,Int}, p::I) where {N, I <: Real}
	Pow{DomainType, N, I}(DomainDim, p)
end

Pow(DomainDim::NTuple{N,Int}, p::I) where {N, I <: Real} = Pow{Float64,N,I}(DomainDim,p)

function mul!(y::AbstractArray{T,N}, L::Pow{T,N,I}, x::AbstractArray{T,N}) where {T,N,I}
	y .= x.^L.p
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{Pow{T, N, I},TT}},
              b::AbstractArray) where {T, N, I, TT <: AbstractArray{T,N}}
    L = J.A
    y .= conj.(L.A.p.*(L.x).^(L.A.p-1)).*b
end

fun_name(L::Pow) = "『"

size(L::Pow) = (L.dim, L.dim)

domainType(L::Pow{T,N}) where {T,N} = T
codomainType(L::Pow{T,N}) where {T,N} = T


export area

"""
    area(polygon::Polygon, radius::Float64=1.0)

Return the area of the spherical polygon. Without given radius a unit sphere is assumed.

Source: en.wikipedia.org/wiki/Spherical_trigonometry # Area and spherical excess

The method has been altered to allow for the usage of an internal point given with the Polygon.
This method only works with polygons that fit within half a sphere as it uses great circle distances between
subsequent points.
"""
function area(polygon::Polygon, radius::Float64=1.0)
    point₁ = polygon.points[1]
    area_polygon_unit_sphere = 0.0
    for point₂ in polygon.points[2:end]
        n_intersectionsᵢ = length(intersection_points(Arc(polygon.inside_point,
        midpoint(point₁, point₂)), polygon))
        isodd(n_intersectionsᵢ) ? area_polygon_unit_sphere +=
        area(polygon.inside_point, point₁, point₂) : area_polygon_unit_sphere -=
        area(polygon.inside_point, point₁, point₂)

        point₁ = point₂
    end
    return area_polygon_unit_sphere * radius^2
end

"""
    area(angular_distance₁₂::Float64, angular_distance₂₃::Float64, angular_distance₁₃::Float64,
    radius::Float64=1.0)

Return the area of the spherical triangle 123 based on the angular distances [deg] 12-23-13. Without given
radius a unit sphere is assumed.

Source: mathworld.wolfram.com/SphericalTriangle.html
"""
function area(angular_distance₁₂::Float64, angular_distance₂₃::Float64, angular_distance₁₃::Float64,
    radius::Float64=1.0)
    return deg2rad(spherical_excess(angular_distance₁₂, angular_distance₁₃, angular_distance₂₃))*radius^2
end

"""
    area(point₁::Point, point₂::Point, point₃::Point, radius::Float64=1.0)

Return the area of the spherical triangle 123 based on the points point₁ - point₂ - point₃. Without given radius
a unit sphere is assumed.

Source: mathworld.wolfram.com/SphericalTriangle.html
"""
area(point₁::Point, point₂::Point, point₃::Point, radius::Float64=1.0) =
area(angular_distance(point₁, point₂), angular_distance(point₂, point₃), angular_distance(point₃, point₁),
radius)


using JuMP, Gurobi, Distributions, StatsFuns

const gurobi_env = Gurobi.Env()

# DIMENSIONS
const numVariables = 100
const numJCC = 100
const conRHS = numVariables^2

# CASES
# number of samples, replicates for scenario approximations
const NumSamples = ceil.(Int64,logspace(1,log10(50000.0),50))
const NumReplicates = 20

# FILENAMES
const objFile = "objectiveValue.txt"
const solnFile = "solution.txt"
const riskFile = "riskLevel.txt"
const riskTimeFile = "riskTime.txt"
const solnTimeFile = "solutionTime.txt"
const checkingTimeFile = "checkingTime.txt"
const sortingTimeFile = "sortingTime.txt"
const callbackFile = "numCallbacks.txt"
const numConstraintsFile = "numConstraintsEnforced.txt"

xi_mean = zeros(Float64,numVariables)
xi_covariance = 0.5*ones(Float64,numJCC,numJCC)

for j = 1:numVariables
	xi_mean[j] = j*1.0/numVariables
end

for i = 1:numJCC
	xi_covariance[i,i] = 1.0
end

xi_cov_chol_tmp = cholfact(xi_covariance)
xi_cov_chol = xi_cov_chol_tmp[:L]

xi_mean_mat = zeros(Float64,numJCC,numVariables)
for j = 1:numVariables
	xi_mean_mat[:,j] = xi_mean[j]*ones(numJCC)
end

# tailored implementation of multivariate normal distribution
function myMvNormal(numSamples::Int64)

	xi = zeros(Float64,numJCC,numVariables,numSamples)

	xi_tmp = rand(Normal(0.0,1.0),numJCC,numVariables,numSamples)
	for samp = 1:numSamples
		xi[:,:,samp] = xi_cov_chol*xi_tmp[:,:,samp] + xi_mean_mat
	end

	return xi
end

# generate random samples from multivariate normal distribution
function generateRandomSamples(numSamples::Int64)

	xi = myMvNormal(numSamples)

	return xi
end

srand(1234)

# generate samples to estimate actual risk level of solution
const numAnalyticalSamples = 20000
const xi_analytical = generateRandomSamples(numAnalyticalSamples)

srand()

# compute negative probability of all constraints being satisfied
# for given scenarios of demands d and a given decision vector x
function computeRiskLevel(x::Array{Float64},reliabilityLevel::Float64=1E-06)

	const numSamples = size(xi_analytical,3)
	simpleEst::Bool = false

	numViolated::Int64 = checkScenarioConstraints(x,xi_analytical)

	riskLevel::Float64 = Inf

	if(simpleEst)
		riskLevel = numViolated*1.0/numSamples
		return riskLevel
	else
		if(numViolated == 0)
			riskLevel = -log(reliabilityLevel)/numSamples
			return riskLevel
		end

		 # termination criterion for bisection
		gamma_tolerance::Float64 = 0.1/numSamples

		# use tail bounds on binomial distribution to estimate lower and upper bounds
		gamma_low::Float64 = (numViolated - sqrt(-numSamples*log(reliabilityLevel)/2.0))/numSamples
		if(gamma_low < 0.0)
			gamma_low = 0.0
		end
		gamma_up::Float64 = (numViolated + sqrt(-numSamples*log(reliabilityLevel)/2.0))/numSamples
		if(gamma_up > 1.0)
			gamma_up = 1.0
		end

		gamma::Float64 = 0.0
		func_value::Float64 = 0.0

		# use bisection to solve the nonlinear equation
		while (gamma_up - gamma_low > gamma_tolerance)

			gamma = (gamma_low + gamma_up)/2.0
			func_value = -reliabilityLevel
			for i = 1:numViolated
				tmp_func = i*log(gamma) + (numSamples-i)*log(1-gamma) + lfact(numSamples) - lfact(numSamples-i) - lfact(i)
				func_value += exp(tmp_func)
				if(func_value > 0)
					break
				end
			end
			if(func_value > 0)
				gamma_low = (gamma_low + gamma_up)/2.0
			else
				gamma_up = (gamma_low + gamma_up)/2.0
			end

		end

		riskLevel = gamma_up
		return riskLevel
	end

end

# determines if all constraints are satisfied for a given single scenario
# returns one if scenario constraints are all satisfied, zero otherwise
function checkScenarioConstraints(x::Array{Float64},xi::Array{Float64})

	numSamples::Int64 = size(xi,3)

	numViolated::Int64 = 0
	for iter = 1:numSamples
		for i = 1:numJCC
			con::Float64 = evaluateConstraint(i,x,xi[i,:,iter])
			if(con > 0)
				numViolated += 1
				break
			end
		end
	end

	return numViolated
end

# evaluate the constraint values for given decision vector x,
# realization of the random variables xi, and constraint scalings
function evaluateConstraint(index::Int64,x::Array{Float64},xi::Array{Float64})

	con::Float64 = norm(xi.*x)^2 - conRHS

	return con
end

# determines if all constraints are satisfied for a given single scenario
# returns one if scenario constraints are all satisfied, zero otherwise
function getScenarioConstraintViolations(x::Array{Float64},xi::SubArray{Float64,2,Array{Float64,3},Tuple{Int64,Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true},pickFirstSetOfConstraints::Bool,maxNumConstraintsPerIteration::Int64)

	const numSamples = size(xi,2)
	numViolated::Int64 = 0
	constraintViolations = Float64[]
	constraintViolationIndices = Int64[]
	constraintViolationTolerance::Float64 = 1E-06

	for iter = 1:numSamples
		@views con::Float64 = norm(xi[:,iter].*x) - sqrt(conRHS)

		if(con >= constraintViolationTolerance)
			numViolated += 1
			push!(constraintViolations,con)
			push!(constraintViolationIndices,iter)
			if(pickFirstSetOfConstraints && numViolated == maxNumConstraintsPerIteration)
				break
			end
		end
	end

	return numViolated, constraintViolations, constraintViolationIndices
end

# write solution to file
function initializeFiles(details_file::String)

	writeMode::String = "w"

	open(details_file, writeMode) do f
		write(f,"Model: $modelName \n")
		write(f,"numVariables: $numVariables \n")
		write(f,"numJCC: $numJCC \n")
		write(f,"conRHS: $conRHS \n")
		write(f,"NumSamples: $NumSamples \n")
		write(f,"NumReplicates: $NumReplicates \n")
		write(f,"Method: Scenario approximations \n")
	end

end

# write solution to file
function writeSolutionToFile(dirName::String, status::Symbol, objvar_opt::Float64, x_opt::Array{Float64}, riskLevel::Float64, solutionTime::Float64, checkingTime::Float64, sortingTime::Float64, numCallbacks::Int64, numConstraintsAdded::Int64, riskTime::Float64)

	if(status == Symbol("Optimal"))
		writeMode::String = "a"

		obj_file::String = dirName * objFile
		open(obj_file, writeMode) do f
			write(f,"$objvar_opt \n")
		end

		solution_file::String = dirName * solnFile
		open(solution_file, writeMode) do f
			for i = 1:numVariables
				write(f,"$(x_opt[i]) ")
			end
			write(f,"\n")
		end

		risk_file::String = dirName * riskFile
		open(risk_file, writeMode) do f
			write(f,"$riskLevel \n")
		end

		risktime_file::String = dirName * riskTimeFile
		open(risktime_file, writeMode) do f
			write(f,"$riskTime \n")
		end

		solntime_file::String = dirName * solnTimeFile
		open(solntime_file, writeMode) do f
			write(f,"$solutionTime \n")
		end

		checktime_file::String = dirName * checkingTimeFile
		open(checktime_file, writeMode) do f
			write(f,"$checkingTime \n")
		end

		sorttime_file::String = dirName * sortingTimeFile
		open(sorttime_file, writeMode) do f
			write(f,"$sortingTime \n")
		end

		callback_file::String = dirName * callbackFile
		open(callback_file, writeMode) do f
			write(f,"$numCallbacks \n")
		end

		constraints_file::String = dirName * numConstraintsFile
		open(constraints_file, writeMode) do f
			write(f,"$numConstraintsAdded \n")
		end
	end

end

# solve the scenario approximation model
function solveScenarioApproximationModel(numScenarios::Int64,dirName::String)

	tic()

	# sample the random variables
	xi = generateRandomSamples(numScenarios)

	maxNumConstraintsPerIteration::Int64 = 10
	numConstraintsAdded::Int64 = 0

	pickFirstSetOfConstraints::Bool = false

	sortingTime::Float64 = 0.0
	checkingTime::Float64 = 0.0
	numCallbacks::Int64 = 0

	objvar_opt::Float64 = Inf
	y_opt = zeros(Float64,numVariables)
	status::Symbol = Symbol("Error")

	numViolations::Int64 = numScenarios

	constraintIndices = [Int64[] for i=1:numJCC]

	while(numViolations > 0)

		numCallbacks += 1
		numViolations = 0

		numConstraintsConsidered = zeros(Int64,numJCC)
		for i = 1:numJCC
			numConstraintsConsidered[i] = size(constraintIndices[i])[1]
		end

		mod = Model(solver=GurobiSolver(gurobi_env,Presolve=0,OutputFlag=0))

		@variable(mod, objvar)
		@variable(mod, 0 <= y[1:numVariables] <= 10^6)

		@objective(mod, Min, objvar)
		@constraint(mod, -sum(y[j] for j = 1:numVariables) <= objvar)
		@constraint(mod, [i=1:numJCC, scen=1:numConstraintsConsidered[i]], norm(xi[i,:,constraintIndices[i][scen]].*y) <= sqrt(conRHS))

		status = solve(mod)
		objvar_opt = getvalue(objvar)
		y_opt = getvalue(y)

		for i = 1:numJCC

			numViolatedConstraints::Int64 = 0
			constraintViolations = Float64[]
			constraintViolationIndices = Int64[]

			tic()

			@views numViolatedConstraints, constraintViolations, constraintViolationIndices = getScenarioConstraintViolations(y_opt,xi[i,:,:],pickFirstSetOfConstraints,maxNumConstraintsPerIteration)

			checkingTime += toq()

			numConstraintsEnforced::Int64 = min(size(constraintViolations,1),maxNumConstraintsPerIteration)

			tic()
			relevantIndices = 1:1:numConstraintsEnforced
			if(!pickFirstSetOfConstraints)
				relevantIndices = sortperm(constraintViolations, rev=true)
			end
			sortingTime += toq()

			for iter = 1:numConstraintsEnforced
				push!(constraintIndices[i],constraintViolationIndices[relevantIndices[iter]])
			end

			numConstraintsAdded += numConstraintsEnforced

			numViolations += numViolatedConstraints
		end

	end

	tic()

	riskLevel::Float64 = computeRiskLevel(y_opt)

	riskTime = toq()

	solutionTime::Float64 = toq()

	@printf "  Objective: %2.5f,  Risklevel: %.6f,  Time: %4.2f,  NumCallbacks: %d,  NumConstraints: %d \n" objvar_opt riskLevel solutionTime numCallbacks numConstraintsAdded

	writeSolutionToFile(dirName, status, objvar_opt, y_opt, riskLevel, solutionTime, checkingTime, sortingTime, numCallbacks, numConstraintsAdded, riskTime)

	return status, objvar_opt, y_opt, riskLevel, solutionTime, checkingTime, sortingTime, numCallbacks, numConstraintsAdded, riskTime
end


function eigmaxreal(A::Matrix)
  if size(A,1)<= 2
    return maximum(real(eigvals(A)))
  else
    return eigs(A,nev=1,which = :LR)[1][1]
  end
end
function eigminreal(A::Matrix)
  if size(A,1)<= 2
    return minimum(real(eigvals(A)))
  else
    return eigs(A,nev=1,which = :SR)[1][1]
  end
end


# This file calculates the first application in Y.-S. Chan, A. C. Fannjiang, and G. H. Paulino,
# Integral equations with hypersingular kernels -- theory and applications to fracture mechanics,
# Int. J. Eng. Sci., 41:683--720, 2003.

using ApproxFun, SingularIntegralEquations

x = Fun(identity)
w = 1/sqrt(1-x^2)
d = domain(x)
d2 = d^2
B = dirichlet(d)
H2 = Hilbert(d,2)
Σ = DefiniteIntegral(d)
ϵ = 2.0
K = LowRankFun((x,y)->-1./(x+y+2ϵ).^2+12(x+ϵ)./(x+y+2ϵ).^3-12(x+ϵ).^2./(x+y+2ϵ).^4,d2)
L = H2[w] + Σ[K*(w/π)]
f = -Fun(one)

uSIE = [B;L]\[zeros(2);f]
@time uSIE = [B;L]\[zeros(2);f]
println("The rank of K is: ",rank(K))
println("The length of uSIE is: ",length(uSIE))
println("The extrema of uSIE are: ",extrema(uSIE))
println("The normalized Stress Intensify Factors are: ",(uSIE/(1-x^2))(-1),"  ",(uSIE/(1-x^2))(1))


abstract AbstractMeasure{T}

abstract SymmetricMeasure{T} <: AbstractMeasure{T}

# Entropy
# =======

"""
Shannon entropy (H)
"""
immutable Entropy{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{Entropy{T}}) = Entropy(T(Base.e))

## Estimate Entropy using ResidueProbability

"""
`estimate(Entropy(base), p)`

`p` should be a `ResidueProbability` table. The result type is determined by `base`.
"""
function estimate{B, T, N, UseGap}(measure::Entropy{B}, p::ResidueProbability{T, N, UseGap})
  H = zero(B)
  for i in 1:length(p)
    @inbounds pi = B(p[i])
    if pi != 0.0
      H += pi * log(pi)
    end
  end
  -H/log(measure.base)
end

"""
`estimate_on_marginal(Entropy{T}(base), p, marginal)`

This function estimate the entropy H(X) if marginal is 1, H(Y) for 2, etc.
The result type is determined by `base`.
"""
function estimate_on_marginal{B, T, N, UseGap}(measure::Entropy{B}, p::ResidueProbability{T, N, UseGap}, marginal::Int)
  H = zero(B)
  for i in 1:nresidues(p)
    @inbounds pi = B(p.marginals[i, marginal])
    if pi != 0.0
      H += pi * log(pi)
    end
  end
  -H/log(measure.base)
end

## Estimate Entropy using ResidueCount

"""
`estimate(Entropy{T}(base), n::ResidueCount)`

It's the fastest option (you don't spend time on probability calculations).
The result type is determined by the `base`.
"""
function estimate{T}(measure::Entropy{T}, n::ResidueCount)
  H = zero(T)
  total = T(n.total)
  for i in 1:length(n)
    @inbounds ni = T(n[i])
    if ni != 0.0
      H += ni * log(ni/total)
    end
  end
  (-H/total)/log(measure.base)
end

function estimate_on_marginal{T}(measure::Entropy{T}, n::ResidueCount, marginal::Int)
  H = zero(T)
  total = T(n.total)
  for i in 1:nresidues(n)
    @inbounds ni = T(n.marginals[i, marginal])
    if ni != 0.0
      H += ni * log(ni/total)
    end
  end
  (-H/total)/log(measure.base)
end

# Kullback-Leibler
# ================

"""
Kullback-Leibler (KL). This `SymmetricMeasure` has two fields.
The first is the base of the logarithm and the second is the backgroud frequency.
"""
immutable KullbackLeibler{T} <: SymmetricMeasure{T}
  base::T
  background::AbstractArray{T}
end

call{T}(::Type{KullbackLeibler{T}}, background::AbstractArray{T}) = KullbackLeibler(T(Base.e), background)

"""
`estimate(KullbackLeibler(base, background), p)`

`p` should be a `ResidueProbability` table, and `background` must have the size of `p`.
The result type is determined by `base` and `background`.
"""
function estimate{B, T, N, UseGap}(measure::KullbackLeibler{B}, p::ResidueProbability{T, N, UseGap})
  q = measure.background
  if size(q) != size(p) || length(q) != length(p)
    throw(ErrorException("p and background should have the same size and length."))
  end
  KL = zero(B)
  @inbounds for i in 1:length(p)
    pi = B(p[i])
    if pi != 0.0
      KL += pi * log(pi/q[i])
    end
  end
  KL/log(measure.base)
end

# Mutual Information
# ==================

"""
Mutual Information (MI)
"""
immutable MutualInformation{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{MutualInformation{T}}) = MutualInformation(T(Base.e))

@inline _mi{T}(::Type{T}, pij, pi, pj) = ifelse(pij > zero(T) && pi > zero(T), T(pij * log(pij/(pi*pj))), zero(T))

"""
`estimate(MutualInformation(), pxy::ResidueProbability [, base])`

Calculate Mutual Information from `ResidueProbability`. The result type is determined by `base`.
"""
function estimate{B, T, UseGap}(measure::MutualInformation{B}, pxy::ResidueProbability{T, 2,UseGap})
  MI = zero(B)
  marginals = pxy.marginals
  @inbounds for j in 1:nresidues(pxy)
    pj = marginals[j,2]
    if pj > 0.0
      @inbounds @simd for i in 1:nresidues(pxy)
        MI +=  _mi(B, pxy[i,j], marginals[i,1], pj)
      end
    end
  end
  MI/log(measure.base)
end

@inline _mi{T}(N::T, nij, ni, nj) = ifelse(nij > zero(T) && ni > zero(T), T(nij * log((N * nij)/(ni * nj))), zero(T))

"""
`estimate(MutualInformation(), pxy::ResidueCount [, base])`

Calculate Mutual Information from `ResidueCount`. The result type is determined by the `base`.
It's the fastest option (you don't spend time on probability calculations).
"""
function estimate{B, T, UseGap}(measure::MutualInformation{B}, nxy::ResidueCount{T, 2,UseGap})
  MI = zero(B)
  N = B(nxy.total)
  marginals = nxy.marginals
  @inbounds for j in 1:nresidues(nxy)
    nj = marginals[j,2]
    if nj > 0.0
      @inbounds @simd for i in 1:nresidues(nxy)
        MI += _mi(N, nxy[i,j], marginals[i,1], nj)
      end
    end
  end
  (MI/N)/log(measure.base)
end

function estimate{B, T, UseGap}(measure::MutualInformation{B}, pxyz::ResidueContingencyTables{T, 3, UseGap})
  pxy = delete_dimensions(pxyz,3)
  return( estimate_on_marginal(Entropy(measure.base), pxyz, 1) + # H(X)
  estimate_on_marginal(Entropy(measure.base), pxyz ,2) + # H(Y)
  estimate_on_marginal(Entropy(measure.base), pxyz, 3) - # H(Z)
  estimate(Entropy(measure.base), pxy) - # H(X, Y)
  estimate(Entropy(measure.base), delete_dimensions!(pxy, pxyz, 2)) - # H(X, Z)
  estimate(Entropy(measure.base), delete_dimensions!(pxy, pxyz, 1)) + # H(Y, Z)
  estimate(Entropy(measure.base), pxyz) ) # H(X, Y, Z)
end

# Normalized Mutual Information by Entropy
# ========================================

"""
Normalized Mutual Information (nMI) by Entropy.

`nMI(X, Y) = MI(X, Y) / H(X, Y)`
"""
immutable MutualInformationOverEntropy{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{MutualInformationOverEntropy{T}}) = MutualInformationOverEntropy(T(Base.e))

function estimate{B}(measure::MutualInformationOverEntropy{B}, table)
  H = estimate(Entropy(measure.base), table)
  if H != zero(B)
    MI = estimate(MutualInformation(measure.base), table)
    return(MI/H)
  else
    return(zero(B))
  end
end

# Pairwise Gap Percentage
# =======================

"""
`GapUnionPercentage`
"""
immutable GapUnionPercentage{T} <: SymmetricMeasure{T} end

"""
`GapIntersectionPercentage`
"""
immutable GapIntersectionPercentage{T} <: SymmetricMeasure{T} end

function estimate{B, T}(measure::GapIntersectionPercentage{B}, nxy::ResidueCount{T, 2, true})
  B(100.0) * B(nxy[21, 21]) / B(sum(nxy))
end

function estimate{B, T}(measure::GapUnionPercentage{B}, nxy::ResidueCount{T, 2, true})
  B(100.0) * B(nxy.marginals[21, 1] + nxy.marginals[21, 2] - nxy[21, 21]) / B(sum(nxy))
end


Base.:+(p::Pol) = p
Base.:-(p::Pol) = Pol([-c for c in p.coeffs])

function Base.:+(p1::Pol, p2::Pol)
    maxdeg = max(deg(p1), deg(p2))
    coeffs = [p1[i]+p2[i] for i in 1:maxdeg+1]
    return Pol(coeffs)
end

Base.:+(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = +(promote(p1, p2)...)
Base.:-(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = p1 + (-p2)

function Base.:*(p1::Pol{T}, p2::Pol{T}) where {T<:Number}
    coeffs = zeros(T, deg(p1)+deg(p2)+1)
    @inbounds for (i, c1) in enumerate(p1.coeffs)
        @inbounds for (j, c2) in enumerate(p2.coeffs)
            coeffs[i+j-1] += c1*c2
        end
    end
    return Pol(coeffs)
end

Base.:*(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = *(promote(p1, p2)...)

Base.:/(p::Pol, n::Number)  = Pol([c/n for c in p.coeffs])

function Base.:^(p::Pol, n::Integer)
    iszero(n) && return Pol([1])
    if iseven(n)
        return (p*p)^(n÷2)
    else
        return p*(p*p)^((n-1)÷2)
    end
end

function Base.divrem(a::Pol, b::Pol)
    r = a
    q = zero(a)
    d = deg(b)
    c = lc(b)
    while !iszero(r) && deg(r) >= d
        s = lc(r)/c
        coeffs = zeros(typeof(s), deg(r)-d+1)
        coeffs[end] = s
        s = Pol(coeffs)
        q, r = q + s, r - s*b
    end
    return q, r
end

function Base.div(a::Pol, b::Pol)
    q, _ = divrem(a, b)
    return q
end

function Base.rem(a::Pol, b::Pol)
    _, r = divrem(a, b)
    return r
end

function Base.gcd(a::Pol, b::Pol)
    r0 = a
    r1 = b
    while !iszero(r1)
        r1, r0 = rem(r0, r1), r1
    end
    return r0/lc(r0)
end

# comparisons

function Base.:(==)(p1::Pol, p2::Pol)
    deg(p1) == deg(p2) || return false
    @inbounds for i in 1:length(p1.coeffs)
        p1.coeffs[i] == p2.coeffs[i] || return false
    end
    return true
end

for op in (:>, :<, :>=, :<=)
    @eval Base.$op(p1::Pol, p2::Pol) = $op(deg(p1), deg(p2))
end

Base.iszero(p::Pol) = iszero(p.coeffs)


using Unitful#, UnitfulPlots
using OrdinaryDiffEq, DiffEqBase

const UNITS_TEST_ALGS = [Euler(),Midpoint(),RK4(),SSPRK104(),SSPRK22(),SSPRK33(),
    SSPRK432(),BS3(),BS5(),DP5(),DP5Threaded(),DP8(),Feagin10(),Feagin12(),Feagin14(),
    TanYam7(),Tsit5(),TsitPap8(),Vern6(),Vern7(),Vern8(),Vern9()]

f = (y,p,t) -> 0.5*y / 3.0u"s"
u0 = 1.0u"N"
prob = ODEProblem(f,u0,(0.0u"s",1.0u"s"))

sol =solve(prob,ExplicitRK())

for alg in UNITS_TEST_ALGS
  if !(typeof(alg) <: DP5Threaded)
    @show alg
    sol = solve(prob,alg,dt=1u"s"/10)
  end
end

println("Units for Number pass")

u0 = [1.0u"N" 2.0u"N"
      3.0u"N" 1.0u"N"]
f = (dy,y,p,t) -> (dy .= 0.5.*y ./ 3.0u"s")
prob = ODEProblem(f,u0,(0.0u"s",1.0u"s"))

sol =solve(prob,ExplicitRK())

for alg in UNITS_TEST_ALGS
  @show alg
  sol = solve(prob,alg,dt=1u"s"/10)
end

println("Units for 2D pass")

using Unitful, RecursiveArrayTools, DiffEqBase, OrdinaryDiffEq
using LinearAlgebra
r0 = [1131.340, -2282.343, 6672.423]u"km"
v0 = [-5.64305, 4.30333, 2.42879]u"km/s"
Δt = 86400.0*365u"s"
μ = 398600.4418u"km^3/s^2"
rv0 = ArrayPartition(r0,v0)

function f(dy, y, μ, t)
    r = norm(y.x[1])
    dy.x[1] .= y.x[2]
    dy.x[2] .= -μ .* y.x[1] / r^3
end

prob = ODEProblem(f,rv0,(0.0u"s",1.0u"s"),μ)
sol = solve(prob,Tsit5())

# coordinate: u = [position, momentum]
# parameters: p = [mass, force constanst]
function f_harmonic!(du,u,p,t)
  du[1] = u[2]/p[1]
  du[2] = -p[2]*u[1]
end

mass = 1.0u"kg"
k = 1.0u"N/m"
p = [mass, k]

u0 = [1.0u"m", 0.0u"kg*m/s"] # initial values (position, momentum)
tspan = (0.0u"s", 10.0u"s")
prob = ODEProblem(f_harmonic!, u0, tspan, p)
sol = solve(prob)