function standard_normal_gausshermite(n::Int)
    ϵᵢ, wᵢ = gausshermite(n) # approximates exp(-x²)
    ϵᵢ   .*= sqrt(2.)        # Normalize ϵᵢ and wᵢ nodes to approximate standard normal
    wᵢ   ./= sqrt(π)

    return ϵᵢ, wᵢ
end

"""
```
gausshermite_expectation(f, μ, σ, n = 10)
gausshermite_expectation(f, μ, Σ, n = 10)
gausshermite_expectation(f, μ, Σ, ns)
```

calculates the expectation of a function of a Gaussian random variable/vector.
The first method evalulates ``\\mathbb{E}[f(X)]`` where ``X \\sim N(\\mu, \\sigma)``,
while the other two methods evaluate ``\\mathbb{E}[f(X)]`` where
``X \\sim \\mathcal{N}(\\mu, \\Sigma)`` and ``\\Sigma`` is diagonal.
The latter two methods differ in that the first assumes the same number of
quadrature points in every dimension while the second does not.

### Inputs
- `f::Function`: some function of a random variable. If `f(x) = x`, then
    `gausshermite_expectation(f, μ, σ)` calculates the mean of ``N(\\mu, \\sigma)``
    using 10-point Gauss-Hermite quadrature.
- `μ::Number` or `μ::AbstractVector`: mean of the Gaussian random variable/vector.
- `σ::Number`: standard deviation of the Gaussian random variable.
- `Σ::AbstractVector`: diagonal of the variance-covariance matrix of
     the Gaussian random vector.
- `n::Int`: number of quadrature points to use
- `ns::AbstractVector{Int}` or `ns::NTuple{N, Int} where N`: number of quadrature points to use
    in each dimension of the Gaussian random vector.
"""
function gausshermite_expectation(f::Function, μ::Number, σ::Number, n::Int = 10)
    ϵᵢ, wᵢ = gausshermite(n)
    ϵᵢ   .*= sqrt(2.)  # Normalize ϵᵢ and wᵢ nodes to approximate standard normal
    # wᵢ   ./= sqrt(π) # This step done later to reduce number of computations

    if μ ≈ 0.
        return sum([wᵢ[i] * f(ϵᵢ[i] * σ) for i in 1:n]) / sqrt(π)
    else
        return sum([wᵢ[i] * f(ϵᵢ[i] * σ + μ) for i in 1:n]) / sqrt(π)
    end

end

function gausshermite_expectation(f::Function, μ::AbstractVector{S},
                                  Σ::AbstractVector{<: Number}, n::Int = 10) where {S <: Number}

    d = length(μ)
    @assert length(Σ) == d "The length of μ and Σ must be the same."
    ϵ, w = gausshermite(n)
    ϵ   .*= sqrt(2.)  # Normalize ϵ and w nodes to approximate standard normal
    # w   ./= sqrt(π) # This step done later to reduce number of computations

    # Evaluate over the tensor grid
    feval = Array{S}(undef, (n for i in 1:d)...)
    allCI = CartesianIndices(feval)
    if all(μ .≈ 0.)
        @simd for CI in allCI
            feval[CI] = f([ϵ[i] for i in Tuple(CI)] .* Σ)
        end
    else
        @simd for CI in allCI
            feval[CI] = f([ϵ[i] for i in Tuple(CI)] .* Σ + μ)
        end
    end

    for n_dim in 1:(d - 1)
        # Iteratively integrate out each dimension, i.e. law of iterated expectations
        iter = CartesianIndices(tuple(Tuple(1:n for i in 1:(d - n_dim))...,
                                      Tuple(1:1 for i in 1:n_dim)...)) # Create CartesianIndices for all remaining dimensions
        # ((1:n for i in 1:(d - n_dim + 1))..., (1 for i in 1:(n_dim - 1))...) creates a Tuple of 1:n for the dimensions
        # that are not to be integrated out and uses 1s for the remaining dimensions. We want to use each dimension of feval
        # from 1 to (d - n_dim) (inclusive). So on the first iteration, the tuple should be (1:n, 1:n).
        # We then assign it to the dimensions of feval from 1 to (d - n_dim - 1) (inclusive) to avoid allocations
        feval[iter] .= dropdims(sum(mapslices(fᵢ -> fᵢ .* w, (@view feval[((1:n for i in 1:(d - n_dim + 1))...,
                                                                           (1 for i in 1:(n_dim - 1))...)...]),
                                              dims = (d - n_dim) + 1), dims = (d - n_dim) + 1), dims = (d - n_dim) + 1)
    end

    # Handle final integration on its own
    return sum(w .* (@view feval[:, (1 for i in 1:(d - 1))...])) / π^(d / 2)
end

function gausshermite_expectation(f::Function, μ::AbstractVector{S},
                                  Σ::AbstractVector{<: Number}, ns::AbstractVector{Int}) where {S <: Number}

    d = length(μ)
    @assert length(Σ) == d "The length of μ and Σ must be the same."
    ϵ = Dict{Int, Vector{S}}()
    w = Dict{Int, Vector{S}}()
    for i in 1:d
        ϵ[i], w[i] = gausshermite(ns[i])
        ϵ[i]     .*= sqrt(2.)  # Normalize ϵ and w nodes to approximate standard normal
        # w[i]   ./= sqrt(π) # This step done later to reduce number of computations
    end

    # Evaluate over the tensor grid
    feval = Array{S}(undef, (n for n in ns)...)
    allCI = CartesianIndices(feval)
    if all(μ .≈ 0.)
        @simd for CI in allCI
            feval[CI] = f([ϵ[n_dim][gridᵢ] for (n_dim, gridᵢ) in enumerate(Tuple(CI))] .* Σ)
        end
    else
        @simd for CI in allCI
            feval[CI] = f([ϵ[n_dim][gridᵢ] for (n_dim, gridᵢ) in enumerate(Tuple(CI))] .* Σ + μ)
        end
    end

    # Iteratively integrate out each dimension, i.e. law of iterated expectations
    for n_dim in 1:(d - 1)
        iter = CartesianIndices(tuple(Tuple(1:ns[i] for i in 1:(d - n_dim))...,
                                      Tuple(1:1 for i in 1:n_dim)...))
        feval[iter, 1] .= dropdims(sum(mapslices(fᵢ -> fᵢ .* w[d - n_dim + 1], (@view feval[((1:ns[i] for i in 1:(d - n_dim + 1))...,
                                                                                (1 for i in 1:(n_dim - 1))...)...]),
                                                 dims = (d - n_dim) + 1), dims = (d - n_dim) + 1), dims = (d - n_dim) + 1)
    end

    # Handle final integration on its own
    return sum(w[1] .* (@view feval[:, (1 for i in 1:(d - 1))...])) / π^(d / 2)
end

function gausshermite_expectation(f::Function, μ::AbstractVector{S},
                                  Σ::AbstractVector{<: Number}, ns::NTuple{N, Int}) where {S<: Number, N}

    d = length(μ)
    @assert length(Σ) == d "The length of μ and Σ must be the same."
    ϵ = Dict{Int, Vector{S}}()
    w = Dict{Int, Vector{S}}()
    for i in 1:d
        ϵ[i], w[i] = gausshermite(ns[i])
        ϵ[i]     .*= sqrt(2.)  # Normalize ϵ and w nodes to approximate standard normal
        # w[i]   ./= sqrt(π) # This step done later to reduce number of computations
    end

    # Evaluate over the tensor grid
    feval = Array{S}(undef, (n for n in ns)...)
    allCI = CartesianIndices(feval)
    if all(μ .≈ 0.)
        @simd for CI in allCI
            feval[CI] = f([ϵ[n_dim][gridᵢ] for (n_dim, gridᵢ) in enumerate(Tuple(CI))] .* Σ)
        end
    else
        @simd for CI in allCI
            feval[CI] = f([ϵ[n_dim][gridᵢ] for (n_dim, gridᵢ) in enumerate(Tuple(CI))] .* Σ + μ)
        end
    end

    # Iteratively integrate out each dimension, i.e. law of iterated expectations
    for n_dim in 1:(d - 1)
        iter = CartesianIndices(tuple(Tuple(1:ns[i] for i in 1:(d - n_dim))...,
                                      Tuple(1:1 for i in 1:n_dim)...))
        feval[iter, 1] .= dropdims(sum(mapslices(fᵢ -> fᵢ .* w[d - n_dim + 1], (@view feval[((1:ns[i] for i in 1:(d - n_dim + 1))...,
                                                                                (1 for i in 1:(n_dim - 1))...)...]),
                                                 dims = (d - n_dim) + 1), dims = (d - n_dim) + 1), dims = (d - n_dim) + 1)
    end

    # Handle final integration on its own
    return sum(w[1] .* (@view feval[:, (1 for i in 1:(d - 1))...])) / π^(d / 2)
end


"
    Polynomial{T<:Number} <: Function

A Polynomial struct implements the mathematical object P(x) = ∑ cᵢ xⁱ by
storing the coefficients cᵢ (i ∈ {0,..., deg(P)}) into a 1-dimensional array of
numbers. The **degree** of a polynomial is the highest power of the xᵢ terms.
"
struct Polynomial{T<:Number} <: Function
    coeffs::Array{T,1}
    degree::Int

    function Polynomial{T}(c::Array{T,1}) where {T<:Number}
        n = length(c) - 1
        new(c, n)
    end

end

Polynomial(c::Array{T,1}) where {T<:Number} = Polynomial{T}(c)

deg(poly::Polynomial{T}) where {T<:Number} = poly.degree

function (poly::Polynomial{T})(x::Number, bases=zeros(T, deg(poly), 1)) where {T<:Number}
    #= Method 1: The first and most straightforward approach:
    result = 0
    for (power, c) in enumerate(poly.coeffs)
        result += c * x^(power - 1)
    end
    =#

    #= Method 2: Find the powers and store them for future uses:
    result = 0
    power = 1
    for c in poly.coeffs
        result += c * power
        power *= x
    end
    =#

    #= Method 3 (Horner): Nest multiplication =#
    result = poly.coeffs[end]
    for i = deg(poly):-1:1
        result *= x - bases[i]
        result += poly.coeffs[i]
    end

    result
end


using Distributions: Normal, Uniform

is_vector(x) = size(x,2) == 1
nrows(x) = size(x,1)
dim_error(message:: String) = throw(DimensionMismatch(message))

"""Sample(X,Y)

Sample holds X array of size n*k (explanatory variables,
independent variables, features or regressor) and Y, a n*1 sized
vector, for observed data (response, dependent variable or regressand).
"""
struct Sample
    X::Array
    Y::Vector
    function Sample(X, Y)
        nrows(X) == nrows(Y) || dim_error("X and Y must have same number of rows")
        is_vector(Y) || dim_error("Y must be a vector: $Y")
        new(X, Y)
    end
end

function add_intercept(X::Array)::Array
    X0 = ones(nrows(X), 1)
    return [X0 X]
end

# TODO: do not add intercept if it is already there.
function add_intercept(sample::Sample)
    return Sample(add_intercept(sample.X), sample.Y)
end

function sample_factory(;x_process, y_process, error_process)
    function sampler(n:: Int)
        X = x_process(n)
        Y = vec(y_process(X) + error_process(X))
        return Sample(X, Y)
    end
end

function normal_sampler(;a, b, β_0, β, sd_e)
    k = length(β)
    du = Uniform(a, b) # this is squares for x only
    dn = Normal(0, sd_e)
    sample_factory(x_process = n -> rand(du, n, k),
                   y_process = X -> β_0 .+ X * β,
                   error_process = X -> rand(dn, nrows(X), 1)
                   )
end


module AngleBetweenVectors

import Base: angle

import LinearAlgebra: norm

@inline unitize(p) = p ./ norm(p)

"""
    angle(point1::T, point2::T) where {T}

Accurately ascertains the undirected angle (0 <= radians < pi)
between two points given in N-dimensional Cartesian coordinates.

Prefer this to `acos` alternatives
- more reliably accurate
- more consistently stable

Suggested when any |coordinate| of either point may be outside 2^±20 or [1/1_000_000, 1_000_000].
Strongly recommended when any |coordinate| is outside 2^±24 or [1/16_000_000, 16_000_000].

If one of the points is at the origin, the result is zero.

You *must* define a tuple constructor `Tuple(x::YourPointType) = ...` if one does not already exist.
"""
function angle(point1::A, point2::A) where {N,T<:Real,NT<:NTuple{N,T}, V<:Vector{T}, A<:Union{NT,V}}
    unitpoint1 = unitize(point1)
    unitpoint2 = unitize(point2)

    y = unitpoint1 .- unitpoint2
    x = unitpoint1 .+ unitpoint2

    a = 2 * atan(norm(y) / norm(x))

    !(signbit(a) || signbit(T(pi) - a)) ? a : (signbit(a) ? zero(T) : T(pi))
end

@inline angle(point1::T, point2::T) where {T} = angle(Tuple(point1), Tuple(point2))

end # AngleBetweenVectors


using LinearAlgebra
using SparseArrays
using Test
import Random

using JuMP
using GLPK
using LightGraphs
using GraphPlot
import SCS
using Colors: @colorant_str

# TODO explain problem

function compute_cut_value(g::LightGraphs.AbstractGraph, w::AbstractMatrix, vertex_subset)
    return sum(w[i,j] for i in vertices(g) for j in neighbors(g, i) if in(i, vertex_subset)  != in(j, vertex_subset))
end

g = SimpleGraph(6)
add_edge!(g, 1, 2)
add_edge!(g, 1, 5)
add_edge!(g, 2, 3)
add_edge!(g, 2, 5)
add_edge!(g, 3, 4)
add_edge!(g, 4, 5)
add_edge!(g, 4, 6);

GraphPlot.gplot(g, nodelabel=1:6)

w = spzeros(6, 6)

for e in edges(g)
   (i, j) = Tuple(e)
   w[i,j] = 1
end

@test compute_cut_value(g, w, (2, 5, 6)) ≈ 5
@show compute_cut_value(g, w, (2, 5, 6));

n = nv(g)

linear_max_cut = Model(GLPK.Optimizer)

@variable(linear_max_cut, z[1:n,1:n], Bin)
@variable(linear_max_cut, x[1:n], Bin)
@constraint(linear_max_cut, [i = 1:n, j = 1:n; (i, j) in edges(g)], z[i,j] <= x[i] + x[j])
@constraint(linear_max_cut, [i = 1:n, j = 1:n; (i, j) in edges(g)], z[i,j] <= 2 - (x[i] + x[j]))

@objective(linear_max_cut, Max, dot(w, z))

optimize!(linear_max_cut)

@test objective_value(linear_max_cut) ≈ 6.0
@show objective_value(linear_max_cut);

x_linear = value.(x)
@show x_linear;
@show [(i,j) for i in 1:n-1 for j in i+1:n if JuMP.value.(z)[i,j] > 0.5];

nodecolor = [colorant"lightseagreen", colorant"orange"]
all_node_colors = [nodecolor[round(Int, xi + 1)]  for xi in x_linear]
GraphPlot.gplot(g, nodelabel=1:6, nodefillc=all_node_colors)

sdp_max_cut = Model(optimizer_with_attributes(SCS.Optimizer, "verbose" => 0))

@variable(sdp_max_cut, Y[1:n,1:n] in PSDCone())
@constraint(sdp_max_cut, [i = 1:n], Y[i,i] == 1)
@objective(sdp_max_cut, Max, 1/4 * sum(w[i,j] * (1 - Y[i,j]) for i in 1:n for j in 1:n))

optimize!(sdp_max_cut)

@test objective_value(linear_max_cut) >= 6.0
@show objective_value(linear_max_cut);

F = svd(value.(Y))
U = F.U * Diagonal(sqrt.(F.S))

Random.seed!(33)

x = randn(size(U, 2))
xhat = sign.(U * x) .> 0

@show collect(zip(round.(Int, x_linear), xhat))
@test LinearAlgebra.norm1(round.(Int, x_linear) .- xhat) == 0 || LinearAlgebra.norm1(round.(Int, x_linear) .- xhat) == n

nodecolor = [colorant"lightseagreen", colorant"orange"]
all_node_colors = [nodecolor[xi + 1] for xi in xhat]
GraphPlot.gplot(g, nodelabel=1:6, nodefillc=all_node_colors)


## Preloads
module DHC_2DUtils

    using FFTW
    using LinearAlgebra
    using Statistics

    export fink_filter_bank
    export fink_filter_list
    export fink_filter_hash
    export DHC_compute

    function fink_filter_bank(c, L; nx=256, wd=1, pc=1, shift=false)
        #c     - sets the scale sampling rate (1 is dyadic, 2 is half dyadic)
        #L     - number of angular bins (usually 8*pc or 16*pc)
        #wd    - width of the wavelets (default 1, wd=2 for a double covering)
        #pc    - plane coverage (default 1, full 2pi 2)
        #shift - shift in θ by 1/2 of the θ spacing

        # -------- set parameters
        dθ   = pc*π/L
        wdθ  = wd*dθ
        θ_sh = shift ? dθ/2 : 0.0
        dx   = nx/2-1
        norm = 1.0/sqrt(wd)

        im_scale = convert(Int8,log2(nx))
        # -------- number of bins in radial direction (size scales)
        J = (im_scale-2)*c

        # -------- allocate output array of zeros
        filt      = zeros(nx, nx, J*L+1)
        psi_index = zeros(Int32, J, L)
        theta     = zeros(Float64, L)
        j_value   = zeros(Float64, J)

        # -------- allocate theta and logr arrays
        θ    = zeros(nx, nx)
        logr = zeros(nx, nx)

        # -------- loop over l
        for l = 0:L-1
            θ_l        = dθ*l+θ_sh
            theta[l+1] = θ_l

        # -------- allocate anggood BitArray
            anggood = falses(nx, nx)

        # -------- loop over pixels
            for x = 1:nx
                sx = mod(x+dx,nx)-dx -1    # define sx,sy so that no fftshift() needed
                for y = 1:nx
                    sy = mod(y+dx,nx)-dx -1
                    θ_pix  = mod(atan(sy, sx)+π -θ_l, 2*π)
                    θ_good = abs(θ_pix-π) <= wdθ

                    # If this is a pixel we might use, calculate log2(r)
                    if θ_good
                        anggood[y, x] = θ_good
                        θ[y, x]       = θ_pix
                        r2            = sx^2 + sy^2
                        logr[y, x]    = 0.5*log2(max(1,r2))
                    end
                end
            end
            angmask = findall(anggood)
        # -------- compute the wavelet in the Fourier domain
        # -------- the angular factor is the same for all j
            F_angular = norm .* cos.((θ[angmask].-π).*(L/(2*wd*pc)))

        # -------- loop over j for the radial part
        #    for (j_ind, j) in enumerate(1/c:1/c:im_scale-2)
            for j_ind = 1:J
                j = j_ind/c
                j_value[j_ind] = j  # store for later
                jrad  = im_scale-j-1
                Δj    = abs.(logr[angmask].-jrad)
                rmask = (Δj .<= 1/c)

        # -------- radial part
                F_radial = cos.(Δj[rmask] .* (c*π/2))
                ind      = angmask[rmask]
        #      Let's have these be (J,L) if you reshape...
        #        f_ind    = (j_ind-1)*L+l+1
                f_ind    = j_ind + l*J
                filt[ind, f_ind] = F_radial .* F_angular[rmask]
                psi_index[j_ind,l+1] = f_ind
            end
        end

        # -------- phi contains power near k=0 not yet accounted for
        filter_power = (sum(filt.*filt, dims=3))[:,:,1]

        # -------- for plane half-covered (pc=1), add other half-plane
        if pc == 1
            filter_power .+= circshift(filter_power[end:-1:1,end:-1:1],(1,1))
        end

        # -------- compute power required to sum to 1.0
        i0 = round(Int16,nx/2-2)
        i1 = round(Int16,nx/2+4)
        center_power = 1.0 .- fftshift(filter_power)[i0:i1,i0:i1]
        zind = findall(center_power .< 1E-15)
        center_power[zind] .= 0.0  # set small numbers to zero
        phi_cen = zeros(nx, nx)
        phi_cen[i0:i1,i0:i1] = sqrt.(center_power)

        # -------- before adding ϕ to filter bank, renormalize ψ if pc=1
        if pc==1 filt .*= sqrt(2.0) end  # double power for half coverage

        # -------- add result to filter array
        phi_index  = J*L+1
        filt[:,:,phi_index] .= fftshift(phi_cen)

        # -------- metadata dictionary
        info=Dict()
        info["npix"]         = nx
        info["j_value"]      = j_value
        info["theta_value"]  = theta
        info["psi_index"]    = psi_index
        info["phi_index"]    = phi_index
        info["pc"]           = pc
        info["wd"]           = wd

        return filt, info
    end

    ## Make a list of non-zero pixels for the (Fourier plane) filters
    function fink_filter_list(filt)
        (ny,nx,Nf) = size(filt)

        # Allocate output arrays
        filtind = fill(CartesianIndex{2}[], Nf)
        filtval = fill(Float64[], Nf)

        # Loop over filters and record non-zero values
        for l=1:Nf
            f = @view filt[:,:,l]
            ind = findall(f .> 1E-13)
            val = f[ind]
            filtind[l] = ind
            filtval[l] = val
        end
        return [filtind, filtval]
    end

    function fink_filter_hash(c, L; nx=256, wd=1, pc=1, shift=false)
        # -------- compute the filter bank
        filt, hash = fink_filter_bank(c, L; nx=nx, wd=wd, pc=pc, shift=shift)

        # -------- list of non-zero pixels
        flist = fink_filter_list(filt)

        # -------- pack everything you need into the info structure
        hash["filt_index"] = flist[1]
        hash["filt_value"] = flist[2]
        return hash
    end

    function DHC_compute(image::Array{Float64,2}, filter_hash, filter_hash2)
        # Use 2 threads for FFT
        FFTW.set_num_threads(2)

        # array sizes
        (Nx, Ny)  = size(image)
        (Nf, )    = size(filter_hash["filt_index"])

        # allocate coeff arrays
        out_coeff = []
        S0  = zeros(Float64, 2)
        S1  = zeros(Float64, Nf)
        S20 = zeros(Float64, Nf, Nf)
        S12 = zeros(Float64, Nf, Nf)
        S2  = zeros(Float64, Nf, Nf)  # traditional 2nd order

        # allocate image arrays for internal use
        im_fdf_0_1 = zeros(Float64,           Nx, Ny, Nf)   # this must be zeroed!
        im_rd_0_1  = Array{Float64, 3}(undef, Nx, Ny, Nf)

        ## 0th Order
        S0[1]   = mean(image)
        norm_im = image.-S0[1]
        S0[2]   = sum(norm_im .* norm_im)/(Nx*Ny)
        norm_im ./= sqrt(Nx*Ny*S0[2])

        append!(out_coeff,S0[:])

        ## 1st Order
        im_fd_0 = fft(norm_im)  # total power=1.0

        # unpack filter_hash
        f_ind   = filter_hash["filt_index"]  # (J, L) array of filters represented as index value pairs
        f_val   = filter_hash["filt_value"]

        f_ind2   = filter_hash2["filt_index"]  # (J, L) array of filters represented as index value pairs
        f_val2   = filter_hash2["filt_value"]

        zarr = zeros(ComplexF64, Nx, Ny)  # temporary array to fill with zvals

        # make a FFTW "plan" for an array of the given size and type
        P   = plan_ifft(im_fd_0)   # P is an operator, P*im is ifft(im)

        ## Main 1st Order and Precompute 2nd Order
        for f = 1:Nf
            S1tot = 0.0
            f_i = f_ind[f]  # CartesianIndex list for filter
            f_v = f_val[f]  # Values for f_i
            # for (ind, val) in zip(f_i, f_v)   # this is slower!
            if length(f_i) > 0
                for i = 1:length(f_i)
                    ind       = f_i[i]
                    zval      = f_v[i] * im_fd_0[ind]
                    S1tot    += abs2(zval)
                    zarr[ind] = zval
                    im_fdf_0_1[ind,f] = abs(zval)
                end
                S1[f] = S1tot/(Nx*Ny)  # image power
                im_rd_0_1[:,:,f] .= abs2.(P*zarr)
                zarr[f_i] .= 0
            end
        end
        append!(out_coeff, S1[:])

        # we stored the abs()^2, so take sqrt (this is faster to do all at once)
        im_rd_0_1 .= sqrt.(im_rd_0_1)

        ## 2nd Order
        #Amat = reshape(im_fdf_0_1, Nx*Ny, Nf)
        #S12  = Amat' * Amat
        #Amat = reshape(im_rd_0_1, Nx*Ny, Nf)
        #S20  = Amat' * Amat

        #append!(out_coeff, S20)
        #append!(out_coeff, S12)

        ## Traditional second order
        for f1 = 1:Nf
            thisim = fft(im_rd_0_1[:,:,f1])  # Could try rfft here
            # println("  f1",f1,"  sum(fft):",sum(abs2.(thisim))/Nx^2, "  sum(im): ",sum(abs2.(im_rd_0_1[:,:,f1])))
            # Loop over f2 and do second-order convolution
            for f2 = 1:Nf
                f_i = f_ind2[f2]  # CartesianIndex list for filter
                f_v = f_val2[f2]  # Values for f_i
                # sum im^2 = sum(|fft|^2/npix)
                S2[f1,f2] = sum(abs2.(f_v .* thisim[f_i]))/(Nx*Ny)
            end
        end
        append!(out_coeff, S2)

        return out_coeff
    end

end # of module


using Test, Astro

# solve_kepler(mean_anomaly, eccentricity, desired_accuracy = 1e-6)

println("Kepler's equation")
print("Duffett-Smith PA with your Computer, page 116 ")

eccentric_anomaly = solve_kepler(deg2rad(201.7292), 0.016718) |> rad2deg
@test isapprox(eccentric_anomaly, 201.38, atol=.01)

eccentric_anomaly = solve_kepler(deg2rad(43.7172), 0.965) |> rad2deg
@test isapprox(eccentric_anomaly, 98.41, atol=.01)

# circular orbit
eccentric_anomaly = solve_kepler(deg2rad(60), 0.0) |> rad2deg
@test isapprox(eccentric_anomaly, 60, atol=.001)

println("passed")


using Test
using GreyDecision

# Testing Core Library
@info "Testing Grey Numbers and their operations"
include("./testgreynumber.jl")

@info "Testing Stats"
include("./teststats.jl")

@info "Testing utility functions"
include("./testutility.jl")

@info "Testing Random module"
include("./testrandom.jl")

@info "Linear Algebra operations"
include("./testlinearalgebra.jl")

# Testing Grey Decision Methods
@info "Testing topsis"
include("./testtopsis.jl")

@info "Testing Vikor"
include("./testvikor.jl")

@info "Testing Saw"
include("./testsaw.jl")

@info "Testing NDS"
include("./testnds.jl")

@info "Testing Moora"
include("./testmoora.jl")

@info "Testing electre"
include("./testelectre.jl")

@info "Testing WPM"
include("./testwpm.jl")

@info "Testing Marcos"
include("./testmarcos.jl")

@info "Testing Aras"
include("./testaras.jl")


#=
jinc.jl
=#

export jinc

using SpecialFunctions: besselj1 # grab bessel function

"""
    jinc(x)

Return `jinc(x) = J1(pi*x)/(2x)`,
where `J1` is a Bessel function of the first kind.

Units of `x` are typically cycles/m.

Return type is `promote_type(typeof(x), Float32)`.
"""
function jinc(x::Real)
    T = promote_type(typeof(x), Float32)
    if (x == 0) return convert(T, pi/4) end
    y = abs(x)
    convert(T, besselj1(pi*y) / (2*y))
end


# Test file of variance.jl in https://github.com/TheAlgorithms/Julia/tree/main/src/statistics
using Test
a = 1:10
@test var(a) == 5.5


######### CmdStan sample example  ###########

using StanSample

bernoulli_model = "
data {
  int<lower=1> N;
  int<lower=0,upper=1> y[N];
}
parameters {
  real<lower=0,upper=1> theta;
}
model {
  theta ~ beta(1,1);
  y ~ bernoulli(theta);
}
";

bernoulli_data = (N = 10, y = [0, 1, 0, 1, 0, 0, 0, 0, 0, 1],)

# Keep tmpdir identical across multiple runs to avoid re-compilation
stanmodel = SampleModel(
  "bernoulli", bernoulli_model; tmpdir = tmpdir,
  method = StanSample.Sample(adapt=StanSample.Adapt(delta=0.85)))

stan_sample(stanmodel, data=bernoulli_data)

# Convert to an MCMCChains.Chains object
chns = read_samples(stanmodel)

# Show the same output in DataFrame format
sdf = read_summary(stanmodel)


import Base: gcd, lcm

function gcd(a::SimplePolynomial, b::SimplePolynomial)
    #try
    d = integerize(_gcd(a, b))
    return d
    # catch
    #     aa = big(a)
    #     bb = big(b)
    #     dd = _gcd(aa,bb)
    #     d = integerize(dd)
    #     return d
    # end
end

function _gcd(a::SimplePolynomial, b::SimplePolynomial)

    if a == 0 && b == 0
        error("The two arguments cannot both be zero")
    end

    if a == 0
        return b
    end

    if b == 0
        return a
    end

    q, r = divrem(a, b)
    return monic(_gcd(b, monic(r)))
end

function lcm(a::SimplePolynomial, b::SimplePolynomial)
    q, r = divrem(a * b, gcd(a, b))
    return monic(q)
end


function cholesky(mat::Matrix)
    # if the matrix is no a square matrix, return one with zeros
    if size(mat, 1) != size(mat, 2)
        return zeros(size(mat, 1), size(mat, 1))
    end
    # since verifing that the matrix is positive definite is so
    # time-consuming, the step is skipped
    # it's assumed that the matrix has $n$ rows and columns
    n = size(mat, 1)
    # $L$ is the matrix to be returned
    L = zeros(n, n)
    # the formula is proved by induction, basically
    for itr1 in 1 : n, itr2 in itr1 : n
        if itr2 == itr1
            L[itr2, itr1] = mat[itr2, itr1]
            for itr3 in 1 : (itr1 - 1)
                L[itr2, itr1] -= L[itr2, itr3] ^ 2
            end
            L[itr2, itr1] = sqrt(L[itr2, itr1])
        else
            L[itr2, itr1] = mat[itr2, itr1]
            for itr3 in 1 : (itr1 - 1)
                L[itr2, itr1] -= L[itr2, itr3] * L[itr1, itr3]
            end
            L[itr2, itr1] /= L[itr1, itr1]
        end
    end
    return L
end


using StatsBase, JuMP, GLPK, Test

function optimal_price_probabilities(
    price, demand, capacity; verbose = false)
    revenue = price .* demand
    model = Model(with_optimizer(GLPK.Optimizer))
    @variable(model, x[1:length(price)], lower_bound=0)
    # Objective: maximize profit
    @objective(model, Max, revenue' * x)
    # Constraint: can carry all
    @constraint(model, sum(x) <= 1)
    #=
    for i in 1:length(prices)
        @constraint(model, x[i] <= demand[i])
    end
    =#
    # Solve problem using MIP solver
    JuMP.optimize!(model)
    if verbose
        println("Objective is: ", JuMP.objective_value(model))
        println("Solution is:")
        for i in 1:length(price)
            print("x[$i] = ", JuMP.value(x[i]))
            println(", revenue = ", price[i] * JuMP.value(x[i]))
        end
    end
    @test JuMP.termination_status(model) == MOI.OPTIMAL
    @test JuMP.primal_status(model) == MOI.FEASIBLE_POINT
    Weights(JuMP.value.(x))
end


using RHEOS
using BenchmarkTools
using InverseLaplace
import MittagLeffler
using FunctionWrappers: FunctionWrapper

t=RheoFloat.(0.0:0.001:1.0);
t0=1.0

println("=========================")
println("Test on maths functions ")
println("=========================")

println()
println("Test on the MittalgLeffler function")
println()
println("Version from Julia repo")
@btime MittagLeffler.mittleff(0.5, t0)
println("Version from RHEOS (should be the same as above)")
@btime RHEOS.mittleff(0.5, t0)

println()
println()

println()
println("Test on the inverse Laplace transform")
println()
cₐ = 1.0; a=0.5; kᵦ = 2.0; kᵧ=3.0
println("Talbot algorithm on FractSLS_Zener creep function")
Ĵ(s) = (1/s)*(cₐ*s^a + kᵦ)/(cₐ*s^a*kᵦ + kᵧ*(cₐ*s^a + kᵦ))
print("  scalar value: ")
@btime InverseLaplace.talbot(s->(1/s)*(cₐ*s^a + kᵦ)/(cₐ*s^a*kᵦ + kᵧ*(cₐ*s^a + kᵦ)), t0)
print("  vector using talbot.: ")
@btime InverseLaplace.talbot.(s->(1/s)*(cₐ*s^a + kᵦ)/(cₐ*s^a*kᵦ + kᵧ*(cₐ*s^a + kᵦ)), t)
print("  vector using talbotarr: ")
@btime InverseLaplace.talbotarr(s -> Ĵ(s), t)
println()

println("Weeks (80) algorithm on FractSLS_Zener creep function")
print("  Initialisation: ")
@btime InverseLaplace.Weeks(Ĵ,80)
ft = InverseLaplace.Weeks(Ĵ,80)
print("  Scalar value: ")
@btime ft(t0)
print("  Vector values: ")
@btime ft.(t)
println()
println()

println("=========================")
println("Test on Moduli functions ")
println("=========================")

println()
println("Test on the Maxwell model")
println()
@btime relaxmod(Maxwell, t0, [2.0,3.0])
@btime relaxmod(Maxwell, t, [2.0,3.0]);

m = RheoModel(Maxwell,  (η=2.0, k=3.0))
@btime relaxmod(m, t0)
@btime relaxmod(m, t);

println()
println("Test on the FractSLS_Zener model relaxation modulus - with mittleff")
println()
cₐ = 1.0; a=0.5; kᵦ = 2.0; kᵧ=3.0
println("Direct function evaluation")
@btime kᵦ*MittagLeffler.mittleff(a, -(kᵦ/cₐ)*t0^a) + kᵧ
@btime kᵦ.*MittagLeffler.mittleff.(a, -(kᵦ./cₐ).*t.^a) .+ kᵧ;
println("From the RheoModelClass")
@btime relaxmod(FractSLS_Zener, t0, [1.0,0.5,2.0,3.0])
@btime relaxmod(FractSLS_Zener, t, [1.0,0.5,2.0,3.0])
m = RheoModel(FractSLS_Zener,  (cₐ = 1.0, a=0.5, kᵦ = 2.0, kᵧ=3.0))
println("From the RheoModel")
@btime relaxmod(m, t0)
@btime relaxmod(m, t)

println()
println("Test on the FractSLS_Zener model creep compliance - with inverse laplace")
println()

fwp = ((t,p)->(InverseLaplace.Weeks(s->(1/s)*(p[1]*s^p[2] + p[3])/(p[1]*s^p[2]*p[3] + p[4]*(p[1]*s^p[2] + p[3])),80))(t)) |> FunctionWrapper{RheoFloat,Tuple{RheoFloat,Vector{RheoFloat}}}
fwpa = ((t,p)->(InverseLaplace.Weeks(s->(1/s)*(p[1]*s^p[2] + p[3])/(p[1]*s^p[2]*p[3] + p[4]*(p[1]*s^p[2] + p[3])),80)).(t)) |> FunctionWrapper{Vector{RheoFloat},Tuple{Vector{RheoFloat},Vector{RheoFloat}}}
fw = InverseLaplace.Weeks(s->(1/s)*(1.0*s^0.5 + 2.0)/(1.0*s^0.5*2.0 + 3.0*(1.0*s^0.5 + 2.0)),80) |> FunctionWrapper{RheoFloat,Tuple{RheoFloat}}
fwa = t->fw.(t) |> FunctionWrapper{Vector{RheoFloat},Tuple{Vector{RheoFloat}}}
println()
println("function wrappers using Weeks")
@btime fwp(t0,[1.0,0.5,2.0,3.0])
@btime fwpa(t,[1.0,0.5,2.0,3.0])
@btime fw(t0)
@btime fwa(t)

println()
println("From the RheoModelClass")
@btime creepcomp(FractSLS_Zener, t0, [1.0,0.5,2.0,3.0])
@btime creepcomp(FractSLS_Zener, t, [1.0,0.5,2.0,3.0])
m = RheoModel(FractSLS_Zener,  (cₐ = 1.0, a=0.5, kᵦ = 2.0, kᵧ=3.0))
println("From the RheoModel")
@btime creepcomp(m, t0)
@btime creepcomp(m, t)
nothing

#=

Hardware:
Intel Core i5-7200U Processor (3MB Cache, up to 3.1Ghz)
8GB LPDDR3 1866

=========================
Test on maths functions
=========================

Test on the MittalgLeffler function

Version from Julia repo
  57.567 ns (1 allocation: 16 bytes)
Version from RHEOS (should be the same as above)
  58.195 ns (1 allocation: 16 bytes)

Test on the inverse Laplace transform

Talbot algorithm on FractSLS_Zener creep function
  scalar value:   21.206 μs (642 allocations: 18.75 KiB)
  vector using talbot.:   21.383 ms (642143 allocations: 18.33 MiB)
  vector using talbotarr:   3.418 ms (173413 allocations: 3.61 MiB)

Weeks (80) algorithm on FractSLS_Zener creep function
  Initialisation:   105.563 μs (2634 allocations: 110.75 KiB)
  Scalar value:   1.616 μs (1 allocation: 16 bytes)
  Vector values:   1.596 ms (2 allocations: 8.03 KiB)

=========================
Test on Moduli functions
=========================

Test on the Maxwell model

  67.185 ns (2 allocations: 112 bytes)
  7.559 μs (2 allocations: 8.09 KiB)
  33.592 ns (1 allocation: 16 bytes)
  8.153 μs (1 allocation: 8.00 KiB)

Test on the FractSLS_Zener model relaxation modulus - with mittleff

Direct function evaluation
  207.000 ns (7 allocations: 112 bytes)
  68.707 μs (15 allocations: 8.53 KiB)
From the RheoModelClass
  166.950 ns (4 allocations: 160 bytes)
  64.459 μs (2 allocations: 8.11 KiB)
From the RheoModel
  104.457 ns (1 allocation: 16 bytes)
  59.557 μs (1 allocation: 8.00 KiB)

Test on the FractSLS_Zener model creep compliance - with inverse laplace

function wrappers using Weeks
  62.121 μs (71 allocations: 30.80 KiB)
  1.660 ms (72 allocations: 38.81 KiB)
  1.636 μs (1 allocation: 16 bytes)
  1.610 ms (5 allocations: 8.12 KiB)

From the RheoModelClass
  10.674 μs (101 allocations: 2.64 KiB)
  10.721 ms (99102 allocations: 2.47 MiB)
From the RheoModel
  9.535 μs (1 allocation: 16 bytes)
  9.580 ms (1 allocation: 8.00 KiB)

=#


function prompt()
    println("Please enter length of short side A if possible, if not leave blank")
    a = readline()
    println("Please enter length of short side B if possible, if not leave blank")
    b = readline()
    println("Please enter length of Hypotinous C if possible, if not leave blank")
    c = readline()

    all_val = a,b,c

    all_val = [all_val[i] == "" ?  missing : all_val[i] for i in 1:length(all_val)]

    if sum(x -> ismissing(x), all_val) == 1
        nothing
    else
        println("Error: too many or too few missing values.")
    end

    all_v = []
    for i in 1:size(all_val,1)
        if  ismissing(all_val[i])
            push!(all_v, missing)
        else
            push!(all_v, parse(Float64,all_val[i]) )
        end
    end

    return all_v

end

all_v = prompt()

function Pythag(inp::Array{Any,1})
    if ismissing(inp[3])
        result = sqrt(inp[1]^2 + inp[2]^2)
    else
        vals = collect(skipmissing(inp))
        result = sqrt( vals[2]^2 - vals[1]^2 )
    end
    return result
end

println("The length of the missing side is: $(Pythag(all_v))")


function mahalanobisSquaredBetweenPairs(pairs::Matrix; covmatrix=nothing)
    n, p = size(pairs)
    newmat = zeros(Float64, n, n)
    if covmatrix === nothing
        covmatrix = cov(pairs)
    end
    try
        invm = inv(covmatrix)
        for i in 1:n
            @inbounds for j in i:n
                newmat[i, j] = ((pairs[i,:] .- pairs[j,:])' * invm * (pairs[i,:] .- pairs[j,:]))
                newmat[j, i] = newmat[i,j]
            end
        end
        return newmat
    catch e
        @warn e
        if det(covmatrix) == 0
            @warn "singular covariance matrix, mahalanobis distances can not be calculated"
        end
        return zeros(Float64, (n, n))
    end
end

"""

    asm2000(setting)

Perform the Setan, Halim and Mohd (2000) algorithm for the given regression setting.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.

# Examples
```julia-repl
julia> reg0001 = createRegressionSetting(@formula(calls ~ year), phones);
julia> asm2000(reg0001)
Dict{Any,Any} with 1 entry:
  "outliers" => [15, 16, 17, 18, 19, 20]
```

# References
Setan, Halim, and Mohd Nor Mohamad. "Identifying multiple outliers in
linear regression: Robust fit and clustering approach." (2000).
"""
function asm2000(setting::RegressionSetting)
    X = designMatrix(setting)
    y = responseVector(setting)
    return asm2000(X, y)
end

function asm2000(X::Array{Float64,2}, y::Array{Float64,1})
    n, p = size(X)
    h = floor((n + p - 1) / 2)
    ltsreg = lts(X, y)

    betas = ltsreg["betas"]
    hsubset = ltsreg["hsubset"]

    predicteds = [sum(X[i,:] .* betas) for i in 1:n]
    resids = y .- predicteds
    stdres = standardize(ZScoreTransform, resids, dims=1)
    stdfit = standardize(ZScoreTransform, predicteds, dims=1)
    pairs = hcat(stdfit, stdres)

    pairs = hcat(resids, predicteds)

    covmatrix = cov(pairs[hsubset, :])
    mahdist = mahalanobisSquaredBetweenPairs(pairs, covmatrix=covmatrix)

    outlierset = Array{Int,1}()

    hcl = hclust(mahdist, linkage=:single)
    majonacrit = majona(hcl)
    clustermappings = cutree(hcl, h=majonacrit)
    uniquemappings = unique(clustermappings)
    for clustid in uniquemappings
        cnt = count(x -> x == clustid, clustermappings)
        if cnt >= h
            outlierset = filter(i -> clustermappings[i] != clustid, 1:n)
        end
    end

    result = Dict()
    result["outliers"] = outlierset
    return result
end


# calculate required particle num for kld sampling
# by wilson-hilferty transformation

module ParticleNumWh
  using Distributions

  function num(epsilon, delta, bin_num)
    return ceil(quantile(Chisq(bin_num-1), 1-delta)/(2*epsilon))
  end

  function num_wh(epsilon, delta, bin_num)
    dof = bin_num - 1
    z = quantile(Normal(), 1 - delta)
    return ceil(dof/(2*epsilon)*(1-2/(9*dof)+sqrt(2/(9*dof))*z)^3)
  end

  function main()
    bins = [2, 4, 8, 1000, 10000, 100000]
    for bin in bins
      println("bin:$(bin), ϵ=0.1, δ=0.01, chisq=$(num(0.1, 0.01, bin)), norm=$(num_wh(0.1, 0.01, bin))")
      println("bin:$(bin), ϵ=0.5, δ=0.01, chisq=$(num(0.5, 0.01, bin)), norm=$(num_wh(0.5, 0.01, bin))")
      println("bin:$(bin), ϵ=0.5, δ=0.05, chisq=$(num(0.5, 0.05, bin)), norm=$(num_wh(0.5, 0.05, bin))")
    end
  end
end


# mathematical constants related to statistics

@irrational loghalf -0.6931471805599453094 log(big(0.5))
@irrational logtwo 0.6931471805599453094 log(big(2.))
@irrational logπ   1.1447298858494001741 log(big(π))
@irrational log2π  1.8378770664093454836 log(big(2.)*π)
@irrational log4π  2.5310242469692907930 log(big(4.)*π)


function getCoefficientsSymplecticEulerForward()
    a = [[0.0 0.0]
         [1.0 0.0]]
    b = [1.0, 0.0]
    c = [0.0, 1.0]

    o = 1

    CoefficientsRK(:symplectic_euler_forward, o, a, b, c)
end

function getCoefficientsSymplecticEulerBackward()
    a = [[0.0 0.0]
         [0.0 1.0]]
    b = [0.0, 1.0]
    c = [0.0, 1.0]

    o = 1

    CoefficientsRK(:symplectic_euler_backward, o, a, b, c)
end

"Tableau for symplectic Euler-A method"
function getTableauSymplecticEulerA()
    TableauEPRK(:symplectic_euler_a, 1, getCoefficientsSymplecticEulerForward(), getCoefficientsSymplecticEulerBackward())
end

"Tableau for symplectic Euler-B method"
function getTableauSymplecticEulerB()
    TableauEPRK(:symplectic_euler_b, 1, getCoefficientsSymplecticEulerBackward(), getCoefficientsSymplecticEulerForward())
end

"Tableau for Gauss-Lobatto IIIAIIIB method with s=2 stages"
function getTableauLobattoIIIAIIIB2()
    TableauEPRK(:lobatto_IIIA_IIIB_2, 2, getCoefficientsLobIIIA2(), getCoefficientsLobIIIB2())
end

"Tableau for Gauss-Lobatto IIIBIIIA method with s=2 stages"
function getTableauLobattoIIIBIIIA2()
    TableauEPRK(:lobatto_IIIB_IIIA_2, 2, getCoefficientsLobIIIB2(), getCoefficientsLobIIIA2())
end


# Linear fractional transformation "LFT"

module LinearFractionalTransformations

import Base.inv, Base.isequal, Base.show, Base.hash
import Base.==, Base.*, Base.getindex
export LFT, isequal, call

const complex_infinity = Inf + Inf * im

struct LFT
    M::Array{Complex{Float64},2}
    function LFT(a, b, c, d)
        if isinf(a) || isinf(b) || isinf(c) || isinf(d)
            error("Arguments must be finite: " * string((a, b, c, d)))
        end

        if a * d - b * c == 0
            error("Singularity detected: " * string((a, b, c, d)))
        end

        new([a b; c d])
    end
end

"""
    LFT
A linear fractional transformation `f(z)=(a*z+b)/(c*z+d)` where `a,b,c,d` are complex numbers.

Constructors:
* `LFT(a,b,c,d)`: create the function described above.
* `LFT()`: create the identity function `f(z) = z`. Equivalent to `LFT(1,0,0,1)`.
* `LFT(a,b,c)`: create the function with `a ↦ 0`, `b ↦ 1`, and `c ↦ ∞`.
* `LFT(a,aa,b,bb,c,cc)`: creates the function with `a ↦ aa`, `b ↦ bb`, and `c ↦ cc`.
"""
function LFT(M::Array)
    return LFT(M[1, 1], M[1, 2], M[2, 1], M[2, 2])
end

function LFT()
    return LFT(1, 0, 0, 1)
end

# a --> 0, b-->1, and c--> oo
function LFT(a::Number, b::Number, c::Number)
    if a == b || b == c || a == c
        error("Three arguments must be distinct: " * string((a, b, c)))
    end

    if isinf(a)
        return LFT(0, b - c, 1, -c)
    end

    if isinf(b)
        return LFT(1, -a, 1, -c)
    end

    if isinf(c)
        return LFT(1, -a, 0, b - a)
    end

    # if all args are finite
    aa = (b - c)
    bb = (-a) * (b - c)
    cc = (b - a)
    dd = (-c) * (b - a)
    return LFT(aa, bb, cc, dd)
end

# a-->aa, b-->bb, c-->cc
function LFT(a::Number, aa::Number, b::Number, bb::Number, c::Number, cc::Number)
    f = LFT(a, b, c)
    g = LFT(aa, bb, cc)
    return inv(g) * f
end

#### Equality checking ####

# These need to be rewritten

function isequal(f::LFT, g::LFT)
    return f[0] == g[0] && f[1] == g[1] && f[Inf] == g[Inf]
    # h = f * inv(g)
    # I = h.M
    # return I[1,1]==I[2,2] && I[1,2]==I[2,1]==0
end

==(f::LFT, g::LFT) = isequal(f, g)

# Inverse transformation
function inv(L::LFT)
    a = L.M[1, 1]
    b = L.M[1, 2]
    c = L.M[2, 1]
    d = L.M[2, 2]
    return LFT(d, -b, -c, a)
end

# Composition
*(A::LFT, B::LFT) = LFT(A.M * B.M)

# Function application
function getindex(A::LFT, x::Number)
    if isinf(x)
        a = A.M[1, 1]
        b = A.M[2, 1]
        if b == 0
            return complex_infinity
        end
        return a / b
    end
    w = A.M * [x + 0im; 1.0 + 0im]
    if w[2] == 0
        return complex_infinity
    end
    return w[1] / w[2]
end

# call(A::LFT, x::Number) = A[x]

(A::LFT)(x::Number) = A[x]

function show(io::IO, L::LFT)
    print(
        io,
        "LFT( ",
        L.M[1, 1],
        " , ",
        L.M[1, 2],
        " , ",
        L.M[2, 1],
        " , ",
        L.M[2, 2],
        " )",
    )
end

function hash(f::LFT, h::UInt64 = UInt64(0))
    z = 0.0 + 0.0 * im # kludge to make -0.0 and -0.0im into +versions
    a = f[0] + z
    b = f[1] + z
    c = f[Inf] + z
    return hash(a, hash(b, hash(c, h)))
end

include("projection.jl")

end # end of module "LFTs"


## based on functions in the tests directory of the MUMPS package
using Base.Test
using SuperLU

ddx(n) = spdiagm((-ones(n), ones(n)),[0,1],n,n+1)

function getDivGrad(n1,n2,n3)
    Div = hcat(kron(speye(n3),kron(speye(n2),ddx(n1))),
               kron(speye(n3),kron(ddx(n2),speye(n1))),
               kron(ddx(n3),kron(speye(n2),speye(n1))))
    Div*Div';
end

dgcube(n) = getDivGrad(n,n,n)

for n in (8,16,24)
    A = dgcube(n)
    b = randn(size(A,1))
    xumf = A\b
    xslu = NCMat(A)\b
    @test_approx_eq (A*xumf) b
    @test_approx_eq (A*xslu) b
    gc()
    umftimes = [@elapsed A\b for i in 1:10]
    gc()
    slutimes = [@elapsed NCMat(A)\b for i in 1:10]
    println("Times for solving divgrad system of size $(size(A)), nnz = $(nfilled(A))")
    showcompact(umftimes)
    println()
    showcompact(slutimes)
    println()
    println("UMFPACK: mean = $(mean(umftimes)), median = $(median(umftimes))")
    println("SuperLU: mean = $(mean(slutimes)), median = $(median(slutimes))")
end


function factor(n)
    answer = []
    hw = []
    for i in eratosphen(n)
        if n%i == 0
            push!(answer,i)
            push!(hw,1)
            n = n/i
            while n%i == 0
                hw[end] += 1
                n = n/i
            end
        end
    end
    return answer,hw
end

function eratosphen(n)
    ser = fill(true,n)
    ser[1] = false
    k = 2
    while k !== nothing && k<n
        ser[k^2:k:end] .= false
        k=findnext(ser, k+1)
    end
    return findall(ser)
end


#=
Starting with the number 1 and moving to the right in a clockwise direction a 5 by 5 spiral is formed as follows:

21 22 23 24 25
20  7  8  9 10
19  6  1  2 11
18  5  4  3 12
17 16 15 14 13

It can be verified that the sum of the numbers on the diagonals is 101.

What is the sum of the numbers on the diagonals in a 1001 by 1001 spiral formed in the same way?
=#
#=
Notes:

#[1],2,[3],4,[5],6,[7],8,[9],10,11,12,[13],14,15,16,[17],18,19,20,[21],22,23,24,[25]
#1,3,5,7,9,13,17,21,25,31,37,43,49
#2, 4, 6
#3x3, 5x5, 7x7
=#
function calc()
  total,start = 1,1
  step = 2
  iterations = 3
  currentsquaresize = 3
  while currentsquaresize <= 1001
    start += step
    for i in start:step:(start+(iterations*step))
      total += i
    end
    start += (iterations*step)
    currentsquaresize += 2
    step += 2
  end
  total
end
@time println(calc())


# Activating Julia environment
using Pkg
Pkg.activate(".")

# Packages
using JuMP, Gurobi, Random, Distributions, LinearAlgebra, DataFrames, Plots

n = 10 # Number of facilities
m = 50 # Number of customers

# Generating random data (please don't change the seeds.)
facilities = 0.6.*rand(MersenneTwister(5), n,2) .+ 0.2;
customers = rand(MersenneTwister(2), m, 2);
c = [LinearAlgebra.norm(customers[i, :] .- facilities[j, :])[1] for j=1:n, i=1:m];
f = rand(MersenneTwister(3), n)*1 .+ 5;
s = rand(MersenneTwister(4), n)*2 .+ 15;
d = rand(MersenneTwister(5), m)*0.5 .+ 0.75

# P matrix
R_D = 0.25
P = [0.2*exp(-1/R_D .*LinearAlgebra.norm(customers[i, :] .- customers[j, :])[1]) for j=1:m, i=1:m];
P = (P .>= 0.2*exp(-1/R_D .* R_D)) .* P

""" Plots the solution of the facility location model.
Blue circles are active facilities with different capacities.
Orange plus signs are other potential facility locations.
Rays describe connections between facilities and demand nodes.
"""
function plot_solution(model, x, y, cost = nothing)
    plt = scatter(facilities[:, 1], facilities[:, 2], markersize = 0.4 .* s .* value.(x))
    scatter!(facilities[:, 1], facilities[:, 2], markersize = 0.4 .* s, markershape = :+)
    for i=1:n
        for j=1:m
            if value(y[i,j]) >= 1e-10
                plot!([customers[j, 1], facilities[i,1]], [customers[j,2], facilities[i,2]], linewidth = value(y[i,j]), legend=false)
            end
        end
    end
    if cost == nothing
        scatter!(customers[:, 1], customers[:, 2], markersize = 3*d,
                title = "Total cost: $(round(objective_value(model), sigdigits=5))")
    else
        scatter!(customers[:, 1], customers[:, 2], markersize = 3*d,
        title = "Total cost: $(round(cost,sigdigits=5))")
    end
    println("Facility cost: $(value(sum(f[j] * x[j] for j = 1:n)))")
    println("Transportation cost: $(value(sum(c[i, j] * y[i, j] for i=1:n, j=1:m)))")
    return plt
end


include("compute_hermite.jl")
function hermitekernel(n,x)
    #Compute the Hermite kernel matrix at points x
    #including degrees from 0 to n inclusive

    # # Method 1: Direct Computation
       phi=compute_hermite(n,x);
       K=phi'*phi;

    # # Method 2: Christoffel-Darboux
    #   phinp1 = compute_hermite(n+1,x,false)
    #   phin = compute_hermite(n,x,false)
    #   K = phinp1'*phin*sqrt((n+1)/2)
    #   x = x[:]*ones(1,length(x))
    #   K = (K-K')./(x-x')

    K
end


using Distributions, Plots; pyplot()

L, K, n  = 500, [450, 400, 250, 100, 50], 30
hyperDists = [Hypergeometric(k,L-k,n) for k in K]
xGrid = 0:1:n
pmfs = [ pdf.(dist, xGrid) for dist in hyperDists ]
labels = "Successes = " .* string.(K)

bar( xGrid, pmfs,
	alpha=0.8, c=[:orange :purple :green :red :blue ],
	label=hcat(labels...), ylims=(0,0.25),
	xlabel="x", ylabel="Probability", legend=:top)


δ=0.10; ρ=0.15; # Param
z= ρ + δ +.02     # Need: z > ρ + δ
I_SS = (z-ρ-δ)/(ρ+δ)
K_SS = I_SS/δ

if 1==1
    a_0(s;δ=δ) = δ*s
    min_s = eps(); max_s = 10.0*K_SS; n_s = 800;
    min_a = eps(); max_a = K_SS; n_a = 200;
    "states:"
    states = range(min_s, max_s, length=n_s)
    ceil_state(s) = states[searchsortedfirst(states, s)]
    "actions:"
    valid_actions()  = range(min_a, max_a, length=n_a)        # possible actions any state.
    valid_actions(s) = filter(>=(-(1-δ)*s), valid_actions())  # valid actions @ state=s
    "Transition:"
    μ(s,a;δ=δ)      = a +(1-δ)*s
    "reward:"
    r(s,a;z=z) = z*s -a -0.5*(a^2)
    "discount:"
    β = 1/(1+ρ);
    using QuickPOMDPs: QuickMDP           #QuickMDP()
    using POMDPModelTools: Deterministic
    m = QuickMDP(
        states     = states,
        actions    = valid_actions,
        transition = (s, a) -> Deterministic( ceil_state( μ(s,a) ) ),
        reward     = (s, a) -> r(s,a),
        discount   = β
    )
    # DiscreteValueIteration: Both work fast!
    using DiscreteValueIteration
    s1 = DiscreteValueIteration.SparseValueIterationSolver()
    s2 = DiscreteValueIteration.ValueIterationSolver()
    @time sol1 = solve(s1, m) #
    @time sol2 = solve(s2, m) #
    # value(sol1, states[2]), action(sol1, states[2])
    # value(sol2, states[2]), action(sol2, states[2])
    value(sol1, states[end])

    using Plots
    # Value
    plot(legend=:bottomright, title="Value Functions");
    #plot!(states[2:end], i->A0 + A1 * i, lab="closed form")
    plot!(states[2:end], i->value(sol1, i),   lab="sol1")
    plot!(states[2:end], i->value(sol2, i),   lab="sol2")

    # Simulation
    Tsim=150; s0=0.5*K_SS; sim1 = []; push!(sim1, s0);
    for tt in 1:Tsim
        s = sim1[tt]
        a = valid_actions()[sol1.policy[searchsortedfirst(states, s)]]
        sp = μ(s,a)
        sp = ceil_state(sp)
        push!(sim1, sp)
    end

    plot(legend=:bottomright, title="Simulation");
    #plot!(simcf,   lab="closed form")
    plot!(sim1,   lab="sol1")
end


module NumericalAnalysis
using Documenter
using ForwardDiff
using LinearAlgebra
using PyCall
using Conda
using SymPy
using Latexify

include("basic.jl")
export my_f, derivative_my_f, ξ, absolute_error, relative_error, NthDerivative, TaylorPolynomials

# Write your package code here.

include("SEq1.jl")

export Bisection, fixed_point, Newton, Secant, FalsePos, ModifiedNewton

include("Polynomial.jl")

export Lagrange, Neville, NDDF, NCSpline, CCSpline

include("NCalculus.jl")

export ThreePoint, FivePoint, Trapezoidal, Simpson, Newton_cotes, Romberg,Gaussian_Quad, SimpsonDoubleIntegral, MutipleIntegralSimple, GaussianDoubleIntegral

end


"""
    X = sylvckr(A,B,C)

Solve the continuous Sylvester matrix equation

                AX + XB = C

using the Kronecker product expansion of equations. `A` and `B` are
square matrices, and `A` and `-B` must not have common eigenvalues.
This function is not recommended for large order matrices.
"""
function sylvckr(A, B, C)
    m, n = size(C)
    [m; n] == LinearAlgebra.checksquare(A, B) ||
             throw(DimensionMismatch("A, B and Q have incompatible dimensions"))
    reshape((kron(Array{eltype(A),2}(I, n, n), A) +
            kron(transpose(B), Array{eltype(B),2}(I, m, m))) \ (C[:]),m,n)
end

"""
    X = sylvdkr(A,B,C)

Solve the discrete Sylvester matrix equation

                AXB + X = C

using the Kronecker product expansion of equations. `A` and `B` are
square matrices, and `A` and `-B` must not have common reciprocal eigenvalues.
This function is not recommended for large order matrices.
"""
function sylvdkr(A, B, C)
    m, n = size(C)
    [m; n] == LinearAlgebra.checksquare(A, B) ||
            throw(DimensionMismatch("A, B and C have incompatible dimensions"))
    reshape((kron(transpose(B), A) + I) \ (C[:]), m, n)
end

"""
    X = gsylvkr(A,B,C,D,E)

Solve the generalized Sylvester matrix equation

                AXB + CXD = E

using the Kronecker product expansion of equations. `A`, `B`, `C` and `D` are
square matrices. The pencils `A-λC` and `D+λB` must be regular and
must not have common eigenvalues.
This function is not recommended for large order matrices.
"""
function gsylvkr(A, B, C, D, E)
   m, n = size(E)
   [m; n; m; n] == LinearAlgebra.checksquare(A, B, C, D) ||
                  throw(DimensionMismatch("A, B, C, D and E have incompatible dimensions"))
   reshape((kron(transpose(B), A) + kron(transpose(D), C)) \ (E[:]), m, n)
end

"""
    sylvsyskr(A,B,C,D,E,F) -> (X,Y)

Solve the Sylvester system of matrix equations

                AX + YB = C
                DX + YE = F

using the Kronecker product expansion of equations. `(A,D)`, `(B,E)` are
pairs of square matrices of the same size.
The pencils `A-λD` and `-B+λE` must be regular and must not have common eigenvalues.
This function is not recommended for large order matrices.
"""
function sylvsyskr(A, B, C, D, E, F)
   m, n = size(C)
   (m == size(F, 1) && n == size(F, 2)) ||
         throw(DimensionMismatch("C and F must have the same dimensions"))
   [m; n; m; n] == LinearAlgebra.checksquare(A, B, D, E) ||
                  throw(DimensionMismatch("A, B, C, D, E and F have incompatible dimensions"))
   z = [ kron(Array{eltype(A),2}(I, n, n), A) kron(transpose(B), Array{eltype(B),2}(I, m, m)) ;
         kron(Array{eltype(D),2}(I, n, n), D) kron(transpose(E), Array{eltype(E),2}(I, m, m))] \ [C[:];F[:]]
   (reshape(z[1:m * n], m, n), reshape(z[m * n + 1:end], m, n))
end

"""
    dsylvsyskr(A,B,C,D,E,F) -> (X,Y)

Solve the dual Sylvester system of matrix equations

       AX + DY = C
       XB + YE = F

using the Kronecker product expansion of equations. `(A,D)`, `(B,E)` are
pairs of square matrices of the same size.
The pencils `A-λD` and `-B+λE` must be regular and must not have common eigenvalues.
This function is not recommended for large order matrices.
"""
function dsylvsyskr(A, B, C, D, E, F)
   m, n = size(C)
   (m == size(F, 1) && n == size(F, 2)) ||
         throw(DimensionMismatch("C and F must have the same dimensions"))
   [m; n; m; n] == LinearAlgebra.checksquare(A, B, D, E) ||
                  throw(DimensionMismatch("A, B, C, D, E and F have incompatible dimensions"))
   z = [ kron(Array{eltype(A),2}(I, n, n), A) kron(Array{eltype(D),2}(I, n, n), D);
         kron(transpose(B), Array{eltype(B),2}(I, m, m)) kron(transpose(E), Array{eltype(E),2}(I, m, m))] \ [C[:];F[:]]
   (reshape(z[1:m * n], m, n), reshape(z[m * n + 1:end], m, n))
end


module YaDistances

using LinearAlgebra

export cosine_similarity

const AVT = AbstractArray{T, N} where {T <: Any, N}

function cosine_similarity(v₁::AVT, v₂::AVT)::Float64 # where {T <: Any}
	dot(v₁, v₂) / (norm(v₁) * norm(v₂))
end

end  ## Module

Data Science From Scratch/src/YaDistances.jl


# Author: Lukasz Bratos

using LinearAlgebra

function hilb(n::Int)
    # Function generates the Hilbert matrix  A of size n,
    #  A (i, j) = 1 / (i + j - 1)
    # Inputs:
    #	n: size of matrix A, n>=1
    #
    #
    # Usage: hilb(10)
    #
    # Pawel Zielinski
    if n < 1
        error("size n should be >= 1")
    end
    return [1 / (i + j - 1) for i in 1:n, j in 1:n]
end

function matcond(n::Int, c::Float64)
# Function generates a random square matrix A of size n with
# a given condition number c.
# Inputs:
#	n: size of matrix A, n>1
#	c: condition of matrix A, c>= 1.0
#
# Usage: matcond(10, 100.0)
#
# Pawel Zielinski
    if n < 2
        error("size n should be > 1")
    end
    if c < 1.0
        error("condition number  c of a matrix  should be >= 1.0")
    end
    (U, S, V) = svd(rand(n, n))
    return U * diagm(0 => [LinRange(1.0, c, n);]) * V'
end

function calculate(matrix, x, size)
    b = matrix * x
    gauss = matrix \ b
    inversion = inv(matrix) * b

    gauss_err = norm(gauss - x) / norm(x)
    inv_err = norm(inversion - x) / norm(x)

    println("$size & $(rank(matrix)) & $(cond(matrix)) & $gauss_err & $inv_err \\\\ \\hline")
end

for i in 1:20
    matrix = hilb(i)
    x = ones(Float64, i)
    calculate(matrix, x, i)
end

for i in [5, 10, 20]
    for j in [1.0, 10.0, 10.0^3, 10.0^7, 10.0^12, 10.0^16]
        matrix = matcond(i, j)
        x = ones(Float64, i)
        calculate(matrix, x, i)
    end
end


fix_literal(id) = if id isa Symbol; Meta.quot(id) else id end
(~)(id) = fix_literal(id)

commutativity(op) = :( ($op)(a, b) == ($op)(b, a) ) |> Rule
right_associative(op) = :( ($op)(a, $(op)(b,c)) => ($op)($(op)(a,b), c) ) |> Rule
left_associative(op) = :( ($op)($(op)(a,b), c) => ($op)(a, $(op)(b,c)) ) |> Rule

associativity_left(op) = :( ($op)(a, $(op)(b,c)) => ($op)($(op)(a,b), c) ) |> Rule
associativity_right(op) = :(  ($op)($(op)(a,b), c) => ($op)(a, $(op)(b,c)) ) |> Rule

associativity(op) = [associativity_left(op), associativity_right(op)]

identity_left(op, id) = let id = ~id; :( ($op)($id, a) => a ) |> Rule end
identity_right(op, id) = let id = ~id; :( ($op)(a, $id) => a ) |> Rule end

inverse_left(op, id, invop) = let id = ~id; :( ($op)(($invop)(a), a) => $id ) |> Rule end
inverse_right(op, id, invop) = let id = ~id; :( ($op)(a, ($invop)(a)) => $id ) |> Rule end

# distributivity of two operations
# example: `@distrib (⋅) (⊕)`
function distrib_left(outop, inop)
	@assert Base.isbinaryoperator(outop)
	@assert Base.isbinaryoperator(inop)
	:( ($outop)(a, ($inop)(b,c)) == ($inop)(($outop)(a,b),($outop)(a,c)) ) |> Rule
end

function distrib_right(outop, inop)
	@assert Base.isbinaryoperator(outop)
	@assert Base.isbinaryoperator(inop)
	:( ($outop)(($inop)(a,b), c) == ($inop)(($outop)(a,c),($outop)(b,c)) ) |> Rule
end

function monoid(op, id)
	let id = ~id
		@assert Base.isbinaryoperator(op)
		[associativity_left(op), associativity_right(op),
		 identity_left(op, id), identity_right(op,id)]
	end
end
macro monoid(op, id) monoid(op, id) end

function commutative_monoid(op, id)
	let id = ~id;
		@assert Base.isbinaryoperator(op)
		[commutativity(op), associativity_left(op),
		associativity_right(op), identity_left(op, id)]
	end
end
macro commutative_monoid(op, id) commutative_monoid(op, id) end

# constructs a semantic theory about a an abelian group
# The definition of a group does not require that a ⋅ b = b ⋅ a
# for all elements a and b in G. If this additional condition holds,
# then the operation is said to be commutative, and the group is called an abelian group.
function commutative_group(op, id, invop)
	let id = ~id;
		@assert Base.isbinaryoperator(op)
		# @assert Base.isunaryoperator(invop)
		commutative_monoid(op, id) ∪ [inverse_right(op, id, invop)]
	end
end
abelian_group(op, id, invop) = commutative_group(op, id, invop)
macro commutative_group(op, id, invop) commutative_group(op, id, invop) end
macro abelian_group(op, id, invop) commutative_group(op, id, invop) end

distrib(outop, inop) = [
	distrib_left(outop, inop), distrib_right(outop, inop),
]


n = 2
m = 1

A = [1.0 1.0; 0.0 1.0]
B = [0.0; 1.0]

Q = [1.0 0.0; 0.0 1.0]
R = [1.0]

x_init = [1.0; 0.0]

nz = 2 * n + m + 2 * n

function unpack(z)
	x2 = z[1:n]
	u1 = z[n .+ (1:m)]
	x1 = z[n + m .+ (1:n)]
	y2 = z[n + m + n .+ (1:n)]
	y1 = z[n + m + n + n .+ (1:n)]

	return x2, u1, x1, y2, y1
end

z0 = rand(nz)
unpack(z0)
ones(1)' * R * ones(1)
function lagrangian(z, x_init)
	x2, u1, x1, y2, y1 = unpack(z)

	L = 0.0

	# objective
	L += transpose(x2) * Q * x2
	L += (transpose(u1) * R * u1)[1]
	L += transpose(x1) * Q * x1

	# constraints
	L += transpose(y2) * (A * x1 + B * u1[1] - x2)
	L += transpose(y1) * (x_init - x1)

	return L
end

@variables z_sym[1:nz]
@variables x_sym[1:n]

L = lagrangian(z_sym, x_sym)
L = simplify.(L)

dL = Symbolics.gradient(L, z_sym)
ddL = Symbolics.hessian(L, z_sym)
ddL_x = Symbolics.jacobian(dL, x_sym)

L_grad = eval(Symbolics.build_function(dL, z_sym, x_sym)[1])
L_hess = eval(Symbolics.build_function(ddL, z_sym, x_sym)[1])
Lx_hess = eval(Symbolics.build_function(ddL_x, z_sym, x_sym)[1])

function solve(z0, x_init)
	z = copy(z0)

	∇L = L_grad(z, x_init)

	println()
	for i = 1:10
		r_norm = norm(∇L)
		println("iter ($i)")
		println("	grad norm: $(r_norm)")
		r_norm < 1.0e-8 && break

		∇²L = L_hess(z, x_init)

		Δ = ∇²L \ ∇L

		α = 1.0
		ẑ = z - α * Δ

		∇L̂ = L_grad(ẑ, x_init)

		iter = 1
		while norm(∇L̂) > norm(∇L)
			α *= 0.5
			ẑ = z - α * Δ
			∇L̂ = L_grad(ẑ, x_init)
			iter += 1

			if iter > 100
				break
			end
		end

		z = ẑ
		∇L = ∇L̂
	end

	return z, ∇L, L_hess(z, x_init)
end

z_sol, ∇L_sol, ∇²L_sol = solve(z0, x_init)
K_direct = (∇²L_sol \ Lx_hess(z_sol, x_init))[n .+ (1:m), :]

# LQR solution
K, P = tvlqr([A], [B], [Q, Q], [R[1,1]])

@assert norm(K_direct - K[1]) < 1.0e-8


## In this notebook, we'll look at least squares on biometric data

##
using Plots
pyplot()
theme(:dark)
##
# https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv
using CSV
df = CSV.read(download("https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv"))
X = Float64.([df[:weight] df[:height]])
# Filter the data
goodpts = X[:,1] .<= 115
Xf = X[goodpts,:]
##
scatter(Xf[:,1],Xf[:,2],label="",xlabel="weight (kg)", ylabel="height (cm)")
##
x = Xf[:,1]
A = [x ones(length(x))]
b = Xf[:,2]
ab = A\b # solve the least squares problem

##
using Printf
scatter(Xf[:,1],Xf[:,2],color=2,size=(300,300), label="")
plot!(x, x*ab[1] .+ ab[2], label="",color=5)
title!(@sprintf("height ≈ %.2f weight + %.2f    ",ab[1],ab[2]))
xlabel!("Weight (kg)")
ylabel!("Height (cm)")
savefig("weight-height-fit.pdf")
## Show the

## Show the quadratic surface
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))
##
plot!(colorbar=false,size=(300,300),dpi=300)
anim = @animate for i=1:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic.gif")

##
## Show the quadratic surface with alpha-blending
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot()
    plot!(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))

##
## Show k random samples of the quadratic surface
plotlyjs()
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf!(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot!(x, y, Z, st=:surface, alpha=0.1)
end
orig = ezsurf(-3:0.25:3,-50:1:250,
  quadratic_leastsq(A,b))
nsamp = 25
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:25] # choose 25 of 200 samples.
  ezsurf!(-3:0.1:3,-50:0.5:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
## Show k random samples of the quadratic surface
using Random
using LinearAlgebra
plotlyjs()
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf!(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot!(x, y, Z, st=:surface, alpha=0.4)
end
plot(framestyle=:none,colorbar=false,size=(300,300),dpi=300)
nsamp = 10
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:nsamp] # choose 25 of 200 samples.
  ezsurf!(-3:0.25:3,-50:2:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
##
#plot!(colorbar=false,size=(300,300),dpi=300)
plot!(colorbar=false,size=(900,900))
anim = @animate for i=2:2:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic-samples.gif")

## Show the same thing with a log10
using Random
using LinearAlgebra
plotlyjs()
quadratic_leastsq(A,b) = x-> log10(0.5*norm(A*x - b)^2)/size(A,1)
plot(framestyle=:none,colorbar=false,size=(300,300),dpi=300)
nsamp = 10
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:nsamp] # choose 25 of 200 samples.
  ezsurf!(-3:0.1:3,-50:0.5:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
##
#plot!(colorbar=false,size=(300,300),dpi=300)
plot!(colorbar=false,size=(900,900))
anim = @animate for i=2:2:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic-samples-log.gif")
##


# Local Tangent Space Alignment (LTSA)
# ---------------------------
# Principal Manifolds and Nonlinear Dimension Reduction via Local Tangent Space Alignment,
# Zhang, Zhenyue; Hongyuan Zha (2004),  SIAM Journal on Scientific Computing 26 (1): 313–338.
# doi:10.1137/s1064827502419154.

#### LTSA type
struct LTSA{T <: Real} <: AbstractDimensionalityReduction
    k::Int
    λ::AbstractVector{T}
    proj::Projection{T}

    LTSA{T}(k::Int, λ::AbstractVector{T}, proj::Projection{T}) where T = new(k, λ, proj)
end

## properties
outdim(R::LTSA) = size(R.proj, 1)
eigvals(R::LTSA) = R.λ
neighbors(R::LTSA) = R.k

## show
summary(io::IO, R::LTSA) = print(io, "LTSA(outdim = $(outdim(R)), neighbors = $(neighbors(R)))")

## interface functions
function fit(::Type{LTSA}, X::AbstractMatrix{T}; maxoutdim::Int=2, k::Int=12) where {T<:Real}
    n = size(X, 2)

    # Construct NN graph
    D, E = find_nn(X, k)
    S = ones(k)./sqrt(k)
    B = spzeros(T, n,n)
    for i=1:n
        II = @view E[:,i]

        # re-center points in neighborhood
        μ = mean(X[:, II], dims=2)
        δ_x = X[:, II] .- μ

        # Compute orthogonal basis H of θ'
        θ_t = svd(δ_x).V[:,1:maxoutdim]

        # Construct alignment matrix
        G = hcat(S, θ_t)
        B[II, II] .+= diagm(0 => fill(one(T), k)) .- G*transpose(G)
    end

    # Align global coordinates
    λ, V = decompose(B, maxoutdim)
    return LTSA{T}(k, λ, transpose(V))
end

transform(R::LTSA) = R.proj


using LinearAlgebra

function so3ToVec(so3mat)
    if( size(so3mat)!=(3,3) )
        println("!Warning! : Input matrix should be 3x3.")
        return
    end
    if( !((so3mat[1,1]==so3mat[2,2]==so3mat[3,3]==0)) )
        println("!Warning! : Diagonals should be zero.")
        return
    end
    if( so3mat[2,1]!=-so3mat[1,2] ||
        so3mat[3,1]!=-so3mat[1,3] ||
        so3mat[3,2]!=-so3mat[2,3] )
        println("!Warning! : Entered matrix isn't a skew symmetric matrix.")
        return
    end
    ret_vec = [so3mat[3,2];
               so3mat[1,3];
               so3mat[2,1] ]
        return ret_vec
end

function vecToSo3(omg)
    if( size(omg)!=(3,) )
        println("!Warning! : Input matrix should be 3x1.")
        return
    end

    ret_ss_matrix = [0      -omg[3]   omg[2];
                     omg[3]    0     -omg[1];
                    -omg[2]  omg[1]      0];
    return ret_ss_matrix
end


"""
 Tests for  https://link.springer.com/article/10.1007/s10543-007-0130-3 with test problems as in the paper.
 RS1, RS2
 and for https://www.sciencedirect.com/science/article/pii/S0377042706003906
 NON, COM, NON2
"""

import Statistics # for mean values of trajectories
import LinearAlgebra # for the normn
using StochasticDiffEq
using Test
using Random
using DiffEqDevTools
#using DiffEqGPU

seed = 137475

function prob_func(prob, i, repeat)
    remake(prob,seed=seeds[i])
end

"""
 Test non-commutative noise SDEs (iip)
"""

@info "Non-commutative noise"

u₀ = [0.1,0.1]
function f2!(du,u,p,t)
  du[1] = 5//4*u[2]-5//4*u[1]
  du[2] = 1//4*u[1]-1//4*u[2]
end
function g2!(du,u,p,t)
  du[1,1] = sqrt(3)/2*(u[1]-u[2])
  du[1,2] = 1//2*(u[1]+u[2])
  #du[2,1] = 0
  du[2,2] = u[1]
end
dts = 1 .//2 .^(4:-1:1)
tspan = (0.0,1.0)

h2(z) = z^2 # E(x_i) = 1/10 exp(1/2t) or E(x_1* x_2) = 1/100 exp(2t)

prob = SDEProblem(f2!,g2!,u₀,tspan,noise_rate_prototype=zeros(2,2))
ensemble_prob = EnsembleProblem(prob;
        output_func = (sol,i) -> (h2(sol[end][1]),false),
        prob_func = prob_func
        )

numtraj = Int(3e7)
Random.seed!(seed)
seeds = rand(UInt, numtraj)

sim = test_convergence(dts,ensemble_prob,RS1(),
    save_everystep=false,trajectories=numtraj,save_start=false,adaptive=false,
    weak_timeseries_errors=false,weak_dense_errors=false,
    expected_value=1//100*exp(2)
    )
@test abs(sim.𝒪est[:weak_final]-2) < 0.3
println("RS1:", sim.𝒪est[:weak_final])

sim = test_convergence(dts,ensemble_prob,RS2(),
    save_everystep=false,trajectories=numtraj,save_start=false,adaptive=false,
    weak_timeseries_errors=false,weak_dense_errors=false,
    expected_value=1//100*exp(2)
    )
@test abs(sim.𝒪est[:weak_final]-2) < 0.3
println("RS2:", sim.𝒪est[:weak_final])

sim = test_convergence(dts,ensemble_prob,NON(),
    save_everystep=false,trajectories=numtraj,save_start=false,adaptive=false,
    weak_timeseries_errors=false,weak_dense_errors=false,
    expected_value=1//100*exp(2)
    )
@test abs(sim.𝒪est[:weak_final]-2) < 0.3
println("NON:", sim.𝒪est[:weak_final])

sim = test_convergence(dts,ensemble_prob,NON2(),
    save_everystep=false,trajectories=numtraj,save_start=false,adaptive=false,
    weak_timeseries_errors=false,weak_dense_errors=false,
    expected_value=1//100*exp(2)
    )
@test abs(sim.𝒪est[:weak_final]-2) < 0.3
println("NON2:", sim.𝒪est[:weak_final])

numtraj = Int(2e7)
Random.seed!(seed)
seeds = rand(UInt, numtraj)

sim = test_convergence(dts,ensemble_prob,COM(),
    save_everystep=false,trajectories=numtraj,save_start=false,adaptive=false,
    weak_timeseries_errors=false,weak_dense_errors=false,
    expected_value=1//100*exp(2)
    )
@test abs(sim.𝒪est[:weak_final]-2) < 0.3
println("COM:", sim.𝒪est[:weak_final])
# COM tests are passing; problem might be not hard enough..


# An example of the Burgers code for the
# propagation of Burgers equation in time
# studies the pointwise convergence order of a solution
# to Burgers equation as a function of resolution

using Burgers
using Printf
using PyPlot
include("Utils.jl")
using .Utils

function convergence_ratio(it::I, sc::SlopeCalc) where {T, I<:Int}
	# calculates the pointwise convergence ratio function
	# it     : the index for the time at which to slice u(x, t)
	#          must satisfy 0 <= t0 <= Ttot
	# sc     : the type of slope calculation (linear, minmod, constant)

	Ttot     = 2.0   # total duration
	Nt_b     = 512   # number of time divisions for lowest resolution
	Ncells_b = 128   # number of position divisions for lowest resolution
	Nghost   = 2     # padding on position divisions for boundary conditions

	if !(1 <= it <= Nt_b)
		error("The input time index must be between 1 and $Nt_base")
	end

	# The scaling factors
	scale1 = 8
	scale2 = 2*scale1
	scale3 = 2*scale2

	# Creating the Resolution instances
	res1 = Resolution{Float64, Int64}(Ttot, scale1*Nt_b, scale1*Ncells_b, Nghost)
	res2 = Resolution{Float64, Int64}(Ttot, scale2*Nt_b, scale2*Ncells_b, Nghost)
	res3 = Resolution{Float64, Int64}(Ttot, scale3*Nt_b, scale3*Ncells_b, Nghost)

	# Evaluating the solution to Burgers equation at t = t0
	ufun1 = get_evolved_slice(get_sinewave, res1, scale1*it, sc)
	ufun2 = get_evolved_slice(get_sinewave, res2, scale2*it, sc)
	ufun3 = get_evolved_slice(get_sinewave, res3, scale3*it, sc)

	# Calculating the absolute difference between solutions
	# of successive resolutions
	udiff12 = absolute_difference(ufun1, ufun2, ufun1.xdat)
	udiff23 = absolute_difference(ufun2, ufun3, ufun1.xdat)

	# stabilize the solutions, prevents the denominator from being too small
	eps = 1e-6
	uratio = (eps + udiff12)/(eps + udiff23)

	return uratio
end

function plot_convergence(cfun::FunArr{T, U}, imagename::AbstractString) where {T, U}
	# plots and saves the binary logarithm of the pointwise convergence function
	PyPlot.clf()

	xlabel("x")
	xlim(0, 1)
	xticks(range(0, 1, length = 10 + 1))

	ylabel("C(x)")
	ylim(0, 6)
	yticks(range(0, 6, length = 6 + 1))

	plot(cfun.xdat, log.(2, cfun.udat))
	savefig("images/" * imagename * ".png", dpi = 300)
end

# collect the command line arguments
# calculate and plot the convergence ratio
it, sc, imagename = commandlineparser(true)
convergence       = convergence_ratio(it, sc)
plot_convergence(convergence, imagename)


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ 4ea41550-9313-11eb-1cdf-df6c5527f80f
begin
	using PlutoUI
	PlutoUI.TableOfContents(aside=true)
end

# ╔═╡ 4c360009-f01e-45bb-b458-29b15f8cac8f
begin
	using LinearAlgebra, SpecialMatrices, Statistics
	import Random
	Random.seed!(125)
	x=rand(-9:9,5)
end

# ╔═╡ 7d3ed16e-50ea-4ede-b968-e32baa56f76f
md"""
# Eigenvalue Decomposition - Perturbation Theory

__Prerequisites__

The reader should be familiar with basic linear algebra concepts and facts about eigenvalue decomposition.

__Competences__

The reader should be able to understand and check the facts about perturbations of eigenvalues and eigenvectors.

"""

# ╔═╡ da0d7094-fa3e-4f1e-a6f1-9d1b9316f65a
md"""
# Norms

In order to measure changes, we need to define norms. For more details and the proofs of the Facts below, see
[R. Byers and B. N. Datta, Vector and Matrix Norms, Error Analysis, Efficiency, and Stability, pp. 50.1-50.24](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

## Definitions

__Norm__ on a vector space $X$ is a real-valued function $\| \phantom{x} \| : X\to \mathbb{R}$ with the following properties:

1. __Positive definiteness.__ $\| x\|\geq 0$ and $\|x\|=0$ if and only if $x$ is the zero vector.
2. __Homogeneity.__ $\| \lambda x\|=|\lambda| \|x\|$
3. __Triangle inequality.__ $\| x+y\| \leq \|x\|+\|y\|$

Commonly encountered vector norms for $x\in\mathbb{C}^n$ are:

* __Hölder norm__ or $p$-__norm__: for $p\geq 1$, $\|x\|_p=\big(|x_1|^p+|x_2|^p+\cdots |x_n|^p)^{1/p}$,
* __Sum norm__ or $1$-__norm__: $\|x\|_1=|x_1|+|x_2|+\cdots |x_n|$,
* __Euclidean norm__ or $2$-__norm__: $\|x\|_2=\sqrt{|x_1|^2+|x_2|^2+\cdots |x_n|^2}$,
* __Sup-norm__ or $\infty$-__norm__: $\|x\|_\infty = \max\limits_{i=1,\ldots,n} |x_i|$.

Vector norm is __absolute__ if $\||x|\|=\|x\|$.

Vector norm is __monotone__ if $|x|\leq |y|$ implies $\|x\|\leq \|y\|$.

From every vector norm we can derive a corresponding __induced__ matrix norm (also, __operator norm__ or __natural norm__):

$$\|A\| = \max\limits_{x\neq 0} \frac{\|Ax\|}{\|x\|}=\max\limits_{\|x\|=1} \|Ax\|.$$

For matrix $A\in\mathbb{C}^{m\times n}$ we define:

* __Maximum absolute column sum norm__: $\|A\|_1=\max\limits_{1\leq j \leq n} \sum_{i=1}^m |a_{ij}|$,
* __Spectral norm__: $\|A\|_2=\sqrt{\rho(A^*A)}=\sigma_{\max}(A)$  (largest singular value of $A$),
* __Maximum absolute row sum norm__: $\|A\|_{\infty}=\max\limits_{1\leq i \leq m} \sum_{j=1}^n |a_{ij}|$,
* __Euclidean norm__ or __Frobenius norm__:
$\|A\|_F =\sqrt{\sum_{i,j} |a_{ij}|^2}=\sqrt{\mathop{\mathrm{tr}}(A^*A)}$.

Matrix norm is __consistent__ if $\|A\cdot B\|\leq \|A\| \cdot \| B\|$, where $A$ and $B$ are compatible for matrix multiplication.

Matrix norm is __absolute__ if $\||A|\|=\|A\|$.
"""

# ╔═╡ f8ba4230-9313-11eb-170c-ab58597826f8
md"
### Examples
"

# ╔═╡ 7bd6f898-b86c-4670-96e8-9016666ee14f
norm(x,1), norm(x), norm(x,Inf)

# ╔═╡ 8e1ced3a-d23d-4f5f-8c65-dc13ab392504
A=rand(-4:4,5,4)

# ╔═╡ d9ef7b4b-4954-464a-96ac-e3f49800e0b7
norm(A,1), norm(A), norm(A,2), norm(A,Inf),
opnorm(A), opnorm(A,1), opnorm(A,Inf)

# ╔═╡ 90fcd9f5-d98c-4db6-bf30-21e377209681
md"""
## Facts

1.  $\|x\|_1$, $\|x\|_2$, $\|x\|_\infty$ and $\|x\|_p$ are absolute and monotone vector norms.
2. A vector norm is absolute iff it is monotone.
3. __Convergence.__ $x_k\to x_*$ iff for any vector norm $\|x_k-x_*\|\to 0$.
4. Any two vector norms are equivalent in the sense that, for all $x$ and some $\alpha,\beta>0$

$$\alpha \|x\|_\mu \leq \|x\|_\nu \leq \beta \|x\|_\mu.$$

In particular:

   *  $\|x\|_2 \leq \|x\|_1\leq \sqrt{n}\|x\|_2$,
   *  $\|x\|_\infty \leq \|x\|_2\leq \sqrt{n}\|x\|_\infty$,
   *  $\|x\|_\infty \leq \|x\|_1\leq n\|x\|_\infty$.

5. __Cauchy-Schwartz inequality.__ $|x^*y|\leq \|x\|_2\|y\|_2$.
6. __Hölder inequality.__ If $p,q\geq 1$ and $\displaystyle\frac{1}{p}+\frac{1}{q}=1$, then

$$|x^*y|\leq \|x\|_p\|y\|_q.$$

7.  $\|A\|_1$, $\|A\|_2$ and $\|A\|_\infty$ are induced by the corresponding vector norms.
8.  $\|A\|_F$ is not an induced norm.
9.  $\|A\|_1$, $\|A\|_2$, $\|A\|_\infty$ and $\|A\|_F$ are consistent.
10.  $\|A\|_1$, $\|A\|_\infty$ and $\|A\|_F$ are absolute. However, $\||A|\|_2\neq \|A\|_2$.
11. Any two matrix norms are equivalent in the sense that, for all $A$ and some $\alpha,\beta>0$

$$\alpha \|A\|_\mu \leq \|A\|_\nu \leq \beta \|A\|_\mu.$$

In particular:
   *  $\frac{1}{\sqrt{n}}\|A\|_\infty \leq \|A\|_2\leq \sqrt{m}\|A\|_\infty$,
   *  $\|A\|_2 \leq \|A\|_F\leq \sqrt{n}\|A\|_2$,
   *  $\frac{1}{\sqrt{m}}\|A\|_1 \leq \|A\|_2\leq \sqrt{n}\|A\|_1$.

12.  $\|A\|_2\leq \sqrt{\|A\|_1 \|A\|_\infty}$.
13.  $\|AB\|_F\leq \|A\|_F\|B\|_2$ and $\|AB\|_F\leq \|A\|_2\|B\|_F$.
14. If $A=xy^*$, then $\|A\|_2=\|A\|_F=\|x\|_2\|y\|_2$.
15.  $\|A^*\|_2=\|A\|_2$ and $\|A^*\|_F=\|A\|_F$.
16. For a unitary matrix $U$ of compatible dimension,

$$\|AU\|_2=\|A\|_2,\quad \|AU\|_F=\|A\|_F,\quad
\|UA\|_2=\|A\|_2,\quad  \|UA\|_F=\|A\|_F.$$

17. For $A$ square, $\rho(A)\leq\|A\|$.
18. For $A$ square, $A^k\to 0$ iff $>\rho(A)<1$.
"""

# ╔═╡ 8261b5ce-9315-11eb-2378-af63c3eb4c02
md"
### Random matrix
"

# ╔═╡ bb6eee61-c3ee-4b6a-ab44-a638fba776b8
A

# ╔═╡ 060b8d2f-e6f3-4625-9cf6-d4cc504145fa
# Absolute norms. Spectral norm is not absolute.
opnorm(A,1), opnorm(abs.(A),1), norm(A,1), opnorm(A,Inf), opnorm(abs.(A),Inf), norm(A), norm(abs.(A)),  opnorm(A),opnorm(abs.(A))

# ╔═╡ f55a3c60-9315-11eb-027a-dd008031f6f9
m,n=size(A)

# ╔═╡ c4a1b888-9497-4eb4-aa1c-23671123f426
# Equivalence of norms
opnorm(A,Inf)\ √n,opnorm(A), √m*opnorm(A,Inf)

# ╔═╡ b46cbb6f-36f4-4a66-b970-8fe9b0506b6a
opnorm(A), norm(A), √n*opnorm(A)

# ╔═╡ 254b2df4-f57e-4284-a735-6fd5d4546c52
opnorm(A,1)\ √m,opnorm(A), √n*opnorm(A,1)

# ╔═╡ 09fd0923-1987-4a2e-be28-78451aa2c11a
# Fact 12
opnorm(A), √(opnorm(A,1)*opnorm(A,Inf))

# ╔═╡ dcba6892-b90e-41ad-8d58-53e72d40e882
begin
	# Fact 13
	B=rand(n,rand(1:9))
	norm(A*B), norm(A)*opnorm(B), opnorm(A)*norm(B)
end

# ╔═╡ c765c02e-9316-11eb-146c-ff5d3ba81323
x

# ╔═╡ d948741f-6748-44d7-ae8c-cf074f7549bd
begin
	# Fact 14
	y=rand(ComplexF64,10)
	opnorm(x*y'), norm(x*y'), norm(x)*norm(y)
end

# ╔═╡ ec0746a1-41bd-4d61-9deb-8e227106e592
begin
	# Fact 15
	A₁=rand(-4:4,7,5)+im*rand(-4:4,7,5)
	opnorm(A₁), opnorm(A₁'), norm(A₁), norm(A₁')
end

# ╔═╡ d92799f7-1813-450b-8353-9d03078f69f0
# Unitary invariance - generate random unitary matrix U
U=qr(rand(ComplexF64,size(A₁)));

# ╔═╡ 0d58d4c4-16c6-4a5b-bdbd-f3f599d0d648
opnorm(A₁), opnorm(U.Q*A₁), norm(A₁), norm(U.Q*A₁)

# ╔═╡ b1a90c59-8c3e-4530-83ef-6b3e6c6fa687
begin
	# Spectral radius
	A₂=rand(ComplexF64,7,7)
	maximum(abs,eigvals(A₂)), opnorm(A₂,Inf), opnorm(A₂,1), opnorm(A₂), norm(A₂)
end

# ╔═╡ 31e73f82-93e7-448c-940b-5c9f5e573e5c
begin
	# Fact 18
	B₂=A₂/(maximum(abs,eigvals(A₂))+2)
	maximum(abs,eigvals(B₂)), norm(B₂^100)
end

# ╔═╡ 909033d9-bb27-4f0e-b07d-574f7fcee700
md"""
# Errors and condition numbers

We want to answer the question:

__How much the value of a function changes with respect to the change of its argument?__

## Definitions

For function $f(x)$ and argument $x$, the __absolute error__ with respect to the __perturbation__ of the argument
$\delta x$ is

$$
\| f(x+\delta x)-f(x)\| = \frac{\| f(x+\delta x)-f(x)\|}{\| \delta x \|} \|\delta x\| \equiv \kappa \|\delta x\|.$$

The  __condition__ or  __condition number__ $\kappa$ tells how much does the perturbation of the argument increase. (Its form resembles derivative.)

Similarly, the __relative error__ with respect to the relative perturbation of the argument is

$$
\frac{\| f(x+\delta x)-f(x)\|}{\| f(x) \|}= \frac{\| f(x+\delta x)-f(x)\|\cdot  \|x\| }{\|\delta x\| \cdot\| f(x)\|}
\cdot \frac{\|\delta x\|}{\|x\|} \equiv \kappa_{rel} \frac{\|\delta x\|}{\|x\|}.$$
"""

# ╔═╡ 7ef162c0-5c80-4e96-bcc3-125b302b1d5e
md"""
# Peturbation bounds

## Definitions

Let $A\in\mathbb{C}^{n\times n}$.

Pair $(\lambda,x)\in\mathbb{C}\times\mathbb{C}^{n\times n}$ is an __eigenpair__ of $A$ if $x\neq 0$ and $Ax=\lambda x$.

Triplet $(y,\lambda,x)\in\times\mathbb{C}^{n}\times\mathbb{C}\times\mathbb{C}^{n}$ is an __eigentriplet__ of $A$ if $x,y\neq 0$ and $Ax=\lambda x$ and $y^*A=\lambda y^*$.

__Eigenvalue matrix__ is a diagonal matrix $\Lambda=\mathop{\mathrm{diag}}(\lambda_1,\lambda_2,\ldots,\lambda_n)$.

If all eigenvalues are real, they can be increasingly ordered. $\Lambda^\uparrow$ is the eigenvalue matrix of increasingly ordered eigenvalues.

 $\tau$ is a __permutation__ of $\{1,2,\ldots,n\}$.

 $\tilde A=A+\Delta A$ is a __perturbed matrix__, where $\Delta A$ is __perturbation__. $(\tilde \lambda,\tilde x)$ are the eigenpairs of $\tilde A$.

__Condition number__ of a nonsingular matrix $X$ is $\kappa(X)=\|X\| \|X^{-1}\|$.

Let $X,Y\in\mathbb{C}^{n\times k}$ with $\mathop{\mathrm{rank}}(X)=\mathop{\mathrm{rank}}(Y)=k$. The __canonical angles__ between their column spaces, $\theta_i$, are defined by $\cos \theta_i=\sigma_i$, where $\sigma_i$ are the singular values of the matrix

$$(Y^*Y)^{-1/2}Y^*X(X^*X)^{-1/2}.$$

The __canonical angle matrix__ between $X$ and $Y$ is

$$\Theta(X,Y)=\mathop{\mathrm{diag}}(\theta_1,\theta_2,\ldots,\theta_k).$$

"""

# ╔═╡ 3c8f71a5-0c2f-43a0-bf71-db9df707f84e
md"""
## Facts

Bounds become more strict as matrices have more structure.
Many bounds have versions in spectral norm and Frobenius norm.
For more details and the proofs of the Facts below, see
[R.-C. Li, Matrix Perturbation Theory, pp 21.1-21.20](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897), and the references therein.

1. There exists $\tau$ such that

$$\|\Lambda- \tilde\Lambda_\tau\|_2\leq 4(\|A\|_2+\|\tilde A\|_2)^{1-1/n}\|\Delta A\|_2^{1/n}.$$

2. __First-order perturbation bounds.__ Let $(y,\lambda,x)$ be an eigentriplet of a simple $\lambda$. $\Delta A$ changes $\lambda$ to $\tilde\lambda=\lambda+ \delta\lambda$, where _(for proof see GVL p. 359)_

$$\delta\lambda=\frac{y^*(\Delta A)x}{y^*x}+O(\|\Delta A\|_2^2).$$

3. Let $\mu$ be a semisimple eigenvalue of $A$ with multiplicitiy $k$, and let $X,Y\in \mathbb{C}^{n\times k}$ be the matrices of the corresponding right and left eigenvectors, that is, $AX=\lambda X$ and $Y^*A=\lambda Y^*$, such that $Y^*X=I_k$. $\Delta A$ changes the $k$ copies of $\mu$ to $\tilde \mu=\mu+\delta\mu_i$, where $\delta\mu_i$ are the eigenvalues of $Y^*(\Delta A) X$ up to $O(\|\Delta A\|_2^2)$.

4. Perturbations and the inverse: if $\|A\|_p<1$, then $I-A$ is nonsingular and _(for proof see GVL p. 74)_

$$(I-A)^{-1}=\sum\limits_{k=0}^\infty A^k,$$

$$\|(I-A)^{-1}\|_p \leq \frac{1}{1-\|A\|_p},\qquad
\|(I-A)^{-1}-I\|_p \leq \frac{\|A\|_p}{1-\|A\|_p}.$$

5. __Geršgorin Circle Theorem.__ If $X^{-1} A X=D+F$, where $D=\mathop{\mathrm{diag}}(d_1,\ldots,d_n)$ and $F$ has zero diagonal entries, then _(for proof see GVL p. 357)_

$$\sigma(A) \subseteq \bigcup\limits_{i=1}^n D_i,$$

where

$$D_i=\big\{z\in\mathbb{C} : |z-d_i| \leq \sum\limits_{j=1}^n |f_{ij}| \big\}.$$

Moreover, by continuity, if a connected component of $D$ consists of $k$ circles, it contains $k$ eigenvalues.

6. __Bauer-Fike Theorem.__ If $A$ is diagonalizable and $A=X\Lambda X^{-1}$ is its eigenvalue decomposition, then _(for proof see GVL p. 357)_

$$\max_i\min_j |\tilde \lambda_i -
\lambda_j|\leq \|X^{-1}(\Delta A)X\|_p\leq \kappa_p(X)\|\Delta A\|_p.$$

7. If $A$ and $\tilde A$ are diagonalizable, there exists $\tau$ such that

$$\|\Lambda-\tilde\Lambda_\tau\|_F\leq \sqrt{\kappa_2(X)\kappa_2(\tilde X)}\|\Delta A\|_F.$$

If $\Lambda$ and  $\tilde\Lambda$ are real, then

$$\|\Lambda^\uparrow-\tilde\Lambda^\uparrow\|_{2,F} \leq \sqrt{\kappa_2(X)\kappa_2(\tilde X)}\|\Delta A\|_{2,F}.$$

8. If $A$ is normal, there exists $\tau$ such that $\|\Lambda-\tilde\Lambda_\tau\|_F\leq\sqrt{n}\|\Delta A\|_F$.

9. __Hoffman-Wielandt Theorem.__ If $A$ and $\tilde A$ are normal, there exists $\tau$ such that $\|\Lambda-\tilde\Lambda_\tau\|_F\leq\|\Delta A\|_F$.

10. If $A$ and $\tilde A$ are Hermitian, for any unitarily invariant norm $\|\Lambda^\uparrow-\tilde\Lambda^\uparrow\| \leq \|\Delta A\|$. In particular,

$$\begin{aligned}
\max_i|\lambda^\uparrow_i-\tilde\lambda^\uparrow_i|&\leq \|\Delta A\|_2,\\
\sqrt{\sum_i(\lambda^\uparrow_i-\tilde\lambda^\uparrow_i)^2}&\leq \|\Delta A\|_F.
\end{aligned}$$

11. __Residual bounds.__ Let $A$ be Hermitian. For some $\tilde\lambda\in\mathbb{R}$ and $\tilde x\in\mathbb{C}^n$ with $\|\tilde x\|_2=1$, define __residual__ $r=A\tilde x-\tilde\lambda\tilde x$. Then $|\tilde\lambda-\lambda|\leq \|r\|_2$ for some $\lambda\in\sigma(A)$.

12. Let, in addition,  $\tilde\lambda=\tilde x^* A\tilde x$, let $\lambda$ be closest to $\tilde\lambda$ and $x$ be its unit eigenvector, and let

$$\eta=\mathop{\mathrm{gap}}(\tilde\lambda)= \min_{\lambda\neq\mu\in\sigma(A)}|\tilde\lambda-\mu|.$$

If $\eta>0$, then

$$|\tilde\lambda-\lambda|\leq \frac{\|r\|_2^2}{\eta},\quad \sin\theta(x,\tilde x)\leq \frac{\|r\|_2}{\eta}.$$

13. Let $A$ be Hermitian, $X\in\mathbb{C}^{n\times k}$ have full column rank, and $M\in\mathcal{H}_k$ having eigenvalues $\mu_1\leq\mu_2\leq\cdots\leq\mu_k$. Set $R=AX-XM$. Then there exist $\lambda_{i_1}\leq\lambda_{i_2}\leq\cdots\leq\lambda_{i_k}\in\sigma(A)$ such that

$$
\begin{aligned}
\max_{1\leq j\leq k} |\mu_j-\lambda_{i_j}|& \leq \frac{\|R\|_2}{\sigma_{\min}(X)},\\
\sqrt{\sum_{j=1}^k (\mu_j-\lambda_{i_j})^2}&\leq \frac{\|R\|_F}{\sigma_{\min}(X)}.
\end{aligned}$$

(The indices $i_j$ need not be the same in the above formulae.)

14. If, additionally, $X^*X=I$ and $M=X^*AX$, and if all but $k$ of $A$'s eigenvalues differ from every one of $M$'s eigenvalues by at least $\eta>0$, then

$$\sqrt{\sum_{j=1}^k (\mu_j-\lambda_{i_j})^2}\leq \frac{\|R\|_F^2}{\eta\sqrt{1-\|R\|_F^2/\eta^2}}.$$

15. Let $A=\begin{bmatrix} M & E^* \\ E & H \end{bmatrix}$ and $\tilde A=\begin{bmatrix} M & 0 \\ 0 & H \end{bmatrix}$ be Hermitian, and set $\eta=\min |\mu-\nu|$ over all $\mu\in\sigma(M)$ and $\nu\in\sigma(H)$. Then

$$\max |\lambda_j^\uparrow -\tilde\lambda_j^\uparrow| \leq \frac{2\|E\|_2^2}{\eta+\sqrt{\eta^2+4\|E\|_2^2}}.$$

16. Let

$$\begin{bmatrix} X_1^*\\ X_2^* \end{bmatrix} A \begin{bmatrix} X_1 & X_2 \end{bmatrix}=
\begin{bmatrix} A_1 &  \\ & A_2 \end{bmatrix}, \quad \begin{bmatrix} X_1 & X_2 \end{bmatrix} \quad \textrm{unitary},
\quad X_1\in\mathbb{C}^{n\times k}.$$

Let $Q\in\mathbb{C}^{n\times k}$ have orthonormal columns and for a Hermitian $k\times k$ matrix $M$ set
$R=AQ-QM$. Let $\eta=\min|\mu-\nu|$ over all $\mu\in\sigma(M)$ and $\nu\in\sigma(A_2)$. If $\eta > 0$, then

$$\|\sin\Theta(X_1,Q)\|_F\leq \frac{\|R\|_F}{\eta}.$$
"""

# ╔═╡ 98fade4c-cdcb-495e-9328-96e86ca8e90e
md"""
## Examples

### Nondiagonalizable matrix
"""

# ╔═╡ 8f064645-cf1f-4226-add4-53bc72665bf6
A₃=[-3 7 -1; 6 8 -2; 72 -28 19]

# ╔═╡ 568366c7-3eee-4948-a096-7bf87537412d
# (Right) eigenvectors
X₃=eigen(A₃)

# ╔═╡ fe46c02b-74bd-4303-8314-3b6e96b9e51a
cond(X₃.vectors)

# ╔═╡ c48aa813-8012-409f-9b08-ce9805da9cb3
# Left eigenvectors
Y₃=eigen(Matrix(A₃'))

# ╔═╡ f0bcadb5-882c-4e46-9079-b36097650435
cond(Y₃.vectors)

# ╔═╡ 21b412f3-f460-4f60-8b31-85c839566d62
begin
	# Try k=2,3
	k=1
	Y₃.vectors[:,k]'*A₃-Y₃.values[k]*Y₃.vectors[:,k]'
end

# ╔═╡ 6654b8d5-0c31-4a44-a16a-3f71e714ce31
begin
	ΔA₃=rand(3,3)/20
	B₃=A₃+ΔA₃
end

# ╔═╡ 3fbcbecd-6e69-49d0-94bf-06a4f6704858
norm(ΔA₃)

# ╔═╡ 210137a6-1b7b-4741-84d2-9d72e18cd999
Z₃=eigen(B₃)

# ╔═╡ e946019f-5e75-4195-8991-da371300e9a2
begin
	# Fact 2
	l=1
	Z₃.values[l]-X₃.values[l], Y₃.vectors[:,l]'*ΔA₃*X₃.vectors[:,l] /(Y₃.vectors[:,l]⋅X₃.vectors[:,l])
end

# ╔═╡ 4723a4f9-af98-4f2d-8628-0ffc58488e62
md"""
### Jordan form
"""

# ╔═╡ f3d63eb6-0174-458c-aeef-46ed6d62db7f
begin
	n₄=6
	c=0.5
	J=Bidiagonal(c*ones(n₄),ones(n₄-1),'U')
end

# ╔═╡ b9770791-4fa6-4d3f-9688-ac79b7ae7f22
# Accurately defined eigenvalues
λ₄=eigvals(J)

# ╔═╡ 8112bf44-3146-4b9f-953a-54de6f4c8c5b
# Only one eigenvector
eigvecs(J)

# ╔═╡ 6b26afc1-e427-496e-b9a5-4618551aeede
begin
	x₄=eigvecs(J)[:,1]
	y₄=eigvecs(J')[:,1]
end

# ╔═╡ b2a07659-cb08-4c53-99ac-32f817485062
y₄'*J-0.5*y₄'

# ╔═╡ 989b7c60-c673-4b77-ada8-909db2904fa2
# Just one perturbed element in the lower left corner
ΔJ=√eps()*[zeros(n₄-1);1]*Matrix(I,1,n₄)

# ╔═╡ 1da2bfee-6792-4aed-8798-91cc058cd540
μ₄=eigvals(J+ΔJ)

# ╔═╡ a6769e47-59f7-430c-9c20-a121a98f19bf
# Fact 2
maximum(abs,λ₄-μ₄)

# ╔═╡ fab42c40-7d01-4787-a491-3d37cae1768d
y₄'*ΔJ*x₄/(y₄⋅x₄)

# ╔═╡ 2f783e6e-5b29-477f-8b45-00461013c66f
md"""
However, since $J+\Delta J$ is diagonalizable, we can apply Bauer-Fike theorem to it:
"""

# ╔═╡ 9385e7e0-17d6-4840-98a2-fa69d53ebea3
Y₄=eigvecs(J+ΔJ)

# ╔═╡ 18e6478b-bb8e-482c-bd02-0c85bca6fc7f
cond(Y₄)

# ╔═╡ ae524f6a-3e64-49ea-92e3-671559f85a5f
opnorm(inv(Y₄)*ΔJ*Y₄), cond(Y₄)*opnorm(ΔJ)

# ╔═╡ 9c0c9fcf-9581-479f-8ec5-d885dd3e6665
md"""
### Normal matrix
"""

# ╔═╡ 300e0ddd-3ab0-4803-9528-665e754e54ca
begin
	n₅=5
	C=Circulant(rand(-5:5,n₅))
end

# ╔═╡ fdecade7-4d5c-460f-a566-d52868c75ec0
eigvals(Matrix(C))

# ╔═╡ 01250371-f8ab-4a0a-b283-ffbcffdc2929
ΔC=randn(n₅,n₅)*0.0001

# ╔═╡ 0787f637-44bf-4c5c-b8d2-92df842a91b5
opnorm(ΔC), eigvals(C+ΔC)

# ╔═╡ 5645f7ea-50e5-43f7-afc3-1c46fbdd7c5b
md"""
### Hermitian matrix
"""

# ╔═╡ 51cd2fc5-e295-4b76-9fd1-fdd2b92ce05a
begin
	# Random matrix with strong column scaling
	m₆=10
	n₆=6
	D₆=Diagonal(exp.(10*randn(n₆)))
	A₆=cor(randn(m₆,n₆)*D₆)
end

# ╔═╡ bc5a0ee0-931f-11eb-204c-b9e03584c787
D₆

# ╔═╡ 1eb12479-0662-410b-9569-c4e8939a291f
ΔA₆=cor(rand(m₆,n₆)*D₆)*1e-5

# ╔═╡ 821f6b8b-0e12-404b-b1dc-57d809ebf79b
begin
	λ₆,U₆=eigen(A₆)
	μ₆=eigvals(A₆+ΔA₆)
	[λ₆ μ₆]
end

# ╔═╡ ac3d3677-1160-4a79-a25f-8faa7fd89940
norm(ΔA₆)

# ╔═╡ f4051f63-a0d3-4b43-8c61-2bbd08338a72
# ?round

# ╔═╡ 612870ba-75df-4729-a16e-432f70707fb4
begin
	# Residual bounds - how close is μ, y to λ[2],X[:,2]
	k₆=3
	ζ₆=round(λ₆[k₆],sigdigits=2)
	y₆=round.(U₆[:,k₆],sigdigits=2)
	normalize!(y₆)
end

# ╔═╡ 3ba55b4b-5445-48a3-b327-b7d3af28397a
ζ₆

# ╔═╡ 6f3da676-c953-4d23-8b9a-bb62b8b8ac94
# Fact 9
r₆=A₆*y₆-ζ₆*y₆

# ╔═╡ 886ce4f8-73a0-4b57-8d5e-082bcfc1f9aa
minimum(abs,ζ₆.-λ₆), norm(r₆)

# ╔═╡ 82a4adaa-3663-4200-9f15-d09fd4cd0259
begin
	# Fact 10 - μ is Rayleigh quotient
	ξ₆=y₆⋅(A₆*y₆)
	ρ₆=A₆*y₆-ξ₆*y₆
end

# ╔═╡ b1025690-4244-4933-9693-9bbd3ad8aedb
η₆=min(abs(ξ₆-λ₆[k₆-1]),abs(ξ₆-λ₆[k₆+1]))

# ╔═╡ 5e69cab3-712c-4754-b6b1-579bc3c8e114
ξ₆-λ₆[k₆], norm(ρ₆)^2/η₆

# ╔═╡ 3792c478-fbaa-4683-849a-ca0d0cb751f3
begin
	# Eigenvector bound
	# cos(θ)
	cosθ=dot(y₆,U₆[:,k₆])
	# sin(θ)
	sinθ=sqrt(1-cosθ^2)
	sinθ,norm(ρ₆)/η₆
end

# ╔═╡ 0dd080f1-575d-43e0-9285-20279f390d2f
begin
	# Residual bounds - Fact 13
	Q₆=round.(U₆[:,1:3],sigdigits=2)
	# Orthogonalize
	F=qr(Q₆)
	X=Matrix(F.Q)
	# Make sure M₆ is hermitian
	M₆=Hermitian(X'*A₆*X)
	μₘ=eigvals(M₆)
	R₆=A₆*X-X*M₆
end

# ╔═╡ ea07eea3-bab7-4add-90cc-43acd7c2ec89
λ₆

# ╔═╡ 53f2860a-db33-49cd-b95e-3086f76c2592
μₘ

# ╔═╡ 8cdffcc2-c8c1-410e-95db-6327931782a1
M₆

# ╔═╡ 0e871e72-47ee-4492-9f73-fcfd2ef1aada
# The entries of μ are not ordered - which algorithm was called?
issymmetric(M₆)

# ╔═╡ 108f73bc-b1cf-47ed-80f0-281cb5ebd048
ηₘ=λ₆[4]-λ₆[3]

# ╔═╡ dbc219f9-9fcf-4aac-b7d5-feb8fcafe3e9
norm(λ₆[1:3]-μₘ), norm(R₆)^2/ηₘ

# ╔═╡ 7cae5501-8a51-4874-8b23-5825f79624e3
A₆

# ╔═╡ 7cb49041-f326-4b2c-90f8-99c4d58178eb
begin
	# Fact 15
	M₇=A₆[1:3,1:3]
	H=A₆[4:6,4:6]
	E=A₆[4:6,1:3]
	# Block-diagonal matrix
	B₇=cat(M₇,H,dims=(1,2))
end

# ╔═╡ 75c00141-e691-4f9c-8b7a-ed262b88a62d
begin
	η₇=minimum(abs,eigvals(M₇)-eigvals(H))
	μ₇=eigvals(B₇)
	[λ₆ μ₇]
end

# ╔═╡ 433a0652-912f-4f59-8494-4d0381e22115
2*norm(E)^2/(η₇+√(η₇^2+4*norm(E)^2))

# ╔═╡ ad6ca597-65b2-4403-a6aa-1394399c3ed8
begin
	# Eigenspace bounds - Fact 16
	B₆=A₆+ΔA₆
	μ,V=eigen(B₆)
end

# ╔═╡ 4727c0ba-124e-41a4-861a-57eb5a6d849d
begin
	# sin(Θ(U[:,1:3],V[:,1:3]))
	X₆=U₆[:,1:3]
	Q=V[:,1:3]
	cosθ₆=svdvals(√(Q'*Q)*Q'*X₆*√(X₆'*X₆))
	sinθ₆=sqrt.(1 .-cosθ₆.^2)
end

# ╔═╡ dd00cbef-e1f6-43bc-95b0-1b6694c23583
# Bound
M₈=Q'*A₆*Q

# ╔═╡ 99f44a10-b3e1-442c-a042-b2f0afad1797
R₈=A₆*Q-Q*M₈

# ╔═╡ b6f8f48e-2e80-4999-95a9-701d491411bd
eigvals(M₈), λ₆

# ╔═╡ 7a6554bb-0518-4068-9784-678f846d3b3f
begin
	η₈=abs(eigvals(M₈)[3]-λ₆[4])
	norm(sinθ₆), norm(R₈)/η₈
end

# ╔═╡ Cell order:
# ╟─4ea41550-9313-11eb-1cdf-df6c5527f80f
# ╟─7d3ed16e-50ea-4ede-b968-e32baa56f76f
# ╟─da0d7094-fa3e-4f1e-a6f1-9d1b9316f65a
# ╟─f8ba4230-9313-11eb-170c-ab58597826f8
# ╠═4c360009-f01e-45bb-b458-29b15f8cac8f
# ╠═7bd6f898-b86c-4670-96e8-9016666ee14f
# ╠═8e1ced3a-d23d-4f5f-8c65-dc13ab392504
# ╠═d9ef7b4b-4954-464a-96ac-e3f49800e0b7
# ╟─90fcd9f5-d98c-4db6-bf30-21e377209681
# ╟─8261b5ce-9315-11eb-2378-af63c3eb4c02
# ╠═bb6eee61-c3ee-4b6a-ab44-a638fba776b8
# ╠═060b8d2f-e6f3-4625-9cf6-d4cc504145fa
# ╠═f55a3c60-9315-11eb-027a-dd008031f6f9
# ╠═c4a1b888-9497-4eb4-aa1c-23671123f426
# ╠═b46cbb6f-36f4-4a66-b970-8fe9b0506b6a
# ╠═254b2df4-f57e-4284-a735-6fd5d4546c52
# ╠═09fd0923-1987-4a2e-be28-78451aa2c11a
# ╠═dcba6892-b90e-41ad-8d58-53e72d40e882
# ╠═c765c02e-9316-11eb-146c-ff5d3ba81323
# ╠═d948741f-6748-44d7-ae8c-cf074f7549bd
# ╠═ec0746a1-41bd-4d61-9deb-8e227106e592
# ╠═d92799f7-1813-450b-8353-9d03078f69f0
# ╠═0d58d4c4-16c6-4a5b-bdbd-f3f599d0d648
# ╠═b1a90c59-8c3e-4530-83ef-6b3e6c6fa687
# ╠═31e73f82-93e7-448c-940b-5c9f5e573e5c
# ╟─909033d9-bb27-4f0e-b07d-574f7fcee700
# ╟─7ef162c0-5c80-4e96-bcc3-125b302b1d5e
# ╟─3c8f71a5-0c2f-43a0-bf71-db9df707f84e
# ╟─98fade4c-cdcb-495e-9328-96e86ca8e90e
# ╠═8f064645-cf1f-4226-add4-53bc72665bf6
# ╠═568366c7-3eee-4948-a096-7bf87537412d
# ╠═fe46c02b-74bd-4303-8314-3b6e96b9e51a
# ╠═c48aa813-8012-409f-9b08-ce9805da9cb3
# ╠═f0bcadb5-882c-4e46-9079-b36097650435
# ╠═21b412f3-f460-4f60-8b31-85c839566d62
# ╠═6654b8d5-0c31-4a44-a16a-3f71e714ce31
# ╠═3fbcbecd-6e69-49d0-94bf-06a4f6704858
# ╠═210137a6-1b7b-4741-84d2-9d72e18cd999
# ╟─e946019f-5e75-4195-8991-da371300e9a2
# ╟─4723a4f9-af98-4f2d-8628-0ffc58488e62
# ╠═f3d63eb6-0174-458c-aeef-46ed6d62db7f
# ╠═b9770791-4fa6-4d3f-9688-ac79b7ae7f22
# ╠═8112bf44-3146-4b9f-953a-54de6f4c8c5b
# ╠═6b26afc1-e427-496e-b9a5-4618551aeede
# ╠═b2a07659-cb08-4c53-99ac-32f817485062
# ╠═989b7c60-c673-4b77-ada8-909db2904fa2
# ╠═1da2bfee-6792-4aed-8798-91cc058cd540
# ╠═a6769e47-59f7-430c-9c20-a121a98f19bf
# ╠═fab42c40-7d01-4787-a491-3d37cae1768d
# ╟─2f783e6e-5b29-477f-8b45-00461013c66f
# ╠═9385e7e0-17d6-4840-98a2-fa69d53ebea3
# ╠═18e6478b-bb8e-482c-bd02-0c85bca6fc7f
# ╠═ae524f6a-3e64-49ea-92e3-671559f85a5f
# ╟─9c0c9fcf-9581-479f-8ec5-d885dd3e6665
# ╠═300e0ddd-3ab0-4803-9528-665e754e54ca
# ╠═fdecade7-4d5c-460f-a566-d52868c75ec0
# ╠═01250371-f8ab-4a0a-b283-ffbcffdc2929
# ╠═0787f637-44bf-4c5c-b8d2-92df842a91b5
# ╟─5645f7ea-50e5-43f7-afc3-1c46fbdd7c5b
# ╠═51cd2fc5-e295-4b76-9fd1-fdd2b92ce05a
# ╠═bc5a0ee0-931f-11eb-204c-b9e03584c787
# ╠═1eb12479-0662-410b-9569-c4e8939a291f
# ╠═821f6b8b-0e12-404b-b1dc-57d809ebf79b
# ╠═ac3d3677-1160-4a79-a25f-8faa7fd89940
# ╠═f4051f63-a0d3-4b43-8c61-2bbd08338a72
# ╠═612870ba-75df-4729-a16e-432f70707fb4
# ╠═3ba55b4b-5445-48a3-b327-b7d3af28397a
# ╠═6f3da676-c953-4d23-8b9a-bb62b8b8ac94
# ╠═886ce4f8-73a0-4b57-8d5e-082bcfc1f9aa
# ╠═82a4adaa-3663-4200-9f15-d09fd4cd0259
# ╠═b1025690-4244-4933-9693-9bbd3ad8aedb
# ╠═5e69cab3-712c-4754-b6b1-579bc3c8e114
# ╠═3792c478-fbaa-4683-849a-ca0d0cb751f3
# ╠═0dd080f1-575d-43e0-9285-20279f390d2f
# ╠═ea07eea3-bab7-4add-90cc-43acd7c2ec89
# ╠═53f2860a-db33-49cd-b95e-3086f76c2592
# ╠═8cdffcc2-c8c1-410e-95db-6327931782a1
# ╠═0e871e72-47ee-4492-9f73-fcfd2ef1aada
# ╠═108f73bc-b1cf-47ed-80f0-281cb5ebd048
# ╠═dbc219f9-9fcf-4aac-b7d5-feb8fcafe3e9
# ╠═7cae5501-8a51-4874-8b23-5825f79624e3
# ╠═7cb49041-f326-4b2c-90f8-99c4d58178eb
# ╠═75c00141-e691-4f9c-8b7a-ed262b88a62d
# ╠═433a0652-912f-4f59-8494-4d0381e22115
# ╠═ad6ca597-65b2-4403-a6aa-1394399c3ed8
# ╠═4727c0ba-124e-41a4-861a-57eb5a6d849d
# ╠═dd00cbef-e1f6-43bc-95b0-1b6694c23583
# ╠═99f44a10-b3e1-442c-a042-b2f0afad1797
# ╠═b6f8f48e-2e80-4999-95a9-701d491411bd
# ╠═7a6554bb-0518-4068-9784-678f846d3b3f

src/Module B - Eigenvalue and Singular Value Decompositions/L3b Eigenvalue Decomposition - Perturbation Theory.jl


# Ilija Spasić PR 34-2020

"""
    generateArray(a,b,n=30)

Funkcija za generisanje zadatog niza brojeva iz opsega **[1 ... 100]**,
sa najmanje 30 elementa. Gde parametar **a** predstavlja početak opsega, parametar **b** kraj
opsega, dok je parametar **n** opcioni parametar za broj elemenata generisanog niza.
"""
function generateArray(a::Int64, b::Int64, n::Int64=30)::Array{Int64}
    if a < 1 || b > 100
        error("Neispravan opseg funkcije. Za vise informacija upisite ?generateArray")
    end
    X = [ ]
    for _ in 1:n
        push!(X, Int64(rand(a:b)))
    end
    return X
end

"""

    bucketSort(X::Array, smer::String)

Funkcija sortiranja niza primenom Bucket Sort algoritma. Gde smer uzima vrednosti
**"o"** za sortiranje u opadajućem poretku i
**"r"** za sortiranje u rastućem poretku
"""
function bucketSort(X::Array{Int64},smer::String="r")
    if smer != "r" && smer != "o"
        error("Pogrešan parametar funkcije! Proverite ? bucketSort")
    end
    rez = [ ]
    M = maximum(X)
    n = M/length(X)
    buckets = [ ]
    for _ in 1:length(X)
        push!(buckets,[ ])
    end
    for i in 1:length(X)
        j = Int64(floor((X[i]/n)))
        j = j == 0 ? 1 : j
        if j != length(X)
            push!(buckets[j],X[i])
        else
            push!(buckets[length(X)-1],X[i])
        end
    end
    for i in 1:length(X)
        insertSortBucket(buckets[i],smer)
    end
    if smer == "o"
        for i in 1:length(X)
            rez = vcat(buckets[i],rez)
        end
    else
        for i in 1:length(X)
            rez = vcat(rez,buckets[i])
        end
    end
    return rez
end

"""

    insertSortBucket(x::Array)

Pomoćna funkcija za sortiranja podeljenih delova (bucket-a) kod
Bucket Sort algoritma koja koristi Insertion Sort algoritam.
"""
function insertSortBucket(x,smer)
    for j in 2:length(x)
        key = x[j]
        i = j-1
        while i>0 && (smer == "o" ? x[i] < key : x[i] > key)
            x[i+1] = x[i]
            i = i-1
        end
        x[i+1] = key
    end
end


# lasso.jl - Lasso solvers based on FISTA and ADMM using ProximalOperators
#
#   minimize 0.5*||A*x - b||^2 + lam*||x||_1
#

using LinearAlgebra
using Random
using ProximalOperators

Random.seed!(0)

# Define solvers

function lasso_fista(A, b, lam, x; tol=1e-3, maxit=50000)
  x_prev = copy(x)
  g = NormL1(lam)
  gam = 1.0/norm(A)^2
  for it = 1:maxit
    # extrapolation step
    x_extr = x + (it-2)/(it+1)*(x - x_prev)
    # compute least-squares residual
    res = A*x_extr - b
    # compute gradient (forward) step
    y = x_extr - gam*(A'*res)
    # store current iterate
    x_prev .= x
    # compute proximal (backward) step
    prox!(x, g, y, gam)
    # stopping criterion
    if norm(x_extr-x, Inf)/gam <= tol*(1+norm(x, Inf))
      break
    end
  end
  return x
end

function lasso_admm(A, b, lam, x; tol=1e-8, maxit=50000)
  u = zero(x)
  z = copy(x)
  f = LeastSquares(A, b)
  g = NormL1(lam)
  gam = 100.0/norm(A)^2
  for it = 1:maxit
    # perform f-update step
    prox!(x, f, z - u, gam)
    # perform g-update step
    prox!(z, g, x + u, gam)
    # stopping criterion
    if norm(x-z, Inf) <= tol*(1+norm(u, Inf))
      break
    end
    # dual update
    u .+= x - z
  end
  return z
end

# Generate random problem

println("Generating random lasso problem")

m, n, k, sig = 500, 2500, 100, 1e-3
A = randn(m, n)
x_true = [randn(k)..., zeros(n-k)...]
b = A*x_true + sig*randn(m)
lam = 0.1*norm(A'*b, Inf)

# Call solvers

println("Calling solvers")

x_fista = lasso_fista(A, b, lam, zeros(n))
println("FISTA")
println("      nnz(x)    = $(norm(x_fista, 0))")
println("      obj value = $(0.5*norm(A*x_fista-b)^2 + lam*norm(x_fista, 1))")

x_admm = lasso_admm(A, b, lam, zeros(n))
println("ADMM")
println("      nnz(x)    = $(norm(x_admm, 0))")
println("      obj value = $(0.5*norm(A*x_admm-b)^2 + lam*norm(x_admm, 1))")

@test x_fista ≈ x_admm rtol=1e-3