using Unitful, UnitfulAstro
using PhysicalConstants.CODATA2018: h, c_0, k_B

export blackbody

"""
    blackbody(wave::Vector{<:Quantity}, T::Quantity)
    blackbody(wave::Vector{<:Real}, T::Real)

Create a blackbody spectrum using Planck's law. The curve follows the mathematical form

``B_\\lambda(T) = \\frac{2hc^2}{\\lambda^5}\\frac{1}{e^{hc/\\lambda k_B T} - 1}``

If `wave` and `T` are not `Unitful.Quantity`, they are assumed to be in angstrom and Kelvin, and the returned flux will be in units `W m^-2 Å^-1`.

The physical constants are calculated using [PhysicalConstants.jl](https://github.com/juliaphysics/physicalconstants.jl), specifically the CODATA2018 measurement set.

# References
[Planck's Law](https://en.wikipedia.org/wiki/Planck%27s_law)

# Examples
```jldoctest
julia> using Spectra, Unitful, UnitfulAstro

julia> wave = range(1, 3, length=100)u"μm"
(1.0:0.020202020202020204:3.0) μm

julia> bb = blackbody(wave, 2000u"K")
UnitfulSpectrum (100,)
  λ (μm) f (W μm^-1 m^-2)
  T: 2000 K
  name: Blackbody

julia> blackbody(ustrip.(u"angstrom", wave), 6000)
Spectrum (100,)
  T: 6000
  name: Blackbody

julia> bb.wave[argmax(bb)]
1.4444444444444444 μm

julia> 2898u"μm*K" / bb.T # See if it matches up with Wien's law
1.449 μm
```
"""
function blackbody(wave::AbstractVector{<:Quantity}, T::Quantity)
    out_unit = u"W/m^2" / unit(eltype(wave))
    flux = _blackbody(wave, T) .|> out_unit
    return spectrum(wave, flux, name = "Blackbody", T = T)
end

function blackbody(wave::AbstractVector{<:Real}, T::Real)
    flux = ustrip.(u"W/m^2/angstrom", _blackbody(wave * u"angstrom", T * u"K"))
    return spectrum(wave, flux, name = "Blackbody", T = T)
end

_blackbody(wave::AbstractVector{<:Quantity}, T::Quantity) = blackbody(T).(wave)

"""
  blackbody(T::Quantity)

Returns a function for calculating blackbody curves.
"""
blackbody(T::Quantity) = w->2h * c_0^2 / w^5 / (exp(h * c_0 / (w * k_B * T)) - 1)


## PART 2

# Define the error functions

function σf1(v̄̂,σ̂)
    ddvs = zeros(5);
    ddvs[1] = 1 / v̄̂[2]^2;
    ddvs[2] = v̄̂[1]^2 / v̄̂[2]^4;
    return sqrt( dot( ddvs, σ̂.^2) );
end

function σf2(v̄̂,σ̂)
    ddvs = zeros(5);
    ddvs[3] = exp(v̄̂[3] - v̄̂[4])^2;
    ddvs[4] = exp(v̄̂[3] - v̄̂[4])^2;
    return sqrt( dot( ddvs, σ̂.^2) );
end

function σf3(v̄̂,σ̂)
    lv5s = log(v̄̂[5])^2;
    ddvs = zeros(5);
    ddvs[1] = lv5s / v̄̂[2]^2;
    ddvs[2] = lv5s * v̄̂[1]^2 / v̄̂[2]^4;
    ddvs[3] = lv5s / v̄̂[4]^2;
    ddvs[4] = lv5s * v̄̂[3]^2 / v̄̂[4]^4;
    ddvs[5] = ( ( v̄̂[1]/v̄̂[2] + v̄̂[3]/v̄̂[4] ) / v̄̂[5] )^2;
    return sqrt( dot( ddvs, σ̂.^2) );
end

println("Calculating σ̂f1, σ̂f2, σ̂f3 for sample sizes N1, N2");
σ̂f1N1 = σf1(v̄̂,σ̂1);
σ̂f2N1 = σf2(v̄̂,σ̂1);
σ̂f3N1 = σf3(v̄̂,σ̂1);

σ̂f1N2 = σf1(v̄̂,σ̂2);
σ̂f2N2 = σf2(v̄̂,σ̂2);
σ̂f3N2 = σf3(v̄̂,σ̂2);

println("\n\tNaive standard deviations for sample size 1,000");
println("\tσ̂f1N1 = $σ̂f1N1");
println("\tσ̂f2N1 = $σ̂f2N1");
println("\tσ̂f3N1 = $σ̂f3N1");

println("\n\tNaive standard deviations for sample size 10,000");
println("\tσ̂f1N2 = $σ̂f1N2");
println("\tσ̂f2N2 = $σ̂f2N2");
println("\tσ̂f3N2 = $σ̂f3N2");

println("\n\tTrue standard deviations for sample size 1,000");
println("\tσ̂truef1N1 = $σ̂truef1N1");
println("\tσ̂truef2N1 = $σ̂truef2N1");
println("\tσ̂truef3N1 = $σ̂truef3N1");

println("\n\tTrue standard deviations for sample size 10,000");
println("\tσ̂truef1N2 = $σ̂truef1N2");
println("\tσ̂truef2N2 = $σ̂truef2N2");
println("\tσ̂truef3N2 = $σ̂truef3N2");


"""
	cracks2moduli(E₀, nu₀, α1, α3, β11, β13, β33)

Compute 6×6 stiffness matrix (in Voight notation) for a cracked material characterised by normalised crack density parameters α1, α3, β11, β13 and β33, with solid matrix moduli E₀ and nu₀. Crack density parameters are normalised by h = 3E₀(2-nu₀)/(32(1-nu₀²)). From Sayers and Kachanov 1995.
"""
function cracks2moduli(E₀, nu₀, α1, α3, β11, β13, β33)
    S011 = 1/E₀
    S012 = -nu₀/E₀
    h = cracknormfactor(E₀, nu₀)

    D = (S011 + α3/h + β33/h)*(S011 + S012 + α1/h + 4β11/h/3) -
        2*(S012 + β13/h)^2

    c11 = (1/2)*((S011+α3/h+β33/h)/D + 1/(S011-S012+α1/h+2β11/h/3))
    c33 = (S011 + S012 + α1/h + 4β11/h/3)/D
    c44 = 1/(2S011 - 2S012 + α1/h + α3/h + 4β13/h)
    c13 = -(S012 + β13/h)/D
    c66 = 1/(S011-S012+α1/h+2β11/h/3)/2

    return stiffmatrixTI(c11, c13, c33, c44, c66)
end

"""
	cracknormfactor(E₀, ν₀)

Compute normalisation factor h=3E₀*(2-nu₀)/(32(1-nu₀²)).
"""
function cracknormfactor(E₀, ν₀)
    return 3E₀*(2-ν₀)/(32(1-ν₀^2))
end

"""
	findcracks(p::Vector{VMeasure},
                    sv::Vector{VMeasure},
                    sh::Vector{VMeasure},
                    ρ::Real,
                    E₀::Real,
                    nu₀::Real,
                    α₁₀::Tuple{Real,Real},
                    α₃₀::Tuple{Real,Real},
                    β₁₁₀::Tuple{Real,Real},
                    β₁₃₀::Tuple{Real,Real},
                    β₃₃₀::Tuple{Real,Real})

Estimate normalised crack density parameters and phase angles from measurements of group P, SV ans SH wave velocities, using quasinewton method. A priori values ()₀ are tuples with value and standard deviation (a priori gaussian).

Return α₁₁, α₃₃, β₁₁₁₁, β₁₁₃₃, β₃₃₃₃, θp, θsv, θsh, CMpost.
"""
function findcracks(p::Vector{VMeasure},
                    sv::Vector{VMeasure},
                    sh::Vector{VMeasure},
                    ρ::Real,
                    E₀::Real,
                    nu₀::Real,
                    α₁₀::Tuple{Real,Real},
                    α₃₀::Tuple{Real,Real},
                    β₁₁₀::Tuple{Real,Real},
                    β₁₃₀::Tuple{Real,Real},
                    β₃₃₀::Tuple{Real,Real})

    dobs, CDi, np, nsv, nsh = inputdobs(p,sv,sh)

    #make vector of model parameters (first guess)
    m₀ = [α₁₀[1],α₃₀[1],β₁₁₀[1],β₁₃₀[1],β₃₃₀[1]]
    σm = [α₁₀[2],α₃₀[2],β₁₁₀[2],β₁₃₀[2],β₃₃₀[2]]

    C₀ = cracks2moduli(E₀, nu₀, m₀...)
    ϵ₀, δ₀, γ₀ = moduli2thomsen(C₀)
    vp,vs = velocities0(C₀,ρ) # here density does not matter

    for x in p
        push!(m₀, phaseangleP(x.angle, vp/vs, ϵ₀, δ₀))
        push!(σm, 100)
    end
    for x in sv
        push!(m₀, phaseangleSV(x.angle, vp/vs, ϵ₀, δ₀))
        push!(σm, 100)
    end
    for x in sh
        push!(m₀, phaseangleSH(x.angle, γ₀))
        push!(σm, 100)
    end

    CM = Diagonal(σm.^2)
    CMi = inv(CM)

    m, CMpost = quasinewton(dobs, _gf, m₀, CMi, CDi, (E₀, nu₀, ρ,np,nsv,nsh))

    return m[1:5], m[6:5+np], m[6+np:5+np+nsv], m[6+np+nsv:5+np+nsv+nsh], CMpost

end

function _gf(m,E₀,nu₀,ρ,np,nsv,nsh)
    C₀ = cracks2moduli(E₀, nu₀, m[1:5]...)
    ϵ, δ, γ = moduli2thomsen(C₀)
    vp0,vs0 = velocities0(C₀,ρ)
    mt = vcat([vp0,vs0,ϵ,δ,γ], m[6:end])
    return _gt(mt, np, nsv, nsh)
end


include("setting.jl")

# jacobian of lorenz96
function jac_lorenz(x::Vector{Float64})::Matrix{Float64}
    # jac[i, j] = -1 when j = i
    # jac[i, j] = x[i-1] when j = i+1
    # jac[i, j] = x[i+1] - x[i-2] when j = i-1
    # jac[i, j] = -x[i-1] when j = i-2
    len = length(x)
    lenm = len - 1;
    jac = zeros(len, len)

    for i = 1:len
        ip1 = (i+1+lenm)%len+1
        im1 = (i-1+lenm)%len+1
        im2 = (i-2+lenm)%len+1
        @inbounds jac[i, i] = -1
        @inbounds jac[i, ip1] = x[im1]
        @inbounds jac[i, im1] = x[ip1] - x[im2]
        @inbounds jac[i, im2] = -x[im1]
    end
    jac
end

a = repeat([1., 3, 6, 4, 5], 3)
da = repeat([.2, .2, .2, .2, .2], 3)
b = a + da
fa = lorenz96(a, Missing, Missing)
fb = lorenz96(b, Missing, Missing)
jfa = jac_lorenz(a) * da
display(jac_lorenz(a))
println()
@show fa + jfa - fb
@assert all(abs.(fa+jfa-fb) .< 1e-6)


# Converts infix expression into reverse Polish notation
# (Only 1-digit numbers are allowed.)

operators = {
    # opname => (operator, precedence)
    '+' => (+, 6),
    '-' => (-, 6),
    '*' => (*, 7),
    '/' => (/, 7),
}

is_operator(op) = haskey(operators, op)
get_operator(ch) = operators[ch][1];
get_precedence(ch) = operators[ch][2];
PREC_OFFSET = 100

function in2post(expr::ASCIIString)
    opstack = Char[]
    precedence_stack = Int[]
    output = Char[]
    prec_offset = 0

    for ch in expr
        if ch == '('
            prec_offset += PREC_OFFSET
        elseif ch == ')'
            prec_offset -= PREC_OFFSET
        elseif is_operator(ch)
            precedence = get_precedence(ch) + prec_offset
            while !isempty(opstack) &&
                precedence <= precedence_stack[end]
                push!(output, pop!(opstack))
                pop!(precedence_stack)
            end
            push!(opstack, ch)
            push!(precedence_stack, precedence)
        else
            push!(output, ch)
        end
    end

    if prec_offset != 0
        warn("Unbalanced parentheses!")
    end
    [output, reverse(opstack)]
end

function eval_post(postexp)
    numstack = Int[]
    for ch in postexp
        if is_operator(ch)
            b = pop!(numstack);
            a = pop!(numstack);
            push!(numstack, get_operator(ch)(a, b))
        else
            push!(numstack, ch - '0');
        end
    end
    assert(length(numstack) == 1)
    numstack[end]
end

expr = strip(readline())
postexp = in2post(expr)
@printf("The converted RPN is: %s,\nwhich = %d",
    join(postexp), eval_post(postexp))

# Conclusion: Julia is far from mature enough to use in production.


@doc raw"""
```
binarize(Balanced(), img)
```
Binarizes the image using the balanced histogram thresholding method.

# Output

Returns the binarized image as an `Array{Gray{Bool},2}`.

# Details
In balanced histogram thresholding, one interprets a  bin as a  physical weight
with a mass equal to its occupancy count. The balanced histogram method involves
iterating the following three steps: (1) choose the midpoint bin index as a
"pivot",  (2) compute the combined weight to the left and right of the pivot bin
and (3) remove the leftmost bin if the left side is the heaviest, and the
rightmost bin otherwise. The algorithm stops when only a single bin remains. The
last bin determines the sought-after threshold with which the image is
binarized.

Let ``f_n`` (``n = 1 \ldots N``) denote the number of observations in the ``n``th
bin of the image histogram. The balanced histogram method constructs a sequence
of nested intervals

```math
[1,N] \cap \mathbb{Z} \supset I_2 \supset I_3 \supset \ldots \supset I_{N-1},
```
where for ``k = 2 \ldots N-1``
```math
I_k = \begin{cases}
   I_{k-1} \setminus \{\min \left( I_{k-1} \right) \} &\text{if } \sum_{n = \min \left( I_{k-1} \right)}^{I_m}f_n \gt   \sum_{n =  I_m + 1}^{ \max \left( I_{k-1} \right)} f_n, \\
   I_{k-1} \setminus \{\max \left( I_{k-1} \right) \} &\text{otherwise},
\end{cases}
```
and ``I_m = \lfloor \frac{1}{2}\left(  \min \left( I_{k-1} \right) +  \max \left( I_{k-1} \right) \right) \rfloor ``.
The final interval ``I_{N-1}`` consists of a single element which is the bin index
corresponding to the desired threshold.

If one interprets a bin as a physical weight with a mass equal to its occupancy
count, then each step of the algorithm can be conceptualised as removing the
leftmost or rightmost bin to "balance" the resulting histogram on a pivot. The
pivot is defined to be the midpoint between the start and end points of the
interval under consideration.

If it turns out that the single element in ``I_{N-1}`` equals ``1`` or ``N`` then
the original histogram must have a single peak and the algorithm has failed to
find a suitable threshold. In this case the algorithm will fall back to using
the `UnimodalRosin` method to select the threshold.

# Arguments

The function argument is described in more detail below.

##  `img`

An `AbstractArray` representing an image. The image is automatically converted
to `Gray` in order to construct the requisite graylevel histogram.

# Example

Binarize the "cameraman" image in the `TestImages` package.

```julia
using TestImages, ImageBinarization

img = testimage("cameraman")
img_binary = binarize(Balanced(), img)
```

# Reference

1. “BI-LEVEL IMAGE THRESHOLDING - A Fast Method”, Proceedings of the First International Conference on Bio-inspired Systems and Signal Processing, 2008. Available: [10.5220/0001064300700076](https://doi.org/10.5220/0001064300700076)
"""
function binarize(algorithm::Balanced,  img::AbstractArray{T,2}) where T <: Colorant
  img₀₁ = zeros(Gray{Bool}, axes(img))
  edges, counts = build_histogram(img,  256)
  t = find_threshold(HistogramThresholding.Balanced(), counts[1:end], edges)
  for i in CartesianIndices(img)
    img₀₁[i] = img[i] < t ? 0 : 1
  end
  img₀₁
end


function boxcox(z, λ)
    reduce(&, z .> 0) || throw(DomainError("boxcox requires strictly positive data"))
	if λ == 0
		return log.(z)
	else
		return (z.^λ .- 1) ./ λ
	end
end

function inv_boxcox(z, λ)
    if λ == 0
        return exp.(z)
    else
        return (λ .* z .+ 1).^(1/λ)
    end
end

function guerrero(z; lower=-1, upper=2, s=2)
    s >= 2 || throw(ArgumentError("guerrero requires seasonality to be higher or equal to 2"))
    N = length(z)
    rz = reshape(view(z, (N % s + 1):N), N ÷ s, s)
    m = mean(rz, dims=1)
    sd = std(rz, mean=m, dims=1)
    function cv(λ)
        w = sd ./ (m.^(1 - λ))
        mw = mean(w)
        return std(w, mean=mw) / mw
    end
    return optimize(cv, lower, upper).minimizer
end

function difference(z::AbstractVector{T}; d::Integer=1,s::Integer=1) where T
    N = length(z)
    differenced = Vector{T}(undef, N)
    @inbounds @simd for i in 1:N
        differenced[i] = z[i]
    end
    @inbounds for j in 1:d
        for i in 1:N - j * s
            differenced[N - i + 1] -= differenced[N - i + 1 - s]
        end
    end
    return differenced[1 + d * s:N]
end

function integrate(z::AbstractVector{T}, z0::AbstractVector; d::Integer=1,s::Integer=1) where T
    N = length(z)
    integrated = Vector{T}(undef, N + d * s)
    N0 = length(z0)
    for i in 1:N + d * s
        integrated[i] = i > d * s ? z[i - d * s] : i > N0 ? zero(T) : z0[i]
    end
    for j in 1:d
        for i in (1 + s):N + d * s
            integrated[i] += integrated[i - s]
        end
    end
    return integrated
end

integrate(z::AbstractVector{T}; d::Integer=1,s::Integer=1) where T = integrate(z, T[], d=d, s=s)


using Plots,ApproxFun;pyplot()

x = Fun(identity,[0.,10.])
f = sin(x^2)
g = cos(x)

h = f + g^2
r = roots(h)
rp = roots(differentiate(h))
plot(h;legend=false,grid=false)
scatter!(r,h(r))
scatter!(rp,h(rp))
xlabel!("\$x\$")
ylabel!("\$h(x)\$")
xlims!(0,10)
PyPlot.savefig("extrema.png",dpi=300)
println("First image done")

x = Fun(identity,[-1000.,200.])
d = domain(x)
D = Derivative(d)
B = dirichlet(d)
L = D^2 - x
u = [B;L] \ [airyai(d.a),airyai(d.b)]
plot(u;legend=false,grid=false)						    # Requires Gadfly or PyPlot
xlabel!("\$x\$");ylabel!("\${\\rm Ai}(x)\$")
PyPlot.savefig("airy.png",dpi=300)

println("Second image done")

d = Interval(-π,π)
a = Fun(t-> 1+sin(cos(2t)),d)
D = Derivative(d)
L = D + a
f = Fun(t->exp(sin(10t)),d)
B = periodic(d,0)
uChebyshev = [B;L]\[0.,f]

d = PeriodicSegment([-π,π])
a = Fun(t-> 1+sin(cos(2t)),d)
D = Derivative(d)
L = D + a
f = Fun(t->exp(sin(10t)),d)
uFourier = L\f

length(uFourier)/length(uChebyshev),2/π
plot(real(uFourier);legend=false,grid=false)						    # Requires Gadfly or PyPlot
xlims!(d.a,d.b);xlabel!("\$t\$");ylabel!("\$u(t)\$")
PyPlot.savefig("periodic.png",dpi=300)

println("Third image done")

f = abs(Fun(sin,[-5.,5.]))
d = domain(f)
x = ApproxFun.sample(f,10000)
plot(f/sum(f);legend=false,grid=false)                           # Requires Gadfly or PyPlot
histogram!(x;normed=true,nbins=100)
xlims!(leftendpoint(d),rightendpoint(d));ylims!(0.0,0.18);xlabel!("\$x\$");ylabel!("Density")
PyPlot.savefig("Sample.png",dpi=300)

println("Fourth image done")

## Nonlinear BVP

x=Fun()
u0=0x

N=u->[u(-1.)-1.,u(1.)+0.5,0.001u''+6*(1-x^2)*u'+u^2-1.]
u=newton(N,u0)
plot(u;legend=false,grid=false)
PyPlot.savefig("nbvp.png",dpi=300)

println("Fifth image done")

## Multivariate

d = ChebyshevInterval()^2                            # Defines a rectangle

u = [dirichlet(d);lap(d)+100I]\[[1.0,1.0,1.0,1.0]]      # First four entries of rhs are
                                            # boundary conditions
surface(u,grid=false)                                     # contour plot

PyPlot.savefig("helmholtz.png",dpi=300)


__precompile__()

module Vandermonde

export pvand

function pvand(alpha, b)
    # x = pvand(alpha, b)
    #
    # Solves system A*x = b
    # A is Vandermonde matrix, with nonstandard definition
    # A(i,j) = alpha(j)^i
    #
    # Algorithm by Bjorck & Pereyra
    # Mathematics of Computation, Vol. 24, No. 112 (1970), pp. 893-903
    # https://doi.org/10.2307/2004623
    #
    n = length(alpha);
    x = copy(b);
    for k=1:n
        for j=n:-1:k+1
            x[j] = x[j]-alpha[k]*x[j-1];
        end
    end
    for k=n-1:-1:1
        for j=k+1:n
            x[j] = x[j]/(alpha[j]-alpha[j-k]);
        end
        for j=k:n-1
            x[j] = x[j]-x[j+1];
        end
    end
    return x
end

end


using SpecialFunctions

include("spindown_besselzeros.jl")

function u_spindown(r, t, Re)
    # Assuming unit radius, speed and viscosity
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        ds = besselj(1, lambda[n]*r/a) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function Lu_spindown(r, t, Re)
    # Laplacian of the above
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        b = lambda[n]/a
        ds = -b^2*besselj(1, b*r) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function Du_spindown(r, t, Re)
    # r derivative
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        b = lambda[n]/a
        ds = b/2*(besselj(0, b*r)-besselj(2, b*r)) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function plot_spindown(t, Re)
    r = linspace(0,1,200)
    u = map(x -> u_spindown(x, t, Re), r)
    plot(r, u, label="Reference")
end


# Part of submodule Utils of BetaML _ the Beta Machine Learning Toolkit
# Various measures of pairs (x,y) (including vectors or matrix pairs)

# ------------------------------------------------------------------------------
# Some common distance measures

"""L1 norm distance (aka _Manhattan Distance_)"""
l1_distance(x,y)     = sum(abs.(x-y))
"""Euclidean (L2) distance"""
l2_distance(x,y)     = norm(x-y)
"""Squared Euclidean (L2) distance"""
l2²_distance(x,y)    = norm(x-y)^2
"""Cosine distance"""
cosine_distance(x,y) = dot(x,y)/(norm(x)*norm(y))

################################################################################
### VARIOUS ERROR / LOSS / ACCURACY MEASURES
################################################################################

# ------------------------------------------------------------------------------
# Classification tasks...

# Used as neural network loss function
"""
   crossEntropy(ŷ, y; weight)

Compute the (weighted) cross-entropy between the predicted and the sampled probability distributions.

To be used in classification problems.

"""
crossEntropy(ŷ, y; weight = ones(eltype(y),length(y)))  = -sum(y .* log.(ŷ .+ 1e-15) .* weight)
dCrossEntropy(ŷ, y; weight = ones(eltype(y),length(y))) = - y .* weight ./ (ŷ .+ 1e-15)

""" accuracy(ŷ,y;ignoreLabels=false) - Categorical accuracy between two vectors (T vs T). """
function accuracy(ŷ::AbstractArray{T,1},y::AbstractArray{T,1}; ignoreLabels=false)  where {T}
    # See here for better performances: https://discourse.julialang.org/t/permutations-of-a-vector-that-retain-the-vector-structure/56790/7
    if(!ignoreLabels)
        return sum(ŷ .== y)/length(ŷ)
    else
        classes  = unique(y)
        nCl      = length(classes)
        N        = size(y,1)
        pSet     =  collect(permutations(1:nCl))
        bestAcc  = -Inf
        yOrigIdx = [findfirst(x -> x == y[i] , classes) for i in 1:N]
        ŷOrigIdx = [findfirst(x -> x == ŷ[i] , classes) for i in 1:N]
        for perm in pSet
            py = perm[yOrigIdx] # permuted specific version
            acc = sum(ŷOrigIdx .== py)/N
            if acc > bestAcc
                bestAcc = acc
            end
        end
        return bestAcc
    end
end

""" error(ŷ,y;ignoreLabels=false) - Categorical error (T vs T)"""
error(ŷ::AbstractArray{T,1},y::AbstractArray{T,1}; ignoreLabels=false) where {T} = (1 - accuracy(ŷ,y;ignoreLabels=ignoreLabels) )

"""
    accuracy(ŷ,y;tol)
Categorical accuracy with probabilistic prediction of a single datapoint (PMF vs Int).

Use the parameter tol [def: `1`] to determine the tollerance of the prediction, i.e. if considering "correct" only a prediction where the value with highest probability is the true value (`tol` = 1), or consider instead the set of `tol` maximum values.
"""
function accuracy(ŷ::Array{T,1},y_pos::Int64;tol=1,rng=Random.GLOBAL_RNG) where {T <: Number}
    #if  length(Set(ŷ) == 1                         # all classes the same prob
    #    return rand(rng) < (1 / length(y)) ? 1 : 0 # If all values have the same prob, it returns 1 with prob 1/nClasses
    #end
    tol > 1 || return mode(ŷ;rng=rng) == y_pos ? 1 : 0 # if tol is one we delegate the choice of a single prediction to mode, that handles multimodal pmfs
    sIdx = sortperm(ŷ)[end:-1:1]
    if ŷ[y_pos] in ŷ[sIdx[1:min(tol,length(sIdx))]]
        return 1
    else
        return 0
    end
end

"""
    accuracy(ŷ,y;tol)

Categorical accuracy with probabilistic prediction of a single datapoint given in terms of a dictionary of probabilities (Dict{T,Float64} vs T).

# Parameters:
- `ŷ`: The returned probability mass function in terms of a Dictionary(Item1 => Prob1, Item2 => Prob2, ...)
- `tol`: The tollerance to the prediction, i.e. if considering "correct" only a prediction where the value with highest probability is the true value (`tol` = 1), or consider instead the set of `tol` maximum values [def: `1`].
"""
function accuracy(ŷ::Dict{T,Float64},y::T;tol=1,rng=Random.GLOBAL_RNG) where {T}
    if !(y in keys(ŷ)) return 0 end
    tol > 1 || return (mode(ŷ;rng=rng) == y) ? 1 : 0 # if tol is one we delegate the choice of a single prediction to mode, that handles multimodal pmfs
    sIdx  = sortperm(collect(values(ŷ)))[end:-1:1]            # sort by decreasing values of the dictionary values
    sKeys = collect(keys(ŷ))[sIdx][1:min(tol,length(sIdx))]  # retrieve the corresponding keys
    return (y in sKeys) ? 1 : 0
end

@doc raw"""
   accuracy(ŷ,y;tol,ignoreLabels)

Categorical accuracy with probabilistic predictions of a dataset (PMF vs Int).

# Parameters:
- `ŷ`: An (N,K) matrix of probabilities that each ``\hat y_n`` record with ``n \in 1,....,N``  being of category ``k`` with $k \in 1,...,K$.
- `y`: The N array with the correct category for each point $n$.
- `tol`: The tollerance to the prediction, i.e. if considering "correct" only a prediction where the value with highest probability is the true value (`tol` = 1), or consider instead the set of `tol` maximum values [def: `1`].
- `ignoreLabels`: Whether to ignore the specific label order in y. Useful for unsupervised learning algorithms where the specific label order don't make sense [def: false]

"""
function accuracy(ŷ::Array{T,2},y::Array{Int64,1};tol=1,ignoreLabels=false,rng=Random.GLOBAL_RNG) where {T <: Number}
    (N,D) = size(ŷ)
    pSet = ignoreLabels ? collect(permutations(1:D)) : [collect(1:D)]
    bestAcc = -Inf
    for perm in pSet
        pŷ = hcat([ŷ[:,c] for c in perm]...)
        acc = sum([accuracy(pŷ[i,:],y[i];tol=tol,rng=rng) for i in 1:N])/N
        if acc > bestAcc
            bestAcc = acc
        end
    end
    return bestAcc
end

@doc raw"""
   accuracy(ŷ,y;tol)

Categorical accuracy with probabilistic predictions of a dataset given in terms of a dictionary of probabilities (Dict{T,Float64} vs T).

# Parameters:
- `ŷ`: An array where each item is the estimated probability mass function in terms of a Dictionary(Item1 => Prob1, Item2 => Prob2, ...)
- `y`: The N array with the correct category for each point $n$.
- `tol`: The tollerance to the prediction, i.e. if considering "correct" only a prediction where the value with highest probability is the true value (`tol` = 1), or consider instead the set of `tol` maximum values [def: `1`].

"""
function accuracy(ŷ::Array{Dict{T,Float64},1},y::Array{T,1};tol=1,rng=Random.GLOBAL_RNG) where {T}
    N = size(ŷ,1)
    acc = sum([accuracy(ŷ[i],y[i];tol=tol,rng=rng) for i in 1:N])/N
    return acc
end

""" error(ŷ,y) - Categorical error with probabilistic prediction of a single datapoint (PMF vs Int). """
error(ŷ::Array{T,1},y::Int64;tol=1) where {T <: Number} = 1 - accuracy(ŷ,y;tol=tol)
""" error(ŷ,y) - Categorical error with probabilistic predictions of a dataset (PMF vs Int). """
error(ŷ::Array{T,2},y::Array{Int64,1};tol=1) where {T <: Number} = 1 - accuracy(ŷ,y;tol=tol)
""" error(ŷ,y) - Categorical error with with probabilistic predictions of a dataset given in terms of a dictionary of probabilities (Dict{T,Float64} vs T). """
error(ŷ::Array{Dict{T,Float64},1},y::Array{T,1};tol=1) where {T} = 1 - accuracy(ŷ,y;tol=tol)

"""
    ConfusionMatrix

Scores and measures resulting from a comparation between true and predicted categorical variables

Use the function `ConfusionMatrix(ŷ,y;classes,labels,rng)` to build it and `report(cm::ConfusionMatrix;what)` to visualise it, or use the individual parts of interest, e.g. `display(cm.scores)`.

# Fields:
- `labels`: Array of categorical labels
- `accuracy`: Overall accuracy rate
- `misclassification`: Overall misclassification rate
- `actualCount`: Array of counts per lebel in the actual data
- `predictedCount`: Array of counts per label in the predicted data
- `scores`: Matrix actual (rows) vs predicted (columns)
- `normalisedScores`: Normalised scores
- `tp`: True positive (by class)
- `tn`: True negative (by class)
- `fp`: False positive (by class), aka "type I error" or "false allarm"
- `fn`: False negative (by class), aka "type II error" or "miss"
- `precision`: True class i over predicted class i (by class)
- `recall`: Predicted class i over true class i (by class), aka "True Positive Rate (TPR)", "Sensitivity" or "Probability of detection"
- `specificity`: Predicted not class i over true not class i (by class), aka "True Negative Rate (TNR)"
- `f1Score`: Harmonic mean of precision and recall
- `meanPrecision`: Mean by class, respectively unweighted and weighted by actualCount
- `meanRecall`: Mean by class, respectively unweighted and weighted by actualCount
- `meanSpecificity`: Mean by class, respectively unweighted and weighted by actualCount
- `meanF1Score`: Mean by class, respectively unweighted and weighted by actualCount

"""
struct ConfusionMatrix{T}
    classes::Vector{T}                      # Array of categorical labels
    labels::Vector{String}                  # String representation of the categories
    accuracy::Float64                       # Overall accuracy rate
    misclassification::Float64              # Overall misclassification rate
    actualCount::Vector{Int64}              # Array of counts per lebel in the actual data
    predictedCount::Vector{Int64}           # Array of counts per label in the predicted data
    scores::Array{Int64,2}                  # Matrix actual (rows) vs predicted (columns)
    normalisedScores::Array{Float64,2}      # Normalised scores
    tp::Vector{Int64}                       # True positive (by class)
    tn::Vector{Int64}                       # True negative (by class)
    fp::Vector{Int64}                       # False positive (by class)
    fn::Vector{Int64}                       # False negative (by class)
    precision::Vector{Float64}              # True class i over predicted class i (by class)
    recall::Vector{Float64}                 # Predicted class i over true class i (by class)
    specificity::Vector{Float64}            # Predicted not class i over true not class i (by class)
    f1Score::Vector{Float64}                # Harmonic mean of precision and recall
    meanPrecision::Tuple{Float64,Float64}   # Mean by class, respectively unweighted and weighted by actualCount
    meanRecall::Tuple{Float64,Float64}      # Mean by class, respectively unweighted and weighted by actualCount
    meanSpecificity::Tuple{Float64,Float64} # Mean by class, respectively unweighted and weighted by actualCount
    meanF1Score::Tuple{Float64,Float64}     # Mean by class, respectively unweighted and weighted by actualCount
end

# Resources concerning Confusion Matrices:
# https://towardsdatascience.com/confusion-matrix-for-your-multi-class-machine-learning-model-ff9aa3bf7826
# https://en.wikipedia.org/wiki/Confusion_matrix
# https://scikit-learn.org/stable/modules/model_evaluation.html#classification-metrics
# https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html

"""
    ConfusionMatrix(ŷ,y;classes,labels,rng)

Build a "confusion matrix" between predicted (columns) vs actual (rows) categorical values

# Parameters:
- `ŷ`: Vector of predicted categorical data
- `y`: Vector of actual categorical data
- `classes`: The full set of possible classes (useful to give a specicif order or if not al lclasses are represented in `y`) [def: `unique(y)` ]
- `labels`: String representation of the classes [def: `string.(classes)`]
- `rng`: Random number generator. Used only if `ŷ` is given in terms of a PMF and there are multi-modal values, as these are assigned randomply [def: `Random.GLOBAL_RNG`]

# Return:
- a `ConfusionMatrix` object
"""
function ConfusionMatrix(ŷ,y::AbstractArray{T};classes=unique(y),labels=string.(classes),rng=Random.GLOBAL_RNG) where {T}
    nCl              = length(labels)
    ŷ                = typeof(ŷ) <: AbstractVector{T} ? ŷ : mode(ŷ,rng=rng) # get the mode if needed
    N                = length(y)
    length(ŷ) == N || @error "ŷ and y must have the same length in ConfusionMatrix"
    actualCount      = [get(classCountsWithLabels(y),i,0) for i in classes]   # TODO just use classCount
    predictedCount   = [get( classCountsWithLabels(ŷ),i,0) for i in classes]  # TODO just use classCount
    scores           = zeros(Int64,(nCl,nCl))
    normalisedScores = zeros(Float64,(nCl,nCl))
    [scores[findfirst(x -> x == y[i],classes),findfirst(x -> x == ŷ[i],classes)] += 1 for i in 1:N]
    [normalisedScores[r,:] = scores[r,:] ./ actualCount[r] for r in 1:nCl]
    tp = [scores[i,i] for i in 1:nCl]
    tn = [sum(scores[r,c] for r in 1:nCl, c in 1:nCl if r != i && c != i)  for i in 1:nCl]
    fp = [sum(scores[r,c] for r in 1:nCl, c in 1:nCl if r != i && c == i)  for i in 1:nCl]
    fn = [sum(scores[r,c] for r in 1:nCl, c in 1:nCl if r == i && c != i)  for i in 1:nCl]
    precision         = tp ./ (tp .+ fp)
    recall            = tp ./ (tp .+ fn)
    specificity       = tn ./ (tn .+ fp)
    #f1Score           = 2 .* (precision .* recall) ./ (precision .+ recall)
    f1Score           = (2 .* tp) ./ (2 .* tp  .+ fp .+ fn )
    meanPrecision     = (mean(precision), sum(precision .* actualCount) / sum(actualCount) )
    meanRecall        = (mean(recall), sum(recall .* actualCount) / sum(actualCount) )
    meanSpecificity   = (mean(specificity), sum(specificity .* actualCount) / sum(actualCount) )
    meanF1Score       = (mean(f1Score), sum(f1Score .* actualCount) / sum(actualCount) )
    accuracy          = sum(tp)/N
    misclassification = 1-accuracy
    return  ConfusionMatrix(classes,labels,accuracy,misclassification,actualCount,predictedCount,scores,normalisedScores,tp,tn,fp,fn,precision,recall,specificity,f1Score,meanPrecision,meanRecall,meanSpecificity,meanF1Score)
end

import Base.print, Base.println
"""
    print(cm,what)

Print a `ConfusionMatrix` object

The `what` parameter is a string vector that can include "all", "scores", "normalisedScores" or "report" [def: `["all"]`]
"""
function print(io::IO,cm::ConfusionMatrix{T},what="all") where T
   if what == "all" || what == ["all"]
       what = ["scores", "normalisedScores", "report" ]
   end
   nCl = length(cm.labels)

   println("\n-----------------------------------------------------------------\n")
   if( "scores" in what || "normalisedScores" in what)
     println("*** CONFUSION MATRIX ***")
   end
   if "scores" in what
       println("")
       println("Scores actual (rows) vs predicted (columns):\n")
       displayScores = vcat(permutedims(cm.labels),cm.scores)
       displayScores = hcat(vcat("Labels",cm.labels),displayScores)
       show(stdout, "text/plain", displayScores)
   end
   if "normalisedScores" in what
       println(io,"")
       println(io,"Normalised scores actual (rows) vs predicted (columns):\n")
       displayScores = vcat(permutedims(cm.labels),cm.normalisedScores)
       displayScores = hcat(vcat("Labels",cm.labels),displayScores)
       show(stdout, "text/plain", displayScores)
   end
   if "report" in what
     println("\n *** CONFUSION REPORT ***\n")
     labelWidth =  max(8,   maximum(length.(string.(cm.labels)))+1  )
     println("- Accuracy:               $(cm.accuracy)")
     println("- Misclassification rate: $(cm.misclassification)")
     println("- Number of classes:      $(nCl)")
     println("")
     println("  N ",rpad("Class",labelWidth),"precision   recall  specificity  f1Score  actualCount  predictedCount")
     println("    ",rpad(" ",labelWidth), "              TPR       TNR                 support                  ")
     println("")
     # https://discourse.julialang.org/t/printf-with-variable-format-string/3805/4
     print_formatted(fmt, args...) = @eval @printf($fmt, $(args...))
     for i in 1:nCl
        print_formatted("%3d %-$(labelWidth)s %8.3f %8.3f %12.3f %8.3f %12i %15i\n", i, string(cm.labels[i]),  cm.precision[i], cm.recall[i], cm.specificity[i], cm.f1Score[i], cm.actualCount[i], cm.predictedCount[i])
     end
     println("")
     print_formatted("- %-$(labelWidth+2)s %8.3f %8.3f %12.3f %8.3f\n", "Simple   avg.",  cm.meanPrecision[1], cm.meanRecall[1], cm.meanSpecificity[1], cm.meanF1Score[1])
     print_formatted("- %-$(labelWidth+2)s %8.3f %8.3f %12.3f %8.3f\n", "Weigthed avg.",  cm.meanPrecision[2], cm.meanRecall[2], cm.meanSpecificity[2], cm.meanF1Score[2])
   end
   println("\n-----------------------------------------------------------------")
   return nothing
end
println(io::IO, cm::ConfusionMatrix{T}, what="all") where T = begin  print(cm,what);print("\n"); return nothing end

# ------------------------------------------------------------------------------
# Regression tasks...

# Used as neural network loss function
"""
   squaredCost(ŷ,y)

Compute the squared costs between a vector of prediction and one of observations as (1/2)*norm(y - ŷ)^2.

Aside the 1/2 term, it correspond to the squared l-2 norm distance and when it is averaged on multiple datapoints corresponds to the Mean Squared Error ([MSE](https://en.wikipedia.org/wiki/Mean_squared_error)).
It is mostly used for regression problems.
"""
squaredCost(ŷ,y)   = (1/2)*norm(y - ŷ)^2
dSquaredCost(ŷ,y)  = ( ŷ - y)
"""
    mse(ŷ,y)

Compute the mean squared error (MSE) (aka mean squared deviation - MSD) between two vectors ŷ and y.
Note that while the deviation is averaged by the length of `y` is is not scaled to give it a relative meaning.
"""
mse(ŷ,y) = (sum((y-ŷ).^(2))/length(y))

"""
  meanRelError(ŷ,y;normDim=true,normRec=true,p=1)

Compute the mean relative error (l-1 based by default) between ŷ and y.

There are many ways to compute a mean relative error. In particular, if normRec (normDim) is set to true, the records (dimensions) are normalised, in the sense that it doesn't matter if a record (dimension) is bigger or smaller than the others, the relative error is first computed for each record (dimension) and then it is averaged.
With both `normDim` and `normRec` set to `false` the function returns the relative mean error; with both set to `true` (default) it returns the mean relative error (i.e. with p=1 the "[mean absolute percentage error (MAPE)](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error)")
The parameter `p` [def: `1`] controls the p-norm used to define the error.

The _mean relative error_ enfatises the relativeness of the error, i.e. all observations and dimensions weigth the same, wether large or small. Conversly, in the _relative mean error_ the same relative error on larger observations (or dimensions) weights more.

For example, given `y = [1,44,3]` and `ŷ = [2,45,2]`, the _mean relative error_ `meanRelError(ŷ,y)` is `0.452`, while the _relative mean error_ `meanRelError(ŷ,y, normRec=false)` is "only" `0.0625`.

"""
function meanRelError(ŷ,y;normDim=true,normRec=true,p=1)
    ŷ = makeMatrix(ŷ)
    y = makeMatrix(y)
    (n,d) = size(y)
    #ϵ = abs.(ŷ-y) .^ p
    if (!normDim && !normRec) # relative mean error
        avgϵRel = (sum(abs.(ŷ-y).^p)^(1/p) / (n*d)) / (sum( abs.(y) .^p)^(1/p) / (n*d)) # (avg error) / (avg y)
        # avgϵRel = (norm((ŷ-y),p)/(n*d)) / (norm(y,p) / (n*d))
    elseif (!normDim && normRec) # normalised by record (i.e. all records play the same weigth)
        avgϵRel_byRec = (sum(abs.(ŷ-y) .^ (1/p),dims=2).^(1/p) ./ d) ./   (sum(abs.(y) .^ (1/p) ,dims=2) ./d)
        avgϵRel = mean(avgϵRel_byRec)
    elseif (normDim && !normRec) # normalised by dimensions (i.e.  all dimensions play the same weigth)
        avgϵRel_byDim = (sum(abs.(ŷ-y) .^ (1/p),dims=1).^(1/p) ./ n) ./   (sum(abs.(y) .^ (1/p) ,dims=1) ./n)
        avgϵRel = mean(avgϵRel_byDim)
    else # mean relative error
        avgϵRel = sum(abs.((ŷ-y)./ y).^p)^(1/p)/(n*d) # avg(error/y)
        # avgϵRel = (norm((ŷ-y)./ y,p)/(n*d))
    end
    return avgϵRel
end


using Plots

function wheeler(x, a=1.5)
    return exp(-(x[1]*x[2] - a)^2 - (x[2] - a)^2);
end

# plot

heatmap(-10:0.01:25, -3:0.01:6, (x, y)->wheeler([x, y]))


"""
    circumcenter_radius2(pts)

Calculate circumcenter and circumradius squared from points.

Based on https://github.com/hirani/pydec/blob/master/pydec/math/circumcenter.py and
http://www.ics.uci.edu/~eppstein/junkyard/circumcenter.html
"""
# TODO: try this with SMatrix?
function circumcenter_radius2(pts)
    if length(pts) == 1
        return pts[1], 0.0
    elseif length(pts) == 2
        return 0.5 * sum(pts), 0.25 * sum(abs2, pts[1] - pts[2])
    else
        P = to_matrix(pts)
        n = length(pts)

        A = ones(n + 1, n + 1)
        A[end, end] = 0.0
        A[1:(end - 1), 1:(end - 1)] .= 2 * P' * P

        b = vec(sum(abs2, P; dims=1))
        push!(b, 1.0)

        fact = lu(A; check=false)
        if issuccess(fact)
            x = fact \ b
            bary_coords = x[1:(end - 1)]

            center = P * bary_coords
            radius = P[:, 1] - center
            return center, sum(abs2, radius)
        else
            return fill(NaN, length(pts[1])), Inf
        end
    end
end

# TODO: could be faster, takes a long time to compile
function _build_dims!(dicts, triangulation, points, ::Val{D}, verbose) where {D}
    if verbose
        progbar = Progress(size(triangulation, 2); desc="Collecting $D-simplcies... ")
    end
    for face in eachcol(triangulation)
        for σ in IterTools.subsets(face, Val(D + 1))
            @assert issorted(σ; rev=true)
            σ_idx = index(σ)
            if !haskey(dicts[D + 1], σ_idx)
                _, σ_r2 = circumcenter_radius2(points[SVector(σ)])
                if isfinite(σ_r2)
                    dicts[D + 1][σ_idx] = σ_r2
                else
                    continue
                end
            end
            # Propagate birth time to facets.
            σ_r2 = dicts[D + 1][σ_idx]
            for i in 1:(D + 1)
                τ = TupleTools.deleteat(σ, i)
                τ_idx = index(τ)
                if haskey(dicts[D], τ_idx)
                    dicts[D][τ_idx] = min(σ_r2, dicts[D][τ_idx])
                elseif length(τ) > 1
                    τ_c, τ_r2 = circumcenter_radius2(points[SVector(τ)])
                    if sum(abs2, τ_c - points[σ[i]]) < τ_r2
                        dicts[D][τ_idx] = σ_r2
                    end
                end
            end
        end
        verbose && next!(progbar)
    end
end

function _fix_dim!(dicts, threshold, ::Val{D}, verbose) where {D}
    for (idx, birth) in dicts[D + 1]
        σ = Tuple(_vertices(idx, Val(D + 1)))
        σ_idx = index(σ)
        if D > 1
            for i in 1:(D + 1)
                τ = TupleTools.deleteat(σ, i)
                τ_idx = index(τ)
                dicts[D][τ_idx] = min(dicts[D][τ_idx], birth)
            end
        end
        corrected_birth = 2 * √dicts[D + 1][σ_idx]
        dicts[D + 1][σ_idx] = corrected_birth
    end
end

"""
    alpha_simplices(points[, verbose])

Collect all simplices and their birth times in alpha filtration.

Based on https://github.com/scikit-tda/cechmate/blob/master/cechmate/filtrations/alpha.py
"""
function alpha_simplices(points, threshold, verbose, ::Type{I}) where {I}
    @prog_print verbose "Building triangulation... "
    triangulation = I.(delaunay(to_matrix(points)))
    sort!.(eachcol(triangulation), rev=true, alg=InsertionSort)
    @prog_println verbose "done."

    largest_face = tuple(maximum(eachcol(triangulation))...)
    index_overflow_check(largest_face)

    dim = length(points[1])
    dicts = [Dict{I,Float64}() for _ in 0:dim]

    # Build the filtration
    for d in dim:-1:1
        _build_dims!(dicts, triangulation, points, Val(d), verbose)
    end
    for i in 1:length(points)
        dicts[1][i] = 0.0
    end
    if verbose
        progbar = Progress(dim; desc="Fixing birth times...     ")
    end
    # Make sure all simplices are born after their facets and sqrt the birth times.
    for d in dim:-1:1
        _fix_dim!(dicts, threshold, Val(d), verbose)
        verbose && next!(progbar)
    end

    return dicts
end

"""
    Alpha{I, P<:SVector} <: AbstractFiltration{I, Float64}

`Alpha` filtrations are filtrations of the Delaunay complex.

They have much fewer simplices than `Rips`, so they are efficient even with large datasets,
as long as their dimensionality is low.  What "low" means depends on the data, but this is
definitely a good choice for 3D or lower. For high dimensional data, filtration construction
may take a long time.

!!! note
    Unlike most implementations, this one uses circumdiameters instead of
    circumradii. This makes the scale of the results comparable to `Rips`. If you need
    radius based values, divide your data or the resulting interval endpoints by 2.

!!! warning
    This filtration uses [MiniQhull.jl](https://github.com/gridap/MiniQhull.jl). Please see
    the installation instructions if constructions cause errors. MiniQhull currently has
    problems running on Windows. See [this
    issue](https://github.com/gridap/MiniQhull.jl/issues/5) for more info.

# Constructors

* `Alpha(points; threshold, verbose)`: `points` should be a vector of `Tuple`s, `SVector`s
  or similar.
* `Alpha{I}(args...)`: `I` sets the size of integer used to represent simplices. Try using
  `I=Int128` if construction complains about overflow.

# Reference

Edelsbrunner, H. (1993, July). The union of balls and its dual shape. [In Proceedings of the
ninth annual symposium on Computational geometry
(pp. 218-231)](https://dl.acm.org/doi/abs/10.1145/160985.161139).

# Example

```jldoctest
julia> data = [(sin(t), cos(t), (t - π)^2) for t in range(0, 2π, length=101)[1:end-1]];

julia> alpha = Alpha(data)
Alpha{Int64, Float64}(nv=100)

julia> rips = Rips(data)
Rips{Int64, Float64}(nv=100, sparse=false)

julia> length(Ripserer.edges(alpha))
197

julia> length(Ripserer.edges(rips))
3613

julia> sort(ripserer(alpha)[2], by=persistence)[end]
[0.375, 2.01) with:
 birth_simplex: Simplex{1, Float64, Int64}
 death_simplex: Simplex{2, Float64, Int64}

julia> sort(ripserer(rips)[2], by=persistence)[end]
[0.375, 2.01) with:
 birth_simplex: Simplex{1, Float64, Int64}
 death_simplex: Simplex{2, Float64, Int64}
```
"""
struct Alpha{I,P<:SVector} <: AbstractCustomFiltration{I,Float64}
    dicts::Vector{Dict{I,Float64}}
    adj::SparseMatrixCSC{Bool,Int}
    threshold::Float64
    points::Vector{P}
end
function Alpha{I}(points; threshold=nothing, verbose=false) where {I}
    pts = SVector.(points)
    threshold = isnothing(threshold) ? 2radius(pts) : threshold
    dicts = alpha_simplices(pts, threshold, verbose, I)
    adj = _adjacency_matrix(dicts)
    return Alpha{I,eltype(pts)}(dicts, adj, threshold, pts)
end
function Alpha(points; kwargs...)
    return Alpha{Int}(points; kwargs...)
end

adjacency_matrix(alpha::Alpha) = alpha.adj
simplex_dicts(alpha::Alpha) = alpha.dicts
threshold(alpha::Alpha) = alpha.threshold

struct AlphaDist{A} <: AbstractMatrix{Float64}
    alpha::A
end

Base.size(ad::AlphaDist) = (nv(ad.alpha), nv(ad.alpha))
function Base.getindex(ad::AlphaDist, i::Integer, j::Integer)
    return Euclidean()(ad.alpha.points[i], ad.alpha.points[j])
end

distance_matrix(alpha::Alpha) = AlphaDist(alpha)


using OrdinaryDiffEq
using Distributions
using DiffEqFlux, Flux
using Random
using Plots;

Random.seed!(123);

function sir_ode(u,p,t)
    (S,I,C) = u
    (β,γ) = p
    dS = -β*S*I
    dI = β*S*I - γ*I
    dC = β*S*I
    [dS,dI,dC]
end;

solver = RadauIIA3();

N = 1000.0
p = [0.5,0.25]
u0 = [0.99, 0.01, 0.0]
tspan = (0., 40.)
δt = 1;

sir_prob = ODEProblem(sir_ode, u0, tspan, p)
sir_sol = solve(sir_prob, solver, saveat = δt);

plot(sir_sol,
     xlabel = "Time",
     ylabel = "Proportion",
     labels = ["S" "I" "R"])

train_time = 30.0
tsdata = Array(sir_sol(0:δt:train_time))
cdata = diff(tsdata[3,:])
noisy_data = rand.(Poisson.(N .* cdata));

plot(1:δt:train_time, N .* cdata,
     xlabel = "Time",
     ylabel = "New cases per day",
     label = "True value")
scatter!(1:δt:train_time, noisy_data, label="Data")

foi1 = FastDense(1, 1, relu, bias=false)
p1_ = Float64.(initial_params(foi1))
length(p1_)

function sir_ude(u,p_,t,foi)
    S,I,C = u
    β,γ = p
    λ = foi([I],p_)[1]
    dS = -λ*S
    dI = λ*S - γ*I
    dC = λ*S
    [dS, dI, dC]
end;

tspan_train = (0,train_time)
sir_ude1 = (u,p_,t) -> sir_ude(u,p_,t,foi1)
prob_ude1 = ODEProblem(sir_ude1,
                      u0,
                      tspan_train,
                      p1_);

function predict(θ, prob)
    Array(solve(prob,
                solver;
                u0 = u0,
                p = θ,
                saveat = δt,
                sensealg = InterpolatingAdjoint(autojacvec=ReverseDiffVJP())))
end;

function loss(θ, prob)
    pred = predict(θ, prob)
    cpred = abs.(N*diff(pred[3,:]))
    Flux.poisson_loss(cpred, float.(noisy_data)), cpred
end;

loss(prob_ude1.p, prob_ude1);

const losses1 = []
callback1 = function (p, l, pred)
    push!(losses1, l)
    numloss = length(losses1)
    if numloss % 10 == 0
        display("Epoch: " * string(numloss) * " Loss: " * string(l))
    end
    return false
end;

res_ude1 = DiffEqFlux.sciml_train((θ)->loss(θ,prob_ude1),
                                  p1_,
                                  cb=callback1);

res_ude1.minimizer, losses1[end]

plot(losses1, xaxis = :log, xlabel = "Iterations", ylabel = "Loss", legend=false)

prob_ude1_fit = ODEProblem(sir_ude1, u0, tspan, res_ude1.minimizer)
sol_ude1_fit = solve(prob_ude1_fit, solver, saveat = δt)
scatter(sir_sol, label=["True Susceptible" "True Infected" "True Recovered"],title="Fitted true model")
plot!(sol_ude1_fit, label=["Estimated Susceptible" "Estimated Infected" "Estimated Recovered"])

Imax = maximum(tsdata[2,:])
Igrid = 0:0.01:0.5
λ = [foi1([I],res_ude1.minimizer)[1] for I in Igrid]
scatter(Igrid,λ,xlabel="Proportion of population infected, I",ylab="Force of infection, λ",label="Neural network prediction")
Plots.abline!(p[1],0,label="True value")
Plots.vline!([Imax],label="Upper bound of training data")

Random.seed!(1234)
nhidden = 4
foi2 = FastChain(FastDense(1, nhidden, relu),
                     FastDense(nhidden, nhidden, relu),
                     FastDense(nhidden, 1, relu))
p2_ = Float64.(initial_params(foi2))
length(p2_)

sir_ude2 = (u,p_,t) -> sir_ude(u,p_,t,foi2)
prob_ude2 = ODEProblem(sir_ude2,
                      u0,
                      tspan_train,
                      p2_);

const losses2 = []
callback2 = function (p, l, pred)
    push!(losses2, l)
    numloss = length(losses2)
    if numloss % 10 == 0
        display("Epoch: " * string(numloss) * " Loss: " * string(l))
    end
    return false
end;

res_ude2 = DiffEqFlux.sciml_train((θ)->loss(θ,prob_ude2),
                                  p2_,
                                  cb = callback2);

losses1[end],losses2[end]

prob_ude2_fit = ODEProblem(sir_ude2, u0, tspan, res_ude2.minimizer)
sol_ude2_fit = solve(prob_ude2_fit, solver, saveat = δt)
scatter(sir_sol, label=["True Susceptible" "True Infected" "True Recovered"],title="Fitted UDE model")
plot!(sol_ude2_fit, label=["Estimated Susceptible" "Estimated Infected" "Estimated Recovered"])

λ = [foi2([I],res_ude2.minimizer)[1] for I in Igrid]
scatter(Igrid, λ, xlabel="Proportion of population infected, i", ylab="Force of infection, λ", label="Neural network prediction")
Plots.abline!(p[1], 0,label="True value")
Plots.vline!([Imax], label="Upper bound of training data")


#' # Modeling data 2

#' ## Building a model
#' Recall that in notebook 3, we saw that we could use a mathematical function
#' to classify an image as an apple or a banana, based on the average amount of
#' green in an image:
#'
#' ![data_flow](data/data_flow.png)
#'
#' ![what_is_model](data/what_is_model.png)

#' A common function for performing this kind of **classification** is the
#' sigmoid that we saw in the last notebook, and that we will now extend by
#' adding two **parameters**, $w$ and $b$:
#'
#' $$\sigma(x; w, b) := \frac{1}{1 + e^{(-wx + b)}}$$
#'
#' $$x = \mathrm{data}$$
#'
#'
#' $$\begin{align}
#' \sigma(x;w,b) &\approx 0 \implies \mathrm{apple} \\
#' \sigma(x;w,b) &\approx 1 \implies \mathrm{banana}
#' \end{align}$$
#'
#' In our mathematical notation above, the `;` in the function differentiates
#' between the **data** and the **parameters**. `x` is the data and is
#' determined from the image. The parameters, `w` and `b`, are numbers which we
#' choose to make our function match the results it should be modeling.

#' Note that in the code below, we don't distinguish between data and
#' parameters - both are just inputs to our function, σ!

using Images, Statistics

# apple = load("C:\\Users\\Yi-Kun.Yang\\work\\org.allnix.julia\\JuliaAcademy\\FundMachLearn/data/10_100.jpg")
# banana = load("C:\\Users\\Yi-Kun.Yang\\work\\org.allnix.julia\\JuliaAcademy\\FundMachLearn/data/104_100.jpg")
apple = load("data/10_100.jpg")
#'
banana = load("data/104_100.jpg")
#'

apple_green_amount = mean(Float64.(green.(apple)))
banana_green_amount = mean(Float64.(green.(banana)))

println("Average green for apple = $apple_green_amount")
println("Average green for banana = $banana_green_amount")

#+ results="hidden"
σ(x, w, b) = 1 / (1 + exp(-w * x + b))
#+

#' What we want is that when we give σ as input the average green for the apple,
#' roughly `x = 0.3385`, it should return as output something close to 0,
#' meaning "apple". And when we give σ the input `x = 0.8808`, it should output
#' something close to 1, meaning "banana".

#' By changing the parameters of the function, we can change the shape of the
#' function, and hence make it represent, or **fit**, the data better!

#' ## Data fitting by varying parameters

# We can understand how our choice of `w` and `b` affects our model by seeing how our values for `w` and `b` change the plot of the $\sigma$ function.

using Plots
gr()   # GR works better for interactive manipulations
#plotly()
#pyplot()

#' Run the code in the next cell. You should see two "sliders" appear, one for
#' `w` and one for `b`.

#' **Game**:
#' Change w and b around until the blue curve, labeled "model", which is the
#' graph of the `\sigma` function, passes through *both* of the data points at
#' the same time.

w = 10.0 # try manipulating w between -10 and 30
b = 10.0 # try manipulating b between 0 and 20

using Interact

# Function for Interact
function myplot(w,b)
      pt = plot()
      plot!(pt, x -> σ(x, w, b), xlim=(-0,1), ylim=(-0.1,1.1), label="model", legend=(0.1,0.9), lw=3)
      scatter!(pt, [apple_green_amount],  [0.0], label="apple", ms=5)   # marker size = 5
      scatter!(pt, [banana_green_amount], [1.0], label="banana", ms=5)

      return pt
end

#' Notice that the two parameters do two very different things. The **weight**,
#' `w`, determines *how fast* the transition between 0 and 1 occurs. It encodes
#' how trustworthy we think our data  actually is, and in what range we should
#' be putting points between 0 and 1 and thus calling them "unsure". The
#' **bias**, `b`, encodes *where* on the $x$-axis the switch should take place.
#' It can be seen as shifting the function left-right. We'll come to understand
#' these *parameters* more in notebook 6.

#' Here are some parameter choices that work well:

w = 25.58 #20.8
b = 15.6 #12.4
pt = myplot(w,b)

#+ results="hidden"
# Run this in Juno or Jupyter to get interative plot
# @manipulate for w=-10:0.2:30,b = 0:0.2:20
#      vbox(hbox(w,b),myplot(w,b))
# end
#+

#' (Note that in this problem there are many combinations of `w` and `b` that
#' fit the data well.)
apple2 = load("data/107_100.jpg")
#'
green_amount = mean(Float64.(green.(apple2)))
@show green_amount
scatter!(pt, [green_amount], [0.0], label="new apple")

#' Our model successfully says that our new image is an apple! Pat yourself on
#' the back: you've actually just trained your first neural network!

#' #### Exercise 1

#' Load the image of a banana in `data/8_100.jpg` as `mybanana`. Edit the code
#' below to calculate the amount of green in `mybanana` and to overlay data for
#' this image with the existing model and data points.
mybanana = load("data/8_100.jpg")
green_amount = mean(Float64.(green.(mybanana)))
@show green_amount
scatter!(pt, [green_amount], [1.0], label="new banana")
#' ## Closing remarks: bigger models, more data, more accuracy

#' That last apple should start making you think: not all apples are red; some
#' are yellow. "Redness" is one attribute of being an apple, but isn't the
#' whole thing. What we need to do is incorporate more ideas into our model by
#' allowing more inputs. However, more inputs would mean more parameters to
#' play with. Also, we would like to have the computer start "learning" on its
#' own, instead of modifying the parameters ourselves until we think it "looks
#' right". How do we take the next step?

#' The first thing to think about is, if you wanted to incorporate more data
#' into the model, how would you change the sigmoid function? Play around with
#' some ideas. But also, start thinking about how you chose parameters. What
#' process did you do to finally end up at good parameters? These two problems
#' (working with models with more data and automatically choosing parameters)
#' are the last remaining step to understanding deep learning.


using DifferentialEquations
using StochasticDiffEq
using Random
using SparseArrays
using DataFrames
using StatsPlots

function sir_ode!(du,u,p,t)
    (S,I,R) = u
    (β,c,γ) = p
    N = S+I+R
    @inbounds begin
        du[1] = -β*c*I/N*S
        du[2] = β*c*I/N*S - γ*I
        du[3] = γ*I
    end
    nothing
end;

# Define a sparse matrix by making a dense matrix and setting some values as not zero
A = zeros(3,2)
A[1,1] = 1
A[2,1] = 1
A[2,2] = 1
A[3,2] = 1
A = SparseArrays.sparse(A);

# Make `g` write the sparse matrix values
function sir_noise!(du,u,p,t)
    (S,I,R) = u
    (β,c,γ) = p
    N = S+I+R
    ifrac = β*I/N*S
    rfrac = γ*I
    du[1,1] = -sqrt(ifrac)
    du[2,1] = sqrt(ifrac)
    du[2,2] = -sqrt(rfrac)
    du[3,2] = sqrt(rfrac)
end;

δt = 0.1
tmax = 40.0
tspan = (0.0,tmax)
t = 0.0:δt:tmax;

u0 = [990.0,10.0,0.0]; # S,I,R

p = [0.05,10.0,0.25]; # β,c,γ

Random.seed!(1234);

prob_sde = SDEProblem(sir_ode!,sir_noise!,u0,tspan,p,noise_rate_prototype=A)

sol_sde = solve(prob_sde,SRA1());

df_sde = DataFrame(sol_sde(t)')
df_sde[!,:t] = t;

@df df_sde plot(:t,
    [:x1 :x2 :x3],
    label=["S" "I" "R"],
    xlabel="Time",
    ylabel="Number")

include(joinpath(@__DIR__,"tutorials","appendix.jl"))
appendix()


"""`newton1d` performs the interval Newton method on the given function `f`
with its derivative `f′` and initial interval `x`.
Optional keyword arguments give the tolerances `reltol` and `abstol`.
`reltol` is the tolerance on the relative error whereas `abstol` is the tolerance on |f(X)|,
and a `debug` boolean argument that prints out diagnostic information."""

function newton1d{T}(f::Function, f′::Function, x::Interval{T};
                    reltol=eps(T), abstol=eps(T), debug=false)

    L = Interval{T}[]

    R = Root{Interval{T}}[]

    push!(L, x)

    while !isempty(L)
        X = pop!(L)
        m = mid(X)
        if (isempty(X))
            continue
        end

        if 0 ∉ f′(X)
            while true
                m = mid(X)
                N = m - (f(Interval(m)) / f′(X))
                X = X ∩ N

                if isempty(X)
                    break

                elseif 0 ∈ f(Interval(prevfloat(m), nextfloat(m)))
                    push!(R, Root(X, :unique))
                    break
                end
            end

        else
            # 0 ∈ f'(X)
            expansion_pt = Inf
            # expansion point for the newton step might be m, X.lo or X.hi according to some conditions

            if 0 ∈ f(Interval(mid(X)))
                # 0 ∈ fⁱ(x)
                # Step 7

                if 0 ∉ f(Interval(X.lo))
                    expansion_pt = X.lo

                elseif 0 ∉ f(Interval(X.hi))
                    expansion_pt = X.hi

                else
                    x1 = mid(Interval(X.lo, mid(X)))
                    x2 = mid(Interval(mid(X), X.hi))
                    if 0 ∉ f(Interval(x1)) || 0 ∉ f(Interval(x2))
                        push!(L, Interval(X.lo, m))
                        push!(L, Interval(m, X.hi))
                        continue

                    else
                        push!(R, Root(X, :unique))
                        continue
                    end
                end

            else
                # 0 ∉ fⁱ(x)

                if (diam(X)/mag(X)) < reltol && diam(f(X)) < abstol
                    push!(R, Root(X, :unknown))
                    continue
                end
            end
            # Step 8

            if isinf(expansion_pt)
                expansion_pt = mid(X)
            end

            initial_width = diam(X)

            a = f(Interval(expansion_pt))
            b = f′(X)

            if 0 < b.hi && 0 > b.lo && 0 ∉ a
                if a.hi < 0
                    push!(L, X ∩ (expansion_pt - Interval(-Inf, a.hi / b.hi)))
                    push!(L, X ∩ (expansion_pt - Interval(a.hi / b.lo, Inf)))

                elseif a.lo > 0
                    push!(L, X ∩ (expansion_pt - Interval(-Inf, a.lo / b.lo)))
                    push!(L, X ∩ (expansion_pt - Interval(a.lo / b.hi, Inf)))

                end

                continue

            else
                N = expansion_pt - (f(Interval(expansion_pt))/f′(X))
                X = X ∩ N
                m = mid(X)

                if isempty(X)
                    continue
                end
            end

            if diam(X) > initial_width/2
                push!(L, Interval(m, X.hi))
                X = Interval(X.lo, m)
            end

            push!(L, X)
        end
    end

    return R
end

"""`newton1d` performs the interval Newton method on the given function `f` and initial interval `x`.
Optional keyword arguments give the tolerances `reltol` and `abstol`.
`reltol` is the tolerance on the relative error whereas `abstol` is the tolerance on |f(X)|,
and a `debug` boolean argument that prints out diagnostic information."""

newton1d{T}(f::Function, x::Interval{T};  args...) =
    newton1d(f, x->D(f,x), x; args...)


using LinearAlgebra

function err_estimation(xfull, xhalf, order::Integer)
    return (1<<order)/(1<<order-1)*norm(xhalf .- xfull)
end

function richardson_extrapolation(xfull, xhalf, order::Integer)
    return ((1<<order)*xhalf .- xfull)/(1<<order-1)
end

function mixed_step_ode_solver(
    f,
    t0::Number,
    x0,
    stop_cond;
    method = mod_euler_step,
    min_step::AbstractFloat = 1e-3,
    max_step::AbstractFloat = 10,
    target_err::AbstractFloat = 0.1,
    step0::AbstractFloat = 1.0,
    maxit::Integer = 10000,
    debug_dict = nothing
)
    t = Array{typeof(t0 + step0), 1}()
    x = Array{typeof(x0 + step0*f(t0, x0)), 1}()
    step = step0
    push!(t, t0)
    push!(x, x0)
    i = 1
    for j = 1:maxit
        aux = last(x) .+ method(f, last(t), last(x), [step, step/2])
        xfull = aux[:,1]
        xhalf = aux[:,2] + method(f, last(t)+step/2, aux[:,2], step/2)

        err_full = err_estimation(xfull, xhalf, method(:order))
        if err_full < target_err
            push!(t, last(t)+step)
            push!(x, richardson_extrapolation(xfull, xhalf, method(:order)))
            if stop_cond(t, x)
                if debug_dict != nothing
                    debug_dict[:numit] = j
                end
                break
            end
        end

        q = (target_err*step/err_full) ^ (1/(method(:order)))
        q = min(4, max(0.1, q))
        step = min(q*step, max_step)
        if step < min_step
            throw(ErrorException("step went below the min_step($(min_step)) value"))
        end
    end
    return t, x
end

struct ExtButcherTableau
    a
    c
    b1
    b2
    order

    ExtButcherTableau(; a, c, b1, b2, order::Integer) = new(a, c, b1, b2, order)
end

DORPRI5_tableau = ExtButcherTableau(
    a = [          0  1/5        3/40    44/45     19372/6561   9017/3168     35/384;
                   0    0        9/40   -56/15    -25360/2187     -355/33          0;
                   0    0           0     32/9     64448/6561  46732/5247   500/1113;
                   0    0           0        0       -212/729      49/176    125/192;
                   0    0           0        0              0 -5103/18656 -2187/6784;
                   0    0           0        0              0           0      11/84;
                   0    0           0        0              0           0          0],
    c = [          0, 1/5,       3/10,     4/5,           8/9,          1,         1],

    b1 = [    35/384,   0,   500/1113, 125/192,    -2187/6784,      11/84,         0],
    b2 = [5179/57600,   0, 7571/16695, 393/640, -92097/339200,   187/2100,      1/40],
    order = 5
)

RKFehlberg_tableau = ExtButcherTableau(
    a = [       0  1/4        3/32    1932/2197   439/216      -8/27;
                0    0        9/32   -7200/2197        -8          2;
                0    0           0    7296/2197  3680/513 -3544/2565;
                0    0           0            0 -845/4104  1859/4104;
                0    0           0            0         0     -11/40;
                0    0           0            0         0          0],
    c = [       0, 1/4,        3/8,       12/13,        1,       1/2],

    b1 = [ 16/135,   0, 6656/12825, 28561/56430,    -9/50,      2/55],
    b2 = [ 25/216,   0,  1408/2565,   2197/4104,     -1/5,         0],
    order = 4
)

function double_step(tab::ExtButcherTableau, f, t, x, step)
    k = Array{typeof(x + step*f(t, x)), 1}(undef, length(tab.c))
    k[1] = step*f(t + step*tab.c[1], x)
    for j = 2:length(k)
        k[j] = step*f(t + step*tab.c[j], x + tab.a[1:(j-1), j]' * k[1:(j-1)])
    end
    return (x + tab.b1' * k, x + tab.b2' * k)
end

# function double_step(

function adaptative_rk_ode_solver(
    f,
    t0::Number,
    x0,
    stop_cond;
    tableau::ExtButcherTableau = RKFehlberg_tableau,
    min_step::AbstractFloat = 1e-3,
    max_step::AbstractFloat = 10,
    target_err::AbstractFloat = 0.1,
    step0::AbstractFloat = 1.0,
    maxit::Integer = 10000,
    debug_dict = nothing
)
    t = Array{typeof(t0 + step0), 1}()
    x = Array{typeof(x0 + step0*f(t0, x0)), 1}()
    step = step0
    push!(t, t0)
    push!(x, x0)
    i = 1
    for j = 1:maxit
        x1, x2 = double_step(tableau, f, last(t), last(x), step)

        err = norm(x1 .- x2)
        if err < target_err
            push!(t, last(t)+step)
            push!(x, x1)
            if stop_cond(t, x)
                if debug_dict != nothing
                    debug_dict[:numit] = j
                end
                break
            end
        end

        q = (target_err*step/err) ^ (1/(tableau.order))
        q = min(4, max(0.1, q))
        step = min(q*step, max_step)
        if step < min_step
            throw(ErrorException("step went below the min_step($(min_step)) value"))
        end
    end
    return t, x
end


# Program to find GCD using Euclidean Algorithm

function gcd(x,y)
	if x==0
		return y
	else
		return gcd(y%x,x)
	end
end

println("x = ")
x = parse(Int64,readline())
println("\ny = ")
y = parse(Int64,readline())

println("gcd of ($x , $y) = ",gcd(x,y))


export
    sigmoid,
    relu,
    softmax,
    onehot,
    crossentropyerror

function sigmoid(x::T) where T <: Real
    return 1.0 / (1.0 + exp(-x))
end

function relu(x::T) where T <: Real
    return max(zero(x), x)
end

function softmax(x::AbstractVector{T}) where T <: Real
    c = maximum(x)
    exp_a = exp.(x .- c)
    return exp_a ./ sum(exp_a)
end

function softmax(x::AbstractMatrix{T}) where T <: Real
    return mapslices(softmax, x, 1)
end

function onehot(::Type{T}, t::AbstractVector, l::AbstractVector) where T
    r = zeros(T, length(l), length(t))
    for i in 1:length(t)
        r[findfirst(l, t[i]), i] = 1
    end
    return r
end

onehot(t::AbstractVector, l::AbstractVector) = onehot(Int, t, l)

function crossentropyerror(y::Vector, t::Vector)
    δ = 1.0e-7
    # -sum(t .* log.(y .+ δ))
    return -(t ⋅ log.(y .+ δ))
end

function crossentropyerror(y::Matrix, t::Matrix)
    batch_size = size(y, 2)
    δ = 1.0e-7
    # -sum(t .* log.(y .+ δ)) ./ batch_size
    return -vecdot(t, log.(y .+ δ)) ./ batch_size
end


module problem111

description = """
Primes with runs

Considering 4-digit primes containing repeated digits it is clear that they cannot all be the same: 1111 is divisible by 11, 2222 is divisible by 22, and so on. But there are nine 4-digit primes containing three ones:
1117, 1151, 1171, 1181, 1511, 1811, 2111, 4111, 8111

We shall say that M(n, d) represents the maximum number of repeated digits for an n-digit prime where d is the repeated digit, N(n, d) represents the number of such primes, and S(n, d) represents the sum of these primes.
So M(4, 1) = 3 is the maximum number of repeated digits for a 4-digit prime where one is the repeated digit, there are N(4, 1) = 9 such primes, and the sum of these primes is S(4, 1) = 22275. It turns out that for d = 0, it is only possible to have M(4, 0) = 2 repeated digits, but there are N(4, 0) = 13 such cases.

In the same way we obtain the following results for 4-digit primes.
Digit, d 	M(4, d) 	N(4, d) 	S(4, d)
0 	2 	13 	67061
1 	3 	9 	22275
2 	3 	1 	2221
3 	3 	12 	46214
4 	3 	2 	8888
5 	3 	1 	5557
6 	3 	1 	6661
7 	3 	9 	57863
8 	3 	1 	8887
9 	3 	7 	48073

For d = 0 to 9, the sum of all S(4, d) is 273700.

Find the sum of all S(10, d).
"""

function solve(n :: Integer)
  return sum([S(n, d) for d = 0:9])
end

# search for the first r from (n-1) to 2 producing some primes having the digit d repeat r times in n-digit numbers.
function S(n :: Integer, d :: Integer)
  for r = n-1 : -1 : 2
    result = S(n, d, r)
    if result > 0
      return result
    end
  end
end

# Find the sum of primes with n digits, having digit d repeat r times.
function S(n :: Integer, d :: Integer, r :: Integer)
  total = 0
  repeatCombinations = collect(combinations([1 : (d == 0 ? n-1 : n)], r)) # no leading zeros
  for repeated in repeatCombinations
    digits = zeros(n)
    digits[repeated] = d
    changing = setdiff([1:n], repeated)
    total += SPrimes(digits, changing, 1)
  end
  return total
end

function SPrimes(digits :: Vector, changing :: Vector, i :: Integer)

  # Base case: all digits are set, check if result is prime
  if i > length(changing)
    n = sum([int(10^(x-1) * digits[x]) for x in 1:length(digits)])
    return isprime(n) ? n : 0
  end

  # recursive case, set digit at index changing[i], and call with i+1
  total = 0
  for d = (changing[i] == length(digits) ? 1 : 0) : 9
    digits[changing[i]] = d
    total += SPrimes(digits, changing, i+1)
  end
  return total
end

using Base.Test

@test solve(4) == 273700

end


function correlation(sample, k)
    # Stabilized correlation function χ(k) = χ(|i-j|) = ⟨xᵢxⱼ⟩ - ⟨xᵢ⟩⟨xⱼ⟩
    # Following Welford derivation from https://changyaochen.github.io/welford/

    M = length(sample)
    corr = sumx = sumy = 0.0
    for i in 1:M-k
        # ̅xᵢ₊₁ = ̅xᵢ + (xᵢ₊₁ - ̅xᵢ) / (N+1)
        # ̅yᵢ₊₁ = ̅yᵢ + (yᵢ₊₁ - ̅yᵢ) / (N+1)
        # χᵢ₊₁ = χᵢ + [(xᵢ₊₁ - ̅xᵢ)(yᵢ₊₁ - ̅yᵢ₊₁) - χᵢ] / (i + 1)

        invN = 1.0 / i
        sumy += invN * (sample[i + k] - sumy)
        sumx_delta = invN * (sample[i] - sumx)
        corr += sumx_delta * (sample[i + k] - sumy) - invN * corr
        sumx += sumx_delta
    end
    return corr
end

# QMCM eq 3.15
"""
    unbinned_tau(sample[; truncate = true, max_rel_err = 0.0, min_sample_size = 32])

Estimates the autocorrelation time `τ = ∑ₖ (1 - k/N) χₖ/χ₀` from a sample using the
correlation function `χₖ = ⟨xᵢ xᵢ₊ₖ⟩ - ⟨xᵢ⟩⟨xᵢ₊ₖ⟩` where x are values in the sample.

Note that by default this function truncated the sum over k if
`v / τ < max_rel_err / (N - k)` or if the `k > N - min_sample_size`. The former
cuts of fluctuations around 0 and can be turned of with `truncate = false`.
"""
function unbinned_tau(sample; truncate = true, max_rel_err = 0.0, min_sample_size = 32)
    tau = 0.0

    for k in 1:length(sample) - max(1, min_sample_size)
        v = (1 - k/length(sample)) * correlation(sample, k)
        tau += v

        # We assume the worst case is a constant correlation for the tail.
        # There are (length(sample) - k) values left
        # The average prefactor is 0.5 * (1 - k/length(sample))
        # If the sum of the tail is smaller than max_rel_err * tau, i.e. if tau
        # can at most increase by a factor of max_rel_error, we stop the loop
        if truncate && 0.5 * v * (length(sample) - k) < max_rel_err * tau
            @debug "Cancelled unbinned_tau summation after $k iterations."
            break
        end
    end

    return tau / correlation(sample, 0)
end


export batchnorm
using Statistics
using AutoGrad: value
import Knet

"""
    batchnorm(x, mean_estimate, var_estimate, bias, scale;
              epsilon = 1e-5,
              update = Knet.training() ? 0.1 : 0
              use_estimates = !Knet.training())

Return batch normalization applied to `x`:

    ((x .- mean(x; dims)) ./ sqrt.(epsilon .+ var(x; dims))) .* scale .+ bias  # use_estimates=false
    ((x .- mean_estimate) ./ sqrt.(epsilon .+ var_estimate)) .* scale .+ bias  # use_estimates=true

If `use_estimates=true`, the mean_estimate/var_estimate arguments are used in the
normalization calculation, otherwise the actual mean/var of the batch are used. To be
consistent with the CUDNN implementation `var` is called with `corrected=false`.  If `update
> 0`, the mean_estimate/var_estimate arguments will be updated using batch statistics:

    mean_estimate .= mean(x; dims) * update + mean_estimate * (1-update)
    var_estimate  .= var(x; dims)  * update + var_estimate  * (1-update)

By default `use_estimates=false, update > 0` during training and `use_estimates=true,
update=0` during inference.

Bias and scale are trainable parameters, corresponding to beta and gamma in the original
paper.

The relation between the common size of bias/scale/mean/var and the dims value in the
formula can be:

    (1,1,C,1) if size(x)==(W,H,C,N) and dims==(1,2,4) (default, NCHW, per-channel)
    (C,1,1,1) if size(x)==(C,W,H,N) and dims==(2,3,4) (NHWC tensor format)
    (W,H,C,1) if size(x)==(W,H,C,N) and dims==4       (per-activation mode)

Reference: Batch Normalization: Accelerating Deep Network Training by Reducing Internal
Covariate Shift, S. Ioffe, C. Szegedy, 2015.
"""
function batchnorm(
    x, mean_estimate, var_estimate, bias, scale;
    epsilon = 1e-5,
    update = Knet.training() ? 0.1 : 0.0,
    use_estimates = !Knet.training(),
    o...
)
    update,epsilon = eltype(x).((update,epsilon))
    if update > 0 || !use_estimates
        dims = findall(size(mean_estimate) .== 1)
        xmean = mean(x; dims)
        xvar  = var(x; dims, mean=xmean, corrected=false)
    end
    if update > 0
        (m, v, xm, xv) = value.((mean_estimate, var_estimate, xmean, xvar))
        m .= xm * update + m * (1-update)
        v .= xv * update + v * (1-update)
    end
    if use_estimates
        y = ((x .- mean_estimate) ./ sqrt.(epsilon .+ var_estimate)) .* scale .+ bias
    else
        y = ((x .- xmean) ./ sqrt.(epsilon .+ xvar)) .* scale .+ bias
    end
    return y
end


# This function returns the expected score betweena  player and their opponent
#     rating_1: rating of player
#     rating_2: rating of opponent
#     elo_scale: How many points above the other one player needs to be
#        to have a 10x chance of winning the game
function expected_score(rating_1, rating_2; elo_scale = 400)
    return 1/(1+10^((rating_2 - rating_1)/elo_scale))
end

# This function rates a single player and returns the new rating
#     rating_1: rating of player
#     rating_2: rating of opponent
#     outcome: the actual outcome of a given match
#     elo_scale: How many points above the other one player needs to be
#        to have a 10x chance of winning the game
#     outcome_scale: the amount by which each game affects the player's rating
function rate(rating_1, rating_2, outcome; elo_scale=400,
              outcome_scale=15)

    return rating_1 + outcome_scale*(outcome - expected_score(rating_1,
                                                              rating_2;
                                                              elo_scale))

end


function naiveLU( a:: Array{Float64,1}, b :: Array{Float64,1}, c :: Array{Float64,1})
  #= @assert (size(a) == size(b)) =#
  #= @assert (size(a) == size(c)) =#
  l = zeros(a)
  d = zeros(a)
  u = zeros(a)
  d[1] = a[1]
  u[1:end-1] = c[1:end-1]

  for ii = 2:(length(l))
    l[ii] = b[ii]/d[ii-1]
    d[ii] = a[ii] - l[ii]u[ii-1]
  end
  return (l,d,u)
end
export naiveLU

function ForwardThomas( l :: Array{Float64,1}, dataR :: Array{Float64,1})
  y = zeros(dataR)
  y[1] = dataR[1]
  for ii = 2:length(dataR)
    y[ii] = dataR[ii] - l[ii]*y[ii-1]
  end
  return y
end

function BackwardThomas( d :: Array{Float64, 1}, u :: Array{Float64,1}, dataR :: Array{Float64,1})
  x = zeros(dataR)
  x[end] = dataR[end]/d[end]
  for ii = (length(x)-1):-1:1
    x[ii] = (dataR[ii] - u[ii]*x[ii+1])/d[ii]
  end
  return x
end


using QuantEcon
using PyPlot

function get_simultaneous_P(epsilon, N, p)
    P = Array(Float64, (N+1, N+1))
    for i in 1:(N+1)
        if i - 1 < p * N
            P[i, 1:end] = [binomial(N, j)*(epsilon)^j*(1-epsilon)^(N-j) for j in 0:N]
            P[i, end] = 1-sum(P[i, 1:end-1])
        elseif i - 1 == p * N
            P[i, 1:end] = [binomial(N, j)*(1/2)^N for j in 0:N]
            P[i, end] = 1-sum(P[i, 1:end-1])
        else
            P[i, 1:end] = [binomial(N, j)*(epsilon)^(N-j)*(1-epsilon)^j for j in 0:N]
            P[i, end] = 1-sum(P[i, 1:end-1])
        end
    end
    return P
end

function get_sequential_P(epsilon, N, p)
    P = zeros(Float64, (N+1, N+1))
    for i in 1:N+1
        if i == 1
            P[i, i + 1] = (N - (i-1))/N*epsilon/2
            if (i-1) > p * (N-1)
                P[i, i + 1] += 1 - epsilon
            elseif (i-1) == p * (N-1)
                P[i, i + 1] += 1/2
            end
            P[i, i] = 1 - P[i, i + 1]
        elseif i == N+1
            P[i, i-1] = (i-1)/N*epsilon/2
            if i - 2 == p * (N - 1)
                P[i, i-1] = 1/2
            elseif i - 2 < p * (N - 1)
                P[i, i-1] = 1 - epsilon
            end
            P[i, i] = 1 - P[i, i-1]
        else
            P[i, i+1] = (N - (i-1))/N*epsilon/2
            if 1 > p * (N - (i-1))
                P[i, i+1] += 1 - epsilon
            elseif 1 == p * (N - (i-1))
                P[i, i+1] += 1/2
            end
            P[i, i-1] = (i-1)/N*epsilon/2
            if i - 2 == p * (N - 1)
                P[i, i-1] = 1/2
            elseif i - 2 < p * (N - 1)
                P[i, i-1] = 1 - epsilon
            end
            P[i, i] = 1 - P[i, i-1] - P[i, i + 1]
        end
    end
    return P
end

type KMR
    mc
    N
    epsilon
    Xs
end

get_p_2x2(M) = (M[1, 1, 1] - M[2, 1, 1])/(M[1, 1, 1] - M[1, 2, 1] - M[2, 1, 1] + M[2, 2, 1])
KMR(M, N::Int, epsilon::Float64; simul=true) = KMR(simul ? MarkovChain(get_simultaneous_P(epsilon, N, get_p_2x2(M))) : MarkovChain(get_sequential_P(epsilon, N, get_p_2x2(M))), N, epsilon, None)

function simulate!(kmr::KMR, T; init=0)
    kmr.Xs = simulate(kmr.mc, T, init+1) .- 1
end

plot_sample_path(kmr::KMR) = plot(kmr.Xs)#, label="epsilon = $kmr.epsion")

function plot_stationary_dist(kmr::KMR)
    fig, ax = subplots()
    #ax[:hist](mc_compute_stationary(kmr.mc))
    ax[:bar](collect(0:kmr.N), mc_compute_stationary(kmr.mc))
end

function plot_empirical_dist(kmr::KMR)
    fig, ax = subplots()
    ax[:hist](kmr.Xs, normed=true)
end

function plot_sample_path(kmr::KMR)
    plot(kmr.Xs)
end

"""
M = Array(Int, (2, 2, 2))
M[1, 1, :] = [4, 4]
M[1, 2, :] = [0, 3]
M[2, 1, :] = [3, 0]
M[2, 2, :] = [2, 2]

N = 20
T = 100
epsilon = 1/5

kmr = KMR(M, N, epsilon)
simulate!(kmr, T)
#plot_stationary_dist(kmr)
plot_empirical_dist(kmr)
#plot_sample_path(kmr)

for e in linspace(1/2, 0, 7)
    kmr = KMR(M, N, e)
    simulate!(kmr, T)
    #plot_sample_path(kmr)
    #plot_empirical_dist(kmr)
    plot_stationary_dist(kmr)
end
#println(Xs)
#plot(Xs)
#println(P)
#println(mean(X .== 2))
"""


module econ628ps7

using ForwaDiff, LinearAlgebra

function newton(f, f_prime, x_0; tolerance = 1E-7, maxiter = 1000)
    #Setup the algorithm
    x_old = x_0
    error = Inf
    iter = 1
    while error > tolerance && iter <= maxiter
        x_new = x_old - (f(x_old)/f_prime(x_old))
        error = sqrt((x_new - x_old)^2)
        x_old = x_new
        iter = iter + 1
    end
    if iter == maxiter + 1
        return nothing
    else
        return x_old
end
end

f(x) = (x-1)^3
f_prime(x) = 3*(x-1)^2

function newton(f, x_0; tolerance = 1E-7, maxiter = 1000)
f_prime = x -> ForwardDiff.derivative(f, x)
return newton(f,f_prime,x_0;tolerance = tolerance, maxiter = maxiter)
end

export newton, f, f_prime
end


# Import Turing and Distributions.
using Turing, Distributions
# GLM for comparison
using GLM
# Import RDatasets.
using RDatasets
# Import MCMCChains, Plots, and StatPlots for visualizations and diagnostics.
using MCMCChains, Plots, StatsPlots
# Functionality for splitting and normalizing the data.
using MLDataUtils: shuffleobs, splitobs, rescale!
# Functionality for evaluating the model predictions.
using Distances
# set random seeds
using Random
Random.seed!(12212020)

# import RDatasets
df = RDatasets.dataset("datasets","ToothGrowth");
# Dummy code the Supp column
df.Supp_num = convert.(Float64, df.Supp .== "VC")

# split train-test
train, test = splitobs(shuffleobs(df), at=.7)

# put DV and IV into separate vars
function split_iv_dv(df)
    y = Array(df.Len)
    X = Array(df[:,["Supp_num","Dose"]])
    return y, X
end
y_train, X_train = split_iv_dv(train)
y_test, X_test = split_iv_dv(test)

# OLS for comparison
ols = lm(@formula(Len ~ Supp * Dose), train)

# define the generative function
@model function linear_regression(y, X)
    # construct interaction term
    X = [X X[:,1] .* X[:,2]]
    # prior for intercept
    b0 ~ Normal(0,3)
    # prior for betas, mvnormal with 0 means
    bs ~ MvNormal(size(X,2), 3)
    # error variance prior
    e ~ InverseGamma(2,3)

    ## condition on obs
    mu = b0 .+ X * bs
    y ~ MvNormal(mu, e)
end

# prior check, plot sample from prior
prior_sample = sample(
    linear_regression(y_train, X_train),
    Prior(),
    1000
)
plot(prior_sample)
describe(prior_sample)

# NUTS sampling posterior
ch = sample(
    linear_regression(y_train, X_train),
    NUTS(.65),
    1000
)

# visualize the samples
plot(ch)
# summarize the samples
describe(ch)

## check against hold-out data


## Pendulum
# https://github.com/HarvardAgileRoboticsLab/unscented-dynamic-programming/blob/master/pendulum_dynamics.m
function pendulum_dynamics!(xdot,x,u)
    m = 1.
    l = 0.5
    b = 0.1
    lc = 0.5
    I = 0.25
    g = 9.81
    xdot[1] = x[2]
    xdot[2] = (u[1] - m*g*lc*sin(x[1]) - b*x[2])/I
end
n,m = 2,1

#create some empty functions (haven't updated for new model definition)
function blank_error()

end

function blank_expansion()

end

model = Model(pendulum_dynamics!,n,m,blank_error,blank_expansion) # inplace model

# initial conditions
x0 = [0; 0.]

# goal
xf = [pi; 0] # (ie, swing up)

# costs
Q = 1e-3*Matrix(I,n,n)
Qf = 100.0*Matrix(I,n,n)
R = 1e-2*Matrix(I,m,m)

# simulation
tf = 5.

obj_uncon = LQRObjective(Q, R, Qf, tf, x0, xf)

# Constraints
u_bound = 2
u_min = [-u_bound]
u_max = [u_bound]
obj_con = ConstrainedObjective(obj_uncon, u_min=u_min, u_max=u_max) # constrained objective

# Set up problem
pendulum = [model,obj_uncon]
pendulum_constrained = [model, obj_con]


#=
You are given the following information, but you may prefer to do some research for yourself.

- 1 Jan 1900 was a Monday.
- Thirty days has September,
  April, June and November.
  All the rest have thirty-one,
  Saving February alone,
  Which has twenty-eight, rain or shine.
  And on leap years, twenty-nine.
- A leap year occurs on any year evenly divisible by 4, but not on a century unless it is divisible by 400.

How many Sundays fell on the first of the month during the twentieth century (1 Jan 1901 to 31 Dec 2000)?
=#

months = Dict(
  1 => 31,
  2 => 28,
  3 => 31,
  4 => 30,
  5 => 31,
  6 => 30,
  7 => 31,
  8 => 31,
  9 => 30,
  10 => 31,
  11 => 30,
  12 => 31
)

function isleapyear(year)
  if year%100==0
    return year%400==0 ? true : false
  elseif year%4==0
    return true
  end
  false
end

function calc()
  currentyear = 1900
  currentmonth = 1
  totaldays = 1
  totalsundays = 0
  for year in 1901:2000
    months[2] = isleapyear(year) ? 29 : 28 # set feb
    for month in 1:length(months)
      totaldays += months[month]
      if (totaldays+1) % 7 == 0
        totalsundays += 1
      end
    end
  end
  totalsundays
end
@time println(calc())


using SuiteSparseGraphBLAS
function pagerank(
    A,
    d = reduce(Monoids.PLUS_MONOID, A; dims=2),
    α = 0.85,
    maxiters = 100,
    ϵ = 1.0e-4
)
    n = size(A, 1)
    r = GBVector{Float32}(n)
    t = GBVector{Float32}(n)
    d[:, accum=BinaryOps.DIV] = α
    r[:] = 1.0 / n
    teleport = (1 - α) / n
    rdiff = 1.0
    i = 0
    for outer i ∈ 1:maxiters
        temp = t; t = r; r = temp
        w = t ./ d
        r[:] = teleport
        mul!(r, A', w, Semirings.PLUS_SECOND, accum=BinaryOps.PLUS)
        t .-= r
        map!(UnaryOps.ABS, t)
        rdiff = reduce(Monoids.PLUS_MONOID, t)
        if rdiff <= ϵ
            break
        end
    end
    return r, i
end


function integral(x::CellVar, grid::TreeGrid; filter::Function = active)
    int = zero(eltype(x.data))
    for cell ∈ cells(grid, filter = filter)
        @inbounds int += volume(cell) * x[cell]
    end
    return int
end

function inner_product(x::CellVar, y::CellVar, grid::TreeGrid; filter::Function = active)
    int = zero(eltype(x.data))
    for cell ∈ cells(grid, filter = filter)
        @inbounds int += volume(cell) * x[cell] * y[cell]
    end
    return int
end

@inline norm(x::CellVar, grid::TreeGrid; filter::Function = active) = sqrt(inner_product(x, x, grid, filter))


using LinearAlgebra
using SparseArrays
using Plots

"This routine generates a gif movie showing how the heat equation rapidly dissipates
high frequencies."

# m = number of grid points
function build_FD(m,f,α,β)
    x = LinRange(-1,1,m+2)
    xint = x[2:end-1]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m),-1=>-ones(m-1),1=>-ones(m-1))

    function F(t)
        b = f.(xint,t)
        b[1] += α(t)/h^2
        b[m] += β(t)/h^2
        return b
    end
    return A,x,xint,h,F
end

m = 200
T = .025

dt = .005/m

A,x,xint,h,F = build_FD(m,(x,t)->0,t->0,t->0)

u0(x) = sin(pi*x) + .1*sin(2*pi*x) - .33*sin(3*pi*x) + 10*sin(8*pi*x)
u = u0.(xint)
Nsteps = ceil(Int,T/dt)
dt = T/Nsteps
@gif for i = 1:Nsteps
    tprev = (i-1)*dt
    t = i*dt
    u .= (I + .5*dt*A)\((I - .5*dt*A)*u + .5*dt*(F(t)+F(tprev)))
    if i%10==0
        plot(xint,u,mark=:dot,leg=false)
        println("on timestep $i out of $Nsteps")
    end
end every 10


# This file illustrates the recovery of a real signal from its frequency domain.

using Plots
using FFTW

# Define Fourier transform
fs = 100
ts = 1 / 100
l = 2^10
fv = 0 : fs / (l - 1) : fs
xv = sinc.(fv)
ff = 0 : fs / (2l - 1) : fs
xf = [xv; reverse(xv)]

# Define inverse Fourier transform
x = ifft(xf) * fs
t = (0 : 2l - 1) * ts

# Plot the results
plt = plot(layout=2)
plot!(ff, abs.(xf), subplot=1)
xlabel!("f [Hz]", subplot=1)
ylabel!("|X(f)|", subplot=1)
plot!(t, abs.(x), subplot=2)
xlabel!("t [s]", subplot=2)
ylabel!("x(t)", subplot=2)
display(plt)


##############################################################
function softparzen(X, S_centre, r, γ = 1e-6)
##############################################################

  N, D = size(X)
  M = length(S_centre)

  # Usually kernel has also some normalisation constant
  # but in the following calculations it cancels out anyway
  K = zeros(M,N)
  for j in 1:M
    for n in 1:N
      @inbounds K[j,n] = exp(-0.5*norm(X[S_centre[j],:] - X[n,:])^2 / (r*r))
    end
  end

  # Soft version for coefficients
  Q = zeros(M)
  sumK = sum(K)
  for j in 1:M
    @inbounds Q[j] = sum(K[j,:]) / sumK
  end

  # responsibilities - note different to standard mixture models
  # See equation (6)
  resp = zeros(M,N)
  for j in 1:M
    sum_Kj_over_n = sum(K[j,:])
    for n in 1:N
      @inbounds resp[j,n] = K[j,n] / sum_Kj_over_n
    end
  end

  # Soft version for means
  # See equation (7)
  mu = zeros(M, D)
  for j=1:M
    for n=1:N
      for i=1:D
        @inbounds mu[j,i] += resp[j,n] * X[n,i]
      end
    end
  end

  # Soft version for covariances
  C = [zeros(D, D) for j in 1:M]

  for j in 1:M
    for i in 1:D
      for d in 1:D
        for n in 1:N
          @inbounds C[j][i,d] += resp[j,n] * ((X[n,i] - mu[j,i]) * (X[n,d] - mu[j,d]))
        end
      end
    end

    C[j] = (C[j] + C[j]')*0.5 + γ*I

  end

  return Q, mu, C

end


"""
    discretize(domain, N)

Discretize domain with `N` points.
"""
function discretize end

discretize(domain::ClosedIntervalDomain, N) = LinRange(domain.left, domain.right, N + 1)

function discretize(domain::PeriodicIntervalDomain, N)
    LinRange(domain.left + (domain.right - domain.left) / N, domain.right, N)
end


module PCA

    using LinearAlgebra

    function findPCAs(
        X    :: AbstractArray{T, 2};
        num  :: Integer = 1
    ) where T <: AbstractFloat

        dim, N = size(X)

        Σ = Symmetric(X * X')

        println("Dimension of Σ : ", size(Σ))
        println("Any missing data? ", any(isnan.(Σ)))

        if any(isnan.(Σ))
            throw(ErrorException("Data contains NaN!"))
        end

        @time F = eigen(Σ)

        #println(F.values)
        #println(length(F.values))

        order = collect(1:dim)
        sort!(order; by = i -> F.values[i], rev=true)

        evs = zeros(T, dim, num)
        for i = 1:num
            evs[:, i] = normalize(F.vectors[:, order[i]])
        end

        return evs

    end

end


#=

    This is a port of the R2 model CoinBias.cs

    Output from the R2 model:
    ```
    Mean: 0.421294
    Variance: 0.0162177
    Number of accepted samples = 692
    ```

    This model:
  parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
      Symbol   Float64   Float64    Float64   Float64     Float64   Float64       Float64

        bias    0.4166    0.1360     0.0014    0.0020   5021.6561    1.0000     2276.3627

=#

using Turing, StatsPlots, Distributions, StatsBase
using CSV
include("jl_utils.jl")

@model function coin_bias(x)
    n = length(x)
    # Beta(2,5) has mean about 0.2855
    bias ~ Beta(2,5)
    x ~ filldist(Bernoulli(bias),n)
end

x = parse.(Int,split(readline("coin_bias.txt"),","))
println("x:$x")
model = coin_bias(x)

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 1_000)
chns = sample(model, PG(5), 10_000)
# chns = sample(model, SMC(), 1_000)
# chns = sample(model, IS(), 10_000)

# chns = sample(model, HMC(0.1,6), 1_000)
# chns = sample(model, NUTS(), 1_000)

display(chns)

show_var_dist_pct(chns,:bias,20)


# ---
# title: 1652. Defuse the Bomb
# id: problem1652
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Array
# link: <https://leetcode.com/problems/defuse-the-bomb/description/>
# hidden: true
# ---
#
# You have a bomb to defuse, and your time is running out! Your informer will
# provide you with a **circular** array `code` of length of `n` and a key `k`.
#
# To decrypt the code, you must replace every number. All the numbers are
# replaced **simultaneously**.
#
#   * If `k > 0`, replace the `ith` number with the sum of the **next** `k` numbers.
#   * If `k < 0`, replace the `ith` number with the sum of the **previous** `k` numbers.
#   * If `k == 0`, replace the `ith` number with `0`.
#
# As `code` is circular, the next element of `code[n-1]` is `code[0]`, and the
# previous element of `code[0]` is `code[n-1]`.
#
# Given the **circular** array `code` and an integer key `k`, return _the
# decrypted code to defuse the bomb_!
#
#
#
# **Example 1:**
#
#
#
#     Input: code = [5,7,1,4], k = 3
#     Output: [12,10,16,13]
#     Explanation: Each number is replaced by the sum of the next 3 numbers. The decrypted code is [7+1+4, 1+4+5, 4+5+7, 5+7+1]. Notice that the numbers wrap around.
#
#
# **Example 2:**
#
#
#
#     Input: code = [1,2,3,4], k = 0
#     Output: [0,0,0,0]
#     Explanation: When k is zero, the numbers are replaced by 0.
#
#
# **Example 3:**
#
#
#
#     Input: code = [2,4,9,3], k = -2
#     Output: [12,5,6,13]
#     Explanation: The decrypted code is [3+9, 2+3, 4+2, 9+4]. Notice that the numbers wrap around again. If k is negative, the sum is of the **previous** numbers.
#
#
#
#
# **Constraints:**
#
#   * `n == code.length`
#   * `1 <= n <= 100`
#   * `1 <= code[i] <= 100`
#   * `-(n - 1) <= k <= n - 1`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


using Plots, JLD, ProgressBars, Statistics
Directions=[[0,1],[0,-1],[1,0],[-1,0]]
function f(N, Network_,Pos)
    if N==0
        return 1
    end
    Network_[Pos...]=true
    c=0
    for path in Directions
        NextPos=Pos .+ path
        if !Network_[NextPos...]
            c+=f(N-1, Network_, NextPos)
        end
    end
    Network_[Pos...]=false
    return c
end
Totdata=[]
N=16
for n in ProgressBar(0:N)
    L=2n+1
    Network_=falses(L,L)
    FirstPos=[n+1, n+1]
    push!(Totdata, f(n, Network_,FirstPos))
end
plot(0:N,Totdata, legend=false, dpi=400)
scatter!(0:N,Totdata, legend=false, xlabel="N",ylabel="Number of Paths",
    title="Number of Paths versus N for Self-Avoiding Walker", titlefontsize=10, dpi=400)
save("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter5\\4.7\\SAW.jld", "Data", Totdata)
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter5\\4.7\\Figs\\NumberOfPaths_N.png")
plot(0:N, Totdata./ 4 .^(0:N), legend=false, dpi=400)
scatter!(0:N, Totdata./ 4 .^(0:N), legend=false, xlabel="N",ylabel="Ratio of Number of Paths",
    title="Ratio of Number of Paths of SAW and RW ", dpi=400)
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter5\\4.7\\Figs\\Ratio_N.png")


# ---
# title: 883. Projection Area of 3D Shapes
# id: problem883
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/projection-area-of-3d-shapes/description/>
# hidden: true
# ---
#
# On a `N * N` grid, we place some `1 * 1 * 1 `cubes that are axis-aligned with
# the x, y, and z axes.
#
# Each value `v = grid[i][j]` represents a tower of `v` cubes placed on top of
# grid cell `(i, j)`.
#
# Now we view the  _projection_  of these cubes onto the xy, yz, and zx planes.
#
# A projection is like a shadow, that maps our 3 dimensional figure to a 2
# dimensional plane.
#
# Here, we are viewing the "shadow" when looking at the cubes from the top, the
# front, and the side.
#
# Return the total area of all three projections.
#
#
#
# **Example 1:**
#
#
#
#     Input: [[2]]
#     Output: 5
#
#
# **Example 2:**
#
#
#
#     Input: [[1,2],[3,4]]
#     Output: 17
#     Explanation:
#     Here are the three projections ("shadows") of the shape made with each axis-aligned plane.
#     ![](https://s3-lc-upload.s3.amazonaws.com/uploads/2018/08/02/shadow.png)
#
#
# **Example 3:**
#
#
#
#     Input: [[1,0],[0,2]]
#     Output: 8
#
#
# **Example 4:**
#
#
#
#     Input: [[1,1,1],[1,0,1],[1,1,1]]
#     Output: 14
#
#
# **Example 5:**
#
#
#
#     Input: [[2,2,2],[2,1,2],[2,2,2]]
#     Output: 21
#
#
#
#
# **Note:**
#
#   * `1 <= grid.length = grid[0].length <= 50`
#   * `0 <= grid[i][j] <= 50`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end