function isprime(n::Int64)
    if n <= 3
        return n > 1
    elseif n % 2 == 0 || n % 3 == 0
        return false
    end
    i = 5
    while i^2 <= n
        if n % i == 0 || n % (i+2) == 0
            return false
        end
        i += 6
    end
    return true
end

function get_smallest_divisor(n::Int64)
    k = 2
    while n % k ≠ 0
        k += 1
    end
    return k
end

function get_prime_factors(N::Int64)
    S = Vector{Int64}()
    while true
        !isprime(N) || break
        k = get_smallest_divisor(N)
        push!(S, k)
        N::Int64 = N/k
    end
    push!(S, N)
    return S
end

function reverse_int(n::Int64)
    if n <= 9
        return n
    else
        return parse(Int, reduce(string, digits(n)))
    end
end

function totient(n::Int64)
    result::Float64 = n
    p = 2
    while p*p <= n
        if n % p == 0
            while n % p == 0
                n = n//p
            end
            result *= (1.0 - (1.0 / p))
        end
        p += 1
    end

    if n > 1
        result *= (1.0 - (1.0 / n))
    end

    return result
end

function prime_sieve(N::Int64)
    iscoprime(P, i) = !any(x -> i % x == 0, P)

    primes = Int64[]
    for num = 2:N
        if iscoprime(primes, num)
            push!(primes, num)
        end
    end
    primes
end


# The Jacobi method for solving an equation system
# Ax = b
# is an iterative method, where the k:th approximation
# to the solution is calculated as
# x⁽ᵏ⁺¹⁾ = D⁻¹(b - Rx⁽ᵏ⁾),
# where D is the diagonal elements of A, and
# R = A - D.

type JacobiSolver <: Solver
    A
    Dinv
    R
    tol
    ω
    maxiter
    function JacobiSolver(A, tol, ω=2./3, maxiter = 10)
        D = Diagonal(diag(A))
        new(A, inv(D), A-D, tol, ω, maxiter)
    end
end

# This implementation assumes that x is reasonably "close" to b; this
# is the case if A = I + Ξ, with |Ξ| << 1.
function solve!(b::StridedVecOrMat, J::JacobiSolver)
    xp = ones(b)
    x = xp
    for k = 1:J.maxiter
        x = J.ω*(J.Dinv*(b - J.R*xp)) + (1-J.ω)*xp
        if norm(J.A*x-b) < J.tol
            break
        end
        xp = x
    end
    b[:] = x
end

export JacobiSolver, solve!


"""
    ⊗(A, B)

Compute the kronecker product, ``A\\otimes B``.

# Examples
```julia-repl
julia> [1 0; 0 1]⊗[1 0; 0 -1]
4×4 Array{Int64,2}:
 1   0  0   0
 0  -1  0   0
 0   0  1   0
 0   0  0  -1
```
"""
⊗(A, B) = kron(A, B)

"""
    ⊗(A, B, C...)

Compute the kronecker product, ``A\\otimes B\\otimes C \\cdots``.
"""
function ⊗(A, B, C...)
    A = kron(A, B)
    for Ci in C
        A = kron(A, Ci)
    end
    return A
end


nprocs()==1 && addprocs()

@everywhere logis(x) = 1 ./ (1 .+ exp(-x))
@everywhere dlogis(x) = logis(x) .* (1 .- logis(x))

@everywhere function dfa!(w1,w2,w3,B1,B2,inp,out)
    a1 = w1 * inp
    z1 = logis(a1)
    a2 = w2 * z1
    z2 = logis(a2)
    ay = w3 * z2
    y = logis(ay)
    err = y - out
    d_a1 = (B1.*err) .* dlogis(a1)
    d_a2 = (B2.*err) .* dlogis(a2)
    w1 .-= d_a1 * inp'
    w2 .-= d_a2 * z1'
    w3 .-= err * z2'
    return Dict("output"=>y,"err"=>err)
end

@everywhere function bp!(bp_w1,bp_w2,bp_w3,inp,out)
    bp_a1 = bp_w1*inp
    bp_z1 = logis(bp_a1)
    bp_a2 = bp_w2 * bp_z1
    bp_z2 = logis(bp_a2)
    bp_ay = bp_w3 * bp_z2
    bp_y = logis(bp_ay)
    bp_err = bp_y - out
    bp_d_a3 = (bp_err) .* dlogis(bp_ay)
    bp_d_a2 = (bp_w3' * bp_d_a3) .* dlogis(bp_a2)
    bp_d_a1 = (bp_w2' * bp_d_a2) .* dlogis(bp_a1)
    bp_w1 .-= bp_d_a1 * inp'
    bp_w2 .-= bp_d_a2 * bp_z1'
    bp_w3 .-= bp_d_a3 * bp_z2'
    return Dict("output"=>bp_y,"err"=>bp_err)
end

inp = [1 1 ; 0 1 ; 1 0 ; 0 0]'
out = [0 1 1 0]
y=zeros(size(out))
n_hidden = 15; # Number of hidden units
num_iterations = 1000; #Number of learning steps
trials = 50;

e_store = SharedArray(Float64,(num_iterations,trials))
bp_e_store = SharedArray(Float64,(num_iterations,trials))
y_store = SharedArray(Float64,(size(out,2),num_iterations))
bp_y_store = SharedArray(Float64,(size(out,2),num_iterations))

@elapsed @sync @parallel for jj = 1:trials
    w1 = randn(n_hidden,size(inp,1))
    w2 = randn(n_hidden,n_hidden)
    w3 = randn(size(out,1),n_hidden)

    B1=rand(n_hidden,1)
    B2=rand(n_hidden,1)

    bp_w1 = randn(n_hidden,size(inp,1))
    bp_w2 = randn(n_hidden,n_hidden)
    bp_w3 = randn(size(out,1),n_hidden)

    for ii = 1:num_iterations
        dfa_res = dfa!(w1,w2,w3,B1,B2,inp,out)
        bp_res = bp!(bp_w1,bp_w2,bp_w3,inp,out)
        if jj == 1
            y_store[:,ii] = dfa_res["output"]'
            bp_y_store[:,ii] = bp_res["output"]'
        end
        e_store[ii,jj] = sum(abs(dfa_res["err"]))
        bp_e_store[ii,jj] = sum(abs(bp_res["err"]))
    end
end

using Plots
l = @layout [
    a b
    c d
]

gr()
p1=plot(y_store',label=["1,1" "0,1" "1,0" "0,0"],title="DFA")
p2=plot(bp_y_store',label=["1,1" "0,1" "1,0" "0,0"],title="BP")
p3=plot([e_store[:,1] bp_e_store[:,1]],label=["DFA" "BP"],title="First Trial")
p4=plot([mean(e_store,2) mean(bp_e_store,2)],label=["DFA" "BP"],title="Mean Error")
plot(p1,p2,p3,p4,layout=l)


"""
    dominates(p1::Array, p2::Array)

    Return true if each element in p1 is not less than the corresponding element in p2 and at least one element in p1 is bigger than the corresponding element in p2.

# Arguments
- `p1::Array`: Numeric array of n elements.
- `p2::Array`: Numeric array of n elements.

# Examples
```julia-repl
julia> dominates([1,2,3], [1,2,1])
true
julia> dominates([0,0,0,0], [1,0,0,0])
false
```

# References
Deb, Kalyanmoy, et al. "A fast elitist non-dominated sorting genetic algorithm for multi-objective optimization: NSGA-II."
International conference on parallel problem solving from nature. Springer, Berlin, Heidelberg, 2000.
"""
function dominates(p1::Array, p2::Array)::Bool
    n = length(p1)
    notworse = count(i -> p1[i] < p2[i], 1:n)
    better   = count(i -> p1[i] > p2[i], 1:n)
    return (notworse == 0) && (better > 0)
end

"""
    ndsranks(data)

    Sort multidimensional data usin non-dominated sorting algorithm.

# Arguments
- `data::DataFrame`: DataFrame of variables.

# References
Deb, Kalyanmoy, et al. "A fast elitist non-dominated sorting genetic algorithm for multi-objective optimization: NSGA-II."
International conference on parallel problem solving from nature. Springer, Berlin, Heidelberg, 2000.
"""
function ndsranks(data::DataFrame)::Array{Int}

    mat = convert(Matrix, data)

    return ndsranks(mat)

end

"""
    ndsranks(data)

    Sort multidimensional data usin non-dominated sorting algorithm.

# Arguments
- `data::Matrix`: n x k matrix of observations where n is number of observations and k is number of variables.

# Examples
```julia-repl

```

# References
Deb, Kalyanmoy, et al. "A fast elitist non-dominated sorting genetic algorithm for multi-objective optimization: NSGA-II."
International conference on parallel problem solving from nature. Springer, Berlin, Heidelberg, 2000.
"""
function ndsranks(data::Matrix)::Array{Int64}

    n, p = size(data)

    ranks = zeros(Int64, n)

    mat = convert(Matrix, data)

    @inbounds for i in 1:n
        @inbounds for j in 1:n
            if i != j
                if dominates(mat[i,:], mat[j,:])
                    ranks[i] += 1
                end
            end
        end
    end

    return ranks
end

function nds(data::DataFrame)

    ranks = ndsranks(data)

    bestIndex = sortperm(ranks) |> last

    result = NDSResult(
        ranks,
        bestIndex
    )

    return result
end


using LinearAlgebra

A = [2 3 0;
     1 2 4;
     0 2 1]

println("A+A=", A+A)
println("A-A=", A-A)
println("A*A=", A*A)
println("5*A=", 5*A)


# squared Euclidean distance from a set

export SqrDistL2

"""
**Squared distance from a convex set**

    SqrDistL2(ind_S, λ=1.0)

Given `ind_S` the indicator function of a convex set ``S``, and an optional positive parameter `λ`, returns the (weighted) squared Euclidean distance from ``S``, that is function
```math
g(x) = \\tfrac{λ}{2}\\mathrm{dist}_S^2(x) = \\min \\left\\{ \\tfrac{λ}{2}\\|y - x\\|^2 : y \\in S \\right\\}.
```
"""
struct SqrDistL2{R <: Real, T <: ProximableFunction} <: ProximableFunction
  ind::T
  lambda::R
  function SqrDistL2{R,T}(ind::T, lambda::R) where {R <: Real, T<:ProximableFunction}
    if !is_convex(ind) || !is_set(ind)
      error("`ind` must be the indicator of a convex set")
    end
    if lambda < 0
      error("parameter λ must be nonnegative")
    else
      new(ind, lambda)
    end
  end
end

is_prox_accurate(f::SqrDistL2) = is_prox_accurate(f.ind)
is_convex(f::SqrDistL2) = true
is_smooth(f::SqrDistL2) = true
is_quadratic(f::SqrDistL2) = is_affine(f.ind)
is_strongly_convex(f::SqrDistL2) = is_singleton(f.ind)

SqrDistL2(ind::T, lambda::R=1.0) where {R <: Real, T <: ProximableFunction} = SqrDistL2{R, T}(ind, lambda)

function (f::SqrDistL2)(x::AbstractArray{T}) where T <: RealOrComplex
  p, = prox(f.ind, x)
  return (f.lambda/2)*normdiff2(x,p)
end

function prox!(y::AbstractArray{T}, f::SqrDistL2, x::AbstractArray{T}, gamma::Real=1.0) where T <: RealOrComplex
  p, = prox(f.ind, x)
  sqrd = (f.lambda/2)*normdiff2(x,p)
  c1 = 1/(1+f.lambda*gamma)
  c2 = f.lambda*gamma*c1
  for k in eachindex(p)
    y[k] = c1*x[k] + c2*p[k]
  end
  return sqrd*c1^2
end

function gradient!(y::AbstractArray{T}, f::SqrDistL2, x::AbstractArray{T}) where T <: RealOrComplex
  p, = prox(f.ind, x)
  dist2 = normdiff2(x,p)
  y .= f.lambda.*(x .- p)
  return (f.lambda/2)*dist2
end

fun_name(f::SqrDistL2) = "squared Euclidean distance from a convex set"
fun_dom(f::SqrDistL2) = fun_dom(f.ind)
fun_expr(f::SqrDistL2) = "x ↦ (λ/2) inf { ||x-y||^2 : y ∈ S} "
fun_params(f::SqrDistL2) = string("λ = $(f.lambda), S = ", typeof(f.ind))

function prox_naive(f::SqrDistL2, x::AbstractArray{T}, gamma::Real=1.0) where T <: RealOrComplex
  p, = prox(f.ind, x)
  sqrd = (f.lambda/2)*norm(x-p)^2
  gamlam = f.lambda*gamma
  return 1/(1+gamlam)*x + gamlam/(1+gamlam)*p, sqrd/(1+gamlam)^2
end


# ---
# title: 462. Minimum Moves to Equal Array Elements II
# id: problem462
# author: Indigo
# date: 2021-02-04
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/minimum-moves-to-equal-array-elements-ii/description/>
# hidden: true
# ---
#
# Given a **non-empty** integer array, find the minimum number of moves required
# to make all array elements equal, where a move is incrementing a selected
# element by 1 or decrementing a selected element by 1.
#
# You may assume the array's length is at most 10,000.
#
# **Example:**
#
#
#
#     Input:
#     [1,2,3]
#
#     Output:
#     2
#
#     Explanation:
#     Only two moves are needed (remember each move increments or decrements one element):
#
#     [1,2,3]  =>  [2,2,3]  =>  [2,2,2]
#
#
#
## @lc code=start
using LeetCode

function min_moves2(nums::Vector{Int})
    median = partialsort!(nums, (length(nums) + 1) ÷ 2)
    return sum(abs, num - median for num in nums)
end
## @lc code=end


"""
    fm = mergespec_spaceonly(ra,f,kmax)

Spectral merging of coarse field `ra` and fine field `f` at wavenumber `kmax`.
"""
function mergespec_spaceonly(ra,f,kmax)

(nx,ny)=size(f);
(nax,nay)=size(ra);

DFTr=zeros(Complex{Float64},nax,nax);
DFTf=zeros(Complex{Float64},nx,nx);

#fft di tutto il campo f
DFTf=fft(f)

#fft di tutto il campo r
DFTr=fft(ra);

DFTr=fftshift(DFTr); #centro la fft
DFTf=fftshift(DFTf); #centro la fft
DFTfm=zeros(Complex{Float64},nx,nx);
DFTr2=zeros(Complex{Float64},nax+1,nax+1);

#DFTr[nax+1,:,:]=NaN;
#DFTr[:,nax+1,:]=NaN;
DFTr2[1:nax,1:nax]=DFTr[:,:]
DFTr2[nax+1,1:nax]=conj(DFTr[1,:]);
DFTr2[1:nax,nax+1]=conj(DFTr[:,1]);

kmax2=kmax^2;

# We need to fix the phases of the large scale field!
# The first pixel of the large field is centered in dxl/2 with dxl=2pi/nax
# The first pixel of the fine field is centered in dxs/2 with dxf=2pi/nax
ddx=2*pi/nax/2-2*pi/nx/2;

 for j=1:nx
     for i=1:nx
            kx=-div(nx,2)+i-1;
            ky=-div(nx,2)+j-1;
            k2=(kx^2+ky^2);
            ir=div(nax,2)+1+kx;
            jr=div(nax,2)+1+ky;
            if(k2<=kmax2)
                DFTfm[i,j]=DFTr2[ir,jr]*exp(-1im*ddx*kx-1im*ddx*ky);;
            else
                DFTfm[i,j]=DFTf[i,j];
            end
     end
end

DFTfm=ifftshift(DFTfm);
fm=ifft(DFTfm);
fm=real(fm);

end


haversine(lat1, lon1, lat2, lon2) =
    2 * 6372.8 * asin(sqrt(sind((lat2 - lat1) / 2) ^ 2 +
    cosd(lat1) * cosd(lat2) * sind((lon2 - lon1) / 2) ^ 2))

@show haversine(36.12, -86.67, 33.94, -118.4)


module HWfuncapp

using FastGaussQuadrature # to get chebyshevnodes

# you dont' have to use PyPlot. I did much of it in PyPlot, hence
# you will see me often qualify code with `PyPlot.plot` or so.
using PyPlot
import ApproXD: getBasis, BSpline
using Distributions
using ApproxFun
using Plots
export TT, ChebyT, q1

ChebyT(x,deg) = cos(acos(x)*deg)
unitmap(x,lb,ub) = 2 .* (x .- lb) ./ (ub .- lb) .- 1	#[a,b] -> [-1,1]

n=15
map(x,lb,ub) = 0.5 .* (lb .+ ub) .+ 0.5 .* (ub .- lb) .* x

function Cheby(x,deg)
len=length(x)
psi = ones(len,deg)
for i in 1:len
for j in 1:deg
psi[i,j] = ChebyT(x[i],j-1)
end
end
return psi
end

function q1(n=15)
lb, ub, inter, Nod = -3, 3, 100, gausschebyshev(n)[1]#Values for evaluation
x = range(lb,stop = ub,length = n)#
psi = map(Nod, lb, ub) #Evaluation

f(x) = x .+ 2x.^2 - exp.(-x)#Function to evaluate
PSI=inv(Cheby(Nod,n))
ev=f(psi)
ResEv=PSI*ev

testX=range(lb, stop=ub, length=inter)
testY=f(testX)

testMap=map(testX, lb, ub)
testPsi=Cheby(unitmap(testMap), n)

ResEv=testPsi*PSI

testError=ev-ResEv

#PyPlot.plot()
p = Plots.plot(layout = 2, dpi = 400)
Plots.plot!(p[1],testX,testY,label = "Test Y",lw = 1,linecolor = "black")
Plots.plot!(p[1],testX,ResEv, label = "Result Evaluation", lw = 2,linestyle = :dot, linecolor = "pink")
Plots.plot!(p[2],testX,testError, label = "Test Error", lw = 1, linecolor = "blue")

# without using PyPlot, just erase the `PyPlot.` part
PyPlot.savefig(joinpath(dirname(@__FILE__),"..","q1.png"))
return Dict(:error=>maximum(abs,err))
end

end # module


# ## 多項式回帰
# ### 例52
# sin関数に乱数を足した観測データを多項式で回帰します。
# \lineskip
# データの生成
using Plots, Joe, Random
using Plots.PlotMeasures # hide
Plots.reset_defaults() # hide
gr() # hide
default( # hide
    titlefont  = font("JuliaMono", default(:titlefontsize),  ), # hide
    guidefont  = font("JuliaMono",  default(:guidefontsize),  ), # hide
    tickfont   = font("JuliaMono", default(:tickfontsize),   ), # hide
    legendfont = font("JuliaMono",  default(:legendfontsize), ), # hide
    left_margin = 30px, # hide
    bottom_margin = 30px # hide
) # hide
Random.seed!(12)
n = 100; x = randn(n); y = sin.(x) + randn(n);
p52 = scatter(x,y,xlabel="x", ylabel="y", label=false, legend=:topleft);
# 3、5、7次の多項式で回帰します。
# 多項式回帰用にデータセットを作る関数を定義したので、これを使ってみます。
# ```julia
# polynomial(x::Vector, P::Int) = hcat([x.^p for p in 1:P]...)
# polynomial(x::AbstractRange, P::Int)= polynomial(collect(x),P)
# ```
using Joe:polynomial
p_set = [3,5,7]
x_seq = -3:0.1:3
for p in p_set
    global x, y, x_seq, p52 # hide
    X = polynomial(x,p)
    β̂ = multiple_regression(X,y)
    ŷ = insert_ones(polynomial(x_seq,p))*β̂
    plot!(p52, x_seq,ŷ,label="p = $p")
end
savefig(p52,joinpath(@OUTPUT,"fig6-1.svg")) # hide
# \fig{fig6-1}
# ### 例53
# 矩形波に雑音をノイズをのせたデータを作って、フィッティングします。
# まずはデータを生成します。
using Plots, Random
Random.seed!(123)
n = 100; x = randn(n) * π
y = abs.(round.(x) .%2) * 2  .- 1+ randn(n)*0.2
p53 = scatter(x,y,label= false);
# 基底関数が、$f_1(x) = 1$、 $f_2(x) = \cos(x) $、 $f_3(x) = \cos(2x)$、
# $f_4(x) = \cos(3x)$の場合と、$f_1(x) = 1$、 $f_2(x) = \sin(x) $、 $f_3(x) = \sin(2x)$、
# $f_4(x) = \sin(4x)$の場合とでそれぞれ回帰分析します。
# 準備として先ほどのpolynomial関数をべき乗以外にも対応できるように多重ディスパッチで拡張します。
# ```julia
# polynomial(x::Vector, P::Int, f::Function) = hcat([f.(p*x) for p in 1:P]...)
# polynomial(x::AbstractRange, P::Int, f::Function) = polynomial(collect(x), P, f)
# ```
# $(-8,8)$の区間でフィッティングします。
x_seq = -8:0.2:8
# cosの場合
X_cos = polynomial(x,3,cos)
β̂_cos = multiple_regression(X_cos,y)
ŷ_cos = insert_ones(polynomial(x_seq,3,cos))*β̂_cos;
# sinの場合
X_sin = polynomial(x,3,sin)
β̂_sin = multiple_regression(X_sin,y)
ŷ_sin = insert_ones(polynomial(x_seq,3,sin))*β̂_sin;
# 可視化します。
plot!(p53,x_seq,ŷ_cos,label="cos")
plot!(p53,x_seq,ŷ_sin,label="sin")
savefig(p53,joinpath(@OUTPUT,"fig6-2.svg")) # hide
# \fig{fig6-2}
# \lineskip
# cosを基底にとったほうが、上手くフィッティングできることが確認できました。
# ## スプライン回帰
# スプライン回帰については、まずスプラインに関する複合型を作って、
# function-like objectを実装しました。
# spline_matrix関数は、あとで変更できるようにキーワード引数にしました。
#
# ```julia
# using Parameters
# @with_kw mutable struct Spline{T<:Number}
#     xmin::T = -1.0
#     xmax::T = 1.0
#     K::Int = 5
# end
# function spline_matrix(x::Vector,K::Int,xmin,xmax)
#     n = length(x)
#     Knots = range(xmin,stop=xmax,length=K)
#     X = zeros(n,K+4)
#     X[:,1] .= 1
#     X[:,2] .= x
#     X[:,3] .= x.^2
#     X[:,4] .= x.^3
#     for j in 1:K
#         X[:,j+4] = @. max(x - Knots[j], 0)^3
#     end
#     return X
# end
# function (s::Spline)(x::Vector, y::Vector, x_pred::Vector;
#           spline_matrix::Function=spline_matrix)
#     @unpack xmin, xmax, K = s
#     Knots = range(xmin,stop=xmax,length=K)
#     X = spline_matrix(x,K,xmin,xmax)
#     X_pred = spline_matrix(x_pred,K,xmin,xmax)
#     β = (X'X)\X'y
#     ypred = X_pred*β
# end
# (s::Spline)(x::Vector, y::Vector, x_pred::AbstractRange;kwargs...) = s(x,y,collect(x_pred);kwargs...)
# ```
# ### 例54
# sinカーブに乱数を加えたデータにスプライン回帰でフィッティングします。
# 分割点数が7以上でデータに追随していることがわかります。
using Plots, Random
using Joe: Spline
using Joe
Random.seed!(123)
n = 100; x = randn(n)*2π; y = sin.(x) + 0.2randn(n)
p54 = scatter(x,y,xlims=(-5,5),xlabel="x",ylabel="f(x)",label=false)
K_set = 5:2:9
for K in K_set
    spline = Spline(xmin=-2π, xmax=2π, K=K)
    u_seq = -5:0.2:5
    v_seq = spline(x,y,u_seq)
    plot!(p54, u_seq,v_seq, label="K = $K")
end
savefig(p54,joinpath(@OUTPUT,"fig6-5.svg")) # hide
# \fig{fig6-5}
# ### 例56 自然なスプライン曲線
# スプライン曲線の両端が直線になっているものです。
# 原著では、関数dとか関数hを定義して使っていました。
# そんなことはしたくないけど、いい名前もないので、関数内で定義して使うことにしました。
# ```julia
# function natural_spline_matrix(x::Vector, K::Int, xmin, xmax)
#     n = length(x)
#     Knots = range(xmin,stop=xmax,length=K)
#     X = zeros(n,K)
#     X[:,1] .= 1
#     X[:,2] .= x
#     d(x::Vector,a::Number,b::Number) =  @. (max(x-a,0)^3 - max(x-b,0)^3) / (b-a)
#     for j in 1:K-2
#         X[:,j+2] = d(x,Knots[j],Knots[end]) - d(x,Knots[end-1],Knots[end])
#     end
#     return X
# end
# ```
# 例54と同じデータに、ただのスプラインと自然なスプラインの回帰を適用し、
# 結果を可視化します。まずデータの生成とスプライン型を定義します。
using Plots, Random
using Joe: Spline
using Joe
Random.seed!(123)
n = 100; x = randn(n)*2π; y = sin.(x) + 0.2randn(n)
spline6 = Spline(xmin=-5, xmax=5, K=6);
spline11 = Spline(xmin=-5, xmax=5, K=11);
# 次に回帰を行います。自然なスプラインで回帰を行う場合には、キーワード引数を指定します。
u_seq = -6:0.02:6;
v_seq6 = spline6(x,y,u_seq);
v_seq11 = spline11(x,y,u_seq);
w_seq6 = spline6(x,y,u_seq;splinematrix=Joe.natural_spline_matrix);
w_seq11 = spline11(x,y,u_seq;splinematrix=Joe.natural_spline_matrix);
# 可視化します。
p56_1 = scatter(x,y,xlims=(-7,7),xlabel="x",ylabel="f(x),g(x)",label=false,title="K=6")
p56_2 = scatter(x,y,xlims=(-7,7),xlabel="x",ylabel="f(x),g(x)",label=false,title="K=11")
plot!(p56_1,u_seq,v_seq6,label="spline")
plot!(p56_1,u_seq,w_seq6,label="natural spline")
vline!(p56_1,[-5,5];linewidth=1,label=false)
vline!(p56_1,range(-5,stop=5,length=6);linestyle=:dash,label=false)
plot!(p56_2,u_seq,v_seq11,label="spline")
plot!(p56_2,u_seq,w_seq11,label="natural spline")
vline!(p56_2,[-5,5];linewidth=1,label=false)
vline!(p56_2,range(-5,stop=5,length=11);linestyle=:dash,label=false)
p56 = plot(p56_1,p56_2,layout=(2,1))
savefig(p56,joinpath(@OUTPUT,"fig6-7.svg")) # hide
# \fig{fig6-7}
# ## 平滑化スプライン
# 平滑化スプラインは区切り点を指定するときにデータ点自身を使っていたので、
# 以下のように書き直しました。(function-like objectにするまでもなかった。)
# ```julia
# struct SmoothingSpline
#     x::Vector
# end
# function (s::SmoothingSpline)(y::Vector,x_pred::Union{Vector,AbstractRange};λ=0)
#     x = s.x
#     X = smoothing_spline_matrix(x,x)
#     G = green_silverman(x)
#     γ = (X'X + λ*G)\X'y
#     X_pred = smoothing_spline_matrix(x_pred,x)
#     ypred = X_pred*γ
# end
# ```
# ただし、smoothing_spline_matrixとgreen_silverman関数は以下の通りです。
# ```julia
# function smoothing_spline_matrix(x::Vector,Knots::Union{Vector, AbstractRange})
#     n = length(x)
#     K = length(Vector)
#     X = zeros(n,K)
#     X[:,1] .= 1
#     X[:,2] .= x
#     d(x::Vector,a::Number,b::Number) =  @. (max(x-a,0)^3 - max(x-b,0)^3) / (b-a)
#     for j in 1:K-2
#         X[:,j+2] = d(x,Knots[j],Knots[end]) - d(x,Knots[end-1],Knots[end])
#     end
#     return X
# end

# function green_silverman(x::Vector)
#     n = length(x)
#     g = zeros(n,n)
#     for i in 3:n ,j in i:n
#         g[j,i] =(12(x[end]-x[end-1])*(x[end-1]-x[j-2])*(x[end-1]-x[i-2])+
#                 (x[end-1]-x[j-2])^2*(12x[end-1]+6x[j-2]-18x[i-2]))/
#                 (x[end]-x[i-2])/(x[end]-x[j-2])
#         g[i,j] = g[j,i]
#     end
#     return g
# end
# ```
# ### 例57
# $x\in(-5,5)$の一様分布からサンプルを生成します。
# 原著では、サンプル生成コードが間違っているようなので、ipynbの記述を参照しました。
using Plots, Random, Distributions
using Joe: SmoothingSpline
using Joe
Random.seed!(11)
n = 100;
x = rand(Uniform(-5,5),n); y = x .+ 2sin.(x) + randn(n)
index = sortperm(x); x = x[index]; y = y[index]

p57 = scatter(x,y,label=false)

sspline = SmoothingSpline(x)
u_seq = -8:0.02:8
λ_set = [40,400,1000]
for λ in λ_set
    global u_seq, p57 # hide
    v_seq = sspline(y,u_seq,λ=λ)
    plot!(p57,u_seq,v_seq,label="λ = $λ")
end
p57
plot!(p57,u_seq,sspline(y,u_seq,λ=1),label="λ = 1")
savefig(p57,joinpath(@OUTPUT, "fig6-8.svg")) # hide
# \fig{fig6-8}
# \lineskip
# $\lambda$が大きくなると、直線に近くなっていくようです。
# ### 例58 クロスバリデーションによる平滑化スプラインの正則化乗数の最適化
# 3章で使った線形回帰の場合の高速なクロスバリデーションの公式
# $$CV[\lambda] \coloneqq \sum_{S}||(I - H_S[\lambda])^{-1}e_S ||^2$$
# $$H_S[\lambda] \coloneqq X_s(X^\mathsf{T}X+\lambda G)^{-1}X^\mathsf{T}_S$$
# を利用して、$\lambda$の最適化を行います。
# 以下のcv_ss_fast関数を定義して利用します。
# ```julia
# function cv_ss_fast(X::AbstractArray,y::Vector,G::AbstractArray,K::Int;λ=0)
#     n = length(y); m = round(Int,n/K)
#     H = X*((X'X + λ*G)\X')
#     e = (I(n) - H)*y
#     S = 0
#     @views for j in 1:K
#         test = j*m-m+1:j*m;
#         err = (I(m)-H[test,test]) \ e[test]
#         S += dot(err,err)
#     end
#     return S/n,tr(H)
# end
# ```
# 3章の使ったcv_fast関数とほぼ同様ですが、定数項の列を付けないことと、
# 正則化乗数が引数になっていること、クロスバリデーション誤差だけでなく、
# Hのトレースも返していることなどが異なります。
# まずはデータを生成します。
using Random, Distributions
Random.seed!(11)
n = 100; x = rand(Uniform(-5,5),n) ; y = x -0.02sin.(x)-0.1randn(n);
index = sortperm(x); x = x[index]; y = y[index];
# XとGを計算します。
X = Joe.smoothing_spline_matrix(x,x);
G = Joe.green_silverman(x);
# クロスバリデーション誤差とHのトレース(有効自由度)を求めます。
result = hcat([collect(Joe.cv_ss_fast(X,y,G,n,λ=λ)) for λ in 1:50]...)
using Plots
p58 = plot(result[2,:], result[1,:],label= false,
            xlabel="有効自由度", ylabel="CVによる予測誤差",
            title = "有効自由度とCVによる予測誤差")
savefig(p58,joinpath(@OUTPUT,"fig6-9.svg"))
# \fig{fig6-9}
# \lineskip
# (注)ちなみに生成するデータによって、グラフの形はかなり変わります。
# ## 局所回帰
# ### 例60 nadaraya-watson推定量
# nadaraya-watson推定量の計算の実装です。
# カーネルは関数の引数にして、別のものに変更できるようにしました。(ここでは他のカーネルは出てきませんが。)
# ```julia
# epanechnikov(x,y,λ) = norm(x-y)/λ |> c -> max(0.75*(1-c^2),0)
#
# function nadaraya_watson_estimator(x_observed::Vector,y_observed::Vector,kk::Function,x_pred::TP;λ=1.0) where {TP<:Number}
#     n = length(y_observed)
#     S = 0; T = 0;
#     for i in 1:n
#         S += kk(x_observed[i],x_pred,λ)*y_observed[i]
#         T += kk(x_observed[i],x_pred,λ)
#     end
#     result = T == 0 ? 0 : S/T
#     return result
# end
# ベクトル化にも対応しています。
# function nadaraya_watson_estimator(x_observed::Vector,y_observed::Vector,kk::Function,x_pred::Union{Vector,AbstractRange};λ=1.0)
#     y_pred = [nadaraya_watson_estimator(x_observed,y_observed,kk,xp; λ) for xp in x_pred]
# end
# ```
# \note{epanechnikoc関数に絶対値の処理は必要になりますが、abs関数を使うよりもLinearAlgebra.jlのnormを使った方が、多次元化に対応できて良いと思います。}
# トイデータとしてsin関数にノイズをのせたデータを作成します。
using Random
Random.seed!(123)
n=250;x = 2randn(n); y = sin.(2π*x) + randn(n)/4

using Plots
p58 = scatter(x,y,xlims=(-3,3),label=false)
xx = -3:0.1:3
using Joe:nadaraya_watson_estimator, epanechnikov
using LinearAlgebra
# $\lambda=0.05,0.25$の場合にプロットしてみます。
plot!(p58,xx,nadaraya_watson_estimator(x,y,epanechnikov,xx;λ=0.05),
         label="λ = 0.05");
plot!(p58,xx,nadaraya_watson_estimator(x,y,epanechnikov,xx;λ=0.25),
         label="λ = 0.25");
# クロスバリデーションで$\lambda$を最適化してプロットしてみます。
λ_seq = 0.05:0.01:1
SS_min = Inf
λ_best = λ_seq[1]
for λ in λ_seq
    global SS_min, λ_best,n,x,y # hide
    SS = 0
    m = Int(n/10)
    for k in 1:10
        test = k*m-m+1:k*m
        train = setdiff(1:n,test)
        y_pred = nadaraya_watson_estimator(x[train],y[train],epanechnikov,x[test];λ=λ)
        SS += dot(y[test]-y_pred,y[test]-y_pred)
    end
    if SS < SS_min
        SS_min = SS
        λ_best = λ
    end
end

plot!(p58,xx,nadaraya_watson_estimator(x,y,epanechnikov,xx;λ=λ_best), label="λ = λ_best")
savefig(p58,joinpath(@OUTPUT,"fig6-10.svg")) # hide
# \fig{fig6-10}
# \lineskip
# $\lambda_{best}$はこの程度の値になるようです。
@show λ_best;
# ### 例61 局所線形回帰
# カーネル関数$k$を成分に持った対角行列$\bm{W}$を
# $$\bm{W} = \begin{bmatrix}
#  k(x,x_1) & \cdots & 0 \\
#  \vdots & \ddots & \vdots \\
#  0 & \cdots & k(x,x_n) \end{bmatrix} $$
# とした時、
# $$ L \coloneqq (\bm{y} - \bm{Xβ}(x))^\mathsf{T}\bm{W}(\bm{y} - \bm{Xβ}(x)) $$
# を最小化する$\bm{\beta}(x)$を求めます。
# $$ \dfrac{\partial L}{\partial \bm{\beta}} =
# -2\bm{X}^\mathsf{T}\bm{W}(\bm{y} - \bm{Xβ}(x))=0$$とおくと、
# $$ \bm{\beta}(x) = (\bm{X}^\mathsf{T}\bm{WX})^{-1}\bm{X}^\mathsf{T}\bm{Wy}$$
# で係数を求めることができます。この処理を関数で実装するわけですが、原著のlocalという関数名は良くないので、
# local_regressionという名前で実装しました。例では1次元に限定していましたが、
# 多次元データにも対応させています。
# ```julia
# function local_regression(x::Matrix{T},
#                           y::Vector{T},
#                           kernel::Function;
#                           x_pred=x, λ=1) where {T<:Number}
#     N,_ = size(x)
#     @assert N==length(y)
#     X = insert_ones(x)
#     m,_ = size(x_pred)
#     y_pred = Vector{T}(undef,m)
#     for j in 1:m
#         W = [kernel(x[i,:],x_pred[j,:],λ) for i in 1:N] |> diagm
#         β̂ = (X'*W*X)\X'*W*y
#         y_pred[j] = dot(insert_ones(x_pred[j,:]),β̂)
#     end
#     return y_pred
# end
# function local_regression(x::Vector{T}, y::Vector{T},kernel::Function;x_pred=x,λ=1) where {T<:Number}
#     return local_regression(x[:,:],y,kernel;x_pred=x_pred[:,:],λ=λ)
# end
# ```
# トイデータとしてsin関数にノイズをのせたデータを作成します。
using Random,Joe
Random.seed!(123)
n=30; x = 2π*rand(n) .- π; y = sin.(x) .+ randn()
p61 = scatter(x,y,label=false)
m = 200;U = -π:π/m:π;
# 正則化パラメータ$\lambda=1$の条件で回帰分析を行ってプロットします。
V = Joe.local_regression(x,y,Joe.epanechnikov;x_pred=U)
plot!(p61,U,V,label=false,title = "局所線形回帰(p=1, N=30)")
savefig(p61,joinpath(@OUTPUT,"fig6-11.svg")) # hide
# \fig{fig6-11}
# \lineskip
# 予測したい場所の数だけ回帰分析の計算を行わないといけないので、元データの数が多い時は
# 計算が大変そうです。
# ## 一般化加法モデル
# ### 例62
# 基底関数の数が有限個なら線形回帰の手法で係数を求められるけど、
# 数が多すぎると逆行列を解くのが大変でどうしましょうとい３う話。
# バックフィッティングは例63で実装します。
# ### 例63
# 以下のようなpolyfit関数を実装しました。
# ```julia
# function polyfit(x,y;x_pred=x, P::Int=3)
#     X = polynomial(x,P)
#     X_pred=polynomial(x_pred,P)
#     β̂ = multiple_regression(X,y)
#     y_pred = insert_ones(X_pred) * β̂
# end
# ```
# [例６１](#例61)のデータに一般加法モデルを適用します。
# 先に多項式でフィッティングして残りを局所線形回帰で求めていきます。
using Random,Joe
Random.seed!(123)
n=30; x = 2π*rand(n) .- π; sort!(x); y = sin.(x) .+ randn()
y₁ = zeros(n); y₂ = zeros(n)
for k in 1:10
    global x,y,y₁,y₂ # hide
    y₁ = Joe.polyfit(x,y-y₂)
    y₂ = Joe.local_regression(x,y-y₁,Joe.epanechnikov)
end
using Plots
p62_1 = plot(x,y₁,label=false,xlabel="x",ylabel="f(x)",title="多項式回帰(3次)")
p62_2 = plot(x,y₂,label=false,xlabel="x",ylabel="f(x)",title="局所線形回帰")
p62　= plot(p62_1,p62_2,layout=(1,2))
savefig(p62,joinpath(@OUTPUT,"fig6-12.svg")) # hide
# \fig{fig6-12}
# \lineskip
# 大体３次多項式で回帰されていて、残りが局所線形回帰されていることが分かります。
# 先に3次多項式で回帰したためでしょうか。 確認してみることにします。
using Random,Joe
Random.seed!(123)
n=30; x = 2π*rand(n) .- π; sort!(x); y = sin.(x) .+ randn()
y₁ = zeros(n); y₂ = zeros(n)
using Plots
for k in 1:10
    global x,y,y₁,y₂ # hide
    y₂ = Joe.local_regression(x,y-y₁,Joe.epanechnikov)
    y₁ = Joe.polyfit(x,y-y₂)
end
p62_3 = plot(x,y₁,label=false,xlabel="x",ylabel="f(x)",title="多項式回帰(3次)")
p62_4 = plot(x,y₂,label=false,xlabel="x",ylabel="f(x)",title="局所線形回帰")
p62_5　= plot(p62_3,p62_4,layout=(1,2))
savefig(p62_5,joinpath(@OUTPUT,"fig6-12-2.svg")) # hide
# \fig{fig6-12-2}
# \lineskip
# 多項式回帰の影響が減って、局所線形回帰の成分が大きくなることが確認できました。


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Distributions, Plots, LaTeXStrings
end;

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing6.12"

# ╔═╡ 1aa41898-3afa-11eb-2f49-570853ce6435
begin
	mu, sig = 2, 3
	eta = sqrt(3)*sig/pi
	n, N = 15, 10^4
	dNormal   = Normal(mu, sig)
	dLogistic = Logistic(mu, eta)
	alphaUsed = 0.001:0.001:0.1
end;

# ╔═╡ 874034ba-3b2b-11eb-2b77-c1d2d28a1116
function alphaSimulator(dist, n, alpha)
    popVar        = var(dist)
    coverageCount = 0
    for _ in 1:N
        sVar = var(rand(dist, n))
        L = (n - 1) * sVar / quantile(Chisq(n-1),1-alpha/2)
        U = (n - 1) * sVar / quantile(Chisq(n-1),alpha/2)
        coverageCount +=  L < popVar && popVar < U
    end
    1 - coverageCount/N
end

# ╔═╡ 87407402-3b2b-11eb-36de-2b731fe48190
begin
	scatter(alphaUsed, alphaSimulator.(dNormal,n,alphaUsed),
		c=:blue, msw=0, label="Normal")
	scatter!(alphaUsed, alphaSimulator.(dLogistic, n, alphaUsed),
		c=:red, msw=0, label="Logistic")
	plot!([0,0.1],[0,0.1],c=:black, label="1:1 slope",
		xlabel=L"\alpha"*" used", ylabel=L"\alpha"*" actual",
		legend=:topleft, xlim=(0,0.1), ylims=(0,0.2))
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing6.12"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═1aa41898-3afa-11eb-2f49-570853ce6435
# ╠═874034ba-3b2b-11eb-2b77-c1d2d28a1116
# ╠═87407402-3b2b-11eb-36de-2b731fe48190
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


# ============================================================================
# Example 1 from *Reachability analysis of linear systems with uncertain
# parameters and inputs*. TODO: Agregar referencia completa.
# ============================================================================

# initial set
X0 = BallInf([1.0, 1.0], 0.1)

# linear ODE: x' = Ax
A = IntervalMatrix([-1.0 ± 0.05 -4.0 ± 0.05;
                    4.0 ± 0.05 -1.0 ± 0.05])

# IVP(LCS(A), X0) TODO: remove
interval2D_linear = @ivp x' = Ax, x(0) ∈ X0

# affine ODE: x' = Ax + Bu
B = IntervalMatrix(hcat([1.0 ± 0.0; 1.0 ± 0.0])) # why hcat?
U = Interval(-0.05, 0.05)

# IVP(CLCCS(A, B, nothing, U), X0) TODO: remove
interval2D_affine = @ivp x' = Ax + Bu, x(0) ∈ X0, u ∈ U


using AlphaStructures
using ViewerGL
GL =  ViewerGL

"""
Return points from file.
"""
function load_points(filename::String)::Matrix
    io = open(filename, "r")
    point = readlines(io)
    close(io)

    b = [tryparse.(Float64,split(point[i], " ")) for i in 1:length(point)]
    V = hcat(b...)
    return V
end

V = load_points("examples/examples3D/model/griffin.txt")

# view points
GL.VIEW([
	GL.GLPoints(permutedims(V))
])

# points by columns
filtration = AlphaStructures.alphaFilter(V);
VV, EV, FV, TV = AlphaStructures.alphaSimplex(V, filtration, 0.35)

#view all simplices
GL.VIEW(
	[
		GL.GLGrid(V, EV, GL.COLORS[1], 0.6) # White
		GL.GLGrid(V, FV, GL.COLORS[2], 0.8) # Red
		GL.GLGrid(V, TV, GL.COLORS[3], 0.6) # Green
	]
);


# The BiMix distribution samples from a `leftdistribution` distribution with
# probability `p` and the `rightdistribution` with probability `1-p`. this
# is useful for efficiently defining mixture models upon two distributions,
# since it computes a marginalized pdf, thus saving the extra node in the
# execution trace for the mixture assignment.

struct BiMix{T<:Distribution,U<:Distribution}
    prob :: Float64
    leftdistribution :: T
    rightdistribution :: U
end

function Distributions.logpdf(bimix::BiMix, value)
    pleft = log(bimix.prob) + logpdf(bimix.leftdistribution, value)
    pright = log1p(-bimix.prob) + logpdf(bimix.rightdistribution, value)

    m = max(pleft, pright)
    m + log(exp(pleft - m) + exp(pright - m))
end

function Base.rand(bimix::BiMix)
    left = rand() <= bimix.prob
    rand(left ? bimix.leftdistribution : bimix.rightdistribution)
end


# Numerical Integration

struct GaussLegendre
    nip::Int  # number of integral points
    nodes::Vector{Float64}
    weights::Vector{Float64}
    function GaussLegendre(nip::Int)
        nodes, weights = gausslegendre(nip)
        return new(nip, nodes, weights)
    end
end

"""
Integrate on interval
"""
function integrate(func, I1::ClosedInterval{<:Real}, gl1)
    nip1, nodes1, weights1 = gl1.nip, gl1.nodes, gl1.weights
    dnodes1 = (IntervalSets.width(I1) * nodes1 .+ sum(extrema(I1))) / 2

    S1 = weights1[1] * func(dnodes1[1])
    for i1 in 2:nip1
        S1 += weights1[i1] * func(dnodes1[i1])
    end

    return S1 * IntervalSets.width(I1) / 2
end

"""
Integrate on rectangular region.
"""
function integrate(func, I1::ClosedInterval{<:Real}, I2::ClosedInterval{<:Real}, gl1, gl2)
    nip1, nodes1, weights1 = gl1.nip, gl1.nodes, gl1.weights
    nip2, nodes2, weights2 = gl2.nip, gl2.nodes, gl2.weights
    dnodes1 = (IntervalSets.width(I1) * nodes1 .+ sum(extrema(I1))) / 2
    dnodes2 = (IntervalSets.width(I2) * nodes2 .+ sum(extrema(I2))) / 2

    S2 = weights2[1] * func(dnodes1[1], dnodes2[1])
    for i2 in 2:nip2
        S2 += weights2[i2] * func(dnodes1[1], dnodes2[i2])
    end
    S1 = weights1[1] * S2
    for i1 in 2:nip1
        S2 = weights2[1] * func(dnodes1[i1], dnodes2[1])
        for i2 in 2:nip2
            S2 += weights2[i2] * func(dnodes1[i1], dnodes2[i2])
        end
        S1 += weights1[i1] * S2
    end

    return S1 * IntervalSets.width(I1) * IntervalSets.width(I2) / 4
end

"""
Integrate on rectangular solid
"""
function integrate(func, I1::ClosedInterval{<:Real}, I2::ClosedInterval{<:Real}, I3::ClosedInterval{<:Real}, gl1, gl2, gl3)
    nip1, nodes1, weights1 = gl1.nip, gl1.nodes, gl1.weights
    nip2, nodes2, weights2 = gl2.nip, gl2.nodes, gl2.weights
    nip3, nodes3, weights3 = gl3.nip, gl3.nodes, gl3.weights
    dnodes1 = (IntervalSets.width(I1) * nodes1 .+ sum(extrema(I1))) / 2
    dnodes2 = (IntervalSets.width(I2) * nodes2 .+ sum(extrema(I2))) / 2
    dnodes3 = (IntervalSets.width(I3) * nodes3 .+ sum(extrema(I3))) / 2

    S3 = weights3[1] * func(dnodes1[1],dnodes2[1],dnodes3[1])
    for i3 in 2:nip3
        S3 += weights3[i3] * func(dnodes1[1],dnodes2[1],dnodes3[i3])
    end
    S2 = weights2[1] * S3
    for i2 in 2:nip2
        S3 = weights3[1] * func(dnodes1[1],dnodes2[i2],dnodes3[1])
        for i3 in 2:nip3
            S3 += weights3[i3] * func(dnodes1[1],dnodes2[i2],dnodes3[i3])
        end
        S2 += weights2[i2] * S3
    end
    S1 = weights1[1] * S2
    for i1 in 2:nip1
        S3 = weights3[1] * func(dnodes1[i1],dnodes2[1],dnodes3[1])
        for i3 in 2:nip3
            S3 += weights3[i3] * func(dnodes1[i1],dnodes2[1],dnodes3[i3])
        end
        S2 = weights2[1] * S3
        for i2 in 2:nip2
            S3 = weights3[1] * func(dnodes1[i1],dnodes2[i2],dnodes3[1])
            for i3 in 2:nip3
                S3 += weights3[i3] * func(dnodes1[i1],dnodes2[i2],dnodes3[i3])
            end
            S2 += weights2[i2] * S3
        end
        S1 += weights1[i1] * S2
    end

    return S1 * IntervalSets.width(I1) * IntervalSets.width(I2)  * IntervalSets.width(I3) / 8
end


# using Pkg; for p in ("Knet","ArgParse"); haskey(Pkg.installed(),p) || Pkg.add(p); end

"""

LinReg is a simple linear regression example using artificially
generated data. You can run the demo using `julia linreg.jl` on the
command line or `julia> LinReg.main()` at the Julia prompt.  Use
`julia linreg.jl --help` or `julia> LinReg.main("--help")` for a list
of options.  The quadratic loss will be printed at every epoch and
optimized parameters will be returned.

"""
module LinReg
using Knet, CUDA, ArgParse, Random

predict(w,x)=(w*x)

loss(w,x,y)=(sum(abs2,y-predict(w,x)) / size(x,2))

lossgradient = grad(loss)

function train(w, data; lr=.02, epochs=10)
    for epoch=1:epochs
        for (x,y) in data
            g = lossgradient(w, x, y)
            w -= lr * g
        end
    end
    return w
end

function test(w, data)
    sumloss = numloss = 0
    for (x,y) in data
        sumloss += loss(w,x,y)
        numloss += 1
    end
    return sumloss/numloss
end

# Data generator:

struct LRData; w; batchsize; epochsize; noise; rng; atype; end

function LRData(outputdims,inputdims; batchsize=20, epochsize=10000, noise=.01, rng=Random.GLOBAL_RNG, atype=Array)
    LRData(convert(atype, randn(rng,outputdims,inputdims)),batchsize,epochsize,noise,rng,atype)
end

function Base.iterate(l::LRData, n=0)
    if n >= l.epochsize; return nothing; end
    (outputdims, inputdims) = size(l.w)
    x = convert(l.atype, rand(l.rng, inputdims, l.batchsize))
    y = l.w * x + convert(l.atype, l.noise * randn(l.rng, outputdims, l.batchsize))
    return ((x,y), n+l.batchsize)
end

# Main loop:

function main(args=ARGS)
    s = ArgParseSettings()
    s.description="linreg.jl (c) Deniz Yuret, 2016. Linear regression example with artificial data."
    s.exc_handler=ArgParse.debug_handler
    @add_arg_table s begin
        ("--atype"; default="$(Knet.array_type[])"; help="array type: Array for cpu, KnetArray for gpu")
        ("--batchsize"; arg_type=Int; default=20; help="number of instances in a minibatch")
        ("--epochs"; arg_type=Int; default=10; help="number of epochs for training")
        ("--epochsize"; arg_type=Int; default=10000; help="number of instances per epoch")
        ("--fast"; action=:store_true; help="skip loss printing for faster run")
        ("--inputdims"; arg_type=Int; default=100; help="input dimensions")
        ("--lr"; arg_type=Float64; default=0.02; help="learning rate")
        ("--noise"; arg_type=Float64; default=0.01; help="noise in data")
        ("--outputdims"; arg_type=Int; default=10; help="output dimensions")
        ("--seed"; arg_type=Int; default=-1; help="random number seed: use a nonnegative int for repeatable results")
        ("--gcheck"; arg_type=Int; default=0; help="check N random gradients")
    end
    isa(args, AbstractString) && (args=split(args))
    if in("--help", args) || in("-h", args)
        ArgParse.show_help(s; exit_when_done=false)
        return
    end
    o = parse_args(args,s; as_symbols=true)
    println(s.description)
    println("opts=",[(k,v) for (k,v) in o]...)
    o[:seed] > 0 && Knet.seed!(o[:seed])
    atype = eval(Meta.parse(o[:atype]))
    data = LRData(o[:outputdims], o[:inputdims]; batchsize=o[:batchsize], epochsize=o[:epochsize], noise=o[:noise], atype=atype)
    w = convert(atype, 0.1*randn(o[:outputdims], o[:inputdims]))
    println((:epoch,0,:loss,test(w,data)))
    if o[:fast]
        @time w = train(w, data; epochs=o[:epochs], lr=o[:lr])
        println((:epoch,o[:epochs],:loss,test(w,data)))
    else
        @time for epoch=1:o[:epochs]
            w = train(w, data; epochs=1, lr=o[:lr])
            println((:epoch,epoch,:loss,test(w,data)))
            if o[:gcheck] > 0
                gradcheck(loss, w, first(data)...; gcheck=o[:gcheck], verbose=true)
            end
        end
    end
    return w
end

# This allows both non-interactive (shell command) and interactive calls like:
# $ julia linreg.jl --epochs 10
# julia> LinReg.main("--epochs 10")
PROGRAM_FILE == "linreg.jl" && main(ARGS)

end

# SAMPLE RUN 65f57ff+ Wed Sep 14 10:02:30 EEST 2016
#
# linreg.jl (c) Deniz Yuret, 2016. Linear regression example with artificial data.
# opts=(:lr,0.02)(:atype,"KnetArray")(:epochsize,10000)(:inputdims,100)(:noise,0.01)(:epochs,10)(:gcheck,0)(:seed,-1)(:outputdims,10)(:batchsize,20)(:fast,true)
# (:epoch,0,:loss,349.84485494071095)
#   1.320391 seconds (1.65 M allocations: 164.061 MB, 1.85% gc time)
# (:epoch,10,:loss,0.0010327464077591123)


### A Pluto.jl notebook ###
# v0.17.1

using Markdown
using InteractiveUtils

# ╔═╡ 6dd76507-be07-4c3a-9747-99eca74d3519
begin
	filter!(path -> path != "@v#.#", LOAD_PATH)
	using Pkg
	Pkg.activate(Base.current_project())
	Pkg.instantiate()
	using PlutoUI, StatisticsWithJulia
	PlutoUI.TableOfContents(title = "初识 Julia")
end

# ╔═╡ 3baed4bd-1c0c-4d44-b7ab-f7c0627fda1b
begin
	Pkg.add("AbstractTrees") # command + /: 切换为注释形式
	using AbstractTrees
	AbstractTrees.children(x::Type) = subtypes(x)
	with_terminal(() -> print_tree(Number))
end

# ╔═╡ 2edcc72c-bfa4-4623-a34f-220c0df9c4d3
begin
	Pkg.add("HTTP")
	Pkg.add("JSON")
	using HTTP, JSON
end

# ╔═╡ d507db6a-f869-4dc3-b7ef-fd3518fc1660
Pkg.add.(["Plots", "LaTeXStrings", "Measures"])

# ╔═╡ 2fdb3281-7229-4f18-b9d9-bc5e3086d903
using Statistics

# ╔═╡ f924796f-52f3-4b4b-aa47-61a179874647
using Plots, LaTeXStrings, Measures

# ╔═╡ ec3f7b5c-cee9-432f-b33c-a08e16c0171a
StatisticsWithJulia.header(title = @__FILE__, author="")

# ╔═╡ 9d98fa7a-3654-40a0-8a96-75c3fca007ae
md"""
## 本章讲义

- [讲义下载](http://likan.oss-cn-beijing.aliyuncs.com/StatisticsWithJulia/01-%E5%88%9D%E8%AF%86Julia.pdf)

"""

# ╔═╡ 42da8f74-1b44-11ec-32f2-276bdcef1c69
md"""
## 1.1. 你好世界
"""

# ╔═╡ e71aacbb-efff-4b93-b5c4-cbdfa88981ad
md"""
- `command/control + m`: 把当下单元格切换到markdown格式
"""

# ╔═╡ 0660d564-9a26-417d-9a2c-42ef181187c2
with_terminal() do
    println("你好， 世界!")
end

# ╔═╡ 77c5ddd3-ddfc-4a57-ab48-e9db765057d3
with_terminal(() -> println("你好， 世界!"))

# ╔═╡ 8eefb8aa-ae85-4231-abf5-9948cdc92ec1
md"""
`do f() end` 等价于 `（） -> f()`
"""

# ╔═╡ d22fc28f-c591-4db2-8275-54d93fda85e9
f(x) = BigInt(x)^BigInt(x)

# ╔═╡ 4ece5fc8-18d5-408a-851b-31ce58015e72
map(x -> 3x, 1:10)

# ╔═╡ c266f6e2-4e8e-4870-ac38-d4d0e8b9d042
map(1:10) do x
	3x
end

# ╔═╡ 8f3277e9-7905-4beb-9ef1-c3e18b2f449a
map(x -> rand(), 1:10)

# ╔═╡ f6e99ac9-2183-4ea2-991e-f3422b1d1f20
map(__ -> rand(), 1:10)

# ╔═╡ cd0ce1b4-9bc9-4218-95e9-e576abeb1af1
# map(() -> rand(), 1:10) # 不合语法

# ╔═╡ 37fec978-f817-4aa8-85fc-4912eaeb402e
helloArray = ["你好", "👋", "世界"]

# ╔═╡ adc183db-ff23-43de-8cf6-a961d13c2f01
with_terminal() do
	for i ∈ eachindex(helloArray)
		println(helloArray[i])
	end
end

# ╔═╡ 4f1ce95f-2f2f-4817-9d5f-348184a1b800
[i^2 for i in 1:10 for __ in 1:10] # 一维数组

# ╔═╡ dec64158-9161-4b37-b19f-432a5635719e
[i^2 for i in 1:10, __ in 1:10]   # 矩阵

# ╔═╡ 5a401f10-09d2-4939-87fe-621f469a68ee
Square = [i^2 for i in 1:10]

# ╔═╡ c2c7c916-1ba8-4e14-a25b-e15faf676f8a
with_terminal() do
	for i ∈ eachindex(Square)
			println(Square[i])
	end
end

# ╔═╡ 93b0afb2-e9f3-434d-b21a-2ccad6a12d90
[rand() for __ in 1:100]

# ╔═╡ 8bcc15c7-d66a-40cc-9f33-50c6de62fadc
md"""
### 运行速度
"""

# ╔═╡ 428c124a-1052-45c8-a49b-b1047267d644
md"""
- LaTeX Syntax

```math
y = \frac{\displaystyle\sum_{i=1}^\infty}{\epsilon\times\sigma}

```
"""

# ╔═╡ 863624c7-f378-467a-bed2-94182bae73b7
with_terminal() do
	@time begin
		global data = Float64[]
		for __ in 1:10^6
			group = Float64[]
			for __ in 1:5*10^2
				push!(group, rand())
			end
			push!(data, mean(group))
		end
	end
end

# ╔═╡ 0e8b6a32-ba23-4e37-b78c-8ffb3779a43e
"98%的平均值位于 $(quantile(data, 0.01)) 和 $(quantile(data, 0.99)) 之间。"

# ╔═╡ 9e35d6a1-df94-4a27-9e04-234548495780
with_terminal() do
	@time global data2 = [mean(rand(5 * 10 ^ 2)) for __ in 1 : 10 ^ 6]
end

# ╔═╡ 1b45a9bf-56c7-4b7b-bdd5-b4fcaa572ec0
"98%的平均值位于 $(quantile(data2, 0.01)) 和 $(quantile(data2, 0.99)) 之间。"

# ╔═╡ 60b263eb-f786-40c4-b60b-2daabdacba7b
md"""
### 类型和多重派发
"""

# ╔═╡ b0db7a9c-2498-46c5-bf3c-280d3083ba88
# @edit 1 + 2

# ╔═╡ 412ee0c1-26ad-4b9f-9d6e-c552d8431e76
# @edit 1.0 + 2

# ╔═╡ aa5e313d-19d9-4300-a65e-f547f7e571d0
mutable struct zhan
	dt::Int64
end

# ╔═╡ 0ba25fa7-72cb-4d6e-bfaf-eddcf12b9f9c
x = zhan(4)

# ╔═╡ d5251a23-b71e-4faf-9f5a-f6ab1d2ffb7d
y = zhan(8)

# ╔═╡ b717d872-0d16-4556-a6c8-1ccc3c7b76c3
Base.:+(x::zhan, y::zhan) = x.dt * 100 + y.dt * 200

# ╔═╡ a0322171-cff9-48bb-9f73-792c8ea4798f
function g(x)
	x + 1000
end

# ╔═╡ 3711c338-748f-448e-a06d-d3c29094a14b
map(g, 1:10)

# ╔═╡ f77b7762-29ee-4472-a4ee-a5c1a6dfb369
sqrt.(Square) .+ 100

# ╔═╡ e9d5ee6c-de1a-41ee-850e-a6f5471537ea
@. sqrt(Square) + 100

# ╔═╡ 90fe2c2b-d62d-40b5-9a34-0eb1e9836250
[10 + x for x in 1:100]

# ╔═╡ 72b3ed37-3f3f-4a7d-9036-dadb06c3e1e4
x + y

# ╔═╡ 8c6e966d-9185-4a44-8cd9-e69528cee54b
# methods(+)

# ╔═╡ 1e9be5f5-ab15-4a29-8fd6-d20fee084ad5
abstract type LIKANZHAN <: Signed end #抽象类型

# ╔═╡ 59b884e5-a55a-42f6-9ed4-404d55eae5e9
md"""
## 1.2. 几个例子
"""

# ╔═╡ 3cbd7219-7fb0-4c6e-a2ad-724e97b67d94
function bubbleSort!(a)
	n = length(a)
	for i ∈ 1:n-1
		for j ∈ 1:n-i
			if a[j] > a[j+1]
				a[j], a[j+1] = a[j+1], a[j] # Julian Way
			end
		end
	end
	return a
end

# ╔═╡ 93a86467-cf02-4d3c-b127-79247a588250
data3 = [65, 66, 55, 44, 20, 34, 78, 80]

# ╔═╡ 0423223e-90c5-4d86-8370-66ddee6ba496
with_terminal() do
	@time bubbleSort!(rand(10^5))
end

# ╔═╡ 721c67fd-6d40-4171-89c4-21a1c0064880
with_terminal() do
	@time sort!(rand(10^5))
end

# ╔═╡ 5a4dd40b-d917-4634-a4d4-255c63d68d3f
md"""
- 网络接口和 JSON
"""

# ╔═╡ 20097ff6-c516-454c-978f-4c92e6c72d52
data4 = HTTP.request("GET","https://ocw.mit.edu/ans7870/6/6.006/s08/lecturenotes/files/t8.shakespeare.txt");

# ╔═╡ fbbc0558-7916-4308-b8be-4428f47ca4e5
shakespeare = String(data4)

# ╔═╡ 290d3b7f-ee12-476c-a4f2-189671c84398
# shakespeareWords = split(shakespeare)

# ╔═╡ cc728038-b20f-4b93-97a7-883d5734a0b2
# jsonWords = HTTP.request("GET", "https://raw.githubusercontent.com/"*"h-Klok/StatsWithJuliaBook/master/data/jsonCode.json")

# ╔═╡ 2eca9295-370d-41e9-95a4-b56ee2b0e1dc
"julia" * " is good"

# ╔═╡ b4c5a72d-5ce1-4e21-a04f-8344d1023648
string("Julia", " is Good", ". Python", " is slow.")

# ╔═╡ 4f11cc1a-a78e-4d6c-8bd2-857f85f48aa8
ls = 1:7

# ╔═╡ 50039da8-2c71-4a1d-8cc1-2d4bd8126988
typeof(ls)

# ╔═╡ efb2b80c-af6c-4f6e-bcba-e57c7c39cab3
collect(ls)

# ╔═╡ 1ca893dd-2040-47df-8fa0-c3cb4dd0d7dc
md"""
- 数据存储的常见格式：数组
"""

# ╔═╡ dbbcd955-e0c9-4a6c-81bf-fe5c5ad2f018
md"""
- 数据存储的另外格式： 词典
"""

# ╔═╡ 5565b0a4-792b-40f6-82a2-f8f642432129
dct = Dict("x" => collect(ls), "y" => collect(ls .+ 1) )

# ╔═╡ 6d4175a4-af6e-4a08-bcd2-0b1966ae9035
getindex(dct, "x")

# ╔═╡ c4a58e43-2ea8-48fa-a99d-09f83b1b8ba8
md"""
- 数据存储的另外格式： 元组
"""

# ╔═╡ e0e5ef4a-fc35-4d9b-9a34-a9439d87051d
vec = [1, 2, 3]; typeof(vec)

# ╔═╡ c9d041cb-6a29-449a-8909-b529e8c562a6
yz = (1, 2, 4); typeof(yz)

# ╔═╡ c77241fb-bd9e-4270-bd04-e8b5bcb2406b
md"""
## 1.3. 做图初步
"""

# ╔═╡ 87296377-fe6d-4225-b114-a8c3b8bea1bc
with_terminal() do
	Pkg.status()
end

# ╔═╡ aa61efe4-b3a2-4379-a748-f37cd7f9c6fb
f(x, y) = x^2 + y^2

# ╔═╡ 9d9f4d96-7755-4fd1-851a-092929ad1ac7
f(6)

# ╔═╡ 8b0db1dd-5d79-42e5-9763-ca4ef7259060
map(f, 1:10)

# ╔═╡ 182ab07b-a3e6-47f0-8de5-d59113027c43
f0(x) = f(x, 0)

# ╔═╡ 6262451f-4dcf-4f84-8be6-2f8fbf2631bd
f2(x) = f(x, 2)

# ╔═╡ 3d664567-d879-41f7-9e1a-cdfc08e4e7c9
xVals, yVals = -5:0.1:5, -5:0.1:5

# ╔═╡ 5e03a17e-861e-4749-b18e-00e293d24f28
plot([f0, f2], xVals) # command + /

# ╔═╡ 4d5c7119-f2bf-44ed-bc20-5597a795fbcd
plot(xVals, [f0.(xVals), f2.(xVals)],
	color = [:blue :green],
	xlims = (-6, 6),
	ylims = (-5, 30),
	xlabel = "Range of X",
	ylabel = L"f(x, \cdot)",
	label = [L"f(x, 0)" L"f(x, 2)"],
	legends = :bottomleft
)

# ╔═╡ af2e86c1-4c58-4577-9ae9-313f8ae78475
p1 = annotate!(0, -0.2, text("(0, 0) the mainimum \n of f(x, 0)", :left, :top, 10))

# ╔═╡ 9461292e-3658-41e2-8349-45790fa7c9e0
z = [f(x, y) for y in yVals, x in xVals];

# ╔═╡ 7e03b8d1-1ddb-435f-b9ba-b72b0c44da18
p2 = surface(xVals, yVals, z,
	color = cgrad([:blue, :red]),
	legend = :none,
	ylabel = "y",
	zlabel = L"f(x, y)"
)

# ╔═╡ e54420c1-1ff3-4f38-b9b8-0e3a18127c9a
M = z[1:10, 1:10];

# ╔═╡ 458dfaeb-0370-4e61-90a2-f5b54a0fab42
p3 = heatmap(M,
	color = cgrad([:blue, :red]),
	ylabel = "y",
	xticks = ([1:10; ], xVals),
	yticks = ([1:10; ], yVals)
)

# ╔═╡ b7729c93-781a-46ec-b057-65c4e9e15a04
plot(p1, p2, p3, layout = (1, 3), size = (1200, 400), xlabel = "x", margin = 5mm)

# ╔═╡ 543de112-675c-4368-957d-f37647ad85bd
 L"\frac{\delta}{\gamma\times \cdots}"

# ╔═╡ 37f6486e-e03c-4ddb-bca6-d13157d69822
plot([f0, f2], 1:0.5:10,
	seriestype = [:scatter, :line],
	markersize = 6,
	linewidth = 3, linecolor = :green,
	legend = false)

# ╔═╡ 68d89124-d6e9-4609-b12a-d3b508733680
# surface() # 等价于 plot(.., seriestype = :surface,...,)

# ╔═╡ 6003cd52-89ea-4d93-ac00-43ca395357b8
md"""
### 1.3.1 希尔顿序列
"""

# ╔═╡ 92c42555-8550-4c1b-9b1a-502813c40237
md"""
- 条件句
"""

# ╔═╡ 5c3f1925-ebfb-4bef-8425-635262927bf1
function ff1(x)
	if x % 3 == 0
		return "$x 除以 3 的余数是 0"
	elseif x % 3 == 1
		return "$x 除以 3 的余数是 1"
	else
		return "$x 除以 3 的余数是 2"
	end
end

# ╔═╡ ec02b4a7-529b-4d37-ba4b-bf146a33ee0f
[ff1(x) for x in 1:5]

# ╔═╡ 954f166b-e8db-4cf7-a0a2-75ab3b3acbe0
function ff2(x)
	if x % 3 == 0
		return "x 除3的余数是 0"
	end
end

# ╔═╡ e97dcedb-4b53-4ce8-a0e9-ed20b77d8b2d
[ff2(x) for x in 1:5]

# ╔═╡ f662c9f4-8004-43d6-ad1e-872ad3d24e48
md"""
- 类型稳定性： 除号可能会改变数据的类型
"""

# ╔═╡ c72335d5-39e3-4ab9-a398-e3960dcafe0c
xd = Int(2)

# ╔═╡ 8ebae10c-3148-43fe-9b14-f07934c3b2ea
typeof(xd)

# ╔═╡ e49a1426-bcf5-453a-8753-2bb8387e67eb
yd = xd/2

# ╔═╡ 46bd26e6-c1bf-4fa3-890a-2d424865c2dc
typeof(yd)

# ╔═╡ 55e79482-e30e-4eea-b083-cac2a9667348
md"""
- 希尔顿序列
"""

# ╔═╡ 9537df9b-7fc8-4b32-bc7c-d8ee8730d73a
function hailLength1(x::Int)
	n = 0
	while x != 1
		if x % 2 == 0
			x = Int(x / 2)
		else
			x = 3x + 1
		end
		n += 1 # 等价于 n = n + 1, *=, /=
	end
	return n
end

# ╔═╡ 6434a9f1-aa8a-491f-ac7e-c58b3a4b2638
function hailLength(x::Int)
	n = 0
	while x != 1
		x = x % 2 == 0 ? Int(x / 2) : 3x + 1
		n += 1
	end
	return n
end

# ╔═╡ 4abd57c6-1a0c-4e7c-a195-d07ddc299b2c
md"""
注意： `condition ? res1 : res2` 等价于 `if condition res1 else res2`
"""

# ╔═╡ aa2c895f-7219-4b98-bf50-c96d676ead09
x2 = 13

# ╔═╡ 5fb204e1-d150-4a3c-ac19-6fa5b67b072b
if x2 % 2 == 1 "奇数" else "偶数" end

# ╔═╡ c0df29b8-368d-40d1-bad0-73493531903a
lengths = [hailLength(x0) for x0 in 2:10^7];

# ╔═╡ 51046c66-6e14-420e-8a5e-5a58580841a6
# histogram(lengths)

# ╔═╡ 3659f664-3fc3-48bd-94e3-f57ae84e7301
plot(lengths, seriestype = :histogram,
	bins = 1000,
	# fill = (:green, true) # 元组
	fill = true, color = :green,
	normed = true, legend = false,
	linealpha = 0.2
)

# ╔═╡ 9392cdfb-ceef-40ac-a67b-8c7ca33de123
md"""
### 1.3.2 动图
"""

# ╔═╡ 20605ee6-f245-4e77-b014-a8e42a520e8d
n = 10

# ╔═╡ c69f8278-9627-458b-8204-91ae08efbf92
points = [exp(2 * pi * im * k / n) for k in 1:n];

# ╔═╡ a5f4d797-db11-4d6a-8906-b4af6804a936
plot(points, seriestype = :scatter, legend = false);

# ╔═╡ 46f62dad-820b-42d3-8cdd-ab02cacabc47
xPts, yPts = real.(points), imag.(points);

# ╔═╡ 758082fe-038e-4090-b658-111db3ee1f13
edges  = [(v, u) for v in 1:n for u in (v+1):n];

# ╔═╡ 931e4862-c109-4b17-9048-7826f3dd0c6f
animate = Animation();

# ╔═╡ 3017c6db-acee-422d-bb97-692495972631
plot(xPts, yPts, seriestype = :scatter, legend = false) # 等价于 scatter(xPts, yPts)

# ╔═╡ 404716dd-7a05-4f3b-b336-cab7fc679ed1
for i in eachindex(edges)
	u, v = edges[i][1], edges[i][2]
	xpoints = [xPts[u], xPts[v]]
	ypoints = [yPts[u], yPts[v]]
	plot!(xpoints, ypoints, line = :red, legend = false)
	frame(animate)
end

# ╔═╡ 43b3b163-05a7-4c06-a948-bf5c001b7af0
gif(animate)

# ╔═╡ 4c0169d7-b280-4326-aa79-9c979280077a
md"""
## 1.4. 随机数
"""

# ╔═╡ 694d2ceb-233f-49ea-9416-9e8a8f272d38
md"""
## 1.6. 嵌入其他语言
"""

# ╔═╡ 72a095fa-c5cf-4eb1-909d-d8b49af0bdc9
StatisticsWithJulia.list_notebooks(@__FILE__)

# ╔═╡ 8a20fa4c-338c-48c9-a12a-02e6d67153df
StatisticsWithJulia.footer() ## FFFFF

# ╔═╡ Cell order:
# ╟─ec3f7b5c-cee9-432f-b33c-a08e16c0171a
# ╟─6dd76507-be07-4c3a-9747-99eca74d3519
# ╟─9d98fa7a-3654-40a0-8a96-75c3fca007ae
# ╟─42da8f74-1b44-11ec-32f2-276bdcef1c69
# ╟─e71aacbb-efff-4b93-b5c4-cbdfa88981ad
# ╠═0660d564-9a26-417d-9a2c-42ef181187c2
# ╠═77c5ddd3-ddfc-4a57-ab48-e9db765057d3
# ╟─8eefb8aa-ae85-4231-abf5-9948cdc92ec1
# ╠═d22fc28f-c591-4db2-8275-54d93fda85e9
# ╠═9d9f4d96-7755-4fd1-851a-092929ad1ac7
# ╠═4ece5fc8-18d5-408a-851b-31ce58015e72
# ╠═8b0db1dd-5d79-42e5-9763-ca4ef7259060
# ╠═a0322171-cff9-48bb-9f73-792c8ea4798f
# ╠═3711c338-748f-448e-a06d-d3c29094a14b
# ╠═c266f6e2-4e8e-4870-ac38-d4d0e8b9d042
# ╠═8f3277e9-7905-4beb-9ef1-c3e18b2f449a
# ╠═f6e99ac9-2183-4ea2-991e-f3422b1d1f20
# ╠═cd0ce1b4-9bc9-4218-95e9-e576abeb1af1
# ╠═37fec978-f817-4aa8-85fc-4912eaeb402e
# ╠═adc183db-ff23-43de-8cf6-a961d13c2f01
# ╠═4f1ce95f-2f2f-4817-9d5f-348184a1b800
# ╠═dec64158-9161-4b37-b19f-432a5635719e
# ╠═5a401f10-09d2-4939-87fe-621f469a68ee
# ╠═c2c7c916-1ba8-4e14-a25b-e15faf676f8a
# ╠═2fdb3281-7229-4f18-b9d9-bc5e3086d903
# ╠═f77b7762-29ee-4472-a4ee-a5c1a6dfb369
# ╠═e9d5ee6c-de1a-41ee-850e-a6f5471537ea
# ╠═93b0afb2-e9f3-434d-b21a-2ccad6a12d90
# ╠═90fe2c2b-d62d-40b5-9a34-0eb1e9836250
# ╟─8bcc15c7-d66a-40cc-9f33-50c6de62fadc
# ╠═428c124a-1052-45c8-a49b-b1047267d644
# ╠═863624c7-f378-467a-bed2-94182bae73b7
# ╠═0e8b6a32-ba23-4e37-b78c-8ffb3779a43e
# ╠═9e35d6a1-df94-4a27-9e04-234548495780
# ╠═1b45a9bf-56c7-4b7b-bdd5-b4fcaa572ec0
# ╟─60b263eb-f786-40c4-b60b-2daabdacba7b
# ╠═b0db7a9c-2498-46c5-bf3c-280d3083ba88
# ╠═412ee0c1-26ad-4b9f-9d6e-c552d8431e76
# ╠═aa5e313d-19d9-4300-a65e-f547f7e571d0
# ╠═0ba25fa7-72cb-4d6e-bfaf-eddcf12b9f9c
# ╠═d5251a23-b71e-4faf-9f5a-f6ab1d2ffb7d
# ╠═b717d872-0d16-4556-a6c8-1ccc3c7b76c3
# ╠═72b3ed37-3f3f-4a7d-9036-dadb06c3e1e4
# ╠═8c6e966d-9185-4a44-8cd9-e69528cee54b
# ╠═1e9be5f5-ab15-4a29-8fd6-d20fee084ad5
# ╠═3baed4bd-1c0c-4d44-b7ab-f7c0627fda1b
# ╟─59b884e5-a55a-42f6-9ed4-404d55eae5e9
# ╠═3cbd7219-7fb0-4c6e-a2ad-724e97b67d94
# ╠═93a86467-cf02-4d3c-b127-79247a588250
# ╠═0423223e-90c5-4d86-8370-66ddee6ba496
# ╠═721c67fd-6d40-4171-89c4-21a1c0064880
# ╠═5a4dd40b-d917-4634-a4d4-255c63d68d3f
# ╠═2edcc72c-bfa4-4623-a34f-220c0df9c4d3
# ╠═20097ff6-c516-454c-978f-4c92e6c72d52
# ╠═fbbc0558-7916-4308-b8be-4428f47ca4e5
# ╠═290d3b7f-ee12-476c-a4f2-189671c84398
# ╠═cc728038-b20f-4b93-97a7-883d5734a0b2
# ╠═2eca9295-370d-41e9-95a4-b56ee2b0e1dc
# ╠═b4c5a72d-5ce1-4e21-a04f-8344d1023648
# ╠═4f11cc1a-a78e-4d6c-8bd2-857f85f48aa8
# ╠═50039da8-2c71-4a1d-8cc1-2d4bd8126988
# ╠═efb2b80c-af6c-4f6e-bcba-e57c7c39cab3
# ╟─1ca893dd-2040-47df-8fa0-c3cb4dd0d7dc
# ╟─dbbcd955-e0c9-4a6c-81bf-fe5c5ad2f018
# ╠═5565b0a4-792b-40f6-82a2-f8f642432129
# ╠═6d4175a4-af6e-4a08-bcd2-0b1966ae9035
# ╟─c4a58e43-2ea8-48fa-a99d-09f83b1b8ba8
# ╠═e0e5ef4a-fc35-4d9b-9a34-a9439d87051d
# ╠═c9d041cb-6a29-449a-8909-b529e8c562a6
# ╟─c77241fb-bd9e-4270-bd04-e8b5bcb2406b
# ╠═d507db6a-f869-4dc3-b7ef-fd3518fc1660
# ╠═f924796f-52f3-4b4b-aa47-61a179874647
# ╠═87296377-fe6d-4225-b114-a8c3b8bea1bc
# ╠═aa61efe4-b3a2-4379-a748-f37cd7f9c6fb
# ╠═182ab07b-a3e6-47f0-8de5-d59113027c43
# ╠═6262451f-4dcf-4f84-8be6-2f8fbf2631bd
# ╠═3d664567-d879-41f7-9e1a-cdfc08e4e7c9
# ╠═5e03a17e-861e-4749-b18e-00e293d24f28
# ╠═4d5c7119-f2bf-44ed-bc20-5597a795fbcd
# ╠═af2e86c1-4c58-4577-9ae9-313f8ae78475
# ╠═9461292e-3658-41e2-8349-45790fa7c9e0
# ╠═7e03b8d1-1ddb-435f-b9ba-b72b0c44da18
# ╠═e54420c1-1ff3-4f38-b9b8-0e3a18127c9a
# ╠═458dfaeb-0370-4e61-90a2-f5b54a0fab42
# ╠═b7729c93-781a-46ec-b057-65c4e9e15a04
# ╠═543de112-675c-4368-957d-f37647ad85bd
# ╠═37f6486e-e03c-4ddb-bca6-d13157d69822
# ╠═68d89124-d6e9-4609-b12a-d3b508733680
# ╟─6003cd52-89ea-4d93-ac00-43ca395357b8
# ╟─92c42555-8550-4c1b-9b1a-502813c40237
# ╠═5c3f1925-ebfb-4bef-8425-635262927bf1
# ╠═ec02b4a7-529b-4d37-ba4b-bf146a33ee0f
# ╠═954f166b-e8db-4cf7-a0a2-75ab3b3acbe0
# ╠═e97dcedb-4b53-4ce8-a0e9-ed20b77d8b2d
# ╟─f662c9f4-8004-43d6-ad1e-872ad3d24e48
# ╠═c72335d5-39e3-4ab9-a398-e3960dcafe0c
# ╠═8ebae10c-3148-43fe-9b14-f07934c3b2ea
# ╠═e49a1426-bcf5-453a-8753-2bb8387e67eb
# ╠═46bd26e6-c1bf-4fa3-890a-2d424865c2dc
# ╟─55e79482-e30e-4eea-b083-cac2a9667348
# ╠═9537df9b-7fc8-4b32-bc7c-d8ee8730d73a
# ╠═6434a9f1-aa8a-491f-ac7e-c58b3a4b2638
# ╠═4abd57c6-1a0c-4e7c-a195-d07ddc299b2c
# ╠═aa2c895f-7219-4b98-bf50-c96d676ead09
# ╠═5fb204e1-d150-4a3c-ac19-6fa5b67b072b
# ╠═c0df29b8-368d-40d1-bad0-73493531903a
# ╠═51046c66-6e14-420e-8a5e-5a58580841a6
# ╠═3659f664-3fc3-48bd-94e3-f57ae84e7301
# ╟─9392cdfb-ceef-40ac-a67b-8c7ca33de123
# ╠═20605ee6-f245-4e77-b014-a8e42a520e8d
# ╠═c69f8278-9627-458b-8204-91ae08efbf92
# ╠═a5f4d797-db11-4d6a-8906-b4af6804a936
# ╠═46f62dad-820b-42d3-8cdd-ab02cacabc47
# ╠═758082fe-038e-4090-b658-111db3ee1f13
# ╠═931e4862-c109-4b17-9048-7826f3dd0c6f
# ╠═3017c6db-acee-422d-bb97-692495972631
# ╠═404716dd-7a05-4f3b-b336-cab7fc679ed1
# ╠═43b3b163-05a7-4c06-a948-bf5c001b7af0
# ╟─4c0169d7-b280-4326-aa79-9c979280077a
# ╟─694d2ceb-233f-49ea-9416-9e8a8f272d38
# ╟─72a095fa-c5cf-4eb1-909d-d8b49af0bdc9
# ╟─8a20fa4c-338c-48c9-a12a-02e6d67153df


using JuMP, Ipopt, LinearAlgebra

## Example from Nocedal 16.2 (p 452-453)
x̄ = [2;-1;1]
λ = [3;-2]

# set up model
m = JuMP.Model(solver=IpoptSolver(print_level=0))

# set up primal variables
n = 3
@variable(m, x[1:n])

# set up constraints
G = [6. 2. 1.; 2. 5. 2.; 1. 2. 4.]
c = [-8.; -3.; -3]
A = [1. 0. 1.; 0. 1. 1.]
b = [3.; 0.]

@objective(m, Min, 0.5*x'*G*x + x'*c)

@constraint(m, con, A*x .== b)

print(m)

status = JuMP.solve(m)

# Solution
println("Objective value: ", getobjectivevalue(m))
println("x = ", getvalue(x))
println("λ = ", getdual(con))

##

# set up model
m = JuMP.Model(solver=IpoptSolver())

# set up primal variables
n = 3
@variable(m, x[1:n])

# set up constraints
G = [6. 2. 1.; 2. 5. 2.; 1. 2. 4.]
c = [-8.; -3.; -3]
A = [1. 0. 1.; 0. 1. 1.]
b = [3.; 0.]

@objective(m, Min, x'*x)

@constraint(m, con, G*x - c .== 0)
# @constraint(m, con2, A*x .== b)

print(m)

status = JuMP.solve(m)

# Solution
println("Objective value: ", getobjectivevalue(m))
println("x = ", getvalue(x))
println("λ = ", getdual(con))


## Simple PageRank
# This treats everything as a dense matrix :(
A = [0 1 1 1 1 0 1 0 0 0
     1 0 0 0 0 0 0 0 0 0
     0 0 0 0 1 1 1 0 0 0
     1 1 0 0 1 1 0 1 0 0
     1 1 1 1 0 0 0 0 1 0
     0 0 1 0 0 0 1 0 1 1
     0 0 0 0 0 1 0 0 0 1
     0 0 0 0 0 0 0 0 1 0
     0 0 0 0 0 0 0 1 0 0
     0 0 0 0 0 0 0 0 0 0]
d = sum(A,dims=2)
##
using Printf
for i=1:10
  for j=1:10
    if i==j
      @printf("1 & ")
    elseif A[j,i] != 0
      @printf("-\\alpha/%i & ", d[j])
    else
      @printf("0 & ")
    end
  end
  println("\\\\")
end
## Build the I - alpha D^+ A'
alpha = 0.85
M = zeros(size(A)...)
for i=1:10
  for j=1:10
    if i==j
      M[i,j] = 1
    elseif A[j,i] != 0
      M[i,j] = -alpha*A[j,i]/d[j]
    else
      M[i,j] = 0
    end
  end
end
M[:,end] .= -alpha/10
M[end,end] += 1
M[:,1]
##
b = ones(size(A,1))/size(A,1)*(1-alpha)
x = M\b


# ---
# title: 1175. Prime Arrangements
# id: problem1175
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/prime-arrangements/description/>
# hidden: true
# ---
#
# Return the number of permutations of 1 to `n` so that prime numbers are at
# prime indices (1-indexed.)
#
# _(Recall that an integer  is prime if and only if it is greater than 1, and
# cannot be written as a product of two positive integers both smaller than
# it.)_
#
# Since the answer may be large, return the answer **modulo`10^9 + 7`**.
#
#
#
# **Example 1:**
#
#
#
#     Input: n = 5
#     Output: 12
#     Explanation: For example [1,2,5,4,3] is a valid permutation, but [5,2,3,4,1] is not because the prime number 5 is at index 1.
#
#
# **Example 2:**
#
#
#
#     Input: n = 100
#     Output: 682289015
#
#
#
#
# **Constraints:**
#
#   * `1 <= n <= 100`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


@polly function kernel_adi(tsteps, u, v, p, q)
    n = size(u,1)

    DX = 1.0 / n
    DY = 1.0 / n
    DT = 1.0 / tsteps
    B1 = 2.0
    B2 = 1.0
    mul1 = B1 * DT / (DX * DX)
    mul2 = B2 * DT / (DY * DY)

    a = -mul1 /  2.0;
    b = 1.0 + mul1
    c = a
    d = -mul2 / 2.0
    _e = 1.0 + mul2
    f = d

    for t = 1:tsteps
        # Column Sweep
        for i = 2:(n-1)
            v[1,i] = 1.0
            p[i,1] = 0.0
            q[i,1] = v[1,i]
            for j = 2:(n-1)
                p[i,j] = -c / (a * p[i,j-1] + b)
                q[i,j] = (-d*u[j,i-1]+(1.0+2.0*d)*u[j,i] - f*u[j,i+1]-a*q[i,j-1])/(a*p[i,j-1]+b)
            end

            v[n,i] = 1.0
            for j = (n-1):-1:2
                v[j,i] = p[i,j] * v[j+1,i] + q[i,j]
            end
        end

        # Row Sweep
        for i = 2:(n-1)
            u[i,1] = 1.0
            p[i,1] = 0.0
            q[i,1] = u[i,1]
            for j = 2:(n-1)
                p[i,j] = -f / (d*p[i,j-1] + _e)
                q[i,j] = (-a*v[i-1,j]+(1.0+2.0*a)*v[i,j] - c*v[i+1,j]-d*q[i,j-1])/(d*p[i,j-1]+_e)
            end

            u[i,n] = 1.0
            for j = (n-1):-1:2
                u[i,j] = p[i,j] * u[i,j+1] + q[i,j]
            end
        end
    end
end

let
    tsteps = 500
    n = 1000

    u = zeros(Float32, n, n)
    v = zeros(Float32, n, n)
    p = zeros(Float32, n, n)
    q = zeros(Float32, n, n)

    for i = 1:n, j = 1:n
        u[i,j] =  (i + n-j) / n
    end

    SUITE["adi"] = @benchmarkable kernel_adi($tsteps, u, v, p, q) setup = (u = copy($u); v = copy($v); p = copy($p); q = copy($q))
end


using Plots, Distributions, LaTeXStrings

# Evaluates target distribution π(q)
function π_q(q)
    return 0.3*1/sqrt(2*pi)*exp(-(q+2)^2/2) + 0.7*1/sqrt(pi)*exp(-(q-3)^2)
end

# Evaluates conditional probability π(p|q)
function π_pq(p,q)
    return pdf(Normal(0, 1), p)
end

# Evaluates Kinetic enery K(p,q)
function K(p,q)
    return -log(π_pq(p,q))
end

# Evaluates Potential energy V(q)
function V(q)
    return -log(π_q(q))
end

# Evaluates dV/dq at q
function dV_dq(q)
    return 1/π_q(q) * (0.3*1/sqrt(2*pi)*(q+2)*exp(-(q+2)^2/2) +
    0.7*2/sqrt(pi)*(q-3)*exp(-(q-3)^2))
end

# Compute Hamiltonian at q,p
function H(p,q)
    return K(p,q) + V(q)
end

# Gets a trajectory of length L with n steps
function get_trajectory(q,p,L,n)
    # Flip sign of momentum
    # p *= -1

    # Step size
    ϵ = L/n

    q_trajectory = zeros(n)
    p_trajectory = zeros(n)

    q_trajectory[1] = q
    p_trajectory[1] = p

    for j=2:n
        ψ = p_trajectory[j-1] - ϵ/2 * dV_dq(q_trajectory[j-1])
        q_trajectory[j] = q_trajectory[j-1] + ϵ*ψ
        p_trajectory[j] = ψ - ϵ/2*dV_dq(q_trajectory[j])
    end

    return q_trajectory, p_trajectory
end

function hmc_alg(q₀, p₀)
    # Number of samples to return
    n = 20000

    q_samples = zeros(n)
    p_samples = zeros(n)
    q_samples[1] = q₀
    p_samples[1] = p₀

    trajectory_length = 1
    trajectory_partitions = 3

    for j=2:n

        q = q_samples[j-1]
        p = rand(Normal(0,1))
        q_proposal_trajectory, p_proposal_trajectory =
        get_trajectory(q, p, trajectory_length, trajectory_partitions)

        # Proposal
        q_L = last(q_proposal_trajectory)
        p_L = last(p_proposal_trajectory)

        # Compute acceptance probabilty
        α = min(1, exp(H(p,q) - H(-p_L,q_L)))

        if rand() < α
            q_samples[j] = q_L
            p_samples[j] = -p_L
        else
            q_samples[j] = q
            p_samples[j] = p
        end

    end

    return q_samples
end

# Generate HMC samples
samples = hmc_alg(0,1)

# Plot histogram of samples
x_vals = -5:0.01:5
y_vals = [π_q(q) for q in x_vals]
Plots.plot(x_vals, y_vals,
 bg = RGB(247/255, 236/255, 226/255),
 grid = false,
 legend = false,
 xlabel = L"q",
 ylabel = L"\pi(q)")
histogram!(samples, color = "blue", opacity = 0.15, normalize = true)

savefig("../images/hmc_ex_1_histogram.svg")


using LinearAlgebra: Matrix, sqrt
using LinearAlgebra
using FFTW
using MAT

function alpha1D(k, N)
    if k>0
        return sqrt(2/N)
    else
        return sqrt(1/N)
    end
end

function alpha2D(k, l, N)
    if k==0 && l==0
        return 1/N
    elseif l==0 || k==0
        return sqrt(2)/N
    else
        return 2/N
    end
end

function mDCT(v)
    N = length(v)
    c = zeros(1, N)

    for k = 0:(N-1)
        sum = 0.0
        for i = 0:(N-1)
            sum = sum + v[i+1] * cos(k*pi*((2*i+1)/(2*N)))
        end
        c[k+1] = alpha1D(k,N) * sum
    end
    return c
end

function miDCT(c)
    N = length(c)
    v = zeros(1,N)
    for j = 0:(N-1)
        for k = 0:(N-1)
            ak= alpha1D(k,N)
            v[j+1] = v[j+1] +(c[k+1] * ak * cos(pi*k*((2*j)+1)/(2*N)))
        end
    end
    return v
end

function mDCT2n4(matrix)
    N = size(matrix,1)
    M = size(matrix,1)

    cmat = zeros(N, M)

    for k = 0:(N-1)
        for l = 0:(M-1)
            sum = 0
            for i = 0:(N-1)
                for j = 0:(M-1)
                    sum = sum + (matrix[i+1, j+1] * cos(k*pi*((2*i+1)/(2*N))) * cos(l*pi*((2*j+1)/(2*M))))
                end
            end
            cmat[k+1, l+1] = alpha2D(k,l,N) * sum
        end
    end
    return cmat
end

function miDCT2n4(cmat)
    N = size(matrix,1)
    M = size(matrix,2)
    matrix = zeros(N, M)
    sum = 0
    for i = 0:(N-1)
        for j = 0:(M-1)
            for k = 0:(N-1)
                for l = 0:(M-1)
                    ak2 = alpha2D(k,l,N)
                    matrix[i+1, j+1] = matrix[i+1, j+1] + (cmat[k+1, l+1] *ak2 *cos(pi*k*((2*i)+1)/(2*N)) * cos(pi*l*((2*j)+1)/(2*M)))
                end
            end
        end
    end
    return matrix
end

function mDCT2n3(matrix)
    N = size(matrix,1)
    M = size(matrix,2)
    cmat = zeros(N,M)

    for j = 0:(N-1)
        ind = j+1
        cmat[ind, :] = mDCT(matrix[ind,:])
    end
    cmatT = transpose(cmat)
    for k = 0:(M-1)
        cmatT[k+1,:] = mDCT(cmatT[k+1,:])
    end
    cmat = transpose(cmatT)
    return cmat
end

function miDCT2n3(cmat)
    N = size(cmat,1)
    M = size(cmat,2)
    matrix = zeros(N,M)
    matrixT = zeros(N,M)

    for j = 0:(N-1)
        matrix[j+1,:] = miDCT(cmat[j+1,:])
    end
    matrixT = transpose(matrix)
    for k = 0:(M-1)
        matrixT[k+1,:] = miDCT(matrixT[k+1,:])
    end
    matrix = transpose(matrixT)
    return matrix
end


import LinearAlgebra

import RungeKutta.Tableaus: get_lobatto_nodes, get_lobatto_weights, get_gauss_nodes

@doc raw"""
The projective Lobatto-GLRK coefficients are implicitly given by
```math
\sum \limits_{j=1}^{s} a_{ij} c_{j}^{k-1} = \frac{\bar{c}_i^k}{k}  \qquad i = 1 , \, ... , \, \sigma , \; k = 1 , \, ... , \, s ,
```
where $c$ are Gauß-Legendre nodes with $s$ stages and $\bar{c}$ are Gauß-Lobatto nodes with $\sigma$ stages.
"""
function get_lobatto_glrk_coefficients(s, σ=s+1, T=Float64)
    if σ == 1
        @error "Lobatto III coefficients for one stage are not defined."
    end

    c = get_gauss_nodes(s)
    b̄ = get_lobatto_weights(σ)
    c̄ = get_lobatto_nodes(σ)
    M = [ c[j]^(k-1) for k in 1:s, j in 1:s ]

    row(i) = begin
        r = [ c̄[i]^k / k for k in 1:s ]
        M \ r
    end

    ā = vcat([row(i)' for i in 1:σ]...)

    CoefficientsIRK{T}(:LobattoIIIGLRK, s^2, s, σ, ā, b̄, c̄)
end

function get_lobatto_ω_matrix(s)
    as = TableauLobattoIIIA(s).a[2:s,1:s]
    es = zeros(s)
    es[s] = 1

    Q = vcat( hcat(as, zeros(s-1)), hcat(zeros(s)', 1) )
    L = vcat( as, es' )
    ω = inv(L) * Q

    return ω
end


"""
    lorenz_key(x_init, y_init, z_init, num_keys; α=10.0, ρ=28.0, β=2.667, dt=0.01, scaling_factor=10.0^16, upper_bound=256.0)

Generates 3 vectors of pseudo-random numbers using Lorenz system of differential
equations.

The equations -

\$\\frac{dx}{dt} = α * (y - x)\$
\$\\frac{dy}{dt} = x * (ρ - z) - y\$
\$\\frac{dz}{dt} = x * y - β * z\$

# Arguments
- `x_init::Float64`: Initial value of x.
- `y_init::Float64`: Initial value of y.
- `z_init::Float64:` Initial value of z.
- `num_keys::Int64`: Number of keys (in a single list) to be generated.
- `α::Float64`: Constant associated with Lorenz system of differential equations.
- `ρ::Float64`: Constant associated with Lorenz system of differential equations.
- `β::Float64`: Constant associated with Lorenz system of differential equations.
- `scaling_factor::Float64=10.0^16`: Factor to be multiplied to the generated value of pseudo-random
    number. Ideally, the factor should be > upper_bound.
- `upper_bound::Float64=256.0`: Upper bound of keys (not included). Use 256 for encrypting images
    as the RGB values of a pixel varies from 0 to 255.

# Returns
- `x::Vector{Int64}`: Generated pseudo-random keys corresponding to x values.
- `y::Vector{Int64}`: Generated pseudo-random keys corresponding to y values.
- `z::Vector{Int64}`: Generated pseudo-random keys corresponding to z values.

# Example
```jldoctest
julia> lorenz_key(0.01, 0.02, 0.03, 20)
([0, 0, 256, 24, 129, 42, 54, 134, 43, 179, 85, 19, 24, 44, 71, 210, 238, 152, 22, 27], [0, 0, 240, 55, 25, 163, 89, 243, 123, 5, 197, 64, 227, 54, 188, 226, 154, 134, 64, 69], [0, 0, 80, 227, 178, 204, 89, 33, 144, 139, 105, 208, 108, 155, 61, 254, 57, 102, 149, 47])
```
"""
function lorenz_key(
    x_init::Float64,
    y_init::Float64,
    z_init::Float64,
    num_keys::Int64;
    α::Float64=10.0,
    ρ::Float64=28.0,
    β::Float64=2.667,
    dt::Float64=0.01,
    scaling_factor::Float64=10.0^16,
    upper_bound::Float64=256.0
)
    # Initializing 3 empty lists
    x = zeros(Float64, num_keys)
    y = zeros(Float64, num_keys)
    z = zeros(Float64, num_keys)

    # Initializing initial values
    x[1], y[1], z[1] = x_init, y_init, z_init

    # System of equations
    for i = 1:num_keys - 1
        x[i + 1] = x[i] + (α * (y[i] - x[i]) * dt)
        y[i + 1] = y[i] + ((x[i] * (ρ - z[i]) - y[i]) * dt)
        z[i + 1] = z[i] + ((x[i] * y[i] - β * z[i]) * dt)
    end

    for i = 1:length(x)
        x[i] = x[i] * scaling_factor % upper_bound
        y[i] = y[i] * scaling_factor % upper_bound
        z[i] = z[i] * scaling_factor % upper_bound
    end

    x = round.(Int, x)
    y = round.(Int, y)
    z = round.(Int, z)

    return x, y, z
end


#############################################################################
# lieb_ando.jl
# Returns tr(K' * A^{1-t} * K * B^t) where A and B are positive semidefinite
# matrices and K is an arbitrary matrix (possibly rectangular).
#
# Disciplined convex programming information:
#    lieb_ando(A,B,K,t) is concave in (A,B) for t in [0,1], and convex
#    in (A,B) for t in [-1,0] or [1,2]. K is a fixed matrix.
#
# Seems numerically unstable when t is on the endpoints of these ranges.
#
# All expressions and atoms are subtypes of AbstractExpr.
# Please read expressions.jl first.
#
#REFERENCE
#   Ported from CVXQUAD which is based on the paper: "Lieb's concavity
#   theorem, matrix geometric means and semidefinite optimization" by Hamza
#   Fawzi and James Saunderson (arXiv:1512.03401)
#############################################################################

const MatrixOrConstant = Union{AbstractMatrix, Constant}

function lieb_ando(A::MatrixOrConstant, B::MatrixOrConstant, K::MatrixOrConstant, t::Rational)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    return real(tr(K' * A^(1-t) * K * B^t))
end

function lieb_ando(A::MatrixOrConstant, B::AbstractExpr, K::MatrixOrConstant, t::Rational)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    KAK = K' * A^(1-t) * K
    KAK = (KAK+KAK')/2
    return trace_mpower(B, t, KAK)
end

function lieb_ando(A::AbstractExpr, B::MatrixOrConstant, K::MatrixOrConstant, t::Rational)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    KBK = K * B^t * K'
    KBK = (KBK+KBK')/2
    return trace_mpower(A, 1-t, KBK)
end

function lieb_ando(A::AbstractExpr, B::AbstractExpr, K::MatrixOrConstant, t::Rational)
    n = size(A,1)
    m = size(B,1)
    Kvec = reshape(K',n*m,1)
    KvKv = Kvec * Kvec'
    KvKv = (KvKv+KvKv')/2
    Im = Matrix(1.0*I, m, m)
    In = Matrix(1.0*I, n, n)

    is_complex = sign(A) == ComplexSign() || sign(B) == ComplexSign() || sign(Constant(K)) == ComplexSign()
    if is_complex
        T = HermitianSemidefinite(n*m)
    else
        T = Semidefinite(n*m)
    end

    if t >= 0 && t <= 1
        # Concave function
        add_constraint!(T, T in GeomMeanHypoCone(kron(A,Im), kron(In,conj(B)), t, false))
        return real(tr(KvKv * T))
    elseif (t >= -1 && t <= 0) || (t >= 1 && t <= 2)
        # Convex function
        add_constraint!(T, T in GeomMeanEpiCone(kron(A,Im), kron(In,conj(B)), t, false))
        return real(tr(KvKv * T))
    else
        throw(DomainError(t, "t must be between -1 and 2"))
    end
end


function rref(mat::Matrix)
    # make a copy of the original matrix
    ret = float(copy(mat))
    # # ret = copy(mat)
    # initialize an array to memorize the pivots
    pivot = []
    # it's assumed that the matrix has $m$ rows and $n$ columns
    m, n = size(ret)
    # $i$ and $j$ are used to represent iterators of row and column,
    # respectively
    i = j = 1
    # part of `forward elimination'
    while i <= m && j <= n
        # find the entry with largest absolute value
        imax, vmax = 0.0, 0.0
        for itr1 in i : m
            if vmax < abs(ret[itr1, j])
                imax, vmax = itr1, abs(ret[itr1, j])
            end
        end
        # but this one seems to work pretty fine?
        (vmax > 0) ? (k = imax) : (j += 1; continue)
        # # if vmax > 0
        # #     k = imax
        # # else
        # #     j += 1
        # #     continue
        # # end
        # add the pivot position to the array
        push!(pivot, (i, j))
        # exchange the rows
        for itr1 in i : n
            ret[i, itr1], ret[k, itr1] = ret[k, itr1], ret[i, itr1]
        end
        # `forward elimination'
        for itr1 in (i + 1) : m
            scaler = ret[itr1, j] / ret[i, j]
            for itr2 in (j + 1) : n
                ret[itr1, itr2] -= ret[i, itr2] * scaler
            end
            # manually set this entry to 0, in case of using
            # LU factorization
            ret[itr1, j] = 0
        end
        i += 1
        j += 1
    end
    # part of `back subsitution'
    for itr1 in length(pivot) : -1 : 1
        # get the pivot
        i, j = pivot[itr1]
        # `back subsitution`
        for itr2 in 1 : (i - 1)
            scaler = ret[itr2, j] / ret[i, j]
            for itr3 in j : n
                ret[itr2, itr3] -= ret[i, itr3] * scaler
            end
        end
        # make entry on the pivot be 1
        # and a little trick in order not to create a new variable
        for itr2 in n : -1 : j
            ret[i, itr2] /= ret[i, j]
        end
    end
    return ret
end


#!/usr/bin/env julia
# 1.7: Roots of a polynomial
using Roots

function polynomialGenerator(a...)
  n = length(a) - 1
  poly = function (x)
    return sum([a[i + 1] * x^i for i in 0:n])
  end
  return poly
end

polynomial = polynomialGenerator(1, 3, -10)
zeroVals = find_zeros(polynomial, -10, 10)
println("Zeros of the function f(x): ", zeroVals)


##############################################################
"""
$(SIGNATURES)

Calculation of Bernoulli function via Horner scheme based on Taylor
coefficients around 0.
"""

function bernoulli_horner(x)
    y=x/47_900_160
    y=x*y
    y=x*(-1/1_209_600+y)
    y=x*y
    y=x*(1/30_240+y)
    y=x*y
    y=x*(-1/720+y)
    y=x*y
    y=x*(1/12+y)
    y=x*(-1/2+y)
    y=1+y
end

# Bernoulli thresholds optimized for Float64
const bernoulli_small_threshold=0.25
const bernoulli_large_threshold=40.0
##############################################################
"""
$(SIGNATURES)

Bernoulli function ``B(x)=\\frac{x}{e^x-1}`` for exponentially
fitted upwinding.

The name `fbernoulli` has been chosen to avoid confusion
with Bernoulli from JuliaStats/Distributions.jl

Returns a real number containing the result.
"""
function fbernoulli(x)
    if x<-bernoulli_large_threshold
        -x
    elseif x>bernoulli_large_threshold
	zero(x)
    else
        expx=exp(x)
        expxm1=expx-1.0
        if abs(expxm1)>bernoulli_small_threshold
            x/expxm1
        else
            bernoulli_horner(x)
        end
    end
end

##############################################################

"""
$(SIGNATURES)

Bernoulli function ``B(x)=\\frac{x}{e^x-1}`` for exponentially
fitted upwind, joint evaluation for positive and negative
argument

Usually, we need ``B(x), B(-x)`` togehter,
and it is cheaper to calculate them together.

Returns two real numbers containing the result for argument
`x` and argument `-x`.

The error in comparison with the evaluation of the original expression
with BigFloat is less than 1.0e-15
"""
function fbernoulli_pm(x)
    if x< -bernoulli_large_threshold
        return -x, zero(x)
    elseif x>bernoulli_large_threshold
	return zero(x),x
    else
        expx=exp(x)
        expxm1=expx-1.0
        if abs(expxm1)>bernoulli_small_threshold
            bp=x/expxm1
            bm=x/(1.0-1.0/expx)
            return bp,bm
        else
            y=bernoulli_horner(x)
            return y,x+y
        end
    end
end


# --- User Parameters

# Number of samples to use to estimate probability distributions
sample_size = 10000;

# --- Imports

import Distributions
using Distributions: MvNormal, ContinuousUnivariateDistribution, cdf
using HypothesisTests
using LinearAlgebra
using Plots
using Random
using StatsBase: Histogram, fit

# --- Internal Parameters

# ------ Uniformity test parameters

# p-value cutoff
p_value_cutoff = 0.1;

# maximum number of attempts to generate a sample of vectors
max_attempts = 10;

# --- Probability distribution for θ (angle between vector and x_1-axis)

struct ThetaDistribution <: ContinuousUnivariateDistribution
    n::Integer
end

function Distributions.cdf(dist::ThetaDistribution, x::Real)
    # Preparations
    k = dist.n-2

    # Compute pdf
    value = 0
    if k % 2 == 0
        # Precompute re-used coefficients
        sgn = (-1)^(k÷2)
        k_choose_k_over_2 = binomial(big(k), k÷2)

        for j = 0:(k÷2 - 1)
            value += (-1)^j * binomial(big(k), j) * sin((k - 2*j) * x) / (k - 2*j)
        end
        value *= 2
        value += sgn * k_choose_k_over_2 * x  # contribution from i = k÷2 term
        value *= sgn / π / k_choose_k_over_2

    else
        norm_ = 0
        for j = 0:(k-1)÷2
            coef = (-1)^j * binomial(big(k), j) / (k - 2*j)
            value += coef * (1 - cos((k - 2*j) * x))
            norm_ += coef
        end
        value /= 2 * norm_

    end

    return value
end

# --- Tests

# 2D: cdf(x) = x / π
theta_dist_2d = ThetaDistribution(2)
for x = range(0, π; length=10)
    computed_cdf = cdf(theta_dist_2d, x)
    expected_cdf = x / π
    @assert (computed_cdf ≈ expected_cdf) "Expected: $(expected_cdf). Got: $(computed_cdf)"
end

# 3D: cdf(x) = 0.5 * (1 - cos(x))
theta_dist_3d = ThetaDistribution(3)
for x = range(0, π; length=10)
    computed_cdf = cdf(theta_dist_3d, x)
    expected_cdf = 0.5 * (1 - cos(x))
    @assert (computed_cdf ≈ expected_cdf) "Expected: $(expected_cdf). Got: $(computed_cdf)"
end

function generate_blades(n::Integer, s::Integer, sample_size::Integer)::Tuple
    """
    Generate a sample of s-dimensional blades in an n-dimensional space where each blade
    is defined by unit vectors drawn from a uniform distribution over the surface of an
    unit n-sphere.

    Parameters
    ----------
    n: dimension of space

    s: grade of blades

    sample_size: number of blades to include in sample

    Return values
    -------------
    vectors: sample of vectors

    uniformity_stats: results of tests for uniformity of sample on unit n-sphere
    """
    # --- Set up uniformity tests

    theta_dist = ThetaDistribution(n)
    AD_p_value = 0
    KS_p_value = 0

    # --- Generate sample of vectors

    num_attempts = 0
    vectors = undef
    while ( ((AD_p_value < p_value_cutoff) || (KS_p_value < p_value_cutoff)) &&
            (num_attempts < max_attempts) )

        # Generate new sample
        dist = MvNormal(zeros(n), ones(n))
        num_vectors = sample_size * s
        vectors = rand(dist, num_vectors)
        for i = 1:num_vectors
            vectors[:, i] /= norm(vectors[:, i])
        end

        # Perform test for uniformity on unit n-sphere
        thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

        # Anderson-Darling Test
        AD_p_value = pvalue(OneSampleADTest(thetas, theta_dist))

        # Kolmogorov-Smirnov Test
        KS_p_value = pvalue(ExactOneSampleKSTest(thetas, theta_dist))

        # Update attempt count
        num_attempts += 1
    end

    # --- Group vectors into blades

    blades = []
    for i = 1:sample_size
        push!(blades, vectors[:, (i-1)*s+1:i*s])
    end

    # --- Save uniformity test results

    uniformity_stats = (
        success=(num_attempts < max_attempts),
        num_attempts=num_attempts,
        AD_p_value=AD_p_value,
        KS_p_value=KS_p_value,
    )

    return blades, uniformity_stats
end

# --- Generate blades

# Parameters
n_2d = 2;  # dimension of space
s = 2;  # grade of blades

# Generate sample of blades
blades, uniformity_stats = generate_blades(n_2d, s, sample_size)
if !uniformity_stats[:success]
    println("FAILED to generated sample with sufficient uniformity on unit circle.")
    display(uniformity_stats)
end

# --- Compute blade volumes

signed_volumes = Vector{Float64}()
for i = 1:sample_size
    F = qr(blades[i])
    signed_volume = det(F.R)
    push!(signed_volumes, signed_volume)
end

volumes = abs.(signed_volumes);

# --- Compute distributions

# Histogram parameters
num_hist_bins = 50

# Plot histograms
hist_bins_signed = range(-1, 1; length=num_hist_bins)
plt_signed = plot(histogram(signed_volumes; bins=hist_bins_signed, normalize=true))

hist_bins = range(0, 1; length=num_hist_bins)
plt = plot(histogram(volumes; bins=hist_bins, normalize=true))

display(plt_signed)
display(plt)

# Compute empirical volume distributions
hist_signed = fit(Histogram, signed_volumes; nbins=num_hist_bins)
edf_signed = normalize(hist_signed; mode=:pdf);

hist = fit(Histogram, volumes; nbins=num_hist_bins)
edf = normalize(hist; mode=:pdf);

# --- Generate blades

# Parameters
n_3d = 3;  # dimension of space
s = 2;  # grade of blades

# Generate sample of blades
blades, uniformity_stats = generate_blades(n_3d, s, sample_size)
if !uniformity_stats[:success]
    println("FAILED to generated sample with sufficient uniformity on unit sphere.")
    display(uniformity_stats)
end

# --- Compute blade volumes

signed_volumes = Vector{Float64}()
for i = 1:sample_size
    F = qr(blades[i])
    signed_volume = det(F.R)
    push!(signed_volumes, signed_volume)
end

volumes = abs.(signed_volumes);

# --- Compute distributions

# Histogram parameters
num_hist_bins = 50

# Plot histograms
hist_bins_signed = range(-1, 1; length=num_hist_bins)
plt_signed = plot(histogram(signed_volumes; bins=hist_bins_signed, normalize=true))

hist_bins = range(0, 1; length=num_hist_bins)
plt = plot(histogram(volumes; bins=hist_bins, normalize=true))

display(plt_signed)
display(plt)

# Compute empirical volume distributions
hist_signed = fit(Histogram, signed_volumes; nbins=num_hist_bins)
edf_signed = normalize(hist_signed; mode=:pdf);

hist = fit(Histogram, volumes; nbins=num_hist_bins)
edf = normalize(hist; mode=:pdf);

# --- Generate blades

# Parameters
n = 100;  # dimension of space
s = 10;  # grade of blades

# Generate sample of vectors
blades, uniformity_stats = generate_blades(n, s, sample_size)
if !uniformity_stats[:success]
    println("FAILED to generated sample with sufficient uniformity on unit $(n)-sphere.")
    display(uniformity_stats)
end

# --- Compute blade volumes

signed_volumes = Vector{Float64}()
for i = 1:sample_size
    F = qr(blades[i])
    signed_volume = det(F.R)
    push!(signed_volumes, signed_volume)
end

volumes = abs.(signed_volumes);

# --- Compute distributions

# Histogram parameters
num_hist_bins = 50

# Plot histograms
hist_bins_signed = range(-1, 1; length=num_hist_bins)
plt_signed = plot(histogram(signed_volumes; bins=hist_bins_signed, normalize=true))

hist_bins = range(0, 1; length=num_hist_bins)
plt = plot(histogram(volumes; bins=hist_bins, normalize=true))
display(plt_signed)
display(plt)

# Compute empirical volume distributions
hist_signed = fit(Histogram, signed_volumes; nbins=num_hist_bins)
edf_signed = normalize(hist_signed; mode=:pdf);

hist = fit(Histogram, volumes; nbins=num_hist_bins)
edf = normalize(hist; mode=:pdf);

# --- Generate blades

# Parameters
n = 500;  # dimension of space
s = 10;  # grade of blades

# Generate sample of vectors
blades, uniformity_stats = generate_blades(n, s, sample_size)
if !uniformity_stats[:success]
    println("FAILED to generated sample with sufficient uniformity on unit $(n)-sphere.")
    display(uniformity_stats)
end

# --- Compute blade volumes

signed_volumes = Vector{Float64}()
for i = 1:sample_size
    F = qr(blades[i])
    signed_volume = det(F.R)
    push!(signed_volumes, signed_volume)
end

volumes = abs.(signed_volumes);

# --- Compute distributions

# Histogram parameters
num_hist_bins = 50

# Plot histograms
hist_bins_signed = range(-1, 1; length=num_hist_bins)
plt_signed = plot(histogram(signed_volumes; bins=hist_bins_signed, normalize=true))

hist_bins = range(0, 1; length=num_hist_bins)
plt = plot(histogram(volumes; bins=hist_bins, normalize=true))
display(plt_signed)
display(plt)

# Compute empirical volume distributions
hist_signed = fit(Histogram, signed_volumes; nbins=num_hist_bins)
edf_signed = normalize(hist_signed; mode=:pdf);

hist = fit(Histogram, volumes; nbins=num_hist_bins)
edf = normalize(hist; mode=:pdf);


function numcollatz(n)
	numiters = 0
	while n > 1
		#print("$n ")
		if n % 2 == 0
			n = div(n,2)
		else
			n = 3n + 1
		end
		#println("$n")
		numiters += 1
	end
	return numiters
end

function main()
	N = 1_000_000

	bests = 0
	bestl = 0
	i = 0
	while i <= N
		i += 1
		l = numcollatz(i)
		if l > bestl
			println("$i $l")
			bestl = l
			bests = i
		end
	end
	println("start=$bests length=$bestl")
end

main()


#Wigner semicircle distribution

importall Distributions
export Semicircle

immutable Semicircle <: ContinuousUnivariateDistribution
    mean::Float64
    radius::Float64
    Semicircle(mu, r) = r > 0 ? new(float64(mu), float64(r)) :
        error("radius r must be positive") #Constructor
end

#Standardized distribution - mean 0, variance 1
Semicircle(mu) = Semicircle(mu, 2.0)
Semicircle() = Semicircle(0.0, 2.0)

# Distribution function methods
###############################

# cumulative distribution function
function cdf(X::Semicircle, x::Real)
    r, a = X.mean, X.radius
    insupport(x) ? 0.5 + (x-a)/(pi*r^2) * sqrt(r^2 - (x-a)^2) + 1/pi * asin((x-a)/r) : (x>a ? 1.0 : 0.0)
end

# probability density function
function pdf(X::Semicircle, x::Real)
    r, a = X.mean, X.radius
    insupport(x) ? 2/(pi*r^2) * sqrt(r^2 - (x-a)^2) : 0.0
end

# predicate is x in the support of the distribution?
insupport(X::Semicircle, x::Real)=abs(x)<=X.radius

#Entropy methods
################

# entropy of distribution in nats
entropy(X::Semicircle)=log(pi*X.radius) - 0.5

#Measures of central tendency methods
#####################################

# mean of distribution
mean(X::Semicircle)=X.mean

# median of distribution
median(X::Semicircle)=X.mean

# mode(s) of distribution as vector
modes(X::Semicircle)=[X.mean]

# kurtosis of the distribution
kurtosis(X::Semicircle)=2

# skewness of the distribution
skewness(X::Semicircle)=0

# standard deviation of distribution
std(X::Semicircle)=X.radius/2

# variance of distribution
var(X::Semicircle)=std(X)^2

# moment of distribution
if _HAVE_GSL
  function moment(X::Semicircle, order::Integer)
    a, r = X.mean, X.radius
    if X.mean != 0
      a^n*hypergeom([(1-n)/2, -n/2], 2, (r/a)^2)
    else
      order%2 ? (0.5*r)^(2n) * catalan(div(order,2)) : 0
    end
  end
else
  function moment(X::Semicircle, order::Integer)
    error("The moment of a semicircle distribution requires the GSL module to be loaded.")
  end
end

# cumulant of distribution
function cumulant(X::Semicircle, order::Integer)
  if X.mean != 0 error("not supported") end
  if order%2
    order==0 ? 1 : (0.5*r)^(2n) * lassalle(order/2)
  else
    0
  end
end

# free cumulant of distribution
function freecumulant(X::Semicircle, order::Integer)
  if order == 0
    return 1
  elseif order == 1
    return mean(X)
  elseif order == 2
    return var(X)
  else
    return 0
  end
end

#Generating function methods
############################

# characteristic function
function cf(X::Semicircle, t::Real)
  r = t * X.mean
  2 * besselj(1, r)/r
end

# moment generating function
function mgf(X::Semicircle, t::Real)
  r = t * X.mean
  2 * besseli(1, r)/r
end

#Sampling methods
#################

# random sampler
# Use relationship with beta distribution
function rand(X::Semicircle)
  Y = rand(Beta(1.5, 1.5))
  X.mean + 2 * X.radius * Y - X.radius
end


# ---
# title: 264. Ugly Number II
# id: problem264
# author: Indigo
# date: 2021-06-05
# difficulty: Medium
# categories: Math, Dynamic Programming, Heap
# link: <https://leetcode.com/problems/ugly-number-ii/description/>
# hidden: true
# ---
#
# Write a program to find the `n`-th ugly number.
#
# Ugly numbers are **positive numbers** whose prime factors only include `2, 3,
# 5`.
#
# **Example:**
#
#
#
#     Input: n = 10
#     Output: 12
#     Explanation:1, 2, 3, 4, 5, 6, 8, 9, 10, 12 is the sequence of the first 10 ugly numbers.
#
# **Note:**
#
#   1. `1` is typically treated as an ugly number.
#   2. `n` **does not exceed 1690**.
#
#
## @lc code=start
using LeetCode

function nth_ugly_number(n::Int)
    dp = Vector{Int}(undef, n)
    p2 = p3 = p5 = dp[1] = 1
    for i in 2:n
        nums = dp[p2] * 2, dp[p3] * 3, dp[p5] * 5
        dp[i] = minimum(nums)
        p2 += dp[i] == nums[1]
        p3 += dp[i] == nums[2]
        p5 += dp[i] == nums[3]
    end
    return dp[n]
end
## @lc code=end


# safe log(x) function, log 0 = 0
safelog(x) = x > zero(x) ? log2(x) : zero(x)
safediv(x, y) = y == zero(y) ? zero(y) : x / y
# general functions for entropy

"""
    make_joint_distribution(N::NT) where {NT<:AbstractEcologicalNetwork}

Returns a double stochastic matrix from the adjacency or incidence matrix.
Raises an error if the matrix contains negative values. Output in bits.
"""
function make_joint_distribution(N::NT) where {NT<:AbstractEcologicalNetwork}
    !(typeof(N) <: QuantitativeNetwork) || any(N.A .≥ 0) || throw(DomainError("Information only for nonnegative interaction values"))
    return N.A / sum(N.A)
end

"""
   entropy(P::AbstractArray)

Computes the joint entropy of a double stochastic matrix. Does not perform any
checks whether the matrix is normalized. Output in bits.
"""
function entropy(P::AbstractArray)
    return - sum(P .* safelog.(P))
end

"""
   entropy(P::AbstractArray, dims::I)

Computes the marginal entropy of a double stochastic matrix. `dims` indicates
whether to compute the entropy for the rows (`dims`=1) or columns (`dims`=2).
Does not perform any checks whether the matrix is normalized. Output in bits.
"""
function entropy(P::AbstractArray, dims::I) where I <: Int
    return entropy(sum(P', dims=dims))
end

"""
   entropy(N::AbstractEcologicalNetwork)

Computes the joint entropy of an ecological network. Output in bits.
"""
function entropy(N::NT) where {NT<:AbstractEcologicalNetwork}
    P = make_joint_distribution(N)
    return entropy(P)
end

"""
   entropy(N::AbstractEcologicalNetwork, dims::I)

Computes the marginal entropy of an ecological network. `dims` indicates
whether to compute the entropy for the rows (`dims`=1) or columns (`dims`=2).
Output in bits.
"""
function entropy(N::NT, dims::I) where {NT<:AbstractEcologicalNetwork, I <: Int}
    P = make_joint_distribution(N)
    return entropy(P, dims)
end

"""
   conditional_entropy(P::AbstractArray, given::I)

Computes the conditional entropy of double stochastic matrix. If `given = 1`,
it is the entropy of the columns, and visa versa when `given = 2`. Output in bits.
"""
function conditional_entropy(P::AbstractArray, given::I) where I <: Int
    dims = (given % 2) + 1
    return - sum(P .* safelog.(safediv.(P, sum(P, dims=dims))))
end

"""
   conditional_entropy(N::AbstractEcologicalNetwork, given::I)

Computes the conditional entropy of an ecological network. If `given = 1`,
it is the entropy of the columns, and visa versa when `given = 2`.
"""
function conditional_entropy(N::NT, given::I) where {NT<:AbstractEcologicalNetwork, I <: Int}
    P = make_joint_distribution(N)
    return conditional_entropy(P, given)
end

"""
    mutual_information(P::AbstractArray)

Computes the mutual information of a double stochastic matrix. Output in bits.
"""
function mutual_information(P::AbstractArray)
    I = entropy(P, 1) - conditional_entropy(P, 2)
    return max(I, zero(I))  # might give small negative value
end

"""
    mutual_information(P::AbstractArray)

Computes the mutual information of an ecological network. Output in bits.
"""
function mutual_information(N::NT) where {NT<:AbstractEcologicalNetwork}
    P = make_joint_distribution(N)
    return mutual_information(P)
end

"""
    variation_information(P::AbstractArray)

Computes the variation of information of a double stochastic matrix. Output in
bits.
"""
function variation_information(P::AbstractArray)
    return conditional_entropy(P, 1) + conditional_entropy(P, 2)
end

"""
    variation_information(N::AbstractEcologicalNetwork)

Computes the variation of information of an ecological network. Output in
bits.
"""
function variation_information(N::NT) where {NT <: AbstractEcologicalNetwork}
    P = make_joint_distribution(N)
    return variation_information(P)
end

"""
    diff_entropy_uniform(P::AbstractArray)

Computes the difference in entropy of the marginals compared to the entropy of
an uniform distribution. The parameter `dims` indicates which marginals are used,
with both if no value is provided. Output in bits.
"""
function diff_entropy_uniform(P::AbstractArray)
    D = log2(length(P)) - entropy(P, 1) - entropy(P, 2)
    return max(zero(D), D)
end

"""
    diff_entropy_uniform(P::AbstractArray, dims::I)

Computes the difference in entropy of the marginals compared to the entropy of
an uniform distribution. The parameter `dims` indicates which marginals are used,
with both if no value is provided. Output in bits.
"""
function diff_entropy_uniform(P::AbstractArray, dims::I) where {I <: Int}
    D = log2(size(P, dims)) - entropy(P, dims)
    return max(zero(D), D)
end

"""
    diff_entropy_uniform(N::AbstractEcologicalNetwork, dims::I=nothing)

Computes the difference in entropy of the marginals compared to the entropy of
an uniform distribution. The parameter `dims` indicates which marginals are used,
with both if no value is provided. Output in bits.
"""
function diff_entropy_uniform(N::NT, dims=nothing) where {NT <: AbstractEcologicalNetwork}
    if dims == nothing
        P = make_joint_distribution(N)
        return diff_entropy_uniform(P)
    else  # compute marginals
        typeof(dims) <: Int || throw(ArgumentError("dims should be an integer (1 or 2)"))
        P = make_joint_distribution(N)
        return diff_entropy_uniform(P, dims)
    end
end

"""
    information_decomposition(N::AbstractEcologicalNetwork; norm::Bool=false, dims::I=nothing)

Performs an information theory decomposition of a given ecological network, i.e.
the information content in the normalized adjacency matrix is split in:

- `:D` : difference in entropy of marginals compared to an uniform distribition
- `:I` : mutual information
- `:V` : variation of information / conditional entropy

If `norm=true`, the components are normalized such that their sum is equal to 1.
One can optinally give the dimision, indicating whether to compute the indices
for the rows (`dims=1`), columns (`dims=2`) or the whole matrix (default).

Result is returned in a Dict. Outputs in bits.
"""
function information_decomposition(N::NT; norm::Bool=false, dims::I=nothing) where {NT <: AbstractEcologicalNetwork, I <: Union{Int, Nothing}}
    decomposition = Dict{Symbol, Float64}()
    P = make_joint_distribution(N)
    if dims == nothing
        # difference marginal entropy
        decomposition[:D] = diff_entropy_uniform(P)
        # mutual information
        decomposition[:I] = 2mutual_information(P)
        # variance of information
        decomposition[:V] = variation_information(P)
    else
        decomposition[:D] = diff_entropy_uniform(P, dims)
        decomposition[:I] = mutual_information(P)
        decomposition[:V] = conditional_entropy(P, (dims % 2) + 1)
    end
    if norm
        potential_information = sum(values(decomposition))
        for key in keys(decomposition)
            decomposition[key] /= potential_information
        end
    end
    return decomposition
end

"""
    convert2effective(indice::Real)

Convert an information theory indices in an effective number (i.e. number of
corresponding interactions).
"""
function convert2effective(indice::R) where R <: Real
    return 2.0^indice
end

"""
    potential_information(N::NT)

Computes the maximal potential information in a network, corresponding to
every species interacting with every other species. Compute result for the
marginals using the optional parameter `dims`. Output in bits.
"""
function potential_information(N::NT) where NT <: AbstractEcologicalNetwork
    m, n = size(N)
    return log2(m) + log2(n)
end

"""
    potential_information(N::NT, dims::I)

Computes the maximal potential information in a network, corresponding to
every species interacting with every other species. Compute result for the
marginals using the optional parameter `dims`. Output in bits.
"""
function potential_information(N::NT, dims::I) where {NT <: AbstractEcologicalNetwork, I <: Int}
    return log2(size(N)[dims])
end