#############
# This demonstrates the Chebyshev transform and inverse transform,
# explaining precisely the normalization and points
#############

using FastTransforms

# first kind points -> first kind polynomials
n = 20
p_1 = chebyshevpoints(Float64, n; kind=1)
f = exp.(p_1)
f̌ = chebyshevtransform(f; kind=1)

f̃ = x -> [cos(k*acos(x)) for k=0:n-1]' * f̌
f̃(0.1) ≈ exp(0.1)

# first kind polynomials -> first kind points
ichebyshevtransform(f̌; kind=1) ≈ exp.(p_1)

# second kind points -> first kind polynomials
p_2 = chebyshevpoints(Float64, n; kind=2)
f = exp.(p_2)
f̌ = chebyshevtransform(f; kind=2)

f̃ = x -> [cos(k*acos(x)) for k=0:n-1]' * f̌
f̃(0.1) ≈ exp(0.1)

# first kind polynomials -> second kind points
ichebyshevtransform(f̌; kind=2) ≈ exp.(p_2)

# first kind points -> second kind polynomials
n = 20
p_1 = chebyshevpoints(Float64, n; kind=1)
f = exp.(p_1)
f̌ = chebyshevutransform(f; kind=1)
f̃ = x -> [sin((k+1)*acos(x))/sin(acos(x)) for k=0:n-1]' * f̌
f̃(0.1) ≈ exp(0.1)

# second kind polynomials -> first kind points
ichebyshevutransform(f̌; kind=1) ≈ exp.(p_1)

# second kind points -> second kind polynomials
p_2 = chebyshevpoints(Float64, n; kind=2)[2:n-1]
f = exp.(p_2)
f̌ = chebyshevutransform(f; kind=2)
f̃ = x -> [sin((k+1)*acos(x))/sin(acos(x)) for k=0:n-3]' * f̌
f̃(0.1) ≈ exp(0.1)

# second kind polynomials -> second kind points
ichebyshevutransform(f̌; kind=2) ≈ exp.(p_2)


# Computes A^(⊗p) = A ⊗ A ⊗ ... ⊗ A
# where ⊗ is the Kronecker product
# Since p > 0, B will be kron'd at least once so it will be a Matrix{T}
# This is not easy to guess for Julia inference so I annotate the return type
function (kronpow(A::MT, p)::MT) where MT <: AbstractMatrix
    @assert p > 0
    B = 1
    C = A
    bitmap = 1
    while bitmap <= p
        if (bitmap & p) != 0
            B = kron(B, C)
        end
        C = kron(C, C)
        bitmap <<= 1
    end
    B
end


@doc """
A simple bisection solver. Included for backwards compatability with previous scripts.
f is a function
a is one starting point
b is another starting point
f(a) and f(b) must have opposite signs.
n is optional and is the number of iterations
"""->function mybisect(f,a,b,n=50)
## Introduction
# Code from Introduction to Numerical Methods and Matlab Programming for
# Engineers( P.19)
# T. Young & M.J. Mohlenkamp
# Ohio University

# function [x e] = mybisect (f,a,b,n)
# Does n iterations of the bisection method for a function f
# Inputs : f -- a function
# a,b -- left and right edges of the interval
# n -- the number of bisections to do.
# Outputs : x -- the estimated solution of f(x) = 0
# e -- an upper bound on the error
# evaluate at the ends and make sure there is a sign change
## Code
c = f(a)
d = f(b)
if (c*d > 0.0)
    error("Function has same sign at both endpoints." )
end

for i in 1:n
    # find the middle and evaluate there
    x = (a + b )/2;
    y = f(x);

    if (y == 0.0) # solved the equation exactly
        e = 0;
        break # jumps out of the for loop
    end
    # decide which half to keep , so that the signs at the ends differ
    if (c*y < 0)
        b=x;
    else
        a=x;
    end
end
# set the best estimate for x and the error bound
x = (a + b )/2;
e = (b-a )/2;

return x
end


#==============================================================================
    Code for solving the Hamiltonian Jacboi Bellman for
	   an basic model: ρv(a) = max_{c} u(c) + v'(a)[s(a)]
	   Where s(a) = w + ra - c(a)

	Translated Julia code from Matlab code by Ben Moll:
        http://www.princeton.edu/~moll/HACTproject.htm

        Updated to Julia 1.0.0
==============================================================================#

using Parameters, Distributions, Plots, SparseArrays, LinearAlgebra

σ= 2.0 #
ρ = 0.05 #the discount rate
r = 0.045 # the depreciation rate
w = 0.1

H = 500
a_min = -0.02
a_max = 1.0

a = LinRange(a_min, a_max, H)
a = convert(Array, a) # create grid for a values
da = (a_max-a_min)/(H-1)

maxit = 10000
ε = 10e-6

Δ = 1000

dVf, dVb= [zeros(H,1) for i =1:2]

#initial guess for V
v0 = (w.+r.*a).^(1-σ)/(1-σ)/ρ
v= v0

dist=[]

for n=1:maxit
	V=v
    #forward difference
    dVf[1:H-1] = (V[2:H]-V[1:H-1])/da;
    dVf[H]= 0;

	# backward difference
	dVb[2:H] = (V[2:H]-V[1:H-1])/da
	dVb[1] = (w.+r.*a_min).^(-σ) # the boundary condition

	I_concave = dVb .> dVf

    # consumption and savings with forward difference
    cf = dVf.^(-1/σ)
    ssf = w.+r.*a-cf

    # consumption and savings with backward difference
	cb = dVb.^(-1/σ)
	ssb = w.+r.*a-cb

    # consumption and savings at steady state
    c0=w.+r.*a
    dV0 =c0.^(-σ)

    #look at the sign of the drift
        #to choose forward or backward difference
    If = ssf .> 0 # positive drift ⇒ forward difference
    Ib = ssb .< 0  # negative drift ⇒ backward difference
    I0 = (1.0.-If-Ib)

    dV_Upwind = dVf.*If + dVb.*Ib + dV0.*I0

     global c = dV_Upwind.^(-1/σ)
     u = c.^(1-σ)/(1-σ)

     # create the transition matrix
     X = -min.(ssb,0)/da
     Y = -max.(ssf,0)/da + min.(ssb,0)/da
     Z = max.(ssf,0)/da

     A = sparse(Diagonal(Y[:])) + [zeros(1,H); sparse(Diagonal(X[2:H])) zeros(H-1,1)] + [zeros(H-1,1) sparse(Diagonal(Z[1:H-1])); zeros(1,H)]
     B = (ρ + 1/Δ)*sparse(I, H, H) - A

     b = u + V/Δ
     V = B\b
     V_change = V-v
     global v= V

	push!(dist,findmax(abs.(V_change))[1])
	if dist[n] .< ε
		println("Value Function Converged Iteration=")
		println(n)
		break
	end
end

plot(dist, grid=false,
		xlabel="Iteration", ylabel="||V^{n+1} - V^n||",
		ylims=(-0.001,0.030),
		legend=false, title="")
png("Convergence")

v_err = c.^(1-σ)/(1-σ) + dVb.*(w .+ r.*a -c) - ρ.*v

plot(a, v_err, grid=false,
		xlabel="k", ylabel="Error in the HJB equation",
		xlims=(a_min,a_max),
		legend=false, title="")
png("HJB_error")

plot(a, v, grid=false,
		xlabel="a", ylabel="V(a)",
		xlims=(a_min,a_max),
		legend=false, title="")
png("Value_function_vs_a")

plot(a, c, grid=false,
		xlabel="a", ylabel="c(a)",
		xlims=(a_min,a_max),
		legend=false, title="")
png("c(a)_vs_a")

# approximation at the borrowing constraint
a_dot = w.+ r.*a -c

plot(a, a_dot, grid=false,
		xlabel="a", ylabel="s(a)",
		xlims=(a_min,a_max), title="", label="s(a)", legend=:bottomleft)
plot!(a, zeros(H,1), label="")
png("stateconstraint")


"""
    Haversine(radius)

The haversine distance between two locations on a sphere of given `radius`.

Locations are described with longitude and latitude in degrees.
The computed distance has the same units as that of the radius.
"""
struct Haversine{T<:Real} <: Metric
    radius::T
end

const VecOrLengthTwoTuple{T} = Union{AbstractVector{T}, NTuple{2, T}}

function (dist::Haversine)(x::VecOrLengthTwoTuple, y::VecOrLengthTwoTuple)
    length(x) == length(y) == 2 || haversine_error()

    @inbounds begin
        # longitudes
        Δλ = deg2rad(y[1] - x[1])

        # latitudes
        φ₁ = deg2rad(x[2])
        φ₂ = deg2rad(y[2])
    end

    Δφ = φ₂ - φ₁

    # haversine formula
    a = sin(Δφ/2)^2 + cos(φ₁)*cos(φ₂)*sin(Δλ/2)^2

    # distance on the sphere
    2 * dist.radius * asin( min(√a, one(a)) ) # take care of floating point errors
end

haversine(x::VecOrLengthTwoTuple, y::VecOrLengthTwoTuple, radius::Real) = Haversine(radius)(x, y)

@noinline haversine_error() = throw(ArgumentError("expected both inputs to have length 2 in Haversine distance"))


# by Lazaro Alonso
# port from the original code used in:
# L Alonso, et. al. https://doi.org/10.1093/comnet/cnx053

# using CairoMakie # HIDE
using LinearAlgebra, Random, GLMakie
GLMakie.activate!() # hide
function RRGAdjacencyM3D(;radius = 0.17, nodes = 500, rseed = 123)
    Random.seed!(rseed)
    xy = rand(nodes, 3)
    x = xy[:,1]
    y = xy[:,2]
    z = xy[:,3]

    matrixAdjDiag = Diagonal(√2 * randn(nodes))
    matrixAdj = zeros(nodes, nodes)
    for point in 1:nodes-1
        xseps = (x[point + 1:end] .- x[point]).^2
        yseps = (y[point + 1:end] .- y[point]).^2
        zseps = (z[point + 1:end] .- z[point]).^2

        distance = sqrt.(xseps.+ yseps .+ zseps)
        dindx = findall(distance .<= radius) .+ point
        if length(dindx) > 0
            rnd = randn(length(dindx))
            matrixAdj[point, dindx] = rnd
            matrixAdj[dindx, point] = rnd
        end
    end
    return (matrixAdj .+ matrixAdjDiag, x, y, z)
end
adjacencyM3D, x, y, z = RRGAdjacencyM3D()

function getGraphEdges3D(adjMatrix3D, x, y, z)
    xyzos = []
    weights = []
    for i in 1:length(x), j in i+1:length(x)
        if adjMatrix3D[i,j] != 0.0
            push!(xyzos, [x[i], y[i], z[i]])
            push!(xyzos, [x[j], y[j], z[j]])
            push!(weights, adjMatrix3D[i,j])
            push!(weights, adjMatrix3D[i,j])
        end
    end
    return (Point3f0.(xyzos), Float32.(weights))
end

function plotGraph3D(adjacencyM3D, x, y, z)
    cmap = (:Spectral_11, 0.75)
    adjmin = minimum(adjacencyM3D)
    adjmax = maximum(adjacencyM3D)
    diagValues = diag(adjacencyM3D)
    segm, weights = getGraphEdges3D(adjacencyM3D, x, y, z)

    fig, ax, pltobj = linesegments(segm, color = weights, colormap = cmap,
        linewidth = abs.(weights), colorrange = (adjmin, adjmax),
        figure = (;resolution = (1000,800)),
        axis = (; type = Axis3, aspect =(1,1,1), perspectiveness = 0.5))
    meshscatter!(ax, x, y, z, color = diagValues, markersize = abs.(diagValues)./90,
        colorrange = (adjmin, adjmax), colormap = cmap)
    #limits!(ax, -0.02,1.02,-0.02,1.02)
    Colorbar(fig[1,2], pltobj, label = "weights", height = Relative(0.5))
    fig
end
#plotGraph(adjacencyM, x, y)
rrgraph3D = with_theme(theme_black()) do
    plotGraph3D(adjacencyM3D, x, y, z)
end
save(joinpath(@__DIR__, "output", "RRGraph3D.png"), rrgraph3D, px_per_unit = 2.0) # HIDE
using Pkg # HIDE
Pkg.status(["GLMakie", "LinearAlgebra", "Random"]) # HIDE


@testset "FBM" begin
    rng = MersenneTwister(42)
    h = 0.3
    k = FBMKernel(h = h)
    v1 = rand(rng, 3); v2 = rand(rng, 3)
    @test k(v1,v2) ≈ (sqeuclidean(v1, zero(v1))^h + sqeuclidean(v2, zero(v2))^h - sqeuclidean(v1-v2, zero(v1-v2))^h)/2 atol=1e-5

    # kernelmatrix tests
    m1 = rand(rng, 3, 3)
    m2 = rand(rng, 3, 3)
    Kref = kernelmatrix(k, m1, m1)
    @test kernelmatrix(k, m1) ≈ Kref atol=1e-5
    K = zeros(3, 3)
    kernelmatrix!(K, k, m1, m1)
    @test K ≈ Kref atol=1e-5
    fill!(K, 0)
    kernelmatrix!(K, k, m1)
    @test K ≈ Kref atol=1e-5

    x1 = rand(rng)
    x2 = rand(rng)
    @test kernelmatrix(k, x1*ones(1,1), x2*ones(1,1))[1] ≈ k(x1, x2) atol=1e-5

    @test repr(k) == "Fractional Brownian Motion Kernel (h = $(h))"
    test_ADs(FBMKernel, ADs = [:ReverseDiff])
    @test_broken "Tests failing for kernelmatrix(k, x) for ForwardDiff and Zygote"
end


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# ╔═╡ e44af6da-9045-11eb-15ab-ad339f35ca21
using ImageFiltering,FileIO,ImageShow,TestImages,Images

# ╔═╡ 002e63da-9046-11eb-20b5-4f1bf8ff7260
img = load("../hello-world.png");

# ╔═╡ 03a23ec6-9046-11eb-035c-ffb12919833f
imgg = imfilter(img, Kernel.gaussian(3));

# ╔═╡ 09e7aea6-9046-11eb-3d06-c3980898c7bb
imgl = imfilter(img, Kernel.Laplacian());

# ╔═╡ 3ffbb60e-9048-11eb-1b5e-57ef5a52c22d
# imgs = imfilter(img, Kernel.sobel())

# ╔═╡ 54499d7c-9048-11eb-0b1b-13f6373e402c
img_coins = load("../1.jpg")

# ╔═╡ 9d38abe4-905d-11eb-1ba2-958ed4d9e9af
img_gray = Gray.(img_coins)

# ╔═╡ da50f694-905d-11eb-29dd-d160c78bc260
kernel = [1 0 -1; 2 0 -2;1 0 -1];

# ╔═╡ e082aeb8-905d-11eb-047c-5bb3725816ab
sobel_x = imfilter(img_gray, kernel);

# ╔═╡ e42883f6-905d-11eb-07bf-a9f6acecf1a1
grad = imfilter(sobel_x, kernel')

# ╔═╡ acceb0e8-905e-11eb-027a-a1c08f28641c

# ╔═╡ 02069d1a-905e-11eb-3471-9f3545ac23f4
grad .^ 4

# ╔═╡ 10460d34-905e-11eb-15ab-57d2dd4b66ef
begin
	kernel_1 = [1 2 1]';
kernel_2 = [1 0 -1];
grad_sep = imfilter(img_gray, (kernel_1, kernel_2, kernel_2', kernel_1'))
end

# ╔═╡ Cell order:
# ╠═e44af6da-9045-11eb-15ab-ad339f35ca21
# ╠═002e63da-9046-11eb-20b5-4f1bf8ff7260
# ╠═03a23ec6-9046-11eb-035c-ffb12919833f
# ╠═09e7aea6-9046-11eb-3d06-c3980898c7bb
# ╠═3ffbb60e-9048-11eb-1b5e-57ef5a52c22d
# ╠═54499d7c-9048-11eb-0b1b-13f6373e402c
# ╠═9d38abe4-905d-11eb-1ba2-958ed4d9e9af
# ╠═da50f694-905d-11eb-29dd-d160c78bc260
# ╠═e082aeb8-905d-11eb-047c-5bb3725816ab
# ╠═e42883f6-905d-11eb-07bf-a9f6acecf1a1
# ╠═acceb0e8-905e-11eb-027a-a1c08f28641c
# ╠═02069d1a-905e-11eb-3471-9f3545ac23f4
# ╠═10460d34-905e-11eb-15ab-57d2dd4b66ef


"""
        greedy_star1_coloring

Find a coloring of a given input graph such that
no two vertices connected by an edge have the same
color using greedy approach. The number of colors
used may be equal or greater than the chromatic
number `χ(G)` of the graph.

A star coloring is a special type of distance - 1  coloring,
For a coloring to be called a star coloring, it must satisfy
two conditions:

1. every pair of adjacent vertices receives distinct  colors
(a distance-1 coloring)

2. For any vertex v, any color that leads to a two-colored path
involving v and three other vertices  is  impermissible  for  v.
In other words, every path on four vertices uses at least three
colors.

Reference: Gebremedhin AH, Manne F, Pothen A. **What color is your Jacobian? Graph coloring for computing derivatives.** SIAM review. 2005;47(4):629-705.
"""
function color_graph(g::LightGraphs.AbstractGraph, ::GreedyStar1Color)
    v = nv(g)
    colorvec = zeros(Int, v)

    forbidden_colors = zeros(Int, v+1)

    for vertex_i = vertices(g)

        for w in inneighbors(g, vertex_i)
            if colorvec[w] != 0
                forbidden_colors[colorvec[w]] = vertex_i
            end

            for x in inneighbors(g, w)
                if colorvec[x] != 0
                    if colorvec[w] == 0
                        forbidden_colors[colorvec[x]] = vertex_i
                    else
                        for y in inneighbors(g, x)
                           if colorvec[y] != 0
                                if y != w && colorvec[y] == colorvec[w]
                                    forbidden_colors[colorvec[x]] = vertex_i
                                    break
                                end
                            end
                        end
                    end
                end
            end
        end

        colorvec[vertex_i] = find_min_color(forbidden_colors, vertex_i)
    end

    colorvec
end

function find_min_color(forbidden_colors::AbstractVector, vertex_i::Integer)
    c = 1
    while (forbidden_colors[c] == vertex_i)
        c+=1
    end
    c
end


n = 10 # rand(1:10)
# no null vector for 1-by-1 chebspec.
if n == 1
    n = 2
end
A = matrixdepot("chebspec", n)
# A has null vector ones(n)
@test A*ones(n) ≈ zeros(n) atol = 1e-6
B = matrixdepot("chebspec", n, 1)
# B's eigenvalues have negative real parts.
v = real(eigvals(B))
for i = 1:n
    @test v[i] < 0
end
println("'chebspec' passed test...")


using PyPlot
using LinearAlgebra: norm, dot

a = [ 0.5  -1.0   0.25
      1.0   0.5  -1.0 ]
c = (1/3) * sum(a, dims=2)

figure(1)

function level_sets(a, j, ξ)
    prev = [3, 1, 2]
    succ = [2, 3, 1]
    m = length(ξ)
    start = zeros(2, m)
    finish = zeros(2, m)
    for k = 1:m
        η = -0.3
        start[:,k] = ( a[:,prev[j]] + ξ[k] * ( a[:,j] - a[:,prev[j]] )
                      + η * ( a[:,succ[j]] - a[:,prev[j]] ) )
        η = 1.1 - ξ[k]
        finish[:,k] = ( a[:,prev[j]] + ξ[k] * ( a[:,j] - a[:,prev[j]] )
                      + η * ( a[:,succ[j]] - a[:,prev[j]] ) )
    end
    return start, finish
end

function draw_level_sets(a, j)
    offset = [ 0.1   0.0  -0.1
               0.0   0.1  -0.2 ]
    s = latexstring("\\xi_", j, "=")
    vals = ["0", "1/3", "2/3", "1"]
    ha = ["left", "center", "right"]
    va = ["center", "center", "center"]
    ξ = [0, 1/3, 2/3, 1]
    start, finish = level_sets(a, j, ξ)
    for k = 1:4
        plot([ start[1,k], finish[1,k] ], [ start[2,k], finish[2,k] ], "b",
            linewidth=0.5)
        text(start[1,k]+offset[1,j], start[2,k]+offset[2,j],
             latexstring(s, vals[k]), color="b",
             horizontalalignment=ha[j], verticalalignment=va[j])
    end
end

function draw_triangle(a, c, d)
    J = [1, 2, 3, 1]
    plot(a[1,J], a[2,J], "-k")
    for j = 1:3
        s = latexstring("a_", j)
        offset = a[:,j] - c
        offset *= d / norm(offset)
        text(a[1,j]+offset[1], a[2,j]+offset[2], s, color="r",
            horizontalalignment="center", verticalalignment="center")
    end
    plot(a[1,:], a[2,:], "ro", c[1], c[2], "ko")
end

for j = 1:3
   draw_level_sets(a, j)
end
draw_triangle(a, c, 0.2)
text(c[1]+0.06, c[2]-0.02, L"c",
     horizontalalignment="left", verticalalignment="center")
axis("equal")
axis("off")
savefig("barycentric.pdf")

figure(2)

function draw_vectors(a, b, c)
    prev = [3, 1, 2]
    succ = [2, 3, 1]
    quiver([ c[1], c[1], c[1] ], [ c[2], c[2], c[2] ], b[1,:], b[2,:],
           angles="xy", scale_units="xy", scale=1,
           width=0.004, headwidth=5, color="g")
    offset = [ -0.1   0.1   -0.1
                0.05  0.07   0.0 ]
    for j = 1:3
        text(c[1]+b[1,j]+offset[1,j], c[2]+b[2,j]+offset[2,j],
                 latexstring("b_", j), color="g",
                 horizontalalignment="center", verticalalignment="center")
        bhat = b[:,j] / norm(b[:,j])
        λ = dot(c - a[:,succ[j]], bhat)
        foot = c - λ * bhat
        plot([c[1], foot[1]], [c[2], foot[2]], "--k")
        side = a[:,succ[j]] - a[:,prev[j]]
        v = side / norm(side)
        plot(foot[1] .+ 0.1*[ bhat[1], bhat[1]+v[1], v[1]],
             foot[2] .+ 0.1*[ bhat[2], bhat[2]+v[2], v[2]], linewidth=0.5, "-k")
    end
end

A = [ (a[:,1]-a[:,3]) (a[:,2]-a[:,3]) ]
B = inv(A')
b = [ B[:,1] B[:,2] -sum(B, dims=2) ]

draw_vectors(a, b, c)
draw_triangle(a, c, 0.15)
axis("equal")
axis([-2.4, 1.4, -2.4, 2.4])
axis("off")
savefig("b_vectors.pdf")

figure(3)

function draw_quadratic_triangle(a, c, d, sz=12)
    J = [1, 2, 3, 1]
    plot(a[1,J], a[2,J], "-k")
    m = zeros(2, 3)
    m[:,1] = ( a[:,3] + a[:,2] ) / 2
    m[:,2] = ( a[:,1] + a[:,3] ) / 2
    m[:,3] = ( a[:,2] + a[:,1] ) / 2
    for j = 1:3
        s = latexstring("a_", j)
        offset = a[:,j] - c
        offset *= d / norm(offset)
        text(a[1,j]+offset[1], a[2,j]+offset[2], s, color="r",
            horizontalalignment="center", verticalalignment="center",
            fontsize=sz)
        s = latexstring("m_", j)
        offset = m[:,j] - c
        offset *= d / norm(offset)
        text(m[1,j]+offset[1], m[2,j]+offset[2], s, color="r",
            horizontalalignment="center", verticalalignment="center",
            fontsize=sz)
    end
    plot(a[1,:], a[2,:], "ro", m[1,:], m[2,:], "ro")
end

draw_quadratic_triangle(a, c, 0.15)
axis("equal")
axis("off")
savefig("midpoints.pdf")


using SymPy
using Base.Test

## matrices
(x,) = @syms x
A = [x 1; 1 x]
a = convert(SymMatrix, A) ## for [:meth] calls
b = [x, 2]

## These fail for older installations of SymPy, such as are present in the travis test environment

det(A)
det(a)

## we use inverse for A[:inv]()
inv(A) # aliased to use inverse
inverse(A)
a[:inv]() |> u -> convert(Array{Sym}, u)
a[:inv]("LU")                   # pass argument to function
adjoint(A)
dual(A)
cholesky(A)
## other functions, could wrap
b = subs(a, x, 2)
QRdecomposition(b)

@assert is_square(a) == true
@assert is_symmetric(a) == true

eigvals(A)

a = [1 0 0; 0 1 0; 0 0 x]
evecs = eigvecs(a)
@assert float(evecs) == eye(3)

eh = convert(SymMatrix, a)
eh[9]
convert(SymMatrix, reshape([x, 1:23...], (2,3,4)))


function inner_product(x::AbstractExpr, y::AbstractExpr)
    if x.size == y.size && x.size[1] == x.size[2]
        return real(tr(x'*y))
    else
        error("Arguments must be square matrix of same dimension")
    end
end

inner_product(x::Value, y::AbstractExpr) = inner_product(Constant(x),y)
inner_product(x::AbstractExpr, y::Value) = inner_product(x,Constant(y))


m = 21          # The number of 'space' nodes
n = 401         # The number of 'time' nodes
τ = 10/(n-1)    # Time step size
h = 1/(m-1)     # Space step size

λ = 0.3
α = 0.3+0.1*(1-cos(1))         # Integral of g(x) = 0.3 +0.1sin(x) = 0.3+0.1(1-cos1)
x = vec(linspace(0,1,m))
t = vec(linspace(0,10,n))

α = 0.3+0.1*(1-cos(1))         # Integral of g(x) = 0.3 +0.1sin(x) = 0.3+0.1(1-cos1)
μ = 1./(1-x).^2; μ[1] = 0; μ[m] = 0;
β = zeros(m,m)
γ = (1-λ*τ+h*τ/2)

optv = 0.85 + 0.05cos(2π*t)   # The optimal distribution we are aiming at

## Calcuate A
A = zeros(m,m)

for i=2:m-1
    A[i,i-1] = 1 - τ*μ[i] + τ/h
    A[i,i+1] = 1 - τ*μ[i] - τ/h
end

## Calculate g
g = (0.3+0.1*sin(x))

## Quadrature
function mu(x)
    return 1/(1-x)^2
end

function f(x)
    ## Use just the x part
    ## Remaining cos(πt) should be multiplied later
    return 0.1*sin(1-x)*sin(1-x)
end

function fmu(x)
    return f(x)mu(x)
end

weights = [(18+√30)/36,(18-√30)/36]
point1 = √(3-2(√6)/(√5))/√7
point2 = √(3+2(√6)/(√5))/√7

## Integrate wrt x
fx = weights[1]*fmu((point1+1)/2)+weights[1]*fmu((-point1+1)/2)+weights[2]*fmu((point2+1)/2)+weights[2]*fmu((-point2+1)/2)
fx = fx/2
## Multiply by time vector
fxt = fx*cos(π*t)

## Calculate β
β = zeros(n,n)
β[1,1] = α
for j=2:n
    for k = 1:j-1
       a = (A^(j-1-k))'*μ
       β[j,k] = dot(μ,g)[1]
    end
       β[j,j] = α
end

B = zeros(n,n)
B[1,1] = α
for j=2:n
    for k=1:j
        B[j,:] += γ^(j-k)*β[k,:]
    end
end
B = τ*B
d = zeros(n,1)
for i=1:n
  d[i] = γ^(i-1)*0.9
end

estb = pinv(B)*vec(d-optv)
estv = -B*estb + d
errv = norm(estv-optv,2);
c = estb + fxt - 0.3optv

w = zeros(m,n)
w[1,:] = λ*estv
w[1,1] = 0
for i=2:n
    w[:,i] = A*w[:,i]-τ*estb[i]*g
end
M = ones(m)
p1hat = M'*w*h

#Pkg.add("Gadfly")
#Pkg.add("Cairo")
using Gadfly
using Cairo
draw(PDF("estv.pdf",6inch, 3inch), plot(x=t, y=estv, Guide.XLabel("Time"), Guide.YLabel("Estimated p0"),Guide.XTicks(ticks=[0:1:10])))
draw(PDF("error.pdf",6inch, 3inch), plot(x=t, y=abs(estv-optv), Guide.XLabel("Time"), Guide.YLabel("Absolute Error"),Guide.XTicks(ticks=[0:1:10])))
draw(PDF("estb.pdf",6inch, 3inch), plot(x=t,y=estb, Guide.XLabel("Time"), Guide.YLabel("b(t)"),Guide.XTicks(ticks=[0:1:10])))
draw(PDF("p1hat.pdf",6inch, 3inch), plot(x=t,y=p1hat, Guide.XLabel("Time"), Guide.YLabel("p1hat(t)"), Guide.YTicks(ticks=[0:0.0025:0.0225]), Guide.XTicks(ticks=[0:1:10])))


# generate ramps for hasak
# ramps are 257 point arrays which are interpolated
cosine_series1(size, k, a0,  a1) =
    a0 - a1 * cos((2π * k) / (size-1))
cosine_series2( size, k, a0, a1, a2) =
    cosine_series1(size, k, a0, a1) + a2 * cos((2 * 2π * k) / (size-1))
cosine_series3(size, k, a0, a1, a2, a3) =
    cosine_series2(size, k, a0, a1, a2) - a3 * cos((3 * 2π * k) / (size-1))
# hann ramp element
hann(k, size) =
    Integer(round(32767 * cosine_series1(size, k, 0.50, 0.50)))
# blackman harris ramp element
blackman_harris(k, size) =
    Integer(round(32767 * cosine_series3(size, k, 0.3587500, 0.4882900, 0.1412800, 0.0116800)))
# linear ramp element
triangular(k, size) =
    Integer(round(32767 * (1.0 - abs( (k - ((size-1)/2.0)) / ((size-1)/2.0) ))))
# sine table element
sine(k, size) =
    Integer(round(32767 * sin(2π * k / size)))

# format tables
using Printf
format_table(t) = join([@sprintf("%s%6d", i>1 && (i%8)==1 ? "\n" : "", t[i]) for i in 1:length(t)], ",")
@printf("const int16_t hann_ramp[257] = {\n%s\n};\n", format_table([hann(k,512) for k in 0:256]))
@printf("const int16_t blackman_harris_ramp[257] = {\n%s\n};\n", format_table([blackman_harris(k,512) for k in 0:256]))
@printf("const int16_t linear_ramp[257] = {\n%s\n};\n", format_table([triangular(k,512) for k in 0:256]))
@printf("const int16_t sine_table[257] = {\n%s\n};\n", format_table([sine(k,256) for k in 0:256]))


# This file investigates integration error with respect to floating point precison.

using FractalTools
using Makie

# Construct interpolation data
f(x, y) = x^2 + y^2 + 1
precisions = 2 .^ (7 : 11)

# Specify theoretical value
fval = 35 / 48

npts = 100
mse = map(precisions) do prec
    # Set precision
    setprecision(prec) do
        freevar = BigFloat(0.001)

        # Construct interpolation data
        ngon = [
            BigFloat.([0.0, 0.0]),
            BigFloat.([1.0, 0.0]),
            BigFloat.([0.5, 1.0])
        ]
        pts = getdata(f, ngon, npts)

        # Compute integration.
        ival = integrate(pts, Interp2D(freevar))

        # Compute errro
        abs(fval - ival)
    end
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Precision", ylabel="Integration Error", title="2D Integration Error")
stem!(ax, precisions, mse, color=:black)
save(joinpath(@__DIR__, "integ2d_error.png"), fig)
display(fig)


#############################################################################
# lieb_ando.jl
# Returns tr(K' * A^{1-t} * K * B^t) where A and B are positive semidefinite
# matrices and K is an arbitrary matrix (possibly rectangular).
#
# Disciplined convex programming information:
#    lieb_ando(A,B,K,t) is concave in (A,B) for t in [0,1], and convex
#    in (A,B) for t in [-1,0] or [1,2]. K is a fixed matrix.
#
# Seems numerically unstable when t is on the endpoints of these ranges.
#
# All expressions and atoms are subtypes of AbstractExpr.
# Please read expressions.jl first.
#
#REFERENCE
#   Ported from CVXQUAD which is based on the paper: "Lieb's concavity
#   theorem, matrix geometric means and semidefinite optimization" by Hamza
#   Fawzi and James Saunderson (arXiv:1512.03401)
#############################################################################

const MatrixOrConstant = Union{AbstractMatrix,Constant}

function lieb_ando(
    A::MatrixOrConstant,
    B::MatrixOrConstant,
    K::MatrixOrConstant,
    t::Rational,
)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    return real(tr(K' * A^(1 - t) * K * B^t))
end

function lieb_ando(
    A::MatrixOrConstant,
    B::AbstractExpr,
    K::MatrixOrConstant,
    t::Rational,
)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    KAK = K' * A^(1 - t) * K
    KAK = (KAK + KAK') / 2
    return trace_mpower(B, t, KAK)
end

function lieb_ando(
    A::AbstractExpr,
    B::MatrixOrConstant,
    K::MatrixOrConstant,
    t::Rational,
)
    if t < -1 || t > 2
        throw(DomainError(t, "t must be between -1 and 2"))
    end
    KBK = K * B^t * K'
    KBK = (KBK + KBK') / 2
    return trace_mpower(A, 1 - t, KBK)
end

function lieb_ando(
    A::AbstractExpr,
    B::AbstractExpr,
    K::MatrixOrConstant,
    t::Rational,
)
    n = size(A, 1)
    m = size(B, 1)
    Kvec = reshape(K', n * m, 1)
    KvKv = Kvec * Kvec'
    KvKv = (KvKv + KvKv') / 2
    Im = Matrix(1.0 * I, m, m)
    In = Matrix(1.0 * I, n, n)

    is_complex =
        sign(A) == ComplexSign() ||
        sign(B) == ComplexSign() ||
        sign(Constant(K)) == ComplexSign()
    if is_complex
        T = HermitianSemidefinite(n * m)
    else
        T = Semidefinite(n * m)
    end

    if t >= 0 && t <= 1
        # Concave function
        add_constraint!(
            T,
            T in GeomMeanHypoCone(kron(A, Im), kron(In, conj(B)), t, false),
        )
        return real(tr(KvKv * T))
    elseif (t >= -1 && t <= 0) || (t >= 1 && t <= 2)
        # Convex function
        add_constraint!(
            T,
            T in GeomMeanEpiCone(kron(A, Im), kron(In, conj(B)), t, false),
        )
        return real(tr(KvKv * T))
    else
        throw(DomainError(t, "t must be between -1 and 2"))
    end
end


using UncertaintyQuantification

function likelihood(x)
    eigvaln = [1.51 0.33; 4.01 0.30; 3.16 0.27; 3.21 0.18; 2.19 0.33; 1.71 0.23;
 2.73 0.21; 5.51 0.20; 1.95 0.11; 4.48 0.20; 1.43 0.16; 2.91 0.26; 3.81 0.23;
 3.58 0.25; 2.62 0.25]

    sig = [1 0.1]

    la1 = (x[1] + 2*x[2] + sqrt(x[1].^2 + 4*x[2].^2))/2
    la2 = (x[1] + 2*x[2] - sqrt(x[1].^2 + 4*x[2].^2))/2

    model = [la1 la2]
    rv = 0

    for i in 1:2
        for j in 1:15
            rv += ((eigvaln[j,i] - model[i])/sig[i])^2
        end
    end
    return exp(-1/2*rv)
 end

 tuningsig = [0.04 0; 0 0.04]
 propdist = MvNormal([0;0], tuningsig)
 propdistsample() = rand(propdist)
 propdistpdf(x) = pdf(propdist, x)

 priorsample(n) = rand(Uniform(0.001,4),(n, 2))

 function uniformprior(x)
    if x[1] > 0.001 && x[1] < 4 && x[2] > 0.001 && x[2] < 4
        rv = 1
    else
        rv = 0
    end
    return  rv
 end

 function term(sset, j)
    #cov = std(sset)/mean(sset)
    #coefficient of variance is an alternative way to terminate the sampler
    rv = true
    if j >= 1 rv = false end
    return rv
 end

tmcmcexsample = tmcmc(likelihood, uniformprior, priorsample, 1000, 0.001)

smcexsample = smc(likelihood, uniformprior, priorsample, propdistpdf, propdistsample, 1000, term)

mhexsample = mh(likelihood, uniformprior, propdistpdf, propdistsample, [2.84, 2.33], 1000, 0)


@testset "link" begin
    x = rand()
    xs = rand(3)

    # Generic link
    f = sin
    l = Link(f)
    @test l(x) == f(x)

    # Log
    l = LogLink()
    @test l(x) == log(x)
    @test inv(l) == ExpLink()
    @test inv(inv(l)) == l

    # Exp
    l = ExpLink()
    @test l(x) == exp(x)
    @test inv(l) == LogLink()
    @test inv(inv(l)) == l

    # Sqrt
    l = SqrtLink()
    @test l(x) == sqrt(x)
    @test inv(l) == SquareLink()
    @test inv(inv(l)) == l

    # Square
    l = SquareLink()
    @test l(x) == x^2
    @test inv(l) == SqrtLink()
    @test inv(inv(l)) == l

    # Logit
    l = LogitLink()
    @test l(x) == logit(x)
    @test inv(l) isa LogisticLink
    @test inv(inv(l)) == l

    # Logistic
    l = LogisticLink()
    @test l(x) == logistic(x)
    @test inv(l) isa LogitLink
    @test inv(inv(l)) == l

    # Probit
    l = ProbitLink()
    @test l(x) == norminvcdf(x)
    @test inv(l) == NormalCDFLink()
    @test inv(inv(l)) == l

    # NormalCDF
    l = NormalCDFLink()
    @test l(x) == normcdf(x)
    @test inv(l) == ProbitLink()
    @test inv(inv(l)) == l

    # SoftMax
    l = SoftMaxLink()
    @test l(xs) == softmax(xs)
end


# JuMP example for the Julia Working Group

# Reproduce the example HS071.
# HS071
# min x1 * x4 * (x1 + x2 + x3) + x3
# st  x1 * x2 * x3 * x4 >= 25
#     x1^2 + x2^2 + x3^2 + x4^2 = 40
#     1 <= x1, x2, x3, x4 <= 5
# Start at (1,5,5,1)
# End at (1.000..., 4.743..., 3.821..., 1.379...)

# First install the package, and a solver
#Pkg.add("JuMP")
#Pkg.add("Ipopt") #solver

# Load these packages
using JuMP
using Ipopt
#using NLopt

tic()  #start clock

#Create the model object
m = Model()
#m = Model(solver = NLoptSolver(algorithm=:LD_SLSQP))
#m=Model(solver=IpoptSolver(tol=1e-8, max_iter=5000)) # can use the argument solver to pass solver options

#Defining Variables - These are defined using the @defVar macro. The first argument of
# @defVar(m,x) will always be the model to associate the variable with. You can also
# impose the bounds on the variable here; and start values (using the start option)
@defVar(m, 1<=x[1:4]<=5)
# Set start value
setValue(x[1],1)
setValue(x[2],5)
setValue(x[3],5)
setValue(x[4],1)

# One can also create variables manually one by one as:
# x = Variable(m::Model, lower::Number, upper::Number, category::Symbol, name::String)
# Bounds can also be set using setLower(x::Variable, lower) and setUpper(x::Variable, lower)

# Objective function - I will show the syntax for a nonlinear objective function. For the
# syntax for linear objective functions, see the documentation. Essentially one needs to
# use setObjective() or @setObjective() rather than @setNLObjective()
# For nonlinear objectives, these are defined using the macro @setNLObjective()

@setNLObjective(m::Model, :Min, x[1]*x[4]*(x[1]+x[2]+x[3])+x[3] )

# Next, set constraints.
# Linear constraints are added through the macro @addConstraint()
# Nonlinear constraints are added through the @addNLConstraint() macro
@addNLConstraint(m::Model, x[1]*x[2]*x[3]*x[4]>=25 )
@addNLConstraint(m::Model, x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2 == 40)

# Finally, solve the model
solve(m)

# Let's look at the solutions
println("x=", getValue(x))

toc()


module new_logistic_regression

using Distributions

struct data
    y::Vector
    x::Array{Float64}
    m::Vector
end

## Logistic regression nobs
function nobs(data::data)
    nx = size(data.x)[1]
    ny = length(data.y)
    nm = length(data.m)
    if (nx != ny)
        error("number of rows in of x is not equal to the length of y")
    elseif (nx != nm)
        error("number of rows in of x is not equal to the length of m")
    elseif (ny != nm)
        error("length of y is not equal to the length of m")
    end
    return nx
end

function observation(data::data, i::Int64)
    vcat(data.y[i], data.m[i], data.x[i,:])
end

function loglik(theta::AbstractArray,
                obs::AbstractArray)
    y = obs[1]
    m = obs[2]
    x = obs[3:end]
    eta = sum(x .* theta)
    mu = exp.(eta) ./ (1 .+ exp.(eta))
    return y .* log.(mu) + (m - y) .* log.(1 .- mu)
end

function logistic_ef(theta::AbstractArray,
                     obs::AbstractArray)
    y = obs[1]
    m = obs[2]
    x = obs[3:end]
    eta = sum(x .* theta)
    mu = exp.(eta) ./ (1 .+ exp.(eta))
    return x * (y - m * mu)
end

function simulate(theta::Vector,
                  x::Matrix{Float64},
                  m::Vector{Int64})
    n = size(x)[2]
    y = rand.(Binomial.(m, cdf.(Logistic(), x * theta)))
    data(y, x, m);
end

end


# ## Data

import CSV
import Random
import TuringModels

using DataFrames

Random.seed!(1)

data_path = joinpath(TuringModels.project_root, "data", "UCBadmit.csv")
df = CSV.read(data_path, DataFrame; delim=';')

# ## Model

using StatsFuns: logistic
using Turing

@model function m11_5(admit, applications)
  θ ~ truncated(Exponential(1), 0, Inf)
  α ~ Normal(0, 2)

  ## alpha and beta for the BetaBinomial must be provided.
  ## The two parameterizations are related by
  ## alpha = prob * theta, and beta = (1-prob) * theta.
  ## See https://github.com/rmcelreath/rethinking/blob/master/man/dbetabinom.Rd

  prob = logistic(α)
  alpha = prob * θ
  beta = (1 - prob) * θ
  admit .~ BetaBinomial.(applications, alpha, beta)
end

model = m11_5(df.admit, df.applications);

# ## Output

chns = sample(model, NUTS(), 1000)

# \defaultoutput{}

# ## Original output

"""
        mean   sd  5.5% 94.5% n_eff Rhat
theta   2.74 0.96  1.43  4.37  3583    1
a       -0.37 0.31 -0.87  0.12  3210    1
""";


using JuMP
using ECOS
using LinearAlgebra
using Random

Random.seed!(1234);

@testset "Test SecondOrderCone formulations" begin
    u0 = rand(10)
    p = rand(10)
    q = rand();

    optimal_value_soc = 0.7413476010760025
    @testset "Primal formulation" begin
        model = Model(optimizer_with_attributes(ECOS.Optimizer, "printlevel" => 0))
        @variable(model, u[1:10])
        @variable(model, t)
        @objective(model, Min, t)
        @constraint(model, [t, (u - u0)...] in SecondOrderCone())
        @constraint(model, u' * p == q)
        optimize!(model)
        @test JuMP.termination_status(model) == MOI.OPTIMAL
        @test objective_value(model) ≈ optimal_value_soc atol=1e-6
    end
    @testset "Dual formulation" begin
        e1 = [1, zeros(10)...]
        dual_model = Model(optimizer_with_attributes(ECOS.Optimizer, "printlevel" => 0))
        @variable(dual_model, y1 <= 0)
        @variable(dual_model, y2[1:11])
        @objective(dual_model, Max, q * y1 + dot(vcat(0, u0), y2))
        @constraint(dual_model, e1 - [0, p...] .* y1 - y2 .== 0)
        @constraint(dual_model, y2 in SecondOrderCone())
        optimize!(dual_model)
        @test JuMP.termination_status(dual_model) == MOI.OPTIMAL
        @test objective_value(dual_model) ≈ optimal_value_soc atol=1e-6
    end
end

@testset "Test RotatedSecondOrderCone formulation" begin
    u0 = rand(10)
    p = rand(10)
    q = rand();
    model = Model(optimizer_with_attributes(ECOS.Optimizer, "printlevel" => 0))
    @variable(model, u[1:10])
    @variable(model, t)
    @objective(model, Min, t)
    @constraint(model, [t, 0.5, (u - u0)...] in RotatedSecondOrderCone())
    @constraint(model, u' * p == q)
    optimize!(model)
    @test JuMP.termination_status(model) == MOI.OPTIMAL
    @test objective_value(model) ≈ 0.5495962595148015 atol=1e-6
end

@testset "Test ExponentialCone formulations" begin
    n = 15;
    m = 10;
    A = randn(m, n);
    b = rand(m, 1);

    model = Model(optimizer_with_attributes(ECOS.Optimizer, "printlevel" => 0))
    @variable(model, t[1:n])
    @variable(model, x[1:n])
    @objective(model, Max, sum(t))
    @constraint(model, sum(x) == 1)
    @constraint(model, A * x .<= b )
    # Cannot use the exponential cone directly in JuMP, hence we use MOI to specify the set.
    @constraint(model, con[i = 1:n], [t[i], x[i], 1.0] in MOI.ExponentialCone())

    optimize!(model);
    @test objective_value(model) ≈ 2.708050207925179
    @test JuMP.termination_status(model) == MOI.OPTIMAL
end


using Plots;gr()

T = 1000
dt = 0.001
y_0 = 0
g_0 = 1

function euler_method(y_0, g_0, T, dt)
    y = zeros(T)
    y[1] = y_0

    f = y_0
    g = g_0
    for t = 1:T-1
        y[t+1] = y[t] + dt*g
        g = g + dt*(6*g - 9*y[t+1])
    end

    y
end

exact_solution(t) = t*exp(3*t)

t = 0:dt:(T-1)dt
y = euler_method(y_0, g_0, T, dt)

plot(t, exact_solution, color=:cornflowerblue, label="Exact",xlabel="t",ylabel="y")
plot!(t, y, color=:coral, linestyle=:dash, label="Numerical",xlabel="t",ylabel="y")
savefig("2nd_order_euler.png")


include("fft.jl")
include("multigrid.jl")
function get_f{T}(u::AbstractArray{T,2}, eps::T)
"""Get the f with left hand side"""
    mx,my=size(u)
    fft=rFFTWrapper(u)
    lap = similar(u)
    laplacian!(u,lap,fft)
    dx=similar(u)
    dy=similar(u)
    set_not_calculated(fft)
    derivx!(u,dx,fft)
    derivy!(u,dy,fft)
    @inbounds for x in 1:mx
        for y in 1:my
            lap[x,y] += eps * (dx[x,y]^2 + dy[x,y]^2 + lap[x,y] * u[x,y])
        end
    end
    return lap
end

function solver{T}(f::AbstractArray{T,2}, eps::T, tol::T, maxiter=50)
"""Solving the equation with laplacian"""
    mx,my=size(f)
    fft=rFFTWrapper(f)
    u=zeros(f)
    newu=similar(u)
    dx=similar(u)
    dy=similar(u)
    rhs=similar(u)
    iter=0
    err=T[]
    while iter < maxiter
        iter += 1
        derivx!(u,dx,fft)
        derivy!(u,dy,fft)
        @inbounds for x in 1:mx
            for y in 1:my
                rhs[x,y]= (f[x,y] - eps * (dx[x,y]^2 + dy[x,y]^2)) / (one(T) + eps * u[x,y])
            end
        end
        solve_laplacian!(rhs,newu,fft)
        e=vecnorm(newu - u) / vecnorm(f)
        push!(err,e)
        if e < tol
            break
        end
        u,newu=newu,u
    end
    return u,err
end

function iteration{T}(grid::gridWrapper, eps::T, err::Array{T,1}, maxiter=4)
    mx,my=size(grid.f)
    iter=0
    while iter < maxiter
        iter += 1
        derivx!(grid.u,grid.dx,grid.fft)
        derivy!(grid.u,grid.dy,grid.fft)
        @inbounds for x in 1:mx
            for y in 1:my
                grid.rhs[x,y]= (grid.f[x,y] - eps * (grid.dx[x,y]^2 + grid.dy[x,y]^2)) / (one(T) + eps * grid.u[x,y])
            end
        end
        solve_laplacian!(grid.rhs,grid.newu,grid.fft)
        grid.u,grid.newu=grid.newu,grid.u
        push!(err,vecnorm(grid.newu - grid.u) / vecnorm(grid.f))
    end
end

function iteration{T}(grid::gridWrapper, eps::T, maxiter=4)
    mx,my=size(grid.f)
    iter=0
    while iter < maxiter
        iter += 1
        derivx!(grid.u,grid.dx,grid.fft)
        derivy!(grid.u,grid.dy,grid.fft)
        @inbounds for x in 1:mx
            for y in 1:my
                grid.rhs[x,y]= (grid.f[x,y] - eps * (grid.dx[x,y]^2 + grid.dy[x,y]^2)) / (one(T) + eps * grid.u[x,y])
            end
        end
        solve_laplacian!(grid.rhs,grid.newu,grid.fft)
        grid.u,grid.newu=grid.newu,grid.u
    end
end

function multigrid_solver{T}(f::AbstractArray{T,2}, eps::T, tol::T, maxiter=32)
"""Solve the equation with multigrid"""
    mx,my=size(f)
    grid1=gridWrapper(f)
    f2=similar(f,div(mx,2),div(my,2))
    restriction!(f,f2)
    grid2=gridWrapper(f2)
    f3=similar(f,div(mx,4),div(my,4))
    restriction!(f2,f3)
    grid3=gridWrapper(f3)
    f4=similar(f,div(mx,8),div(my,8))
    restriction!(f3,f4)
    grid4=gridWrapper(f4)
    err=T[]
    full_cycle(iteration,grid1,grid2,grid3,grid4,eps,tol)
    iter=0
    while iter < maxiter
        iter += 1
        iteration(grid1,eps,err)
        if vecnorm(grid1.u - grid1.newu) < tol * vecnorm(grid1.f)
            break
        end
    end
    return grid1.u,err
end


using GeneralTridiagonal
using LinearAlgebra: Tridiagonal

const T = Rational{Int64}

β = T[-1, 3, 0, 5]
α = T[0, 8, 2, 1, 8]
γ = T[1, 4, 0, -3]
δ = Array{T}(undef, 3)
p = Array{Bool}(undef, 4)

A = Tridiagonal(β, α, γ)
x = T[-2, 5, 8, 0, -3]
b = A*x

factorize!(β, α, γ, δ, p)
println("Row swaps: ", p)

solve!(b, β, α, γ, δ, p)
println("   Exact solution: x = ", x)
println("Computed solution: x = ", b)


function integral(A::AbstractInterpolation, t::Number)
    bw, fw = samples(A)
    idx = max(1+bw, min(searchsortedlast(A.t, t), length(A.t) - fw))
    _integral(A, idx, t)
end

function integral(A::AbstractInterpolation, t1::Number, t2::Number)
    bw, fw = samples(A)
    # the index less than or equal to t1
    idx1 = max(1+bw, min(searchsortedlast(A.t, t1), length(A.t) - fw))
    # the index less than t2
    idx2 = max(2+bw, min(searchsortedlast(A.t, t2), length(A.t) - fw))
    if A.t[idx2] == t2
        idx2-=1
    end
    total = zero(eltype(A))
    for idx in idx1:idx2
        lt1 = idx == idx1 ? t1 : A.t[idx]
        lt2 = idx == idx2 ? t2 : A.t[idx+1]
        total += _integral(A, idx, lt2)-_integral(A, idx, lt1)
    end
    total
end

samples(A::LinearInterpolation{<:AbstractVector}) = (0, 1)
function _integral(A::LinearInterpolation{<:AbstractVector{<:Number}}, idx::Number, t::Number)
  t1 = A.t[idx]
  t2 = A.t[idx+1]
  u1 = A.u[idx]
  u2 = A.u[idx+1]
  t^2*(u1 - u2)/(2*t1 - 2*t2) + t*(t1*u2 - t2*u1)/(t1 - t2)
end

samples(A::ConstantInterpolation{<:AbstractVector}) = (0, 1)
function _integral(A::ConstantInterpolation{<:AbstractVector}, idx::Number, t::Number)
  if A.dir === :left
    # :left means that value to the left is used for interpolation
    return A.u[idx]*t
  else
    # :right means that value to the right is used for interpolation
    return A.u[idx+1]*t
  end
end

samples(A::QuadraticInterpolation{<:AbstractVector}) = (0, 2)
function _integral(A::QuadraticInterpolation{<:AbstractVector{<:Number}}, idx::Number, t::Number)
  t1 = A.t[idx]
  t2 = A.t[idx+1]
  t3 = A.t[idx+2]
  u1 = A.u[idx]
  u2 = A.u[idx+1]
  u3 = A.u[idx+2]
  (t^3*(-t1*u2 + t1*u3 + t2*u1 - t2*u3 - t3*u1 + t3*u2)/
  (3*t1^2*t2 - 3*t1^2*t3 - 3*t1*t2^2 + 3*t1*t3^2 + 3*t2^2*t3 - 3*t2*t3^2) +
  t^2*(t1^2*u2 - t1^2*u3 - t2^2*u1 + t2^2*u3 + t3^2*u1 - t3^2*u2)/
  (2*t1^2*t2 - 2*t1^2*t3 - 2*t1*t2^2 + 2*t1*t3^2 + 2*t2^2*t3 - 2*t2*t3^2) +
  t*(t1^2*t2*u3 - t1^2*t3*u2 - t1*t2^2*u3 + t1*t3^2*u2 + t2^2*t3*u1 - t2*t3^2*u1)/
  (t1^2*t2 - t1^2*t3 - t1*t2^2 + t1*t3^2 + t2^2*t3 - t2*t3^2))
end

samples(A::QuadraticSpline{<:AbstractVector{<:Number}}) = (0, 1)
function _integral(A::QuadraticSpline{<:AbstractVector{<:Number}}, idx::Number, t::Number)
  t1 = A.t[idx]
  t2 = A.t[idx+1]
  u1 = A.u[idx]
  z1 = A.z[idx]
  z2 = A.z[idx+1]
  t^3*(z1 - z2)/(6*t1 - 6*t2) + t^2*(t1*z2 - t2*z1)/(2*t1 - 2*t2) + t*(-t1^2*z1 - t1^2*z2 + 2*t1*t2*z1 + 2*t1*u1 - 2*t2*u1)/(2*t1 - 2*t2)
end

samples(A::CubicSpline{<:AbstractVector{<:Number}}) = (0, 1)
function _integral(A::CubicSpline{<:AbstractVector{<:Number}}, idx::Number, t::Number)
  t1 = A.t[idx]
  t2 = A.t[idx+1]
  u1 = A.u[idx]
  u2 = A.u[idx+1]
  z1 = A.z[idx]
  z2 = A.z[idx+1]
  h2 = A.h[idx+1]
  (t^4*(-z1 + z2)/(24*h2) + t^3*(-t1*z2 + t2*z1)/(6*h2) +
  t^2*(h2^2*z1 - h2^2*z2 + 3*t1^2*z2 - 3*t2^2*z1 - 6*u1 + 6*u2)/(12*h2) +
  t*(h2^2*t1*z2 - h2^2*t2*z1 - t1^3*z2 - 6*t1*u2 + t2^3*z1 + 6*t2*u1)/(6*h2))
end


using BenchmarkTools
using StaticArrays
# this is a slower but pretty solution. took inspiration from a reddit comment

const start = SVector{300}(map(str -> [parse(Int,str[l]) for l in findall(r"[[:digit:]]+", str)], readlines("inputs/6-10/06.txt"))[1])

function day6(start, days)
  c = zeros(Int, 9)
  for x in start
    c[x] += 1
  end
  for _ in 1:days-1
    c = circshift(c, -1)
    c[7] += c[end]
  end
  sum(c)
end

println("Day 6 Part 1: ",day6(start, 80))
println("Day 6 Part 2: ",day6(start, 256))

const A = SMatrix{9,9}([
 0  1  0  0  0  0  0  0  0
 0  0  1  0  0  0  0  0  0
 0  0  0  1  0  0  0  0  0
 0  0  0  0  1  0  0  0  0
 0  0  0  0  0  1  0  0  0
 0  0  0  0  0  0  1  0  0
 1  0  0  0  0  0  0  1  0
 0  0  0  0  0  0  0  0  1
 1  0  0  0  0  0  0  0  0
])

function day6transition( days)
  c = zeros(Int, 9)
  for x in start
    c[x] += 1
  end
  StC = SVector{9}(c)
  (A^days) * StC |> sum
end

@btime day6transition( 256)


using Turing

@model function two_model(x)
    # Hyper-parameters
    μ0 = 0.0
    σ0 = 1.0

    # Draw weights.
    π1 ~ Beta(1, 1)
    π2 = 1 - π1

    # Draw locations of the components.
    μ1 ~ Normal(μ0, σ0)
    μ2 ~ Normal(μ0, σ0)

    # Draw latent assignment.
    z ~ Categorical([π1, π2])

    # Draw observation from selected component.
    if z == 1
        x ~ Normal(μ1, 1.0)
    else
        x ~ Normal(μ2, 1.0)
    end
end

using Turing.RandomMeasures

# Concentration parameter.
α = 10.0

# Random measure, e.g. Dirichlet process.
rpm = DirichletProcess(α)

# Cluster assignments for each observation.
z = Vector{Int}()

# Maximum number of observations we observe.
Nmax = 500

for i in 1:Nmax
    # Number of observations per cluster.
    K = isempty(z) ? 0 : maximum(z)
    nk = Vector{Int}(map(k -> sum(z .== k), 1:K))

    # Draw new assignment.
    push!(z, rand(ChineseRestaurantProcess(rpm, nk)))
end

using Plots

# Plot the cluster assignments over time
@gif for i in 1:Nmax
    scatter(
        collect(1:i),
        z[1:i];
        markersize=2,
        xlabel="observation (i)",
        ylabel="cluster (k)",
        legend=false,
    )
end

@model function infiniteGMM(x)
    # Hyper-parameters, i.e. concentration parameter and parameters of H.
    α = 1.0
    μ0 = 0.0
    σ0 = 1.0

    # Define random measure, e.g. Dirichlet process.
    rpm = DirichletProcess(α)

    # Define the base distribution, i.e. expected value of the Dirichlet process.
    H = Normal(μ0, σ0)

    # Latent assignment.
    z = tzeros(Int, length(x))

    # Locations of the infinitely many clusters.
    μ = tzeros(Float64, 0)

    for i in 1:length(x)

        # Number of clusters.
        K = maximum(z)
        nk = Vector{Int}(map(k -> sum(z .== k), 1:K))

        # Draw the latent assignment.
        z[i] ~ ChineseRestaurantProcess(rpm, nk)

        # Create a new cluster?
        if z[i] > K
            push!(μ, 0.0)

            # Draw location of new cluster.
            μ[z[i]] ~ H
        end

        # Draw observation.
        x[i] ~ Normal(μ[z[i]], 1.0)
    end
end

using Plots, Random

# Generate some test data.
Random.seed!(1)
data = vcat(randn(10), randn(10) .- 5, randn(10) .+ 10)
data .-= mean(data)
data /= std(data);

# MCMC sampling
Random.seed!(2)
iterations = 1000
model_fun = infiniteGMM(data);
chain = sample(model_fun, SMC(), iterations);

# Extract the number of clusters for each sample of the Markov chain.
k = map(
    t -> length(unique(vec(chain[t, MCMCChains.namesingroup(chain, :z), :].value))),
    1:iterations,
);

# Visualize the number of clusters.
plot(k; xlabel="Iteration", ylabel="Number of clusters", label="Chain 1")

histogram(k; xlabel="Number of clusters", legend=false)

if isdefined(Main, :TuringTutorials)
    Main.TuringTutorials.tutorial_footer(WEAVE_ARGS[:folder], WEAVE_ARGS[:file])
end


using DrWatson
@quickactivate "StatReth"

# %%
"""2M4
card1 = bb
card2 = bw
card3 = ww

ways to get b with card1: 2
ways to get b with card2: 1
ways to get b with card3: 0
total ways to get b: 3
otherside also black means this is card1
probability of card1 = (ways to get b with card1) / (total ways to get b) = 2/3
"""

# %%
"""2M5
card1 = bb
card2 = bw
card3 = ww
card4 = bb

ways to get b with card1: 2
ways to get b with card2: 1
ways to get b with card3: 0
ways to get b with card4: 2
total ways to get b: 5
otherside also black means this is card1 or card4
probability of card1 = (ways to get b with card1) / (total ways to get b) = 2/5
probability of card4 = (ways to get b with card4) / (total ways to get b) = 2/5
probability of either card1 or card4 = (probability of card1) + (probability of card4)
                                     = 2/5 + 2/5 = 4/5
"""

# %%
"""2M6
card1 = bb
card2 = bw
card3 = ww

ways to get b with card1: 2  |  ways to get card1: 1
ways to get b with card2: 1  |  ways to get card2: 2
ways to get b with card3: 0  |  ways to get card3: 3
total ways to get b: 4
otherside also black means this is card1
probability of card1 = (ways to get b with card1) / (total ways to get b) = 2/4
"""

# %%
"""2M7
card1 = bb
card2 = bw
card3 = ww

scenarios:

first draw: side of first draw, second draw: side of first draw -> ok/no

card1: b1, card2: b  -> no, second card not white
card1: b1, card2: w  -> ok
card1: b1, card3: w1 -> ok
card1: b1, card3: w2 -> ok

card1: b2, card2: b  -> no, second card not white
card1: b2, card2: w  -> ok
card1: b2, card3: w1 -> ok
card1: b2, card3: w2 -> ok

card2: b, card1: b1  -> no, second card not white
card2: b, card1: b2  -> no, second card not white
card2: b, card3: w1  -> ok
card2: b, card3: w2  -> ok

card2: w, card1: b1  -> no, first card not black
card2: w, card1: b2  -> no, first card not black
card2: w, card3: w1  -> no, first card not black
card2: w, card3: w2  -> no, first card not black

card3: w1, card1: b1 -> no, first card not black
card3: w1, card1: b2 -> no, first card not black
card3: w1, card2: b  -> no, first card not black
card3: w1, card2: w  -> no, first card not black

card3: w2, card1: b1 -> no, first card not black
card3: w2, card1: b2 -> no, first card not black
card3: w2, card2: b  -> no, first card not black
card3: w2, card2: w  -> no, first card not black

number of possible/ok scenarios: 8
possible scenarios with card1 as first card: 6
probability for card1 = 6/8 = 0.75
"""


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.3'
#       jupytext_version: 0.8.6
#   kernelspec:
#     display_name: Julia 1.0.3
#     language: julia
#     name: julia-1.0
# ---
module SIRModel
using DifferentialEquations

function sir_ode(du, u, p, t)
    #Infected per-Capita Rate
    β = p[1]
    #Recover per-capita rate
    γ = p[2]

    #Susceptible Individuals
    S = u[1]
    #Infected by Infected Individuals
    I = u[2]

    du[1] = -β * S * I
    du[2] = β * S * I - γ * I
    du[3] = γ * I
end

#Pram = (Infected Per Capita Rate, Recover Per Capita Rate)
pram = [0.1,0.05]
#Initial Prams = (Susceptible Individuals, Infected by Infected Individuals)
init = [0.99,0.01,0.0]
tspan = (0.0,200.0)

sir_prob = ODEProblem(sir_ode, init, tspan, pram)

sir_sol = solve(sir_prob, saveat = 0.1);

# Visualization
using Plots
plot(sir_sol,xlabel="Time",ylabel="Number")

function sir_ode2(du,u,p,t)
    S,I,R = u
    b,g = p
    du[1] = -b*S*I
    du[2] = b*S*I-g*I
    du[3] = g*I
end
parms = [0.1,0.05]
init = [0.99,0.01,0.0]
tspan = (0.0,200.0)
sir_prob2 = ODEProblem(sir_ode2,init,tspan,parms)
sir_sol = solve(sir_prob2,saveat = 0.1)
end


# Polynomial covariance function

"""
    Poly <: Kernel

Polynomial kernel (covariance)
```math
k(x,x') = σ²(xᵀx' + c)ᵈ
```
with signal standard deviation ``σ``, additive constant ``c``, and degree ``d``.
"""
mutable struct Poly <: Kernel
    "Constant"
    c::Float64
    "Signal variance"
    σ2::Float64
    "Degree of polynomial"
    deg::Int
    "Priors for kernel parameters"
    priors::Array

    """
        Poly(lc::Float64, lσ::Float64, deg::Int)

    Create `Poly` with signal standard deviation `exp(lσ)`, additive constant `exp(lc)`,
    and degree `deg`.
    """
    Poly(lc::Float64, lσ::Float64, deg::Int) = new(exp(lc), exp(2 * lσ), deg, [])
end

function KernelData(k::Poly, X::MatF64)
    XtX=X'*X
    LinearAlgebra.copytri!(XtX, 'U')
    LinIsoData(XtX)
end
kernel_data_key(k::Poly, X::MatF64) = "LinIsoData"

_cov(poly::Poly, xTy) = poly.σ2*(poly.c.+xTy).^poly.deg
function Statistics.cov(poly::Poly, x::VecF64, y::VecF64)
    K = _cov(poly, dot(x,y))
end
function cov!(cK::MatF64, poly::Poly, X::MatF64, data::LinIsoData)
    cK .= _cov(poly, data.XtX)
end
@inline @inbounds function cov_ij(poly::Poly, X::MatF64, data::LinIsoData, i::Int, j::Int, dim::Int)
    return _cov(poly, data.XtX[i, j])
end
Statistics.cov(poly::Poly, X::MatF64, data::LinIsoData) = _cov(poly, data.XtX)

get_params(poly::Poly) = Float64[log(poly.c), log(poly.σ2) / 2]
get_param_names(poly::Poly) = [:lc, :lσ]
num_params(poly::Poly) = 2

function set_params!(poly::Poly, hyp::VecF64)
    length(hyp) == 2 || throw(ArgumentError("Polynomial function has two parameters"))
    poly.c = exp(hyp[1])
    poly.σ2 = exp(2 * hyp[2])
end

@inline dk_dlc(poly::Poly, xTy::Float64) = poly.c*poly.deg*poly.σ2*(poly.c+xTy).^(poly.deg-1)
@inline dk_dlσ(poly::Poly, xTy::Float64) = 2 * _cov(poly,xTy)
@inline function dKij_dθp(poly::Poly, X::MatF64, i::Int, j::Int, p::Int, dim::Int)
    if p==1
        return dk_dlc(poly, dotij(X,i,j,dim))
    else
        return dk_dlσ(poly, dotij(X,i,j,dim))
    end
end
@inline function dKij_dθp(poly::Poly, X::MatF64, data::LinIsoData, i::Int, j::Int, p::Int, dim::Int)
    if p==1
        return dk_dlc(poly, data.XtX[i,j])
    else
        return dk_dlσ(poly, data.XtX[i,j])
    end
end


using SymPy.Physics
using SymPy: Sym, sqrt, symbols, doit, PI
using SymPy.SpecialFuncs: Ynm
using PyCall: PyError
using Test

@testset "Physics" begin
    @test clebsch_gordan(Sym(3)/2, Sym(1)/2, 2, Sym(3)/2, Sym(1)/2, 2) == 1
    @test clebsch_gordan(Sym(3)/2, Sym(1)/2, 1, Sym(3)/2, -Sym(1)/2, 1) == sqrt(Sym(3))/2
    @test clebsch_gordan(Sym(3)/2, Sym(1)/2, 1, -Sym(1)/2, Sym(1)/2, 0) == -sqrt(Sym(2))/2

    θ, ϕ = symbols("theta, phi")
    @test doit(dot_rot_grad_Ynm(3, 2, 2, 0, θ, ϕ)) == 3*sqrt(Sym(55))*Ynm(5, 2, θ, ϕ)/(11*sqrt(PI))

    @test gaunt(1,0,1,1,0,-1) == -1/(2*sqrt(PI))
    #@test N(gaunt(1000,1000,1200,9,3,-12)) ≈ 0.00689500421922113448 # takes forever to compute

    @test_throws PyError gaunt(1.2,0,1.2,0,0,0)
    @test_throws PyError gaunt(1,0,1,1.1,0,-1.1)

    @test racah(3,3,3,3,3,3) == -1//14

    @test wigner_3j(2, 6, 4, 0, 0, 0) == sqrt(Sym(715))/143
    @test wigner_3j(2, 6, 4, 0, 0, 1) == 0

    @test wigner_6j(3,3,3,3,3,3) == -1//14
    @test wigner_6j(5,5,5,5,5,5) == 1//52

    # Optics
    @test RayTransferMatrix(1,2,3,4) == Sym[1 2; 3 4]
    @test FlatMirror() == Sym[1 0; 0 1]

    # Spin
    @test doit(WignerD(Sym(3), Sym(2), Sym(1), PI, PI/2, -PI)) == -sqrt(Sym(10))/8
    @test doit(WignerD(Sym(1), Sym(1), Sym(0), 0, θ, 0)) == -sin(θ)/sqrt(Sym(2))
end


@inline function two_inv(b::T) where {T}
     hi = inv(b)
     lo = fma(-hi, b, one(T))
     lo /= b
     return hi, lo
end

@inline function two_div(a::T, b::T) where {T}
     hi = a / b
     lo = fma(-hi, b, a)
     lo /= b
     return hi, lo
end

@inline function two_sqrt(a::T) where {T}
    hi = sqrt(a)
    lo = fma(-hi, hi, a)
    lo /= 2
    lo /= hi
    return hi, lo
end

"""
    ad_minus_bc(a, b, c, d)

Computes the determinant of a 2x2 matrix.
"""
function ad_minus_bc(a::T, b::T, c::T, d::T) where {T}
    adhi, adlo = two_prod(a,d)
    bchi, bclo = two_prod(b,c)
    return four_sum(adhi, adlo, -bchi, -bclo)
end

#=
   two_fma from three_fma algorithm from
   Sylvie Boldo and Jean-Michel Muller
   Some Functions Computable with a Fused-mac
=#

"""
   two_fma(a, b, c)

Computes `s = fl(fma(a,b,c))` and `e1 = err(fma(a,b,c))`.
"""
function two_fma(a::T, b::T, c::T) where {T}
     x = fma(a, b, c)
     if isinf(x)
        return (x, zero(T))
     end
     y, z = two_prod(a, b)
     t, z = two_sum(c, z)
     t, u = two_sum(y, t)
     y = ((t - x) + u)
     y = y + z
     return x, y
end

#=
"Concerning the division, the elementary rounding error is
generally not a floating point number, so it cannot be computed
exactly. Hence we cannot expect to obtain an error
free transformation for the division. ...
This means that the computed approximation is as good as
we can expect in the working precision."
-- http://perso.ens-lyon.fr/nicolas.louvet/LaLo05.pdf

While the sqrt algorithm is not strictly an errorfree transformation,
it is known to be reliable and is recommended for general use.
"Augmented precision square roots, 2-D norms and
   discussion on correctly reounding xsqrt(x^2 + y^2)"
by Nicolas Brisebarre, Mioara Joldes, Erik Martin-Dorel,
   Hean-Michel Muller, Peter Kornerup
=#


export takens_best_estimate

"""
    takens_best_estimate(X, εmax, metric = Chebyshev(),εmin = 0) → D_C, D_C_95u, D_C_95l
Use the so-called "Takens' best estimate" [^Takens1985][^Theiler1988]
method for estimating the correlation dimension
`D_C` and the upper (`D_C_95u`) and lower (`D_C_95l`) confidence limit for the given dataset `X`.

The original formula is
```math
D_C \\approx \\frac{C(\\epsilon_\\text{max})}{\\int_0^{\\epsilon_\\text{max}}(C(\\epsilon) / \\epsilon) \\, d\\epsilon}
```
where ``C`` is the [`correlationsum`](@ref) and ``\\epsilon_\\text{max}`` is an upper cutoff.
Here we use the later expression
```math
D_C \\approx - \\frac{1}{\\eta},\\quad \\eta = \\frac{1}{(N-1)^*}\\sum_{[i, j]^*}\\log(||X_i - X_j|| / \\epsilon_\\text{max})
```
where the sum happens for all ``i, j`` so that ``i < j`` and ``||X_i - X_j|| < \\epsilon_\\text{max}``.
In the above expression, the bias in the original paper has already been corrected, as suggested in [^Borovkova1999].

The confidence limits are estimated from the log-likelihood function by finding
the values of `D_C` where the function has fallen by 2 from its maximum, see e.g.
[^Barlow] chapter 5.3
Because the CLT does not apply (no independent measurements), the limits are not
neccesarily symmetric.

According to [^Borovkova1999], introducing a lower cutoff `εmin` can make the
algorithm more stable (no divergence), this option is given but defaults to zero.

If `X` comes from a delay coordinates embedding of a timseries `x`, a recommended value
for ``\\epsilon_\\text{max}`` is `std(x)/4`.

[^Takens1985]: Takens, On the numerical determination of the dimension of an attractor, in: B.H.W. Braaksma, B.L.J.F. Takens (Eds.), Dynamical Systems and Bifurcations, in: Lecture Notes in Mathematics, Springer, Berlin, 1985, pp. 99–106.
[^Theiler1988]: Theiler, [Lacunarity in a best estimator of fractal dimension. Physics Letters A, 133(4–5)](https://doi.org/10.1016/0375-9601(88)91016-X)
[^Borovkova1999]: Borovkova et al., [Consistency of the Takens estimator for the correlation dimension. The Annals of Applied Probability, 9, 05 1999.](https://doi.org/10.1214/aoap/1029962747)
[^Barlow]: Barlow, R., Statistics - A Guide to the Use of Statistical Methods in the Physical Sciences. Vol 29. John Wiley & Sons, 1993
"""
function takens_best_estimate(X, εmax, metric = Chebyshev(); εmin=0)
    n, η, N = 0, zero(eltype(X)), length(X)
    @inbounds for i in 1:N
        for j in i+1:N
            d = evaluate(metric, X[i], X[j])
            if εmin < d < εmax
                n += 1
                η += log(d/εmax)
            end
        end
    end
    # bias-corrected version (log-likelihood function shifted on x-axis)
    α = -(n-1)/η
    # biased version (maximum of original log-likelihood function)
    α_b = -n/η
    # value of maximum of original log-likelihood function
    mxl = n*log(α_b) + α_b * η
    # at the 95%-confidence interval, the log-l function has dropped by 2
    # -> log_l(x) - mxl + 2 = 0
    # this is a result of the invariance of the MLE, a really nice property
    # these limits are not going to be perfectly symmetric (CLT does not apply)
    mn, mx = fzeros(x-> n * log(x) + η * x - mxl +2 , 0,2*α)

    # Since the bias-correction is just a shift of the log-l function on the
    # x-axis, we can easily shift the confidence limits by the bias α-α_b
    α95u = α - α_b + mn
    α95l = α - α_b + mx

    return α, α95u, α95l
end


# # Gradient of a scalar field
#
# This example shows different methods to compute the gradient of a real-valued
# 3D scalar field ``θ(\bm{x})`` in Fourier space, where $\bm{x} = (x, y, z)$.
# It is assumed that the field is periodic with period $L = 2π$ along all
# dimensions.
#
# ## General procedure
#
# The discrete Fourier expansion of ``θ`` writes
# ```math
# θ(\bm{x}) = ∑_{\bm{k} ∈ \Z^3} \hat{θ}(\bm{k}) \, e^{i \bm{k} ⋅ \bm{x}},
# ```
# where $\bm{k} = (k_x, k_y, k_z)$ are the Fourier wave numbers and $\hat{θ}$ is
# the discrete Fourier transform of $θ$.
# Then, the spatial derivatives of $θ$ are given by
# ```math
# \frac{∂ θ(\bm{x})}{∂ x_i} =
# ∑_{\bm{k} ∈ \Z^3} i k_i \hat{θ}(\bm{k}) \, e^{i \bm{k} ⋅ \bm{x}},
# ```
# where the subscript $i$ denotes one of the spatial components $x$, $y$ or
# $z$.
#
# In other words, to compute $\bm{∇} θ = (∂_x θ, ∂_y θ, ∂_z θ)$, one has to:
# 1. transform $θ$ to Fourier space to obtain $\hat{θ}$,
# 2. multiply $\hat{θ}$ by $i \bm{k}$,
# 3. transform the result back to physical space to obtain $\bm{∇} θ$.

# ## Preparation

# In this section, we initialise a random real-valued scalar field $θ$ and compute
# its FFT.
# For more details see the [Tutorial](@ref).

using MPI
using PencilFFTs
using Random

MPI.Init()

## Input data dimensions (Nx × Ny × Nz)
dims = (64, 32, 64)

## Apply a 3D real-to-complex (r2c) FFT.
transform = Transforms.RFFT()

## Automatically create decomposition configuration
comm = MPI.COMM_WORLD
pen = Pencil(dims, comm)

## Create plan
plan = PencilFFTPlan(pen, transform)

## Allocate data and initialise field
θ = allocate_input(plan)
randn!(θ)

## Perform distributed FFT
θ_hat = plan * θ
nothing  # hide

# Finally, we initialise the output that will hold ∇θ in Fourier space.
# Noting that ∇θ is a vector field, we choose to store it as a tuple of
# 3 PencilArrays.

∇θ_hat = allocate_output(plan, Val(3))

## This is equivalent:
## ∇θ_hat = ntuple(d -> similar(θ_hat), Val(3))

summary(∇θ_hat)

# ## Fourier wave numbers

# In general, the Fourier wave numbers are of the form
# ``k_i = 0, ±\frac{2π}{L_i}, ±\frac{4π}{L_i}, ±\frac{6π}{L_i}, …``,
# where ``L_i`` is the period along dimension ``i``.
# When a real-to-complex Fourier transform is applied, roughly half of
# these wave numbers are redundant due to the Hermitian symmetry of the complex
# Fourier coefficients.
# In practice, this means that for the fastest dimension $x$ (along which
# a real-to-complex transform is performed), the negative wave numbers are
# dropped, i.e. ``k_x = 0, \frac{2π}{L_x}, \frac{4π}{L_x}, …``.

# The `AbstractFFTs` package provides a convenient way to generate the Fourier
# wave numbers, using the functions
# [`fftfreq`](https://juliamath.github.io/AbstractFFTs.jl/stable/api/#AbstractFFTs.fftfreq)
# and
# [`rfftfreq`](https://juliamath.github.io/AbstractFFTs.jl/stable/api/#AbstractFFTs.rfftfreq).
# We can use these functions to initialise a "grid" of wave numbers associated to
# our 3D real-to-complex transform:

using AbstractFFTs: fftfreq, rfftfreq

box_size = (2π, 2π, 2π)  # Lx, Ly, Lz
sample_rate = 2π .* dims ./ box_size

## In our case (Lx = 2π and Nx even), this gives kx = [0, 1, 2, ..., Nx/2].
kx = rfftfreq(dims[1], sample_rate[1])

## In our case (Ly = 2π and Ny even), this gives
## ky = [0, 1, 2, ..., Ny/2-1, -Ny/2, -Ny/2+1, ..., -1] (and similarly for kz).
ky = fftfreq(dims[2], sample_rate[2])
kz = fftfreq(dims[3], sample_rate[3])

kvec = (kx, ky, kz)

# Note that `kvec` now contains the wave numbers associated to the global domain.
# In the following, we will only need the wave numbers associated to the portion
# of the domain handled by the local MPI process.

# ## [Method 1: global views](@id gradient_method_global)

# [`PencilArray`](https://jipolanco.github.io/PencilArrays.jl/dev/PencilArrays/#PencilArrays.PencilArray)s, returned for instance by [`allocate_input`](@ref)
# and  [`allocate_output`](@ref), take indices that start at 1, regardless of the
# location of the subdomain associated to the local process on the global grid.
# (In other words, `PencilArray`s take *local* indices.)
# On the other hand, we have defined the wave number vector `kvec` which,
# for each MPI process, is defined over the global domain, and as such it takes
# *global* indices.

# One straightforward way of making data arrays compatible with wave numbers is
# to use global views, i.e. arrays that take global indices.
# These are generated from `PencilArray`s by calling the [`global_view`](https://jipolanco.github.io/PencilArrays.jl/dev/PencilArrays/#PencilArrays.global_view-Tuple{PencilArray})
# function.
# Note that, in general, global indices do *not* start at 1 for a given MPI
# process.
# A given process will own a range of data given by indices in `(i1:i2, j1:j2,
# k1:k2)`.

θ_glob = global_view(θ_hat)
∇θ_glob = global_view.(∇θ_hat)
summary(θ_glob)

# Once we have global views, we can combine data and wave numbers using the
# portion of global indices owned by the local MPI process, as shown below.
# We can use `CartesianIndices` to iterate over the global indices associated to
# the local process.

for I in CartesianIndices(θ_glob)
    i, j, k = Tuple(I)  # unpack indices

    ## Wave number vector associated to current Cartesian index.
    local kx, ky, kz  # hide
    kx = kvec[1][i]
    ky = kvec[2][j]
    kz = kvec[3][k]

    ## Compute gradient in Fourier space.
    ## Note that modifying ∇θ_glob also modifies the original PencilArray ∇θ_hat.
    ∇θ_glob[1][I] = im * kx * θ_glob[I]
    ∇θ_glob[2][I] = im * ky * θ_glob[I]
    ∇θ_glob[3][I] = im * kz * θ_glob[I]
end

# The above loop can be written in a slightly more efficient manner by precomputing
# `im * θ_glob[I]`:

@inbounds for I in CartesianIndices(θ_glob)
    i, j, k = Tuple(I)

    local kx, ky, kz  # hide
    kx = kvec[1][i]
    ky = kvec[2][j]
    kz = kvec[3][k]

    u = im * θ_glob[I]

    ∇θ_glob[1][I] = kx * u
    ∇θ_glob[2][I] = ky * u
    ∇θ_glob[3][I] = kz * u
end

# Also note that the above can be easily written in a more generic way, e.g. for
# arbitrary dimensions, thanks in part to the use of `CartesianIndices`.
# Moreover, in the above there is no notion of the dimension permutations
# discussed in [the tutorial](@ref tutorial:output_data_layout), as it is all
# hidden behind the implementation of `PencilArray`s.
# And as seen later in the [benchmarks](@ref gradient_benchmarks),
# these (hidden) permutations have zero cost, as the speed is identical
# to that of a function that explicitly takes into account these permutations.

# Finally, we can perform a backwards transform to obtain $\bm{∇} θ$ in physical
# space:

∇θ = plan \ ∇θ_hat;

# Note that the transform is automatically broadcast over the three fields
# of the `∇θ_hat` vector, and the result `∇θ` is also a tuple of
# three `PencilArray`s.

# ## [Method 2: explicit global indexing](@id gradient_method_global_explicit)

# Sometimes, one does not need to write generic code.
# In our case, one often knows the dimensionality of the problem and the
# memory layout of the data (i.e. the underlying index permutation).

# Below is a reimplementation of the above loop, using explicit indices instead
# of `CartesianIndices`, and assuming that the underlying index permutation is
# `(3, 2, 1)`, that is, data is stored in $(z, y, x)$ order.
# As discussed in [the tutorial](@ref tutorial:output_data_layout),
# this is the default for transformed arrays.
# This example also serves as a more explicit explanation for what is going on
# in the [first method](@ref gradient_method_global).

## Get local data range in the global grid.
rng = axes(θ_glob)  # = (i1:i2, j1:j2, k1:k2)

# For the loop below, we're assuming that the permutation is (3, 2, 1).
# In other words, the fastest index is the *last* one, and not the first one as
# it is usually in Julia.
# If the permutation is not (3, 2, 1), things will still work (well, except for
# the assertion below!), but the loop order will not be optimal.

@assert permutation(θ_hat) === Permutation(3, 2, 1)

@inbounds for i in rng[1], j in rng[2], k in rng[3]
    local kx, ky, kz  # hide
    kx = kvec[1][i]
    ky = kvec[2][j]
    kz = kvec[3][k]

    ## Note that we still access the arrays in (i, j, k) order.
    ## (The permutation happens behind the scenes!)
    u = im * θ_glob[i, j, k]

    ∇θ_glob[1][i, j, k] = kx * u
    ∇θ_glob[2][i, j, k] = ky * u
    ∇θ_glob[3][i, j, k] = kz * u
end

# ## [Method 3: using local indices](@id gradient_method_local)

# Alternatively, we can avoid global views and work directly on `PencilArray`s
# using local indices that start at 1.
# In this case, part of the strategy is to construct a "local" grid of wave
# numbers that can also be accessed with local indices.
# This can be conveniently done using the
# [`localgrid`](https://jipolanco.github.io/PencilArrays.jl/dev/LocalGrids/#PencilArrays.LocalGrids.localgrid)
# function of the PencilArrays.jl package, which accepts a `PencilArray` (or
# its associated `Pencil`) and the global coordinates (here `kvec`):

grid_fourier = localgrid(θ_hat, kvec)

# Note that one can directly iterate on the returned grid object:

@inbounds for I in CartesianIndices(grid_fourier)
    ## Wave number vector associated to current Cartesian index.
    local k⃗  # hide
    k⃗ = grid_fourier[I]
    u = im * θ_hat[I]
    ∇θ_hat[1][I] = k⃗[1] * u
    ∇θ_hat[2][I] = k⃗[2] * u
    ∇θ_hat[3][I] = k⃗[3] * u
end

# This implementation is as efficient as the other examples, while being
# slightly shorter to write.
# Moreover, it is quite generic, and can be made independent of the number of
# dimensions with little effort.

# ## [Method 4: using broadcasting](@id gradient_method_broadcast)

# Finally, note that the local grid object returned by `localgrid` makes it is
# possible to compute the gradient using broadcasting, thus fully avoiding scalar
# indexing.
# This can be quite convenient in some cases, and can also be very useful if
# one is working on GPUs (where scalar indexing is prohibitively expensive).
# Using broadcasting, the above examples simply become:

@. ∇θ_hat[1] = im * grid_fourier[1] * θ_hat
@. ∇θ_hat[2] = im * grid_fourier[2] * θ_hat
@. ∇θ_hat[3] = im * grid_fourier[3] * θ_hat
nothing  # hide

# Once again, as shown in the [benchmarks](@ref gradient_benchmarks) further
# below, this method performs quite similarly to the other ones.

# ## Summary

# The `PencilArrays` module provides different alternatives to deal with
# MPI-distributed data that may be subject to dimension permutations.
# In particular, one can choose to work with *global* indices (first two
# examples), with *local* indices (third example), or to avoid scalar indexing
# altogether (fourth example).

# If one wants to stay generic, making sure that the same code will work for
# arbitrary dimensions and will be efficient regardless of the underlying
# dimension permutation, methods [1](@ref gradient_method_global), [3](@ref
# gradient_method_local) or [4](@ref gradient_method_broadcast) should be
# preferred.
# These use `CartesianIndices` and make no assumptions on possible dimension
# permutations, which are by default enabled in the output of PencilFFTs
# transforms.
# In fact, such permutations are completely invisible in the implementations.

# The [second method](@ref gradient_method_global_explicit) uses explicit
# `(i, j, k)` indices.
# It assumes that the underlying permutation is `(3, 2, 1)` to loop with `i` as
# the *slowest* index and `k` as the *fastest*, which is the optimal order in
# this case given the permutation.
# As such, the implementation is less generic than the others, and
# differences in performance are negligible with respect to more generic variants.

# ## [Benchmark results](@id gradient_benchmarks)

# The following are the benchmark results obtained from running
# [`examples/gradient.jl`](https://github.com/jipolanco/PencilFFTs.jl/blob/master/examples/gradient.jl)
# on a laptop, using 2 MPI processes and Julia 1.7.2, with an input array of
# global dimensions ``64 × 32 × 64``.
# The different methods detailed above are marked on the right.
# The "lazy" marks indicate runs where the wave numbers were represented by
# lazy `Frequencies` objects (returned by `rfftfreq` and `fftfreq`). Otherwise,
# they were collected into `Vector`s.
# For some reason, plain `Vector`s are faster when working with grids generated
# by `localgrid`.

# In the script, additional implementations can be found which rely on a more
# advanced understanding of permutations and on the internals of the
# [`PencilArrays`](https://jipolanco.github.io/PencilArrays.jl/dev/) package.
# For instance, `gradient_local_parent!` directly works with the raw
# data stored in Julia `Array`s, while `gradient_local_linear!` completely
# avoids `CartesianIndices` while staying generic and efficient.
# Nevertheless, these display roughly the same performance as the above examples.
#
#         gradient_global_view!...                  89.900 μs
#         gradient_global_view! (lazy)...           92.060 μs  [Method 1]
#         gradient_global_view_explicit!...         88.958 μs
#         gradient_global_view_explicit! (lazy)...  81.055 μs  [Method 2]
#         gradient_local!...                        92.305 μs
#         gradient_grid!...                         92.770 μs
#         gradient_grid! (lazy)...                  101.388 μs  [Method 3]
#         gradient_grid_broadcast!...               88.606 μs
#         gradient_grid_broadcast! (lazy)...        151.020 μs  [Method 4]
#         gradient_local_parent!...                 92.248 μs
#         gradient_local_linear!...                 91.212 μs
#         gradient_local_linear_explicit!...        90.992 μs


"""
	eulermethod(f, t0, x0; kwargs...)

Solves the differential equation x'(t) = f(t, x(t)) using the euler method.

# Arguments
- `f`: Derivative function
- `t0`: Initial time
- `x0`: Initial position

# Keywords
- `h`: step of the method
- `iter`: number of iterations

# Returns
- `Array{typeof(x0 + h*f(t0, x0))}`: the approximated solution
"""
function eulermethod(f, t0, x0; h = 1e-3, iter = 10000)
	t = t0
	x = Array{typeof(x0 + h*f(t0, x0))}(undef, iter)
	x[1] = x0
	for j = 2:iter
		x[j] = x[j-1] + h*f(t, x[j-1])
		t += h
	end
	return x
end

"""
	targetting_eulermethod(f, dist, t0, x0; kwargs...)

Solves the differential equation x'(t) = f(t, x(t))
using a modification of the euler method to stop close to a given target.
The algorithm is similar to the original euler method,
but the function dist(x) specifies the signed distance from the new point to the target,
if it is less than 0 then it is discarded and the method's time step divided by two,
otherwise it becomes part of the solution.
The algorithm stops when the distance is positive and less than the admitted error
or when the maximum iterations limit is reached.

# Arguments
- `f`: Derivative function
- `dist`: Function for the signed distance to the target
- `t0`: Initial time
- `x0`: Initial position

# Keywords
- `h`: step of the method
- `iter`: number of iterations

# Returns
- `Array{typeof(t0)}, Array{typeof(x0 + h*f(t0, x0))}`: times and values obtained
"""
function targetting_eulermethod(f, dist, t0, x0; h = 1e-3, error = 1e-4, maxit = 100000)
	t = typeof(t0+h)[t0]
	x = Array{typeof(x0 + h*f(t0, x0)), 1}()
	push!(x, x0)
	while dist(last(x)) >= error && maxit > 0
		nx = last(x) + h*f(last(t), last(x))
		d = dist(nx)
		if (d < zero(d))
			h /= 2
		else
			push!(t, last(t)+h)
			push!(x, nx)
		end
		maxit -= 1
	end
	return t, x
end


using Plots
using LaTeXStrings

function euler(init,pars,dt,tmax)
    N = Int(tmax/dt)
    x = Vector(undef,N+1)
    y = Vector(undef,N+1)
    x[1] = init[1]
    y[1] = init[2]
    for i in 1:N
        dx = a*x[i] - b*x[i]*y[i]
        dy = c*x[i]*y[i] - d*y[i]
        x[i+1] = x[i] + dt*dx
        y[i+1] = y[i] + dt*dy
    end
    t = range(1,length=N+1,step=dt)
    return t,[x,y]
end

a= 2/3; b=4/30; c=1/10; d=1
pars = [a,b,c,d]
init = [10,15]
t,x = euler(init,pars,1e-3,50)

init2 = [15,10]
t2,x2 = euler(init2,pars,1e-3,50);

p1 = plot(t,x, label="",xlabel="Time", ylabel="pop. [au]", title = L"x_0 < y_0")
p2 = plot(t2,x2, label=["Prey","Predator"],legendtitle="Species",xlabel="Time", title = L"x_0 > y_0")
plot(p1,p2)

plot([x[1],x2[1]],[x[2],x2[2]],label=[L"x_0 < y_0",L"x_0 > y_0"],legendtitle="init",title="Phase space",xlabel="x",ylabel="y")

function euler_noise(init,pars,dt,tmax,sd)
    N = Int(tmax/dt)
    x = Vector(undef,N+1)
    y = Vector(undef,N+1)
    x[1] = init[1]
    y[1] = init[2]
    for i in 1:N
        dx = a*x[i] - b*x[i]*y[i]
        dy = c*x[i]*y[i] - d*y[i]
        noisex = sqrt(dt)*randn()*sd[1]
        noisey = sqrt(dt)*randn()*sd[2]
        x[i+1] = x[i] + dt*dx + noisex
        y[i+1] = y[i] + dt*dy + noisey
        if x[i+1]<0 x[i+1] = 0 end
        if y[i+1]<0 y[i+1] = 0 end
    end
    t = range(1,length=N+1,step=dt)
    return t,[x,y]
end

init = [10,10]
t,x = euler_noise(init,pars,1e-3,100,[.3,.3])
t2,x2 = euler_noise(init,pars,1e-3,100,[0,0])

p1 = plot(t,x, label="",xlabel="Time", ylabel="pop. [au]", title = "noise")
p2 = plot(t2,x2, label=["Prey","Predator"],legendtitle="Species",xlabel="Time", title = "no noise")
plot(p1,p2)

p1 = plot(t,[x[1],x2[1]], label="",xlabel="Time", ylabel="pop. [au]", title = "Prey")
p2 = plot(t2,[x[2],x2[2]], label=["noise","no noise"],legendtitle="Species",xlabel="Time", title = "Predator")
plot(p1,p2)

plot([x[1],x2[1]],[x[2],x2[2]],label=["noise","no noise"],title="Phase space",xlabel="x",ylabel="y")

function euler_ext(init,pars,dt,tmax,sd)
    N = Int(tmax/dt)
    x = Vector(undef,N+1)
    y = Vector(undef,N+1)
    x[1] = init[1]
    y[1] = init[2]
    for i in 1:N
        dx = a*x[i]*(1-x[i]/K) - b*x[i]*y[i]/(x[i]+S)
        dy = c*x[i]*y[i]/(x[i]+S) - d*y[i]
        noisex = sqrt(dt)*randn()*sd[1]
        noisey = sqrt(dt)*randn()*sd[2]
        x[i+1] = x[i] + dt*dx + noisex
        y[i+1] = y[i] + dt*dy + noisey
        if x[i+1]<0 x[i+1] = 0 end
        if y[i+1]<0 y[i+1] = 0 end
    end
    t = range(1,length=N+1,step=dt)
    return t,[x,y]
end

pars = a,b,c,d,K,S = [1,1,1,1/3,30,10]
init = [1,1]
t,x1  = euler_ext(init,pars,1e-3,100,[0.5,0.5])
t,x1no = euler_ext(init,pars,1e-3,100,[0,0])

init = [10,10]
t,x10  = euler_ext(init,pars,1e-3,100,[0.5,0.5])
t,x10no = euler_ext(init,pars,1e-3,100,[0,0]);

p1 = plot(t,[x1[1]  ,x10[1]]  ,label="",title="Prey",ylabel="noise")
p2 = plot(t,[x1[2]  ,x10[2]]  ,title="Predator",label=[L"x_0 = y_0 = 1",L"x_0 = y_0 = 10"])
p3 = plot(t,[x1no[1],x10no[1]],label="",xlabel="time",ylabel="no noise")
p4 = plot(t,[x1no[2],x10no[2]],label="",xlabel="time")
plot(p1,p2,p3,p4)

p1 = plot([x1[1],x10[1]],[x1[2],x10[2]],label="",title="nnoise",xlabel="x",ylabel="y")
p2 = plot([x1no[1],x10no[1]],[x1no[2],x10no[2]],label=[L"x_0 = y_0 = 1",L"x_0 = y_0 = 10"],title="no noise",xlabel="x",ylabel="y")
plot(p1,p2)

## Interpretation:
# K = carrying capacity or maximum population size (if population is greater than K it decreases, if population is lower than K it increases)
# S = Michaelis constant: concentration at half reaction rate v/2
# Change of system by multiplying v~x/(x+S):
# reaction rate linear for small x (v~x/S), saturation for large x (v~1)
# Change of system by adding -x/K:
# population maximum evens out (otherwise x or y can go to infinity)
# noise:
# no attractor


# ## ロジスティック回帰
# ### 例29
# シグモイド関数を定義しておきます。
# ```julia
# sigmoid(x::Matrix, β::Vector) = 1 / (1 + exp(-x*β))
# ```
using Joe, Plots
using Plots.PlotMeasures # hide
Plots.reset_defaults() # hide
default( # hide
    titlefont  = font("JuliaMono", default(:titlefontsize),  ),  # hide
    guidefont  = font("JuliaMono",  default(:guidefontsize),  ), # hide
    tickfont   = font("JuliaMono", default(:tickfontsize),   ), # hide
    legendfont = font("JuliaMono",  default(:legendfontsize), ), # hide
    left_margin = 30px, # hide
    bottom_margin = 30px # hide
) # hide
p29 = plot(xlims=(-10,10), xlabel="x",ylabel="P(Y=1|x)",
            legend=:topleft, title="ロジスティック曲線")
x29 = insert_ones(-10:0.1:10)
for i in [0, 0.2, 0.5, 1, 2, 10]
    plot!(p29, x29[:,2], Joe.sigmoid(x29,[0,i]),label="$i")
end

savefig(p29,joinpath(@OUTPUT,"fig2-1.svg")) # hide
# \fig{fig2-1}

# ## ニュートン・ラフソン法
# ### 例31
# $x^2-1=0$を4を初期値として数値計算で解く問題。10回でどの程度近づくか。
# せっかくなので、 微分をあらわに書き下すのではなくて、自動微分パッケージを使ってみることにします。
# 見た目が$x \leftarrow x - \dfrac{f(x)}{f'(x)}$という公式のまんまですね。
using Zygote
f(x) = x^2 -1
let # hide
x=4
for i in 1:10
    x -= f(x)/f'(x)
    println(x)
end
end # hide

# ### 例31
# 2変数関数が2つある場合。 Zygote.jlにjacobianがなかったので、ForwardDiff.jlを使ってみます。

using ForwardDiff

f(x,y) = [x^2+y^2-1, x+y]
let # hide
z = [3,4]
for i in 1:10
    z -= ForwardDiff.jacobian(x->f(x[1], x[2]),z) \ f(z[1],z[2])
    println(z)
end
end # hide

# ### 例33
# 最尤推定によりロジスティック回帰を行う問題。
# $\nabla l(\beta_0, \beta) = 0$を求める問題を解いてみる。
using Joe, Random, LinearAlgebra
using Plots
N=1000; p=2; Random.seed!(1)
X = insert_ones(randn(N,p))
β = randn(p+1)
prob = @. 1/(1 + exp($*(X,β)))
threshold=0.5
y = ifelse.(rand(N) .> prob,1,-1) # ここまでデータ生成
@show y β;
# せっかくなのでデータを可視化してみます。
data1 = X[y .== 1, 2:3]
data2 = X[y .== -1, 2:3]
p33=scatter(data1[:,1],data1[:,2],ylims=(-5,5),marker=:auto, label="y=1")
scatter!(p33,data2[:,1],data2[:,2],marker=:auto,label="y=-1")
plot!(p33,x-> -β[1]/β[3] - β[2]/β[3]*x, label="β")
savefig(p33,joinpath(@OUTPUT,"fig2-2.svg")) # hide
# \fig{fig2-2}
γ = randn(2+1) #初期値
γ2 = copy(γ) #別解用にコピー
@show γ
W(v::Vector) = @.(v/(1+ v)^2) |> diagm
t = true
for i in 1:10
    global γ
    s=X*γ
    v = @. exp(-y*s)
    u = @. y*v/(1+v)
    γ += ((X'*W(v)*X) \ X') * u
    #δ'*δ < 0.001 && (t = false)
    @show γ
end
plot!(x-> -γ[1]/γ[3] - γ[2]/γ[3]*x, label="γ")
# 今度は横着をして、ヘシアンを使って計算してみます。
using Zygote, LinearAlgebra
l(γ,X=X,y=y) = sum(@. log( 1 /(1+exp(*($*(X,γ),-y))))) #対数尤度関数

for i in 1:10
    global γ2
    δ =  Zygote.hessian(l,γ2) \ l'(γ2)
    γ2 -= δ
    @show γ2
end

plot!(p33,x-> -γ2[1]/γ2[3] - γ2[2]/γ2[3]*x, label="γ2")
savefig(p33,joinpath(@OUTPUT,"fig2-2-2.svg")) # hide
# \fig{fig2-2-2}
# \lineskip
# だいたい同じような境界線が得られることが分かりました。
# ### 例34
# データの半分を使ってロジスティック回帰で学習し、残りのデータで検証する練習。
# まずデータを生成します。
using Joe, Random, Distributions, Plots, LinearAlgebra
Random.seed!(123)
n=100
x34 = vcat(randn(n).+1,randn(n).-1) |> insert_ones
y34 = vcat(ones(n),-ones(n));
# 訓練データとテストデータを分けて可視化します。
index = sample(1:2n,2n,replace=false); #ランダムなインデックスを作り
train = index[begin:n];# 学習用とテスト用にインデックスを分ける。
test = index[n+1:end];
X_train = x34[train,:];y_train = y34[train]
X_test = x34[test,:];y_test = y34[test]
p34 = scatter(X_train[:,2],y_train,label="train")
scatter!(p34,X_test[:,2],y_test,label="test")
savefig(p34,joinpath(@OUTPUT,"fig2-2-3.svg")) # hide
# \fig{fig2-2-3}
# \lineskip
# \note{
    # **訓練データとテストデータを分ける方法**
    #
    # 原著のpythonコードでは以下のようになっています。
    # ```python
    # import numpy as np
    # train = np.random.choice(2*n, int(n), replace=false)
    # test = list(set(range(n))-set(train))
    # ```
    # setを使った書き方は便利ですが、juliaでは重複を許さずにランダムに並べたインデックスを二つに分ける
    # 方法を使いました。
    # ```julia
    # using StatsBase
    # index = sample(1:n,n,replace=false);
    # train = index[begin:Int(n/2)];
    # test = index[Int(n/2)+1:end];
    # ```
    # でもよく考えるとsetdiffを使えば十分でした。
    # ```julia
    # train = sample(1:2n,n,replace=false); test = setdiff(1:2n,train)
    # ```
# }
# ループは原著の方法に従いました。
β = [0,0] #初期値
γ = randn(2)
while sum(β-γ)^2 > 0.001
    global β, γ
    local W
    β = γ
    s = X_train*β
    v = @. exp(-s*y_train)
    u = @. y_train*v/(1+v)
    w = @. v/(1+v)^2
    W = diagm(w)
    z = @. s + u/w
    γ = (X_train'*W*X_train)\(X_train'*W*z)
    @show γ;
end
# 次にこのγを使って、テストデータを予測してみます。 正誤をカウントする関数を次のように定義します。
# ```julia
# function table_count(test,pred)
#     @assert length(test) == length(pred)
#     m = unique(test) |> length
#     count = zeros(Int,m,m)
#     for i in 1:length(test)
#         count[test[i],pred[i]] += 1
#     end
#     return count
# end
# ```
using Joe:table_count
y_pred = X_test*γ .|> x -> ifelse(x>0,2,1)
y_answer = y_test .|> x -> ifelse(x>0,2,1)
table = table_count(y_answer,y_pred)
@show table;
正答率 = sum(diag(table)) / sum(table)
# ## 2.3 線形判別と二次判別
# ### 例35
# まずは与えられた平均と分散共分散行列から二次元正規分布のデータを2種類生成します。
using Distributions, Random,Plots, LinearAlgebra, Parameters
μ₁=[2,2];  Σ₁ = [2 0; 0 2]
μ₂=[-3,-3]; Σ₂ = [1 -0.8; -0.8 1]

N = 100;Random.seed!(123)
data1 = rand(MvNormal(μ₁,Σ₁),100) |> transpose
data2 = rand(MvNormal(μ₂,Σ₂),100) |> transpose
p35 = scatter(data1[:,1],data1[:,2])
scatter!(p35, data2[:,1],data2[:,2])

savefig(p35,joinpath(@OUTPUT,"fig2-3.svg")) # hide
# \fig{fig2-3}
# \lineskip
# ここまでデータの生成。
# 生成したデータから平均と分散共分散行列を計算する。
μ̂₁ = mean(data1,dims=1)'; Σ̂₁ = cov(data1,dims=1);
μ̂₂ = mean(data2,dims=1)' ; Σ̂₂ = cov(data2,dims=1);
# \warning{mean関数で行列の列方向の平均をとる時、得られるデータは二次元の横ベクトルになっている。
# 縦ベクトルとして扱いたいので、転置(正確には複素共役）をしている。}
# それぞれの分布の情報を複合型にまとめておきます。
# ```julia
# @with_kw struct QDA
#     μ::Array # 平均
#     Σ::Matrix # 分散共分散行列
#     invΣ::Array = inv(Σ) # 分散共分散行列の逆行列
#     detΣ = det(Σ) # 分散共分散行列の行列式
# end
# ```
# \note{しれっとパラメータを初期化した複合型を使っていますが、ここではParameters.jlとい
# うサードパーティライブラリを使っています。}
# また、多変量正規分布の対数尤度を取得する関数を以下のように定義します。
# ここでは、juliaのfunction-like objectの機能を利用しています。
# また、可変長引数(x...)を利用して、データが3次元以上の場合にも拡張しています。
# キーワード付き引数に事前確率を加えています。
# \note{function-like objectの機能を使うと、複合型のインスタンスを関数のように
# 使うことができ、再利用しやすくなります。}
# ```julia
# function (qda::QDA)(x...,prior=1.0)
#     data = collect(x)
#     @unpack μ,invΣ,detΣ = qda
#     a = -0.5*(data-μ)' * invΣ * (data-μ)
#     a[1] - log(detΣ) - log(prior)
# end
# ```
using Joe: QDA
param1=QDA(μ= μ̂₁,Σ = Σ̂₁);param2=QDA(μ = μ̂₂,Σ = Σ̂₂)

hanbetsu(x,y) = param1(x,y) - param2(x,y)
x35=-5:0.1:5;y35=-5:0.1:5;
scatter(data1[:,1],data1[:,2])
scatter!( data2[:,1],data2[:,2])
p35_2=contour!(x35,y35, hanbetsu.(x35,y35'), title="QDA")
savefig(p35_2,joinpath(@OUTPUT,"fig2-4.svg")) # hide
# \fig{fig2-4}
#
# 線形判別を行う場合は、分散共分散行列が等しいことを仮定します。
# data1とdata2を、それぞれ中心化後に統合したデータセットについて、
# 新たな分散共分散行列を求めることにします。
Σ_L =  vcat(data1 .- μ̂₁' , data2 .- μ̂₂') |> cov
param1_L=QDA(μ= μ̂₁,Σ =Σ_L );param2_L=QDA(μ = μ̂₂,Σ = Σ_L)
hanbetsu_L(x,y) = param1_L(x,y) - param2_L(x,y)
p35_3=contour!(p35,x35,y35, hanbetsu_L.(x35,y35'), title="LDA")
savefig(p35_3,joinpath(@OUTPUT,"fig2-5.svg")) # hide
# \fig{fig2-5}

# ### 例36 (Fisherのあやめ)
# まずRDatasetsからirisのデータセットを読み込みます。
using ScikitLearn, StatsBase, Random
using Joe:QDA
@sk_import datasets: load_iris
iris = load_iris()
x = iris["data"]
y = iris["target"]
# 訓練データとテストデータを分けます。
Random.seed!(123)
n = length(y)
index = sample(1:n,n,replace=false); #ランダムなインデックスを作り
train = index[begin:Int(n/2)];# 学習用とテスト用にインデックスを分ける。
test = index[Int(n/2)+1:end];
X_train= x[train,:]; X_test = x[test,:];
y_train = y[train]; y_test = y[test];
# それぞれのあやめの訓練データの平均と分散共分散行列を求めて保存する。
Params = QDA[]
for i in 1:3
    μ,Σ =  mean_and_cov(X_train[y_train .==i,:])
    push!(Params, QDA(μ=μ', Σ=Σ)) #μは行ベクトルであることに注意
end

# テストデータで検証する。対数尤度を最大化するラベルを選択します。
y_pred = similar(y_test)
for i in 1:length(y_test)
    y_pred[i] = argmax([param(X_test[i,:]...) for param in Params])
end

using Joe:table_count
table_count(y_test,y_pred)

# ### 問29 事前確率が分かっている場合
# あやめの事前確率が[0.5,0.25,0.25]だったときは、対数をとって足せば良いでしょう。
# キーワード付き引数で事前確率を指定できるようにしたので、
# 少し書き換えるだけで対応可能です。今回の場合、予測結果に対して影響は少ないようです。
priors = [0.5,0.25,0.25]
y_pred2 = similar(y_test)
for i in 1:length(y_test)
    y_pred2[i] = argmax([Params[j](X_test[i,:]...;prior=priors[j]) for j in 1:3])
end
table_count(y_test,y_pred2)
# ## K近傍法
# まず、K近傍法による関数を定義します。原著の記法をほぼ踏襲しています。
# タイブレーキングが少し重複した書き方になっているのが残念。
# ```julia
# using LinearAlgebra, StatsBase
# function knn(X_train::Matrix,y_train::Vector,X_test::Vector, k)
#     n = size(X_train)[1]
#     distance = [norm(X_train[i,:]-X_test) for i in 1:n]
#     S = sortperm(distance)[1:k]
#     u = counts(y_train[S], 1:k)
#     u_max = maximum(u)
#     m = findall(c->c==u_max,u)
#     while length(m)!==1 #タイブレーキング
#         k -=1
#         S = S[1:k]
#         u = counts(S, 1:k)
#         u_max = maximum(u)
#         m = findall(c->c==u_max,u)
#     end
#     return m[1]
# end
# ```
# テストデータが複数の場合は多重ディスパッチで対応します。
# ```julia
# function knn(X_train::Matrix,y_train::Vector,X_test::Matrix, k)
#     l = size(X_test)[1]
#     w = Array{Int}(undef,l)
#     for i in 1:l
#         w[i] = knn(X_train,y_train,X_test[i,:], k)
#     end
#     return w
# end
# ```
# ### 例37 Fisherのあやめ returns
# 例36と同じデータを使ってK近傍法でやってみます。
using Joe:knn,table_count
y_pred_knn = knn(X_train,y_train,X_test,3)
table_count(y_test,y_pred_knn)
# この乱数の種の場合だと、二次判別とそれほど精度は変わりません。
# ## ROC曲線
# ### 例38
# x, y という表記はpositive, negativeに
# 添え字を陽性をp(positive), 陰性をn(negative)に変えています。
# また、varはσに変更しています。
using Distributions, Plots
μₚ = 1; μₙ = -1
σₚ=1; σₙ =1
Nₚ = 1000; Nₙ = 10000
positive = rand(Normal(μₚ,σₚ),Nₚ)
negative = rand(Normal(μₙ,σₙ),Nₙ)
θ = exp.(-10:0.1:100);
U = Vector{Float64}(undef,length(θ))
V = Vector{Float64}(undef,length(θ))
for i in 1:length(θ)
    global U,V # hide
    U[i] = sum(@. pdf(Normal(μₚ,σₚ),negative) / pdf(Normal(μₙ,σₙ),negative) > θ[i]) / Nₙ
    V[i] = sum(@. pdf(Normal(μₚ,σₚ),positive) / pdf(Normal(μₙ,σₙ),positive) > θ[i]) / Nₚ
end

AUC = 0
for i in 1:length(θ)-1
    global AUC # hide
    AUC += abs(U[i+1]-U[i])*V[i]
end
p38 = plot(U,V,xlabel="False Positive",ylabel="False Negative",
            title="ROC curve",legend=false,
            ann = (0.5,0.5,"AUC = $(round(AUC,digits=2))"))

savefig(p38,joinpath(@OUTPUT,"fig2-6.svg")) # hide
# \fig{fig2-6}


using Test, Astro

print("12.a Sidereal time (mean)")
# Meeus page 88
# Find mean and apparent sidereal  time at Greenwich on 1987 April 10 at 0 UT:
# Mean sidereal time should be 13h10m46.3668s
# Nutation in right ascension should be -0.2317 degrees
# Apparent sidereal time is 13h10m46.1351s
jd = cal_to_jd(1987, 4, 10) + hms_to_fday(0,0,0)
theta0 = mean_sidereal_time_greenwich(jd)
# theta0 is mean sidereal time in radians (2pi radians = 24 hrs)

theta0_st = fday_to_hms(theta0 / (pi * 2)) # is a tuple!
theta_fday = hms_to_fday(theta0_st...)     # so splat required... :)
# mean sidereal time
@test isapprox(theta0 / (pi * 2), theta_fday, atol=0.1)
@test(theta0_st[1] == 13)
@test(theta0_st[2] == 10)
@test isapprox(theta0_st[3], 46.3668, atol=0.1)
println(" passed")

print("12.a Sidereal time (apparent)")
jd = cal_to_jd(1987, 4, 10) + hms_to_fday(0,0,0)
theta0 = apparent_sidereal_time_greenwich(jd)
# apparent sidereal time
ast_hms = fday_to_hms(theta0/(2 * pi))
@test(ast_hms[1] == 13)
@test(ast_hms[2] == 10)
@test isapprox(ast_hms[3], 46.1351, atol=0.1)
println(" passed")

print("12.b Sidereal time (mean)")
# find the mean sidereal time at Greenwich on 1987 April 10 at 19h21m00s
# jd = cal_to_jd(1987, 4, 10) + hms_to_fday(19,21,0)
msd_midnight = mean_sidereal_time_greenwich(cal_to_jd(1987, 4, 10))
println(msd_midnight)

@test isapprox(theta0 / (pi * 2), 128.7378734 / 360, atol=0.2)
println(" passed")

println(" Mean and apparent sidereal time")
mst_g = fday_to_hms(mean_sidereal_time_greenwich(cal_to_jd(1987, 4, 10)) / (2 * pi))
@test(mst_g[1] == 13)
@test(mst_g[2] == 10)
@test isapprox(mst_g[3] , 46.3668, atol=0.01)

ast_g = fday_to_hms(apparent_sidereal_time_greenwich(cal_to_jd(1987, 4, 10)) / (2 * pi))
@test(ast_g[1] == 13)
@test(ast_g[2] == 10)
@test ast_g[3] ≈  46.1351 atol= 0.3#  good enough?

# mean sidereal time and gmt/ut should be the same around September 21/22
mst = fday_to_hms(mod2pi(mean_sidereal_time_greenwich(cal_to_jd(2014,9,21)))/(2 * pi))
@test(mst[1] == 23)
@test(mst[2] == 59)
println(" passed")

# Meeus Example 12.a
# mean_sidereal_time_greenwich(Dates.datetime2julian(DateTime(1987, 4, 10, 0, 0, 0))) |> radianstime_to_fday |> fday_to_hms
# (13,10,46.366821793410054)

#=

function gmst(dt::DateTime)
    jd = Dates.datetime2julian(dt)
    theta0 = mean_sidereal_time_greenwich(jd) # theta0 is mean sidereal time in radians (2pi radians = 24 hrs)
    theta0_st = fday_to_hms(theta0 / (pi * 2)) # is a tuple!
end

# from https://www.cv.nrao.edu/~rfisher/Ephemerides/times.html

function gmst1(dt::DateTime)
    jd  = Dates.datetime2julian(dt)
    d =   jd - 2451545.0
    T =  d/36525
    gmst = 24110.54841 + 8640184.812866T + 0.093104(T^2) - 0.0000062(T^3)
    # difference from UT1
    gmst += (Dates.hour(dt) * 60 * 60) + (Dates.minute(dt) * 60) + Dates.second(dt)
    return fday_to_hms((gmst % 86400) / 86400)
end

=#


# we write a function that returns a root of func in the
# interval [low, high] with relative error tol, assuming such a root exists
function binarySearch(func, low, high, tol)

	# we repeat until the tolorance condition is met
	while abs(high - low)/2 > tol * abs(high + low)/2

		# we define mid as the midpoint of the interval [low, high]
		mid = (low + high)/2

		# we check if the value of func at high and mid have the same sign
		if func(high) * func(mid) > 0

			# and reassign [low, high] accordingly
		    high = mid
		else
		    low = mid
		end
	end

	# we return the avarage of low and high as the approximate root
	return (low + high)/2;
end


module TestStats

using Test: @test, @test, @testset
using Ensemble
using Ensemble.Stats: logsumexp

function testlogsumexp_random()
    for i in 1:100
        x = randn()
        y = randn()

        @test isapprox(logsumexp(x,y), log(exp(x)+exp(y)))
    end
end

function testlogsumexp_inf()
    @test logsumexp(-Inf, 0.0)==0.0
    @test logsumexp(0.0, -Inf)==0.0
    @test logsumexp(-Inf,-Inf)==-Inf
end

function testall()
    testlogsumexp_random()
    testlogsumexp_inf()
end

end


# https://en.wikipedia.org/wiki/Kernel_(statistics)

## these functions inspired by KernelFunctions.jl
# --- begin
abstract type AbstractKernel end

"""
    (k::AbstractKernel)(x, xg)

Evaluate the kernel for the pair of values `(x,xg)`.
"""
(k::AbstractKernel)(x, xg) = kern(k, x, xg)
kern(k::AbstractKernel, x, xg) = scaling(k) * unscaledkern(k,x,xg)
bandwidth(k::AbstractKernel) = k.h
scaling(k::AbstractKernel) = k.constant / k.h

function Base.show(io::IO, k::T) where {T<:AbstractKernel}
    return print(io, "$T (h = ", k.h, ")")
end
# --- end

# Gaussian
struct GaussianKernel{T<:Real,S<:Real} <: AbstractKernel
    h::T
    constant::S
end
GaussianKernel(h::T; constant::S=1/sqrt(2*pi)) where {T,S} = GaussianKernel{T,S}(h,constant)
unscaledkern(k::GaussianKernel, x, xg) =  exp(-((x-xg)/k.h)^2 / 2)

"""
    NormalKernel(h)

Alias of [`GaussianKernel`](@ref).
"""
const NormalKernel = GaussianKernel

# Uniform
struct UniformKernel{T<:Real,S<:Real} <: AbstractKernel
    h::T
    constant::S
end
UniformKernel(h::T; constant::S=1/2) where {T,S} = UniformKernel{T,S}(h,constant)
unscaledkern(k::UniformKernel, x::T, xg) where {T} = abs((x-xg)/k.h) <= 1 ? one(T) : zero(T)

# Epanechnikov
struct EpanechnikovKernel{T<:Real,S<:Real} <: AbstractKernel
    h::T
    constant::S
end
EpanechnikovKernel(h::T; constant::S=3/4) where {T,S} = EpanechnikovKernel{T,S}(h,constant)
unscaledkern(k::EpanechnikovKernel, x::T, xg) where {T} = abs((x-xg)/k.h) <= 1 ? 1 - ((x-xg)/k.h)^2 : zero(T)


"""
R ではウェルチの方法による一元元配置分散分析がデフォルトなのであるが，Julia にはないので以下の関数を定義しておく。

関数定義
UnequalVarianceOneWayANOVATestOneWayANOVATest(groups::AbstractVector{<:Real}...)

OneWayANOVATest(groups::AbstractVector{<:Real}...) の場合と同じく，引数として各標本の観測データベクトルを列挙する。
"""

using Statistics, Distributions

function UnequalVarianceOneWayANOVATest(groups::AbstractVector{<:Real}...)
    roundn(x, digits) = round(x, digits=digits)
    formatp(p) = p < 0.00001 ? "< 0.00001" : "= " * string(roundn(p, 5))
    ni = Int[]
    mi = Float64[]
    vi = Float64[]
    for x in groups
        append!(ni, length(x))
        append!(mi, mean(x))
        append!(vi, var(x))
    end
    k = length(ni)
    wi = ni ./ vi
    sum_wi = sum(wi)
    tmp = sum((1 .- wi ./ sum_wi).^2 ./ (ni .- 1)) ./ (k^2 - 1)
    m = sum(wi .* mi) / sum_wi
    df1, df2 = k - 1, 1 / 3tmp
    F = sum(wi .* (mi .- m).^2) / (df1 * (1 + 2 * (k - 2) * tmp))
    p = ccdf(FDist(df1, df2), F)
    println("F = $(roundn(F, 5)),  df1 = $df1,  df2 = $(roundn(df2, 5)),  p-value $(formatp(p)))")
end;

"""
使用例

julia> g1 = [27.7, 45.2, 32.8, 49.5, 31.0, 55.5, 31.7, 53.9];

julia> g2 = [38.0, 47.4, 55.3, 52.5, 39.2, 34.9];

julia> g3 = [44.0, 29.8, 42.5, 23.4, 20.7, 57.3, 42.0, 56.8, 44.1, 32.7];

julia> UnequalVarianceOneWayANOVATest(g1, g2, g3)
F = 0.51339,  df1 = 2,  df2 = 13.589,  p-value = 0.60962)

"""


function rtuples(T::Type{<:Number}, D::Int, k::Int)::Vector{NTuple{D,T}}
    return [tuple(rand(T,D)...) for i in 1:k];
end
function rarrays(T::Type{<:Number}, D::Int, k::Int)::Vector{Vector{T}}
    return [rand(T,D) for i in 1:k];
end
function totuples(data)
    return [tuple(e...) for e in data];
end
function get_perfect_data(; n = 100)
    cs = [(-5,-5),(0,5),(5,-5)]
    dt = Vector{Tuple{Float64,Float64}}();
    xr = randn(n);
    yr = randn(n);
    for rd in 1:n
        for c in cs
            push!(dt, (xr[rd] + c[1], yr[rd] + c[2]))
        end
    end
    return dt;
end
function center_coords(ps)
    s = ps[1];
    for i in 2:length(ps)
         s = s .+ ps[i]
    end
    return s ./ length(ps);
end
function assign(p::T, cs::Vector{T}) where T
    findmin(distance(p,cs))[2];
end
function assign(ps::Vector{T}, cs::Vector{T}) where T
    return [findmin(distance(p,cs))[2] for p in ps];
end
function distance(p1, p2)
    return sqrt(sum((p1 .- p2).^2))
end
function distance(p1::T, ps::Vector{T}) where T
    return [distance(p1,p) for p in ps];
end
function ave_distance(p, ps)
    length(ps) == 0 && return zero(eltype(p))
    ds = distance(p,ps)
    return sum(ds)/length(ps);
end
function generate_test_2d_data(k; r = 35, n = 1000)
    cs = [(rand()*r,rand()*r) for i in 1:k];
    dt = Vector{Tuple{Float64,Float64}}();
    xr = randn(n);
    yr = randn(n);
    for rd in 1:n
        for c in cs
            push!(dt, (xr[rd] + c[1], yr[rd] + c[2]))
        end
    end
    return dt;
end
export fake_data
function fake_data()
    return get_perfect_data();
end
function remove(col::Vector, td)
    col[[1:(td-1);(td+1):end]];
end
function remove!(col::Vector, td)
    temp = remove(col,td);
    pop!(col);
    col .= temp;
    return col;
end