"""
    KernelFormulation

Abstract supertype for functors that calculate the probability density based on distance.

Concrete implementations must define functor methods with the form:

```julia
(k::SomeKernel)(distance) = # do something with `distance` and `k`"
```

Using an anonymous function would not allow rebuildable model parameters.
"""
abstract type KernelFormulation end

"""
    ExponentialKernel <: KernelFormulation

    ExponentialKernel(λ)

Probability density function of distance ``d``.

```math
y = e^{-d/λ}
```

where λ is a shape parameter.
"""
Base.@kwdef struct ExponentialKernel{P} <: KernelFormulation
    λ::P = Param(1.0, bounds=(0.0, 2.0))
end
(f::ExponentialKernel)(d) = exp(-d / f.λ)

"""
    GeometricKernel <: KernelFormulation

    GeometricKernel(α)

Probability density function of distance ``d``.

The Geometric kernel has a power-law decrease.

```math
y = (1+d)^α (α+1)(α+2) / (2 π)
```

where α is a shape parameter.
"""
Base.@kwdef struct GeometricKernel{P} <: KernelFormulation
    α::P = Param(1.0, bounds=(-1000.0, 1000.0))
end
(f::GeometricKernel)(d) = (1 + d)^f.α * ((f.α + 1)*(f.α + 2)) / (2 * π)

"""
    GaussianKernel <: KernelFormulation

    GaussianKernel(α)

Probability density function of distance ``d``.

```math
y = 1/ (π α^2) e^{-d^2/α^2}
```

where α is a positive parameter.
"""
Base.@kwdef struct GaussianKernel{P} <: KernelFormulation
    α::P = Param(1.0, bounds=(0.0, 1000.0))
end
(f::GaussianKernel)(d) = 1 / (π * f.α^2) * exp(- d^2 / f.α^2)

"""
    WeibullKernel <: KernelFormulation

    WeibullKernel(α,β)

Probability density function of distance ``d``.

```math
y =β /(2 π α^2) d^{β-2} e^{ -d^β/α^β}
```

where α and β are positive parameters.
"""
Base.@kwdef struct WeibullKernel{A,B} <: KernelFormulation
    α::A = Param(1.0, bounds=(0.0, 1000.0))
    β::B = Param(2.0, bounds=(0.0, 1000.0))
end
(f::WeibullKernel)(d) = f.β / (2 * π * f.α^2) * d^(f.β - 2) * exp(- d^f.β / f.α^f.β)


str = """
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450"""

str = join(split(str), "")
l = map(x->parse(Int, x), collect(str))
println(l)

#kernel1 = [1,1,1]
#results1 = conv(l, kernel1)[2:end-2]
#println(results1) # can be used to find max sum

prods = [prod(l[i:i+3]) for i in 1:length(l)-3]
println(maximum(prods))

prods = [prod(l[i:i+12]) for i in 1:length(l)-12]
println(maximum(prods))


#=
  Coupon collecter's problem, or card collecter's problem.

  There are N different collecter's cards hidden in a package, but we don't
  know which card there is in the package we buy.
  We want to collect all of them, how many packages must one buy to collect
  all the different cards?

  See https://en.wikipedia.org/wiki/Coupon_collector%27s_problem
  """
  In probability theory, the coupon collector's problem describes 'collect all coupons and win'
  contests. It asks the following question: If each box of a brand of cereals contains a
  coupon, and there are n different types of coupons, what is the probability that more
  than t boxes need to be bought to collect all n coupons?

  An alternative statement is: Given n coupons, how many coupons do you expect you need
  to draw with replacement before having drawn each coupon at least once? The mathematical
  analysis of the problem reveals that the expected number of trials needed grows as
  Θ(n log(n).
  For example, when n = 50 it takes about 225[b] trials on average to collect all 50 coupons.

  ...

  [b]: E(50) = 50(1 + 1/2 + 1/3 + ... + 1/50) = 224.9603, the expected number of trials to
  collect all 50 coupons.

   The approximation n*log(n) + γ*n + 1/2 for this expected number gives in this case
   ≈ 195.6011 + 28.8608 + 0.5 ≈ 224.9619.  [log is the natural logarithm]
  """

  Note: The assumptions are that the slots are filled using a Poisson process
  and the parameter will influence the result much.

  p (mean): 5.7496

  Distributions of variable p (num:0)
  7.00000 =>    3172  (0.317200)
  6.00000 =>    2920  (0.292000)
  5.00000 =>    2349  (0.234900)
  4.00000 =>    1317  (0.131700)
  3.00000 =>     242  (0.024200)

  Distributions of variable total_filled (num:0)
  50.00000 =>   10000  (1.000000)

  Distributions of variable theoretical (num:0)
  224.96027 =>   10000  (1.000000)

  mean sumx: 288.7327

  See ~/blog/coupon_collector.blog
      ~/webppl/cookie_problem.wppl

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function coupon_collector()
    n = 50
    m = round(Int,sqrt(n))
    #  "Exact" probability from https:# en.wikipedia.org/wiki/Coupon_collector%27s_problem (footnote [b])
    theoretical ~ Dirac(n*sum([1/i for i in 1:n])) #  N*Math.log(N,Math.e);

    #  p = beta(14,14);
    p ~ DiscreteUniform(1,m)

    #  Fill this slot with cards
    fill ~ filldist(Poisson(p),n)

    #  How many slots are filled (i.e. > 0)?
    total_filled ~ Dirac(sum([fill[c] > 0 for c in 1:n]))

    true ~ Dirac(total_filled == n)

    sumx ~ Dirac(sum(fill))
    sum0s ~ Dirac(sum([fill[c] == 0 for c in 1:n] ))

end

model = coupon_collector()
num_chains = 4

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(5), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))

show_var_dist_pct(chns, :p)
show_var_dist_pct(chns, :total_filled)
show_var_dist_pct(chns, :theoretical)
println(["mean sumx: ", mean(chns[:sumx].data)])

show_var_dist_pct(chns, Symbol("fill[1]"))
show_var_dist_pct(chns, Symbol("fill[2]"))
show_var_dist_pct(chns, Symbol("fill[3]"))
show_var_dist_pct(chns, Symbol("fill[4]"))
show_var_dist_pct(chns, Symbol("fill[5]"))
show_var_dist_pct(chns, Symbol("fill[6]"))
show_var_dist_pct(chns, Symbol("fill[7]"))
show_var_dist_pct(chns, Symbol("fill[8]"))
show_var_dist_pct(chns, Symbol("fill[9]"))
show_var_dist_pct(chns, Symbol("fill[10]"))


module FEOperatorsFromWeakForm

using Test
using Gridap.Helpers
using Gridap.Arrays
using Gridap.Algebra
using Gridap.TensorValues
using Gridap.ReferenceFEs
using Gridap.Geometry
using Gridap.Integration
using Gridap.Fields
using Gridap.FESpaces
using Gridap.CellData

u(x) = x[1] + x[2]
f(x) = -(3.0*x[1]+x[2]+1.0)
ν(u,x) = (u+1.0)*x[1]
dν(du,x) = x[1]*du

domain = (0,1,0,1)
partition = (4,4)
model = CartesianDiscreteModel(domain,partition)

order = 2
reffe = ReferenceFE(lagrangian,Float64,order)
V = FESpace(model,reffe,dirichlet_tags="boundary")
U = TrialFESpace(V,u)

degree = 3*order-1
trian = get_triangulation(model)
dΩ = Measure(trian,degree)

a(u,du,v) = ∫( ∇(v)⋅(ν∘(u,identity)*∇(du)) )*dΩ
res(u,v) = a(u,v,u) - ∫(v*f)*dΩ
jac(u,du,v) = a(u,du,v) + ∫( ∇(v)⋅(dν∘(du,identity)*∇(u)) )*dΩ

v = get_cell_shapefuns(V)
du = get_cell_shapefuns_trial(U)
uh = zero(U)

op = FEOperator(res,jac,U,V)

uh = solve(op)
e = u - uh

el2 = sqrt(sum(∫( e*e )*dΩ))
eh1 = sqrt(sum(∫( e*e + ∇(e)⋅∇(e) )*dΩ))

@test el2 < 1.e-8
@test eh1 < 1.e-7

y = ones(num_free_dofs(U))
uh = FEFunction(U,y)
r = residual(op,uh)
A = jacobian(op,uh)
test_fe_operator(op,y,r,≈,jac=A)

#using Gridap.Visualization
#writevtk(trian,"trian",celldata=["e"=>cont])

# Now with autodiff

op = FEOperator(res,U,V)
test_fe_operator(op,y,r,≈,jac=A)

uh = solve(op)
e = u - uh

el2 = sqrt(sum(∫( e*e )*dΩ))
eh1 = sqrt(sum(∫( e*e + ∇(e)⋅∇(e) )*dΩ))

@test el2 < 1.e-8
@test eh1 < 1.e-7

end # module


#=
145 is a curious number, as 1! + 4! + 5! = 1 + 24 + 120 = 145.

Find the sum of all numbers which are equal to the sum of the factorial of their digits.

Note: as 1! = 1 and 2! = 2 are not sums they are not included.
=#
function calc()
  total = 0
  for i in 3:9_999_999
    if sum(map((x)->factorial(x),digits(i))) == i
      total += i
    end
  end
  total
end
@time println(calc())


# QR-decomposition using the MSG algorithm

R = zeros(n,n)

for k=1:n
    R[k,k] = norm(A[:,k])
    A[:,k] /= R[k,k]
    R[k:k,k+1:n] = A[:,k]’*A[:,k+1:n]
    A[:,k+1:n]  -= A[:,k]*R[k:k,k+1:n]
end

Code/Numerical Analysis & Machine Learning/Numerical Linear Algebra/QRDecomposition.jl


using SciPyDiffEq

# Definition of parameters
const a = 1.0
const b = 0.1
const c = 1.5
const d = 0.75

function diffeq(u,p,t)
    du = zeros(2)
    # u1: rabbits
    # u2: foxes
    du[1] = a*u[1] - b*u[1]*u[2]
    du[2] = -c*u[2] + d*b*u[1]*u[2]

    return du

end

# Simulation time
const tspan = (0.0,18.0)

# Initial condition
u0 = zeros(2)
u0[1] = 10
u0[2] = 5

# Solve ODE
prob = ODEProblem(diffeq,u0,tspan)
sol = solve(prob,SciPyDiffEq.BDF(),saveat=(tspan[end]-tspan[1])/999)

#= Visualization of Results
using PyPlot
figure(figsize=(20,8))
plot(sol.t,sol.u)
show()
=#


#=
The prime 41, can be written as the sum of six consecutive primes:

41 = 2 + 3 + 5 + 7 + 11 + 13
This is the longest sum of consecutive primes that adds to a prime below one-hundred.

The longest sum of consecutive primes below one-thousand that adds to a prime, contains 21 terms, and is equal to 953.

Which prime, below one-million, can be written as the sum of the most consecutive primes?
=#
using Primes

function calc()

  x = 10^6
  p = primes(x-1)
  largestrun = largestprime = 0

  for i in 1:length(p)
    currentsum = 0
    for j in i:length(p)
      currentsum += p[j]
      if currentsum >= x
        break
      end
      if isprime(currentsum) && j-i > largestrun
        largestrun = j-i
        largestprime = currentsum
      end
    end
  end
  largestprime
end
@time println(calc())


# cross ratio for 4 numbers
function crossratio(a,b,c,d)
    d_ab = norm(a - b)
    d_bd = norm(b - d)
    d_ac = norm(a - c)
    d_cd = norm(c - d)
    (d_ab/d_bd)/(d_ac/d_cd)
end
export crossratio

# modulo that starts from one
# imod(1,3) = 1
# imod(2,3) = 2
# imod(3,3) = 3
# imod(4,3) = 1
function imod(i, n)
  return (i-1)%n + 1
end
export imod

#TODO handle zero deviation
# l1 = [(x1,y1), (x2,y2)]
# l2 = [(x3,y3), (x4,y4)]
function intersection_point(l1, l2)
  x1 = l1[1][1]
  y1 = l1[1][2]

  x2 = l1[2][1]
  y2 = l1[2][2]

  x3 = l2[1][1]
  y3 = l2[1][2]

  x4 = l2[2][1]
  y4 = l2[2][2]

  denominator = (x1-x2)*(y3-y4)-(y1-y2)*(x3-x4)
  x = (x1*y2-y1*x2)*(x3-x4)-(x1-x2)*(x3*y4-y3*x4)
  y = (x1*y2-y1*x2)*(y3-y4)-(y1-y2)*(x3*y4-y3*x4)
  return [x,y]/denominator
end
export intersection_point

"""
    calcualte_rigid(source_pts,target_pts)

find (R,T) rigid transformation matrix and translation vector
such that target_pts ≈ R * source_pts + T
in the least-square sense
"""
function rigid_trans(src_pts,dst_pts)
  to_array(vv) = [vv[i][j] for i=1:length(vv) , j=1:length(vv[1])]
  n = length(src_pts)
  dim = size(src_pts[1])[1]
  dst_bar = mean(dst_pts)
  src_bar = mean(src_pts)
  dst_centered = dst_pts .- [dst_bar]
  src_centered = src_pts .- [src_bar]

  C = to_array(src_centered)' * to_array(dst_centered) / n
  U,s,V = svd(C)
  # Rotation R in least squares sense:
  Rr = (U * diagm(vcat(ones(dim-1), det(U*V.'))) * V.' ).'
  Tr = -Rr*src_bar + dst_bar

  Pr = PM(Rr,Tr)
  reproj_error = mean(norm.([Rr] .* src_pts .+ [Tr] - dst_pts))
  (Rr,Tr,reproj_error)
end
export rigid_trans


using KroneckerProducts
using LinearAlgebra
using Test

a = rand(ComplexF64, 3, 2)
b = rand(ComplexF64, 4, 5)

k_big = kron(a,b)

k = a ⊗ b

@test k[1] == a[1]*b[1]
@test k[end] == a[end]*b[end]
@test k[1,1] == a[1,1]*b[1,1]
@test k[end,end] == a[end,end]*b[end,end]

@test k == k_big

@test k' == k_big'
@test transpose(k) == transpose(k_big)

@test size(k) == size(k_big)

@test eltype(k) == eltype(k_big)

y = rand(ComplexF64, size(k,2))
yr = rand(ComplexF64, size(k,1))

@test k*y ≈ k_big*y
@test yr'*k ≈ yr'*k_big

a = rand(ComplexF64, 3, 2)
b = rand(ComplexF64, 2, 3)
k_big = kron(a,b)
k = KroneckerProduct(a,b)

@test det(k) == 0
@test abs(det(k_big)) < eps()

@test tr(k) ≈ tr(k_big)

a = rand(ComplexF64, 3, 3)
b = rand(ComplexF64, 2, 2)
k_big = kron(a,b)
k = KroneckerProduct(a,b)

@test det(k) ≈ det(k_big)

@test tr(k) ≈ tr(k_big)

yr = rand(ComplexF64, size(k,1))
@test k\yr ≈ k_big\yr


"""
    parity(char)

Compute whether a characteristic is even or odd.

# Arguments
- `char::Array{}`: array consisting of two arrays of the same length with 0 or 1 entries.
"""
function parity_char(char::Array{})
    return rem.((transpose(char[1])*char[2]),2);
end

"""
    remainder_char(char)

Compute remainder modulo 2 of all the entries of a characteristic.

# Arguments
- `char::Array{}`: array consisting of two arrays of the same length with 0 or 1 entries.
"""
function remainder_char(char::Array{})
    return [rem.(char[1], 2), rem.(char[2], 2)];
end

"""
    theta_char(g)

Compute all theta characteristics of genus g.

# Arguments
- `g::Integer`: genus

# Examples
```julia
julia> theta_char(4)
```
"""
function theta_char(g::Integer)
    chars = [digits(i, base=2, pad=2*g) for i = 0:2^(2*g)-1];
    return [[c[1:g], c[g+1:2*g]] for c in chars];
end

"""
    even_theta_char(g)

Compute all even theta characteristics of genus g.

# Arguments
- `g::Integer`: genus

# Examples
```julia
julia> even_theta_char(4)
```
"""
function even_theta_char(g::Integer)
    char = theta_char(g);
    even_char = filter(x -> parity_char(x) == 0, char);
    return even_char;
end

"""
    odd_theta_char(g)

Compute all odd theta characteristics of genus g.

# Arguments
- `g::Integer`: genus

# Examples
```julia
julia> odd_theta_char(4)
```
"""
function odd_theta_char(g::Integer)
    char = theta_char(g);
    odd_char = filter(x -> parity_char(x) == 1, char);
    return odd_char;
end

"""
    check_azygetic(chars)

Check if a list of characteristics is azygetic.

# Arguments
- `chars::Array{}`: array where each entry is an array consisting of two arrays of the same length with 0 or 1 entries.

# Examples
```julia
julia> check_azygetic([[[1,0,1,0], [1,0,1,0]], [[0,0,0,1], [1,0,0,0]], [[0,0,1,1], [1,0,1,1]]])
"""
function check_azygetic(chars::Array{})
    for i = 1:length(chars), j = i+1:length(chars), k = j+1:length(chars)
        if parity_char(remainder_char(chars[i] + chars[j] + chars[k])) == 0
            return false
        end
    end
    return true
end


function d1hat(n::Int64, dx::Float64 )
    operator = zeros(n,n) # initialize
    for k in 1:n
        try operator[k, k+1] = 1. catch; Nothing end
        try operator[k, k-1] = -1. catch; Nothing end
    end

    return operator./(2*dx)
end

function d2hat(n::Int64, dx::Float64 )
    operator = zeros(n,n)
    for k in 1:n
        operator[k,k] = -2.
        try operator[k, k+1] = 1. catch; Nothing end
        try operator[k, k-1] = 1. catch; Nothing end
    end

    return operator./(dx^2)
end


using ApproxFun
using PoincareInvariants: compute_integral, compute_derivative1, compute_derivative2

### Test parameters ###

# nx  = 20
# ny  = 20
# dtol = 1E-12
# itol = 1E-11

nx  = 100
ny  = 100
dtol = 1E-10
itol = 1E-16

scale = 1E-00
# scale = 1E-15

### Set up ApproxFun spaces ###

SC = Chebyshev(0..1)^2
SU = Ultraspherical(1, 0..1)^2
CU = Conversion(SC, SU)

### check derivatives ###

# compute x=(cos(2πτ)cos(2πσ)) on SC
pc = points(SC, nx*ny)
xc = [scale * cos(2π*pc[i][1]) * cos(2π*pc[i][2]) for i in eachindex(pc)]

# compute x=(cos(2πτ)cos(2πσ)) on SU
pu = points(SU, nx*ny)
xu = [scale * cos(2π*pu[i][1]) * cos(2π*pu[i][2]) for i in eachindex(pu)]

# create Fun and compute derivatives
fun_xc = Fun(SC, transform(SC, xc))
fun_xu = Fun(SU, transform(SU, xu))

fx1 = compute_derivative1(fun_xc)
fx2 = compute_derivative2(fun_xc)

# compute τ-derivative of x (-2πsin(2πτ)cos(2πσ))
pu = points(SU, length(fx1))
x1 = [- 2π * scale * sin(2π*pu[i][1]) * cos(2π*pu[i][2]) for i in eachindex(pu)]

# compute σ-derivative of x (-2πcos(2πτ)sin(2πσ))
pu = points(SU, length(fx2))
x2 = [- 2π * scale * cos(2π*pu[i][1]) * sin(2π*pu[i][2]) for i in eachindex(pu)]

# compare Fun derivative and analytical derivative
# println(maximum(abs.(fx1 .- x1)))
# println(maximum(abs.(fx2 .- x2)))

# the following test should fail but for some reason does not
# (with nx=ny=20 success is expected for atol=5E-13 or larger)
# @test fx1 ≈ x1 atol=1E-15
# @test fx2 ≈ x2 atol=1E-15

# this works, but is less compact
@test (fx1 .- x1) ≈ zero(fx1) atol=dtol*scale
@test (fx2 .- x2) ≈ zero(fx2) atol=dtol*scale

### check integral ###

# compute x=(cos(2πτ)cos(2πσ))
pu = points(SU, nx*ny)
x  = zeros(length(pu))

for i in eachindex(x,pu)
    x[i] = scale * cos(2π*pu[i][1]) * cos(2π*pu[i][2])
end

@test compute_integral(x) ≈ 0.0 atol=itol*scale


"""
    find_ratbasis(list::AbstractVector{<:StaticVector{N,<:Rational{T}}}) where {N,T}

Given a list of `N`-dimensional rational vectors, return a `N×N` matrix whose columns
are independent vectors from the list.
If no such `N` vectors exist, the returned matrix will not be invertible.
"""
function find_ratbasis(list::AbstractVector{<:StaticVector{N,<:Rational{T}}}) where {N,T}
    U = widen(T)
    n = length(list)
    ratbasis = zero(SizedMatrix{N,N,Rational{U}})
    i = 1
    @inbounds while i <= n
        l = list[i]
        if !iszero(l)
            ratbasis[:,1] .= l
            i += 1
            break
        end
        i += 1
    end
    j = 2
    while i <= n && j <= N
        l = list[i]
        if iszero(l)
            i += 1
            continue
        end
        ratbasis[:,j] .= l
        if rank(@view ratbasis[:,1:j]) == j
            j += 1
        end
        i += 1
    end
    return ratbasis
end

# Specialized version for N = 3
function find_ratbasis(list::AbstractVector{<:StaticVector{3,<:Rational{T}}}) where T
    U = widen(T)
    n = length(list)
    ratbasis = zero(SizedMatrix{3,3,Rational{U}})
    # We start by computing a basis for the i first vectors of v where i (≤ n) is minimal
    i = 1
    @inbounds while i <= n
        if !iszero(list[i])
            ratbasis[:,3] .= list[i]
            i += 1
            break
        end
        i += 1
    end
    @inbounds while i <= n
        v = list[i]
        valid = false
        # v is valid if it is linearly independant of ratbasis[:,3]
        if iszero(v[1])
            valid = (!iszero(ratbasis[1,3]) & (!iszero(v[2]) | !iszero(v[3]))) ||
                    (v[2] * ratbasis[3,3] != v[3] * ratbasis[2,3])
        else
            x = ratbasis[1,3] // v[1]
            valid = ratbasis[2,3] != x * v[2] || ratbasis[3,3] != x * v[3]
        end
        if valid
            ratbasis[:,2] .= v
            i += 1
            break
        end
        i += 1
    end
    d = zero(Rational{U})
    @inbounds while i <= n
        v = list[i]
        ratbasis[:,1] .= v
        d = det(ratbasis)
        iszero(d) || break
        i += 1
    end
    # At this point ratbasis is a basis in which the i-1 first coordinates are
    # [0, 0, 1] followed by a (possibly empty) series of [0, 0, x], then
    # [0, 1, 0] followed by a (possibly empty) series of [0, x, y], then [1, 0, 0]
    @inbounds if d < 0
        ratbasis[:,1] .= .-ratbasis[:,1] # Ensure a direct transformation
    end
    return SMatrix{3,3,Rational{U}}(ratbasis)
end

# Specialized version for N = 2
function find_ratbasis(list::AbstractVector{<:StaticVector{2,<:Rational{T}}}) where T
    U = widen(T)
    n = length(list)
    ratbasis = zero(SizedMatrix{2,2,Rational{U}})
    i = 1
    @inbounds while i <= n
        if !iszero(list[i])
            ratbasis[:,1] .= list[i]
            i += 1
            break
        end
        i += 1
    end
    d = zero(Rational{U})
    @inbounds while i <= n
        v = list[i]
        ratbasis[:,2] .= v
        d = det(ratbasis)
        iszero(d) || break
        i += 1
    end
    @inbounds if d < 0
        ratbasis[:,1] .= .-ratbasis[:,1]
    end
    return SMatrix{2,2,Rational{U}}(ratbasis)
end

"""
    normal_basis_rational(list::AbstractVector{<:StaticVector{N,<:Rational{T}}}) where {N,T}

Given a list of `N`-dimensional rational vectors, return a basis for the space
spanned by integer combinations of these vectors.

This basis is deterministically computed from the input.

It should depend only on the spanned space, not on the exact input (although this
assertion should be considered experimental for now).
"""
function normal_basis_rational(list::AbstractVector{<:StaticVector{N,<:Rational{T}}}) where {N,T}
    U = widen(T)
    n = length(list)
    ratbasis = find_ratbasis(list)
    # ratbasis should always be invertible in our setting: if not, it means that
    # we missed the fact that the dimensionality of the graph was strictly lower
    # than N, which should have been detected much earlier.
    # In this case, the definition of invratbasis will cause a failure.

    lcms = SizedVector{N,T}(ones(T, N))
    # per-column lcm of the vectors expressed in ratbasis
    expressed_in_ratbasis = Vector{SVector{N,Rational{T}}}(undef, n)
    invratbasis = inv(ratbasis) # See comment above in case of error here.
    @inbounds for j in 1:n
        expressed_in_ratbasis[j] = invratbasis * list[j]
        lcms .= lcm.(lcms, denominator.(expressed_in_ratbasis[j]))
    end

    expressed_in_basis = [MVector{N,Int}(undef) for _ in 1:n]
    @inbounds for j in 1:n
        x = expressed_in_ratbasis[j]
        expressed_in_basis[j] .= div.(numerator.(x).*lcms, denominator.(x))
    end
    #= lcms contains the lcm of all denominators of the coordinates of translations
       expressed in ratbasis.
       We now work in an artificial basis in which the coordinates are all integers.
       The integer coordinates are stored in expressed_in_basis.
       expressed_in_basis is now going to be reduced to its normal form.
    =#

    intbasis, _ = PeriodicGraphs.normal_basis(expressed_in_basis)

    # We finally go back from the integer normal form to a rational basis that expresses
    # all the original vectors with integer coordinates
    return SMatrix{N,N,Rational{U}}(ratbasis * [@inbounds(intbasis[i,j] // lcms[i]) for i in 1:N, j in 1:N])

    # newcoords = [SVector{N,Int}(normalization * x) for x in intcoords]
    # @toggleassert (newbasis,) .* newcoords == list

    # return newbasis, newcoords
end

# Specialized version for N = 1
function normal_basis_rational(list::AbstractVector{<:StaticVector{1,<:Rational{T}}}) where T
    nzl = [x[] for x in list if !iszero(x[])]
    den = lcm(denominator.(nzl))
    S = widen(T)
    num = minimum(S[abs(S(den*x)) for x in nzl])
    return SMatrix{1,1,Rational{S}}((num // den,))
end


using SpecialFunctions

"""
Standard deviation of the standard deviation of a Normal sample with size nObs

Source: http://stats.stackexchange.com/questions/631/standard-deviation-of-standard-deviation

IN
  xStd
     std dev of x
  nObs
     no of observations
"""
function std_std(xStdV :: Vector{T1}, nObsV :: Vector{T2}, dbg :: Bool) where
	{T1 <: AbstractFloat, T2 <: Integer}

	n = length(xStdV);

	stdStdV = zeros(n);
	for ix = 1 : n
	   nObs = Float64(nObsV[ix]);
	   if nObs < 300
		   # Gamma function from SpecialFunctions
		  gamma1 = gamma((nObs - 1.0) / 2.0)
		  gamma2 = gamma(nObs / 2.0)
	   else
		  # Approximation
		  #  See https://en.wikipedia.org/wiki/Gamma_function#Approximations
		  z1 = (nObs - 1.0) / 2.0 - 1.0;
		  z2 = nObs / 2.0 - 1.0;
		  gamma2 = 1.0;
		  gamma1 = exp(0.5 * log(z1) + z1 * (log(z1) - 1) - 0.5 * log(z2) - z2 * (log(z2) - 1));
	   end

	   # Ratio of gamma2 / gamma1
	   g21 = gamma2 / gamma1;

	   stdStdV[ix] = xStdV[ix] / g21 * sqrt((nObs-1) / 2 - g21 ^ 2);
	end

	##  Output check
	if dbg
	   @assert (all(stdStdV .>= 0));
	end

	return stdStdV
end


##  SCRIPT NAME:   getEquiprobNormalDeviates.jl
##  DATE:          Jnauary 2015
#
#   This script generates N equiprobable line segments in the stationary
#   univariate normal distribution of income Y:
#          Y ~ N [ alpha/1-rho  ;  sigma_epsilon^2/(1-rho)^2 ]
#   where the mean of the above distribution is declared below as 'Ymean'
#   and the standard deviation as 'Ysd'. It delivers two arrays: 'Bounaries'
#   which holds the N+1 bounds of the N equiprobable segments, and
#   'ExpValues' which holds the expected value in each segment.
#
#
##  ALEXANDROS THELOUDIS, UCL
#
#  Needs: A constant called "normBnd"

##  ---------------------------------------------------------------------------------------------

function getEquiprobNormalDeviates(Ymean, Ysd, N)

  #  Initialise output:
  Boundaries = zeros(N+1)
  ExpValues  = zeros(N)

  #  --------------------------------------------------------------------------------------------
  #  Calculate applicable support of univariate normal distribution (first and
  #  last point in the range that I divide equiprobably). Note: in principle
  #  this is +-infinity but I am ruling some extremely unlikely values out:
  Boundaries[1]   = Ymean - normBnd * Ysd
  Boundaries[N+1] = Ymean + normBnd * Ysd

  #  --------------------------------------------------------------------------------------------
  #  Sequentially calculate the remaining bounds of the equiprobable regions:
  for ixi = 2:1:N
    Boundaries[ixi] = quantile(TruncatedNormal(Ymean,Ysd,Boundaries[1],Boundaries[N+1]), (ixi-1)/N)
  end

  #  --------------------------------------------------------------------------------------------
  #  Calculate the expected value of a random variable X ~ N[Ymean,Ysd^2] which
  #  is bounded by the equiprobable line segment above. Note: this is essentially
  #  the mean of a univariate 2-sided truncated normal distribution; read more
  #  on http://en.wikipedia.org/wiki/Truncated_normal_distribution
  for ixi = 1:1:N
    ExpValues[ixi] = Ymean + Ysd * N * (pdf(Normal(),(Boundaries[ixi]-Ymean)/Ysd) - pdf(Normal(),(Boundaries[ixi+1]-Ymean)/Ysd))
  end

  return (Boundaries, ExpValues)

end


using LinearAlgebra
using Plots

export plotnormal2d

"""
    plotnormal2d(p, μ::AbstractVector, Σ::AbstractMatrix; ncontours=2, args...)

Plot the contours of a 2d Normal density. `ncontours` is the number of
contour line to plots.
"""
function plotnormal2d(p, μ::AbstractVector, Σ::AbstractMatrix; ncontours=2, color="blue", label="", args...)
    λ, U = eigen(Σ)
    for i in 1:ncontours
        B = U * diagm(i * sqrt.(λ))
        θ = range(0, stop=2 * pi, length=1000)
        circle = hcat(sin.(θ), cos.(θ))'
        contour = B * circle .+ μ
        plot!(p, contour[1, :], contour[2, :]; color=color, label=i > 1 ? "" : label, args...)
    end
    p
end


function adaptive_simpsons_inner(f::Function, a::Real, b::Real,
                                 epsilon::Real, S::Real,
                                 fa::Real, fb::Real, fc::Real, bottom::Int)
    c = (a + b) / 2
    h = b - a
    d = (a + c) / 2
    g = (c + b) / 2
    fd = f(d)
    fe = f(g)
    Sleft = (h / 12) * (fa + 4 * fd + fc)
    Sright = (h / 12) * (fc + 4 * fe + fb)
    S2 = Sleft + Sright
    if bottom <= 0 || abs(S2 - S) <= 15 * epsilon
        return S2 + (S2 - S) / 15
    end
    return adaptive_simpsons_inner(f, a, c, epsilon / 2, Sleft,  fa, fc, fd, bottom - 1) +
           adaptive_simpsons_inner(f, c, b, epsilon / 2, Sright, fc, fb, fe, bottom - 1)
end

function adaptive_simpsons_outer(f::Function, a::Real, b::Real, accuracy::Real, max_iterations::Int)
    c = (a + b) / 2
    h = b - a
    fa = f(a)
    fb = f(b)
    fc = f(c)
    S = (h / 6) * (fa + 4 * fc + fb)
  return adaptive_simpsons_inner(f, a, b, accuracy, S, fa, fb, fc, max_iterations)
end
adaptive_simpsons_outer(f::Function, a::Real, b::Real) = adaptive_simpsons_outer(f, a, b, 10e-10, 50)

function monte_carlo(f::Function, a::Real, b::Real, iterations::Int)
    estimate = 0.0
    width = (b - a)
    for i in 1:iterations
        x = width * rand() + a
        estimate += f(x) * width
    end
    return estimate / iterations
end

function integrate(f::Function, a::Real, b::Real, method::Symbol)
    if method == :simpsons
        adaptive_simpsons_outer(f, a, b)
    elseif method == :monte_carlo
        monte_carlo(f, a, b, 10_000)
    else
        error("Unknown method of integration: $(method)")
    end
end
integrate(f::Function, a::Real, b::Real) = integrate(f, a, b, :simpsons)


using NHST

@assert isapprox(p_value(BinomialTest, 0, 10, 0.5), 0.001953125)
@assert isapprox(p_value(BinomialTest, 5, 10, 0.5), 1.0)
@assert isapprox(p_value(BinomialTest, 10, 10, 0.5), 0.001953125)
@assert isapprox(p_value(BinomialTest, 4, 10, 0.2), 0.1208739)
@assert isapprox(p_value(BinomialTest, 2, 10, 0.8), 7.79264e-05)
@assert isapprox(p_value(BinomialTest, 3, 100, 0.5), 2.630867e-25)
@assert isapprox(p_value(BinomialTest, 97, 100, 0.5), 2.630867e-25)
@assert isapprox(p_value(BinomialTest, 3, 10, 0.79), 0.001178273)
@assert isapprox(p_value(BinomialTest, 9, 10, 0.79), 0.6989298)

p_value(BinomialTest, 0, 10, 0.5)
p_value(BinomialTest, 0.0, 10, 0.5)
p_value(BinomialTest, 0, 10.0, 0.5)
p_value(BinomialTest, 0.0, 10.0, 0.5)
p_value(BinomialTest, zeros(10), 0.5)

run(BinomialTest, 0, 10, p = 0.5)
run(BinomialTest, 0.0, 10, p = 0.5)
run(BinomialTest, 0, 10.0, p = 0.5)
run(BinomialTest, 0.0, 10.0, p = 0.5)
run(BinomialTest, zeros(10), p = 0.5)

@assert isapprox(p_value(run(BinomialTest, 0, 10, p = 0.5)), 0.001953125)
@assert isapprox(p_value(run(BinomialTest, 5, 10, p = 0.5)), 1.0)
@assert isapprox(p_value(run(BinomialTest, 10, 10, p = 0.5)), 0.001953125)
@assert isapprox(p_value(run(BinomialTest, 4, 10, p = 0.2)), 0.1208739)
@assert isapprox(p_value(run(BinomialTest, 2, 10, p = 0.8)), 7.79264e-05)


@inline function add_ddfp_dd(x::Tuple{T,T}, y::T) where {T<:IEEEFloat}
    hi, lo = x
    hi, lo = two_sum(hi, y, lo)
    return hi, lo
end

@inline function sub_ddfp_dd(x::Tuple{T,T}, y::T) where {T<:IEEEFloat}
    hi, lo = x
    hi, lo = two_sum(hi, lo, -y)
    return hi, lo
end

#=
@inline function mul_ddfp_dd(x::Tuple{T,T}, y::T) where {T<:IEEEFloat}
    hi, lo = x
    hihi, hilo = two_prod(y, hi)
    lohi, lolo = two_prod(y, lo)
    hi, lo = two_sum(hihi, hilo, lohi, lolo)
    return hi, lo
end
=#

# Algorithm 12 from Tight and rigourous error bounds.  relative error <= 5u²
@inline function mul_ddfp_dd(x::Tuple{T,T}, y::T) where T<:IEEEFloat
    xhi, xlo = x
    hi, lo = two_prod(xhi, y)
    t = xlo * y
    t = t + lo
    hi, lo = two_hilo_sum(hi, t)
    return hi, lo
end

@inline function dvi_ddfp_dd(x::Tuple{T,T}, y::T) where {T<:IEEEFloat}
    xhi, xlo = x
    hi = xhi / y
    uh, ul = two_prod(hi, y)
    lo = ((((xhi - uh) - ul) + xlo))/y
    hi,lo = two_hilo_sum(hi, lo)
    return hi, lo
end


module ols

export tols, summary

# Author: Adam Savitzky
# Email: asavitzky@forio.com
# Github: github.com/adambom

# Ported from the Python implemented by Vincent Nijs
# http://www.scipy.org/Cookbook/OLS?action=AttachFile&do=get&target=ols.0.2.py

# Julia type for multiple (multivariate) regression using OLS
# For least squared regression on linear equations of multiple independent variables
# y = a1 * x1 + a2 * x2 + ... an * xn
# Y = AX + E

# Input
## y = dependent variable
## y_varnm = string with the variable label for y
## x = independent variables, note that a constant is added by default
## x_varnm = list of variable labels for the independent variables

# Usage
## Instantiate a new ols type
### reg = ols(y, x, "y", ["x1", "x2", "x3"])
### Coefficients: reg.b
### R-Squared: reg.R2
### F-Statistic: reg.F
### Summary: summary(reg)

type tols
    y::Array{Float64}
    x::Array{Float64}
    y_varnm::String
    x_varnm::Array{String, 1}
    inv_xx::Array{Float64}
    b::Array{Float64, 1}
    nobs::Int
    ncoef::Int
    df_e::Int
    df_r::Int
    er::Array
    sse::Float64
    se::Array{Float64, 1}
    t::Array{Float64}
    #p::Array
    R2::Float64
    R2adj::Float64
    F::Float64
    #Fpv::Float64

    function tols(y, x, y_varnm, x_varnm)
        x = hcat(ones(size(x, 1)), x)
        xT = transpose(x)

        inv_xx = inv(xT * x)
        xy = xT * y
        b = inv_xx * xy           # estimate coefficients

        nobs = size(y, 1)         # number of observations
        ncoef = size(x, 2)        # number of coefficients
        df_e = nobs - ncoef       # degrees of freedom, error
        df_r = ncoef - 1          # degrees of freedom, regression

        er = y - x * b            # residuals
        sse = e^2/df_e            # SSE
        se = sqrt(diag(sse * inv_xx)) # coef. standard errors
        t = b / se                # coef. t-statistics
        # p = (1 - cdf(abs(t), df_e)) * 2 # coef. p-values

        R2 = 1 - var(er) / var(y) # model R-squared
        R2adj = 1 - (1 - R2) * ((nobs - 1) / (nobs - ncoef)) # adjusted R-square

        F = (R2 / df_r) / ((1 - R2) / df_e) # model F-statistic
        # Fpv = 1 - cdf(F, df_r, df_e) # F-statistic p-value

        new(y, x, y_varnm, x_varnm, inv_xx, b, nobs, ncoef, df_e, df_r, er, sse, se, t, R2, R2adj, F)
    end
end

function dw(self::tols)
    # Calculates the Durbin-Waston statistic
    de = self.er - 1.
    result = dot(de, de) / dot(self.er, self.er)
    return result
end

function ll(self::tols)
    # Calculate model log-likelihood and two information criteria

    # Model log-likelihood, AIC, and BIC criterion values
    loglike = -(self.nobs / 2) * (1 + log(2pi)) - (self.nobs / 2) * log(dot(self.er, self.er) / self.nobs)
    aic = -2loglike / self.nobs + (2 * self.ncoef / self.nobs)
    bic = -2loglike / self.nobs + (self.ncoef * log(self.nobs)) / self.nobs

    return loglike, aic, bic
end

function summary(self::tols)
    # print model output to screen

    t = time()

    # extra stats
    loglike, aic, bic = ll(self)
    #JB, JBpv, skew, kurtosis = self.JB()
    #omni, omnipv = self.omni()

    println("==============================================================================")
    #println("Dependent Variable: " + self.y_varnm)
    println("Method: Least Squares")
    println("Time: $t")
    println("No. obs:               $(self.nobs)")
    println("No. variables:         $(self.ncoef)")
    println("==============================================================================")
    println("variable             coefficient             std. Error              t-statistic")
    println("==============================================================================")
    for i in 1:length(self.x_varnm)
        println("$(self.x_varnm[i])         $(self.b[i])        $(self.se[i])        $(self.t[i])")
    end
    println("========================================================================================")
    println("Model Stats                                    Residual Stats")
    println("========================================================================================")
    println("R-Squared            $(self.R2)        Durbin-Watson Stat: $(dw(self))")
    println("Adjusted R-Squared   $(self.R2adj)        Omnimbus Stat:      ?")
    println("F-Statistic          $(self.F)        Prob(Omnibus stat): ?")
    println("Log-Likelihood       $loglike       Prob(JB):           ?")
    println("AIC Criterion        $aic        Skew:               ?")
    println("BIC Criterion        $bic         Kurtosis:           ?")
    println("========================================================================================")
end

function linreg{T<:Number}(X::StridedVecOrMat{T}, y::Vector{T})
    hcat(ones(T, size(X,1)), X)\y
end

end


using OrdinaryDiffEq, LinearAlgebra, Test

function dynamics!(dx,x,θ,t)
    zmin, zmax, M, g = θ
    z, dz, p = x

    dx[1] = dz
    dx[2] = -M*g -z + p
    dx[3] = p-w(t) # the last state (algebraic) p has to be equal to w

    if z <= zmin
        dx[2] = max(dx[2], 0.0)
    elseif z >= zmax
        dx[2] = min(dx[2], 0.0)
    end
end

w(t) = .1*(7.0+3.0sin(t))
zmin_cond(x,t,integrator) =  x[1]-zmin
zmax_cond(x,t,integrator) =  zmax-x[1]

function zmin_affect_neg!(integrator)
    integrator.u[1] = zmin
    integrator.u[2] = 0.0
end

function zmax_affect_neg!(integrator)
    integrator.u[1] = zmax
    integrator.u[2] = 0.0
end

cbs = CallbackSet(
    ContinuousCallback(zmin_cond,zmin_affect_neg!),
    ContinuousCallback(zmax_cond,zmax_affect_neg!)
)

tf = 20.0
tspan = (0.0,tf)

zmin = 1.0e-3
zmax = 2.0e-2
M = 0.05
g = 9.81
θ = zmin, zmax, M, g

x0 = [(zmin+zmax)/2,0.0,w(0.0)]

E = diagm([1.0,M,0.0])

f = ODEFunction(dynamics!,mass_matrix=E)
prob = ODEProblem(f,x0,tspan,θ)

sol1 = solve(prob,Rodas4(), callback = cbs, reltol=1e-6)
@test sol1(0.06692341688237893)[3] ≈ 0.72 atol=1e-2
sol1 = solve(prob,Rodas5(), callback = cbs, reltol=1e-6)
@test sol1(0.06692341688237893)[3] ≈ 0.72 atol=1e-2
sol1 = solve(prob,Rodas5P(), callback = cbs, reltol=1e-6)
@test sol1(0.06692341688237893)[3] ≈ 0.72 atol=1e-2

#=
sol1 = solve(prob,Rosenbrock23(),callback=cbs, reltol=1e-6)
@test sol1(1.0)[3] ≈ 0.95 atol=1e-2
=#


using PyPlot
using LinearAlgebra

a = [ 0.5  -1.0   0.25
      1.0   0.5  -1.0 ]

function points(λ, offset)
    x = zeros(2,3)
    x[:,1] .= (1-2λ) * a[:,1] +      λ * a[:,2] +      λ * a[:,3]
    x[:,2] .=      λ * a[:,1] + (1-2λ) * a[:,2] +      λ * a[:,3]
    x[:,3] .=      λ * a[:,1] +      λ * a[:,2] + (1-2λ) * a[:,3]
    m = zeros(2,3)
    m[:,1] = 0.5 * a[:,3] + 0.5 * a[:,2]
    m[:,2] = 0.5 * a[:,1] + 0.5 * a[:,3]
    m[:,3] = 0.5 * a[:,2] + 0.5 * a[:,1]
    J = [1, 2, 3, 1]
    plot(a[1,J], a[2,J], "-k")
    c = sum(a, dims=2) / 3
    for j = 1:3
        plot([a[1,j], m[1,j]], [a[2,j], m[2,j]], "--k")
        text(x[1,j] + offset[1,j], x[2,j] + offset[2,j],
             latexstring("x^{\\langle K\\rangle}_", j), color="r",
             verticalalignment="center", horizontalalignment="center")
        v = a[:,j] - c
        v = 0.1 * v / LinearAlgebra.norm(v)
        text(a[1,j] + v[1] , a[2,j] + v[2], latexstring("a_", j),
             verticalalignment="center", horizontalalignment="center")
    end
    plot(x[1,:], x[2,:], "or", markeredgecolor="w")
    plot(a[1,:], a[2,:], "ok", markersize=5.0)
end

figure(1)
subplot(1, 2, 1)
offset = [ -0.10  0.10  0.15
            0.05  0.10  0.05  ]
points(1/6, offset)
axis("equal")
axis((-1.2, 0.7, -1.2, 1.2))
axis("off")

subplot(1, 2, 2)
offset = [ -0.12  0.15  0.0
           -0.10  0.0   0.15  ]
points(1/2, offset)
axis("equal")
axis((-1.2, 0.7, -1.2, 1.2))
axis("off")

savefig("quadrature_points.pdf")


#=
BibText
@TECHREPORT{Benner95acollection,
author = {Peter Benner and Alan J. Laub and Volker Mehrmann},
title = {A Collection of Benchmark Examples for the Numerical Solution of
Algebraic Riccati Equations II: Discrete-Time Case},
institution = {FAK. F. MATHEMATIK, TU CHEMNITZ--ZWICKAU},
year = {1995}
}

=#

# ------------------------------------------------------------------
# ejm1 collection of benchmark examples
A1 = [4. 3.;-4.5 -3.5]
B1 = [1. -1]'
R1 = 1.
Q1 = [9. 6.;6. 4.]
X1 = (1+sqrt(5))/2*[9 6;6 4] # theoretical solucion

X11 = dare(A1,B1,Q1,R1) # computed solution

# --------------------------------------------------------------------
# ejm computed with mathematica
Am = [1. -1 1;0 1 1;0 0 1]
Bm = [1. 0;1 0;0 1]
Rm = [10 0;0 0.1]
Qm = [10 0 0;0 1 0;0 0 0.1]
Xm = [42.2835 -68.5247 -3.94783;-68.5247 154.043 16.0017;-3.94783 16.0017 8.33197] # mathematica solution

Xmm = dare(Am,Bm,Qm,Rm) # computed solution

#####

print("Testing dare: ")
if (sum((X1-X11).^2) < 1e-6) && (sum((Xm-Xmm).^2) < 1e-6)
	println("OK")
end


using SemiclassicalOrthogonalPolynomials, ClassicalOrthogonalPolynomials, ForwardDiff, Plots, StaticArrays
import ForwardDiff: derivative, jacobian, Dual
import SemiclassicalOrthogonalPolynomials: Weighted
import ClassicalOrthogonalPolynomials: associated, affine

Base.floatmin(::Type{Dual{T,V,N}}) where {T,V,N} = Dual{T,V,N}(floatmin(V))
Base.big(d::Dual{T,V,N}) where {T,V,N} = Dual{T}(big(d.value), ForwardDiff.Partials(map(big,d.partials.values)))

####
#
# We first do a single interval.
# Equilibrium measures for a symmetric potential
#  with one interval of support
# a measure w(x) supported on
# [-b,b]
# such that
#       1. H*w == V'
#       2. sum(w) == 1
#       3.  w is bounded
#
#  rescaling x == b*t these becomes find
# a measure w̃(t) supported on
# [-1,1]
# such that
#       1. H*w̃ == V'(b*t)
#       2. sum(w̃) == 1/b
#       3.  w is bounded
#
# Note (1) and (2) can always be satisfied
# thus the constraint comes from (3).
# The following gives the evaluation of the
# unweighted-component of the measure evaluated
# at (a,b)
#####

V = x -> x^2
function equilibriumcoefficients(T, b)
    U = associated(T)
    W = Weighted(T)
    t = axes(W,1)
    H = @. inv(t - t')
    H̃ = U \H*W
    [1/(b*sum(W[:,1])); 2H̃[:,2:end] \ ( U \ derivative.(V, b*t))]
end
function equilibriumendpointvalue(b::Number)
    T = ChebyshevT{typeof(b)}()
    dot(T[end,:], equilibriumcoefficients(T,b))
end

function equilibrium(b::Number)
    T = ChebyshevT{typeof(b)}()
    U = ChebyshevU{typeof(b)}()
    # convert to Weighted(U) to make value at ±b accurate
    Weighted(U)[affine(-b..b,axes(T,1)),:] * ((Weighted(T) \ Weighted(U))[3:end,:] \ equilibriumcoefficients(T,b)[3:end])
end

μ = equilibrium(sqrt(2))

T = Chebyshev()
b = sqrt(2)
μ = Weighted(T) * equilibriumcoefficients(T, b)
x = axes(μ,1)

plot(μ)

xx = 0.7; 2b*(log.(abs.(x .- x'))*μ)[xx] - V(b*xx)

b = 1.0 # initial guess
for _ = 1:10
    b -= derivative(equilibriumendpointvalue,b) \ equilibriumendpointvalue(b)
end
b

plot(equilibrium(b))

#####
# Equilibrium measures for a symmetric potential
# with two intervals of support consists of finding
# a measure w(x) supported on
# [-b,-a] ∪ [a,b]
# such that
#       1. H*w == V'
#       2. sum(w) == 1
#       3.  w is bounded
#
#  rescaling x == b*t these becomes find
# a measure w̃(t) supported on
# [-1,-a/b] ∪ [a/b,1]
# such that
#       1. H*w̃ == V'(b*t)
#       2. sum(w̃) == 1/b
#       3.  w is bounded
#
# Note (1) and (2) can always be satisfied
# thus the two constraints come from (3).
# The following gives the evaluation of the
# unweighted-component of the measure evaluated
# at (a,b)
#####
V = x -> x^4 - 10x^2
function equilibriumcoefficients(P,a,b)
    W = Weighted(P)
    Q = associated(P)
    t = axes(W,1)
    x = axes(Q,1)
    H = @. inv(x - t')
    H̃ = Q \ H*W
    [1/(b*sum(W[:,1])); 2H̃[:,2:end] \( Q \ derivative.(V, b*x))]
end
function equilibriumendpointvalues(ab::SVector{2})
    a,b = ab
    # orthogonal polynomials w.r.t.
    # abs(x) / (sqrt(1-x^2) * sqrt(x^2 - ρ^2))
    P = TwoBandJacobi(a/b, -one(a)/2, -one(a)/2, one(a)/2)
    Vector(P[[a/b,1],:] * equilibriumcoefficients(P,a,b))
end

function equilibrium(ab)
    a,b = ab
    P = TwoBandJacobi(a/b, -1/2, -1/2, 1/2)
    Weighted(P) * equilibriumcoefficients(P,a,b)
end

ab = SVector(2.,3.)
ab -= jacobian(equilibriumendpointvalues,ab) \ equilibriumendpointvalues(ab)
a,b = ab
xx = range(-4,4;length=1000)
μ = equilibrium(ab)
μx = x -> a < abs(x) < b ? μ[x/b] : 0.0
plot!(xx, μx.(xx))

plot(equilibrium(ab))


# Description:
# Dimensions: d

# The Zakharov function has no local minima except the global one. It is shown here in its two-dimensional form.

# Input Domain:
# The function is usually evaluated on the hypercube xi ∈ [-5, 10], for all i = 1, …, d.

# Global Minimum:
# https://www.sfu.ca/~ssurjano/zakharov3.png

function zakharov(xx)
    d = length(xx);
    sum1 = 0;
    sum2 = 0;

    for ii = 1:d
       	xi = xx[ii];
       	sum1 = sum1 + xi^2;
       	sum2 = sum2 + 0.5 * ii * xi;
    end

    y = sum1 + sum2^2 + sum2^4;

    return [y]
end


"""
    LexicographicallyLessThan(row_dim::Int, column_dim::Int)

Ensures that each column of the matrix is lexicographically less than
the next column.

Formally, for two columns:

``\\{(x, y) \\in \\mathbb{R}^\\mathtt{column\\_dim} \\times \\mathbb{R}^\\mathtt{column\\_dim} | \\exists j \\in \\{1, 2 \\dots \\mathtt{column\\_dim}\\}: x_j < y_j, \\forall i < j, x_i = y_i \\}``.

Also called [`lex_less`](https://sofdem.github.io/gccat/gccat/Clex_less.html).

The matrix is encoded by stacking the columns, matching the behaviour of
Julia's `vec` function.
"""
struct LexicographicallyLessThan <: MOI.AbstractVectorSet
    row_dim::Int
    column_dim::Int
end

function LexicographicallyLessThan(column_dim::Int)
    # By default, only two columns.
    return LexicographicallyLessThan(2, column_dim)
end

MOI.dimension(set::LexicographicallyLessThan) = set.row_dim * set.column_dim

"""
    LexicographicallyGreaterThan(row_dim::Int, column_dim::Int)

Ensures that each column of the matrix is lexicographically greater than
the next column.

Formally, for two columns:

``\\{(x, y) \\in \\mathbb{R}^\\mathtt{column\\_dim} \\times \\mathbb{R}^\\mathtt{column\\_dim} | \exists j \\in \\{1, 2 \\dots \\mathtt{column\\_dim}\\}: x_j > y_j, \\forall i < j, x_i = y_i \\}``.

Also called [`lex_greater`](https://sofdem.github.io/gccat/gccat/Clex_greater.html).

The matrix is encoded by stacking the columns, matching the behaviour of
Julia's `vec` function.
"""
struct LexicographicallyGreaterThan <: MOI.AbstractVectorSet
    row_dim::Int
    column_dim::Int
end

function LexicographicallyGreaterThan(column_dim::Int)
    # By default, only two columns.
    return LexicographicallyGreaterThan(2, column_dim)
end

MOI.dimension(set::LexicographicallyGreaterThan) = set.row_dim * set.column_dim

# TODO: bridge to LexicographicallyLessThan.

"""
    DoublyLexicographicallyLessThan(dimension::Int)

Ensures that each column of the matrix is lexicographically less than
the next column, and that each row of the matrix is lexicographically less
than the next row.

Also called [`lex2`](https://sofdem.github.io/gccat/gccat/Clex2.html).

The matrix is encoded by stacking the columns, matching the behaviour of
Julia's `vec` function.
"""
struct DoublyLexicographicallyLessThan <: MOI.AbstractVectorSet
    row_dim::Int
    column_dim::Int
end

MOI.dimension(set::DoublyLexicographicallyLessThan) = set.row_dim * set.column_dim

"""
    DoublyLexicographicallyGreaterThan(dimension::Int)

Ensures that each column of the matrix is lexicographically greater than
the next column, and that each row of the matrix is lexicographically greater
than the next row.

The matrix is encoded by stacking the columns, matching the behaviour of
Julia's `vec` function.
"""
struct DoublyLexicographicallyGreaterThan <: MOI.AbstractVectorSet
    row_dim::Int
    column_dim::Int
end

MOI.dimension(set::DoublyLexicographicallyGreaterThan) = set.row_dim * set.column_dim

# TODO: bridge to LexicographicallyLessThan.

"""
    Sort(dimension::Int)

Ensures that the first `dimension` elements is a sorted copy of the next
`dimension` elements.

## Example

    [a, b, c, d] in Sort(2)
    # Enforces that:
    # - the first part is sorted: a <= b
    # - the first part corresponds to the second one:
    #     - either a = c and b = d
    #     - or a = d and b = c
"""
struct Sort <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::Sort) = 2 * set.dimension

"""
    SortPermutation(dimension::Int)

Ensures that the first `dimension` elements is a sorted copy of the next
`dimension` elements.

The last `dimension` elements give a permutation to get from the original array
to its sorted version.

## Example

    [a, b, c, d, i, j] in SortPermutation(2)
    # Enforces that:
    # - the first part is sorted: a <= b
    # - the first part corresponds to the second one:
    #     - either a = c and b = d: in this case, i = 1 and j = 2
    #     - or a = d and b = c: in this case, i = 2 and j = 1
"""
struct SortPermutation <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::SortPermutation) = 3 * set.dimension

"""
    MinimumAmong(dimension::Int)

Ensures that the first element is the minimum value among the next
`dimension` elements.

## Example

    [a, b, c] in MinimumAmong(2)
    # Enforces that a == min(b, c)
"""
struct MinimumAmong <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::MinimumAmong) = 1 + set.dimension

"""
    ArgumentMinimumAmong(dimension::Int)

Ensures that the first element is the index of the minimum value among the
next `dimension` elements.

## Example

    [a, b, c] in ArgumentMinimumAmong(2)
    # Enforces that a == argmin(b, c)
    # I.e., if b < c, a = 1, if b > c, a = 2
"""
struct ArgumentMinimumAmong <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::ArgumentMinimumAmong) = 1 + set.dimension

"""
    MaximumAmong(dimension::Int)

Ensures that the first element is the maximum value among the next
`dimension` elements.

## Example

    [a, b, c] in MaximumAmong(2)
    # Enforces that a == max(b, c)
"""
struct MaximumAmong <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::MaximumAmong) = 1 + set.dimension

"""
    ArgumentMaximumAmong(dimension::Int)

Ensures that the first element is the index of the maximum value among the
next `dimension` elements.

## Example

    [a, b, c] in ArgumentMaximumAmong(2)
    # Enforces that a == argmax(b, c)
    # I.e., if b > c, a = 1, if b < c, a = 2
"""
struct ArgumentMaximumAmong <: MOI.AbstractVectorSet
    dimension::Int
end

MOI.dimension(set::ArgumentMaximumAmong) = 1 + set.dimension

"""
    Increasing(dimension::Int)

Ensures that the elements of the vector are in increasing order (<= operation).

## Example

    [a, b, c] in Increasing(3)
    # Enforces that a <= b <= c
"""
struct Increasing <: MOI.AbstractVectorSet
    dimension::Int
end

"""
    Decreasing(dimension::Int)

Ensures that the elements of the vector are in decreasing order (>= operation).

## Example

    [a, b, c] in Decreasing(3)
    # Enforces that a >= b >= c
"""
struct Decreasing <: MOI.AbstractVectorSet
    dimension::Int
end

# isbits types, nothing to copy
function copy(
    set::Union{
        LexicographicallyLessThan,
        LexicographicallyGreaterThan,
        DoublyLexicographicallyLessThan,
        DoublyLexicographicallyGreaterThan,
        Sort,
        SortPermutation,
        MinimumAmong,
        ArgumentMinimumAmong,
        MaximumAmong,
        ArgumentMaximumAmong,
        Increasing,
        Decreasing,
    },
)
    return set
end


"""
    CircularArc2(c::Circle2)

Constructs an arc from a full circle.
"""
CircularArc2(c::Circle2)

"""
    CircularArc2(c::Circle2, p::Point2, q::Point2)

Constructs the circular arc supported by `c`, whose source is `p` and whose
target is `q` when traversing the circle in counterclockwise direction.

If `p` and `q` are the same, the arc is in fact a full circle.

!!! info "Precondition"

    `p` and `q` lie on `c`.
"""
CircularArc2(c::Circle2, p::Point2, q::Point2)

"""
    CircularArc2(p::Point2, q::Point2, r::Point2)

Constructs an arc that is supported by the circle of type [`Circle2`](@ref)
passing through points `p`, `q` and `r`.

The source and target are respectively `p` and `r`, when traversing the
supporting circle in the counterclockwise direction.  Note that, depending on
the orientation of the point triple `(p,q,r)`, `q` may not lie on the arc.

!!! info "Precondition"

    `p`, `q`, and `r` are not collinear.
"""
CircularArc2(p::Point2, q::Point2, r::Point2)

"""
    supporting_circle(ca::CircularArc2)

Returns the supporting circle of `ca`.
"""
supporting_circle(ca::CircularArc2)

"""
    center(ca::CircularArc2)

Returns the center of `ca`'s supporting circle.
"""
center(ca::CircularArc2)

"""
    squared_radius(ca::CircularArc2)

Returns the squared radius of `ca`'s supporting circle.
"""
squared_radius(ca::CircularArc2)

"""
    source(ca::CircularArc2)

Returns the arc's starting point, supposing its supporting circle is
traversed in counterclockwise direction.
"""
source(ca::CircularArc2)

"""
    target(ca::CircularArc2)

Returns the arc's ending point, supposing its supporting circle is traversed
in counterclockwise direction.
"""
target(ca::CircularArc2)

"""
    left(ca::CircularArc2)

Returns the arc's leftmost point.

!!! info "Precondition"

    `is_x_monotone(ca)`
"""
left(ca::CircularArc2)

"""
    right(ca::CircularArc2)

Returns the arc's rightmost point.

!!! info "Precondition"

    `is_x_monotone(ca)`
"""
right(ca::CircularArc2)

"""
    bbox(ca::CircularArc2)

Returns a bounding box containing the arc.
"""
bbox(ca::CircularArc2)

"""
    is_x_monotone(ca::CircularArc2)

Tests whether the arc is x-monotone.
"""
is_x_monotone(ca::CircularArc2)

"""
    is_y_monotone(ca::CircularArc2)

Tests whether the arc is y-monotone.
"""
is_y_monotone(ca::CircularArc2)

"""
==(ca₁::CircularArc2, ca₂::CircularArc2)

Test for equality.

Two arcs are equal, iff their non-oriented supporting circles are equal (i.e.
they have the same center and same squared radius) and their endpoints are
equal.
"""
==(ca₁::CircularArc2, ca₂::CircularArc2)


function toCartesian(lon, lat; r = 1, cxyz = (0,0,0) )
    lat, lon = lat*π/180, lon*π/180
    cxyz[1] + r * cos(lat) * cos(lon), cxyz[2] + r * cos(lat) * sin(lon), cxyz[3] + r *sin(lat)
end
function lonlat3D(lon, lat, data; cxyz = (0,0,0))
    xyzw = zeros(size(data)..., 3)
    for (i,lon) in enumerate(lon), (j,lat) in enumerate(lat)
        x, y, z = toCartesian(lon, lat; cxyz = cxyz)
        xyzw[i,j,1] = x
        xyzw[i,j,2] = y
        xyzw[i,j,3] = z
    end
    xyzw[:,:,1], xyzw[:,:,2], xyzw[:,:,3]
end

function SphereBall(; r = 0.995, cxyz = (0,0,0))
    Θ = LinRange(0, 2π, 500) # 50
    Φ = LinRange(0, π, 500)
    x0 = [cxyz[1] + r * cos(θ) * sin(ϕ)      for θ in Θ, ϕ in Φ]
    y0 = [cxyz[2] + r * sin(θ) * sin(ϕ)      for θ in Θ, ϕ in Φ]
    z0 = [cxyz[3] + r * cos(ϕ) for θ in Θ, ϕ in Φ]
    x0, y0, z0
end


#Load the required Packages
using JLD2, Random, LinearAlgebra, Statistics, CSV, DataFrames, FreqTables, Distributions
#Set the seed
Random.seed!(1234)
#(a)Create the following four matrices of random numbers
#(i) A 10 by 7 matrix drawn randomly from a uniform distribution U[-5,10]
A = rand(Uniform(-5,10),10,7)
#(ii) A 10 by 7 matrix drawn randomly from a normal distribution N[-2,15]
B = rand(Normal(-2,15),10,7)


function CroutLU(A::Array)

  #A = the matrix of coefficients 'A' must be squared

    m,n =size(A)

    if m != n
        error("Matrix must be squared")
    end

    # System of equations
    L=zeros(m,m)
    U=Matrix{Float64}(I,m,m)

    for i=1:m
        L[i,1]=A[i,1]
    end

    for j=2:m
       U[1,j]=A[1,j]/L[1,1]
    end

    for j=2:m-1
       for i=j:m
           L[i,j]=A[i,j]- (L[i,1:j-1]'*U[1:j-1,j])
        end

        for k=j:m
           U[j,k]=(A[j,k]-(L[j,1:j-1]'*U[1:j-1,k]))/L[j,j]
        end
    end

       L[m,m]=A[m,m]-(L[m,1:m-1]'*U[1:m-1,m])

    return L, U
end


using LinearAlgebra, StatsBase

# Transition probability matrix
P = [0.5 0.4 0.1;
     0.3 0.2 0.5;
     0.5 0.3 0.2]

# First way
P^100
piProb1 = (P^100)[1,:]

# Second way
A = vcat((P' - Matrix{Float64}(I, 3, 3))[1:2,:],ones(3)')
b = [0 0 1]'
piProb2 = A\b

# Third way
eigVecs = eigvecs(copy(P'))
highestVec = eigVecs[:,findmax(abs.(eigvals(P)))[2]]
piProb3 = Array{Float64}(highestVec)/norm(highestVec,1);

# Fourth way
numInState = zeros(3)
state = 1
N = 10^6
for t in 1:N
    numInState[state] += 1
    global state = sample(1:3,weights(P[state,:]))
end
piProb4 = numInState/N

[piProb1 piProb2 piProb3 piProb4]


using EngEconomics

# Given
aInit = 680
bInit = 1100
aAnnualSavings = 245
bAnnualSavings = 440
aAnnualMaintenance = 35
aAnnualMaintenanceg = 10
bAnnualMaintenance = 60
aServiceLife = 4
bServiceLife = 6
aScrapValue = 100
bScrapValue = 250

# Find
# a.) Use PW Method to determine which is better
MARR = 0.1
years = 12 # Just knew b/c it's the least common multiple of 4 and 6s
aPeriods = years / aServiceLife
bPeriods = years / bServiceLife

# Product A
aMultiplicand = 1 + presentWorthFactor(MARR, aServiceLife) + presentWorthFactor(MARR, 2 * aServiceLife)
aInitPW = -aInit * aMultiplicand
aAnnualSavingsPW = aAnnualSavings *
		seriesPresentAmountFactor(MARR, years)
aAnnualMaintenancePW = -aAnnualMaintenance *
		aAnnualMaintenanceg * sinkingFundPaymentArithmetic(MARR, aServiceLife) *
		aMultiplicand
aScrapValuePW = aScrapValue *
		(presentWorthFactor(MARR, aServiceLife) +
		presentWorthFactor(MARR, 2 * aServiceLife) +
		presentWorthFactor(MARR, 3 * aServiceLife))
aPW = aInitPW + aAnnualSavingsPW + aAnnualMaintenancePW + aScrapValuePW

# Product B

if aPW > bPW
	println("CR1000 is better")
else
	println("CRX is better")
end


using DifferentialEquations
using ModelingToolkit
using DataDrivenDiffEq
using LinearAlgebra, DiffEqSensitivity, Optim
using DiffEqFlux, Flux
using Plots
gr()

# function triNN!(du,u,p,t,dens,cons)
#     # unpack rates and constants
#     nᵣ,nₓ,n₃ = u
#     k₁,k₋₁,k₂,k₋₂ = cons
#     mᵣ,mₗ,mₓ,A = dens
#     z = L(u,p)
#     # model
#     du[1] = dnᵣ = A*k₁*mᵣ*mₗ - k₋₁*nᵣ + z[1]
#     du[2] = dnₓ = A*k₂*mₓ*mₗ - k₋₂*nₓ + z[2]
#     du[3] = dn₃ = z[1] + z[2]
#
# end
function bimolecular!(du,u,p,t,dens,cons)
    # unpack rates and constants
    nᵣ = u[1]
    k₁,k₋₁  = p
    mᵣ,mₗ,A = dens
    # model
    du[1] = dnᵣ = A*k₁*mᵣ*mₗ - k₋₁*nᵣ

end

dens = [100.,10.,1.]
cons = Float64[]
f = (du,u,p,t) -> bimolecular!(du,u,p,t,dens,cons)
# Define the experimental parameter
tspan = (0.0f0,10.0f0)
u0 = Float32[0.]
p_ = Float32[1e-4, 0.5]
prob = ODEProblem(f, u0,tspan, p_)
solution = solve(prob, Tsit5(), saveat = 0.1)

scatter(solution, alpha = 0.25)
plot!(solution, alpha = 0.5)

# Ideal data
X = Array(solution)
# Add noise to the data
println("Generate noisy data")
Xₙ = X + Float32(1e-2)*randn(eltype(X), size(X))

# function dudt_(u, p,t)
#     x = u
#     z = L(u,p)
#     [p_[1]*x + z[1],
#     -p_[4]*y + z[2]]
# end
function biNN!(du,u,p,t,dens,cons)
    # unpack rates and constants
    nᵣ = u[1]
    k₁,k₋₁  = p
    mᵣ,mₗ,A = dens
    z = L(u,p)
    # model
    du[1] = dnᵣ = z[1]

end

L = FastChain(FastDense(1, 32, tanh),FastDense(32, 32, tanh), FastDense(32, 1))
#L = FastChain(FastDense(3, 20, tanh),FastDense(20, 20, tanh), FastDense(20, 2))
p = initial_params(L)

#
fNN = (du,u,p,t) -> biNN!(du,u, p,t,dens,cons)
prob_nn = ODEProblem(fNN,u0, tspan, p)
sol_nn = solve(prob_nn, Tsit5(), saveat =  solution.t)

plot(solution)
plot!(sol_nn)
# summ = reduce(vcat,sum(sol_nn,dims=1))
# h = plot!(sol_nn.t,summ,linecolor=:black)

function predict(θ)

    tmp_prob = remake(prob_nn,u0=u0,p=θ)
    tmp_sol =  solve(tmp_prob, VCABM(), saveat = solution.t,
                  abstol=1e-5, reltol=1e-5)
                  # backsolve=true)
    # Σ_sol = sum(Array(tmp_sol),dims=1) # Note: this returns a row vector!
    Array(tmp_sol)
end

function loss(θ)
    pred = predict(θ)
    sum(abs2, Xₙ .- pred), pred
end

# Test
@time loss(p)

const losses = []

callback(θ,l,pred) = begin
    push!(losses, l)
    @show l
    if length(losses)%50==0
        println("Current loss after $(length(losses)) iterations: $(losses[end])")
    end
    h = plot(solution.t, pred')
    scatter!(h, solution.t, Xₙ')
    display(h)
    false
end

p = initial_params(L)
# First train with ADAM for better convergence
@time res1 = DiffEqFlux.sciml_train(loss, p, ADAM(0.01), cb=callback, maxiters = 40)

# Train with BFGS
res2 = DiffEqFlux.sciml_train(loss, res1.minimizer, BFGS(),
                                cb=callback, maxiters = 100)

# @time res1 = DiffEqFlux.sciml_train(loss, res2.minimizer, ADAM(0.001), cb=callback, maxiters = 20)

println("Final training loss after $(length(losses)) iterations: $(losses[end])")

# Plot the losses
imgloss= plot(losses, yaxis = :log, xaxis = :log, xlabel = "Iterations", ylabel = "Loss",fmt = :svg)
savefig(imgloss,"training_loss")
# Plot the data and the approximation
NNsolution = predict(res2.minimizer)
# Trained on noisy data vs real solution
plot(solution.t, NNsolution',linewidth=3,xlabel = "t (s)", ylabel = "n",fmt = :svg)
scatter!(solution.t, Xₙ')
savefig("fit")

L̂ = L(Xₙ,res2.minimizer)
## Sparse Identification
#
# # Create a Basis
# @variables u[1:2]
# # Lots of polynomials
# polys = Operation[1]
#
# for i ∈ 1:2
#     push!(polys, u[1]^i)
#     push!(polys, u[2]^i)
#     for j ∈ i:2
#         if i != j
#             push!(polys, (u[1]^i)*(u[2]^j))
#             push!(polys, u[2]^i*u[1]^i)
#         end
#     end
# end
#
# # And some other stuff
# h = [cos.(u)...; sin.(u)...; polys...]
# h = [ polys...]
# basis = Basis(h, u)
#
# # Create an optimizer for the SINDy problem
# opt = SR3()
# # Create the thresholds which should be used in the search process
# λ = exp10.(-7:0.1:3)
# # Target function to choose the results from; x = L0 of coefficients and L2-Error of the model
# g(x) = x[1] < 1 ? Inf : norm(x, 2)
# # Test on original data and without further knowledge
# # println("SINDy on full ideal, unavailable data")
# # Ψ = SINDy(Xₙ[:, :], DX[:, :], basis, λ, opt, g = g, maxiter = 10000) # Fail
# # println(Ψ)
# # print_equations(Ψ)
# #
# # # Test on ideal derivative data for unknown function ( not available )
# # println("SINDy on partial ideal, unavailable data")
# # Ψ = SINDy(Xₙ[:, 1:end], L̄[:, 1:end], basis, λ,opt, g = g, maxiter = 10000) # Succeed
# # println(Ψ)
# # print_equations(Ψ)
#
# # Test on uode derivative data
# println("SINDy on learned, partial, available data")
# Ψ = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], basis, λ,  opt, g = g, maxiter = 100000, normalize = true, denoise = true) # Succeed
# println(Ψ)
# print_equations(Ψ)
#
# # Extract the parameter
# p̂ = parameters(Ψ)
# println("First parameter guess : $(p̂)")

@variables u[1:1]
h = Operation[u; u.^2; 1]
basis = Basis(h, u)

# Create the thresholds which should be used in the search process
λ = exp10.(-7:0.1:3)
# Target function to choose the results from; x = L0 of coefficients and L2-Error of the model
g(x) = x[1] < 1 ? Inf : norm(x, 2)

opt = SR3(3e-1, 1.0)
# Ψ = SInDy(X[:, 1:1000], DX[:, 1:1000], basis, maxiter = 10000, opt = opt, normalize = true)
Ψ = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], basis, λ,  opt, g = g, maxiter = 10000, normalize = true) # Succeed
# Ψ = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], basis, maxiter = 10000, opt = opt, normalize = true,denoise = true)
println(Ψ)
print_equations(Ψ)

sys = ODESystem(Ψ)
p = parameters(Ψ)

dudt = ODEFunction(sys)

estimator = ODEProblem(dudt, u0, tspan, p)
estimation = solve(estimator, Tsit5(), saveat = solution.t)

plot!(estimation,linecolor=:red,linewidth=3)
savefig("sindy_raw_estim")

using DiffEqParamEstim
using Optim
# cost_function = build_loss_objective(estimator,t,Xₙ,Tsit5(),maxiters=10000)

cost_function = build_loss_objective(estimator,Tsit5(),L2Loss(solution.t,Xₙ),
                                     maxiters=10000)#,verbose=false)
result = optimize(cost_function, p, BFGS())
tmp_estimator = remake(estimator, p=result.minimizer)
better_estimation = solve(tmp_estimator, Tsit5(), saveat = solution.t)

plot!(better_estimation,linecolor=:black,linestyle=:dash,linewidth=4)
savefig("sindy_better_estim")
#
# # The parameters are a bit off, but the equations are recovered
# # Start another SINDy run to get closer to the ground truth
# # Create function
# unknown_sys = ODESystem(Ψ)
# unknown_eq = ODEFunction(unknown_sys)
#
# # Just the equations
# b = Basis((u, p, t)->unknown_eq(u, [1.; 1.], t), u)
#
# # Retune for better parameters -> we could also use DiffEqFlux or other parameter estimation tools here.
# Ψf = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], b, STRRidge(0.01), maxiter = 100, convergence_error = 1e-18) # Succeed
# println(Ψf)
# p̂ = parameters(Ψf)
# println("Second parameter guess : $(p̂)")
#
# # Create function
# recovered_sys = ODESystem(Ψf)
# recovered_eq = ODEFunction(recovered_sys)
# estimator = ODEProblem(dudt, u0, tspan, p)
# estimation = solve(estimator, Tsit5(), saveat = solution.t)
#
# plot!(estimation)
#
#
#


const helix = let res_init=zeros(3), jac_init=zeros(3,3), x_init=[-1.0, 0.0, 0.0]

    function res(x, r)
        theta = atan(x[2], x[1]) / 2pi

        if x[1] <= 0.0 && x[2] <= 0.0
            theta = theta + 1.0
        end

        r[1] = 10.0 * (x[3] - 10.0 * theta )
        r[2] = 10.0 * (sqrt(x[1]^2 + x[2]^2) - 1.0)
        r[3] = x[3]
        return r
    end

    function jac(x, jac)
        t = x[1]^2 + x[2]^2
        ti = 100.0 / (2pi * t )
        jac[1, 1] = ti * x[2]
        t = 10.0 / sqrt(t)
        jac[2, 1] = x[1] * t
        jac[3, 1] = 0.0
        jac[1, 2] = -ti * x[1]
        jac[2, 2] = x[2] * t
        jac[3, 2] = 0.0
        jac[1, 3] = 10.0
        jac[2, 3] = 0.0
        jac[3, 3] = 1.0
        return jac
    end

    f(;scale=1, verbose=false, print_steps=false) = testone("helix", res, jac,
                                                            res_init, jac_init, x_init;
                                                            scale=scale, verbose=verbose,
                                                            print_steps=print_steps)
end


#######################
# The Bernstein basis
#######################

using BasisFunctions: UnitInterval, PolynomialBasis
using BasisFunctions: hasderivative, hasantiderivative, ordering, support

const BernsteinInterval = UnitInterval
const BernsteinIndex = NativeIndex{:bernstein}

get_bernstein_nodes(T,n) = collect(LinRange{T}(0.0, 1.0, n))

struct Bernstein{T} <: PolynomialBasis{T}
    n :: Int
    nodes :: ScatteredGrid{T}

    function Bernstein(nodes::ScatteredGrid{T}) where {T}
        new{T}(length(nodes.points), nodes)
    end
    function Bernstein(ξ::Vector{T}) where {T}
        Bernstein(ScatteredGrid(ξ, BernsteinInterval{T}()))
    end
end

Bernstein(n::Int) = Bernstein(get_bernstein_nodes(Float64,n))
Bernstein{T}(n::Int) where {T} = Bernstein(get_bernstein_nodes(T,n))

# Convenience constructor: map the Bernstein basis to the interval [a,b]
Bernstein(ξ, a::Number, b::Number) = rescale(Bernstein(ξ), a, b)

nodes(b::Bernstein)  = b.nodes.points
nnodes(b::Bernstein) = b.n
degree(b::Bernstein) = b.n-1

BasisFunctions.native_index(b::Bernstein, idx) = BernsteinIndex(idx)
BasisFunctions.linear_index(b::Bernstein, idx) = BernsteinIndex(idx)
BasisFunctions.ordering(b::Bernstein) = Base.OneTo(nnodes(b))
BasisFunctions.support(b::Bernstein{T}) where {T} = BernsteinInterval{T}()

BasisFunctions.interpolation_grid(b::Bernstein{T}) where {T} = ScatteredGrid(get_bernstein_nodes(T,length(b)), BernsteinInterval{T}())

BasisFunctions.similar(::Bernstein, ::Type{T}, n::Int) where {T} = Bernstein{T}(n)
BasisFunctions.similar(::Bernstein, ::Type{T}, ξ::Vector{T}) where {T} = Bernstein(ξ)

Base.size(b::Bernstein) = (b.n,)

function _bernstein(b::Bernstein{T}, i::BernsteinIndex, n::BernsteinIndex, x::T) where {T}
    if i < 0 || i > n
        return zero(T)
    else
        if n == 0
            return one(T)
        else
            return _bernstein(b, i, n-1, x) * (1-x) + _bernstein(b, i-1, n-1, x) * x
        end
    end
end

_bernstein(b::Bernstein, i, n, x) = _bernstein(b, native_index(b,i), native_index(b,n), x)

function BasisFunctions.unsafe_eval_element(b::Bernstein, i, x)
    _bernstein(b, i-1, nnodes(b)-1, x)
end

function BasisFunctions.unsafe_eval_element_derivative(b::Bernstein, i, x)
    (nnodes(b)-1) * ( _bernstein(b, i-2, nnodes(b)-2, x) - _bernstein(b, i-1, nnodes(b)-2, x) )
end


function f(x::Vector)
    (x[1] - 5.0)^4
end

function g!(x::Vector, storage::Vector)
    storage[1] = 4.0 * (x[1] - 5.0)^3
end

function h!(x::Vector, storage::Matrix)
    storage[1, 1] = 12.0 * (x[1] - 5.0)^2
end

d = TwiceDifferentiableFunction(f, g!, h!)

results = Optim.newton(d, [0.0])
@assert length(results.trace.states) == 0
@assert results.converged
@assert norm(results.minimum - [5.0]) < 0.01

eta = 0.9

function f(x::Vector)
  (1.0 / 2.0) * (x[1]^2 + eta * x[2]^2)
end

function g!(x::Vector, storage::Vector)
  storage[1] = x[1]
  storage[2] = eta * x[2]
end

function h!(x::Vector, storage::Matrix)
  storage[1, 1] = 1.0
  storage[1, 2] = 0.0
  storage[2, 1] = 0.0
  storage[2, 2] = eta
end

d = TwiceDifferentiableFunction(f, g!, h!)
results = Optim.newton(d, [127.0, 921.0])
@assert length(results.trace.states) == 0
@assert results.converged
@assert norm(results.minimum - [0.0, 0.0]) < 0.01


"""
    cartesian2jacobi!(r, r_j, m, m_sum)

Transforms cartesian coordinates into jacobi coordinates.

# Args

* `r`: NxD matrix where N is the number of bodies and D is the number of dimensions, each row contains the positions of the bodies in cartesian coordinates.
* `r_j`: NxD matrix in which the results will be saved.
* `m`: array of N elements, where element number i contains the mass of i-th body.
* `m_sum`: array of N elements, where element with index i contains the mass of the bodies from 1 to i
```math
m_{sum_{i}} = \\sum_{j = 1}^{i} m_{j} \\\\
```
"""
function cartesian2jacobi!(r, r_j, m, m_sum)
    R = m[1]*r[1,:]
    for i in 2:size(r)[1]
        r_j[i,:] = r[i,:] - (R/m_sum[i-1])
        R = R*(1.0+m[i]/m_sum[i-1]) + m[i]*r_j[i,:]
    end
    r_j[1,:] = R/m_sum[size(r)[1]]
end

"""
    jacobi2cartesian!(r, r_j, m, m_sum)

Transforms jacobi coordinates into cartesian coordinates.

# Args

* `r`: NxD matrix in which the results will be saved.
* `r_j`: NxD matrix where N is the number of bodies and D is the number of dimensions, each row contains the positions of the bodies in jacobi coordinates.
* `m`: array of N elements, where element number i contains the mass of i-th body.
* `m_sum`: array of N elements, where element with index i contains the mass of the bodies from 1 to i
```math
m_{sum_{i}} = \\sum_{j = 1}^{i} m_{j} \\\\
```
"""
function jacobi2cartesian!(r, r_j, m, m_sum)
    R = r_j[1,:]*m_sum[size(r)[1]]
    for i in size(r)[1]:-1:2
        R = (R - m[i]*r_j[i,:])/m_sum[i]
        r[i,:] = r_j[i,:] + R
        R *= m_sum[i-1]
    end
    r[1,:] = R/m[1]
end


const score_table = (1 => 10, 5 => 5, 10 => 1)

"""
    score(x, y)

Return the score earned by a dart landing at coordinates (x, y).
"""
function score(x, y)
    z = hypot(x, y)
    for (radius, points) in score_table
        z <= radius && return points
    end
    return 0
end


"""
   shi(x)

   hyperbolic sine integral function

   shi(x) = ₀∫ˣ (sinh(t) / t) δt
""" shi

"""
    chi(x)

    hyperbolic cosine integral

chi(x) = γ + ln(x) + ₀∫ˣ ((cosh(t) - 1) / t) δt
where γ == 0.5772_1566_4901_5328_6060_6512_0901-
           (the Euler-Mascheroni constant)
""" chi

"""
    li(x)

    logarithmic integral

li(x) =  ᵤ∫ˣ (δt / ln(t))

         where u = 1.4513_6923_4883_3810_5028+
                   (Soldner's constant)
""" li

"""
    si(x)

    sine integral

si(x) = ₀∫ˣ (sin(t) / t) δt
""" si


include("util.jl")
include("../../src/CGPO.jl")

function PerformCGFor(
    A::AbstractMatrix,
    b::AbstractVecOrMat;
    epsilon=1e-2,
    exact::Bool=true,
    partial_ortho=nothing,
)
    cg = CGPO(A, b)

    if exact

    else
        if partial_ortho === nothing
            cg |> TurnOffReorthgonalize!
        else
            StorageLimit!(cg, partial_ortho)
        end
    end

    ẋ = A\b
    ė = ẋ - cg.x
    ėAė = dot(ė, A*ė)
    E = Vector{Float64}()
    push!(E, 1)
    RelErr = 1
    while RelErr > epsilon
        cg()
        e = ẋ - cg.x
        RelErr = (dot(e, A*e)/ėAė)|>sqrt
        push!(E, RelErr)
    end
    return E
end

E = PerformCGFor(Diagonal(rand(10)), rand(10))

function PerformExperiment1(filename, p=0.9)
    N = 256
    A = GetNastyPSDMatrix(N, p)
    b = rand(N)
    A = convert(Matrix{Float16}, A)
    b = convert(Vector{Float16}, b)
    # TODO: Make the plot distinguishable without colors.

    # ==========================================================================
    # The exact computations
    # ==========================================================================

    RelErr = PerformCGFor(A, b, epsilon=1e-3, exact=true)
    k = length(RelErr)
    fig1 = plot(
        RelErr,
        label="Relative Energy (exact)",
        legend=:bottomleft,
        yaxis=:log10,
        xlabel="iterations: k",
        ylabel="\$\\frac{\\Vert e_k \\Vert_A}{\\Vert e_0\\Vert_A}\$",
        title="CG Rel Energy Error \$\\rho = $(p)\$",
        size=(750, 500), dpi=300,
        left_margin = 10Plots.mm
    )

    # ==========================================================================
    # No-Orthogonalizations
    # ==========================================================================

    RelErr = PerformCGFor(A, b, exact=false, epsilon=1e-3)
    k = length(RelErr)
    plot!(
        fig1,
        RelErr,
        label="Relative Energy (floats)",
        linestyle=:dash#, markershape=:+
    )

    # ==========================================================================
    # Theoretical Bounds
    # ==========================================================================
    ErrorsBound = [TheoreticalErrorBound(A, idx) for idx in 1: k]
    plot!(
        fig1,
        ErrorsBound,
        label="Theoretical Bound (exact)",
        linestyle=:dot
    )

    # ==========================================================================
    # Floating Points Partially Orthogonalized
    # ==========================================================================
    RelErr = PerformCGFor(A, b, exact=false, epsilon=1e-3, partial_ortho=div(N, 8))
    k = length(RelErr)
    plot!(
        fig1,
        RelErr,
        label="Relative Energy (partial)",
        legend=:bottomleft,
        linestyle=:dashdot # ,  markershape=:x
    )

    display(fig1)
    SaveFigToCurrentScriptDir(fig1, "$(filename).png")

return end

PerformExperiment1("cg_convergence_0.9")
PerformExperiment1("cg_convergence_1", 1)


# Descriptive Statistics

#############################
#
#   Location
#
#############################

# Geometric mean
function geomean(a::RealArray)
    s = 0.0
    n = length(a)
    for i = 1 : n
        @inbounds s += log(a[i])
    end
    return exp(s / n)
end

# Harmonic mean
function harmmean(a::RealArray)
    s = 0.0
    n = length(a)
    for i in 1 : n
        @inbounds s += inv(a[i])
    end
    return n / s
end

# Trimmed mean
function trimmean(x::RealArray, p::Real)
    n = length(x)
    n > 0 || error("x can not be empty.")
    0 <= p < 1 || error("p must be non-negative and less than 1.")
    rn = min(iround(n * p), n-1)

    sx = sort(x)
    nl = rn >> 1
    nh = (rn - nl)
    s = 0.0
    for i = (1+nl) : (n-nh)
        @inbounds s += x[i]
    end
    return s / (n - rn)
end

# compute mode, given the range of integer values
function mode{T<:Integer}(a::AbstractArray{T}, rgn::UnitRange{T})
    isempty(a) && error("mode: input array cannot be empty.")
    len = length(a)
    r0 = rgn[1]
    r1 = rgn[end]
    cnts = zeros(Int, length(rgn))
    mc = 0    # maximum count
    mv = r0   # a value corresponding to maximum count
    for i = 1:len
        @inbounds x = a[i]
        if r0 <= x <= r1
            @inbounds c = (cnts[x - r0 + 1] += 1)
            if c > mc
                mc = c
                mv = x
            end
        end
    end
    return mv
end

function modes{T<:Integer}(a::AbstractArray{T}, rgn::UnitRange{T})
    r0 = rgn[1]
    r1 = rgn[end]
    n = length(rgn)
    cnts = zeros(Int, n)
    # find the maximum count
    mc = 0
    for i = 1:length(a)
        @inbounds x = a[i]
        if r0 <= x <= r1
            @inbounds c = (cnts[x - r0 + 1] += 1)
            if c > mc
                mc = c
            end
        end
    end
    # find all values corresponding to maximum count
    ms = T[]
    for i = 1:n
        @inbounds if cnts[i] == mc
            push!(ms, rgn[i])
        end
    end
    return ms
end

# compute mode over arbitrary array
function mode{T}(a::AbstractArray{T})
    isempty(a) && error("mode: input array cannot be empty.")
    cnts = Dict{T,Int}()
    # first element
    mc = 1
    mv = a[1]
    cnts[mv] = 1
    # find the mode along with table construction
    for i = 2 : length(a)
        @inbounds x = a[i]
        if haskey(cnts, x)
            c = (cnts[x] += 1)
            if c > mc
                mc = c
                mv = x
            end
        else
            cnts[x] = 1
            # in this case: c = 1, and thus c > mc won't happen
        end
    end
    return mv
end

function modes{T}(a::AbstractArray{T})
    isempty(a) && error("modes: input array cannot be empty.")
    cnts = Dict{T,Int}()
    # first element
    mc = 1
    cnts[a[1]] = 1
    # find the mode along with table construction
    for i = 2 : length(a)
        @inbounds x = a[i]
        if haskey(cnts, x)
            c = (cnts[x] += 1)
            if c > mc
                mc = c
            end
        else
            cnts[x] = 1
            # in this case: c = 1, and thus c > mc won't happen
        end
    end
    # find values corresponding to maximum counts
    ms = T[]
    for (x, c) in cnts
        if c == mc
            push!(ms, x)
        end
    end
    return ms
end

#############################
#
#   quantile and friends
#
#############################

percentile{T<:Real}(v::AbstractArray{T}, p) = quantile(v, p * 0.01)

quantile{T<:Real}(v::AbstractArray{T}) = quantile(v, [.0, .25, .5, .75, 1.0])
nquantile{T<:Real}(v::AbstractArray{T}, n::Integer) = quantile(v, (0:n)/n)

#############################
#
#   Dispersion
#
#############################

# Variation coefficient: std / mean
variation{T<:Real}(x::AbstractArray{T}, m::Real) = stdm(x, m) / m
variation{T<:Real}(x::AbstractArray{T}) = variation(x, mean(x))

# Standard error of the mean: std(a) / sqrt(len)
sem{T<:Real}(a::AbstractArray{T}) = sqrt(var(a) / length(a))

# Median absolute deviation
mad{T<:Real}(v::AbstractArray{T}, args...;arg...) = mad!(copy(v), args...;arg...)
mad{T<:Real}(v::Range{T}, args...;arg...) = mad!([v], args...;arg...)

function mad!{T<:Real}(v::AbstractArray{T}, center::Real=median!(v); constant::Real=1.4826)
    for i in 1:length(v)
        @inbounds v[i] = abs(v[i]-center)
    end
    constant * median!(v)
end

# Interquartile range
iqr{T<:Real}(v::AbstractArray{T}) = (q = quantile(v, [.25, .75]); q[2] - q[1])

#############################
#
#   Z-scores
#
#############################

function _zscore!(Z::AbstractArray, X::AbstractArray, μ::Real, σ::Real)
    # Z and X are assumed to have the same size
    iσ = inv(σ)
    if μ == zero(μ)
        for i = 1 : length(X)
            @inbounds Z[i] = X[i] * iσ
        end
    else
        for i = 1 : length(X)
            @inbounds Z[i] = (X[i] - μ) * iσ
        end
    end
    return Z
end

@ngenerate N typeof(Z) function _zscore!{S,T,N}(Z::AbstractArray{S,N}, X::AbstractArray{T,N}, μ::AbstractArray, σ::AbstractArray)
    # Z and X are assumed to have the same size
    # μ and σ are assumed to have the same size, that is compatible with size(X)
    siz1 = size(X, 1)
    @nextract N ud d->size(μ, d)
    if size(μ, 1) == 1 && siz1 > 1
        @nloops N i d->(d>1 ? (1:size(X,d)) : (1:1)) d->(j_d = ud_d ==1 ? 1 : i_d) begin
            v = (@nref N μ j)
            c = inv(@nref N σ j)
            for i_1 = 1:siz1
                (@nref N Z i) = ((@nref N X i) - v) * c
            end
        end
    else
        @nloops N i X d->(j_d = ud_d ==1 ? 1 : i_d) begin
            (@nref N Z i) = ((@nref N X i) - (@nref N μ j)) / (@nref N σ j)
        end
    end
    return Z
end

function _zscore_chksize(X::AbstractArray, μ::AbstractArray, σ::AbstractArray)
    size(μ) == size(σ) || throw(DimensionMismatch("μ and σ should have the same size."))
    for i=1:ndims(X)
        dμ_i = size(μ,i)
        (dμ_i == 1 || dμ_i == size(X,i)) || throw(DimensionMismatch("X and μ have incompatible sizes."))
    end
end

function zscore!{ZT<:FloatingPoint,T<:Real}(Z::AbstractArray{ZT}, X::AbstractArray{T}, μ::Real, σ::Real)
    size(Z) == size(X) || throw(DimensionMismatch("Z and X must have the same size."))
    _zscore!(Z, X, μ, σ)
end

function zscore!{ZT<:FloatingPoint,T<:Real,U<:Real,S<:Real}(Z::AbstractArray{ZT}, X::AbstractArray{T},
                                                            μ::AbstractArray{U}, σ::AbstractArray{S})
    size(Z) == size(X) || throw(DimensionMismatch("Z and X must have the same size."))
    _zscore_chksize(X, μ, σ)
    _zscore!(Z, X, μ, σ)
end

zscore!{T<:FloatingPoint}(X::AbstractArray{T}, μ::Real, σ::Real) = _zscore!(X, X, μ, σ)

zscore!{T<:FloatingPoint,U<:Real,S<:Real}(X::AbstractArray{T}, μ::AbstractArray{U}, σ::AbstractArray{S}) =
    (_zscore_chksize(X, μ, σ); _zscore!(X, X, μ, σ))

function zscore{T<:Real}(X::AbstractArray{T}, μ::Real, σ::Real)
    ZT = typeof((zero(T) - zero(μ)) / one(σ))
    _zscore!(Array(ZT, size(X)), X, μ, σ)
end

function zscore{T<:Real,U<:Real,S<:Real}(X::AbstractArray{T}, μ::AbstractArray{U}, σ::AbstractArray{S})
    _zscore_chksize(X, μ, σ)
    ZT = typeof((zero(T) - zero(U)) / one(S))
    _zscore!(Array(ZT, size(X)), X, μ, σ)
end

zscore{T<:Real}(X::AbstractArray{T}) = ((μ, σ) = mean_and_std(X); zscore(X, μ, σ))
zscore{T<:Real}(X::AbstractArray{T}, dim::Int) = ((μ, σ) = mean_and_std(X, dim); zscore(X, μ, σ))

#############################
#
#   entropy and friends
#
#############################

function entropy{T<:Real}(p::AbstractArray{T})
    s = 0.
    z = zero(T)
    for i = 1:length(p)
        @inbounds pi = p[i]
        if pi > z
            s += pi * log(pi)
        end
    end
    return -s
end

function crossentropy{T<:Real}(p::AbstractArray{T}, q::AbstractArray{T})
    length(p) == length(q) || throw(DimensionMismatch("Inconsistent array length."))
    s = 0.
    z = zero(T)
    for i = 1:length(p)
        @inbounds pi = p[i]
        @inbounds qi = q[i]
        if pi > z
            s += pi * log(qi)
        end
    end
    return -s
end

function kldivergence{T<:Real}(p::AbstractArray{T}, q::AbstractArray{T})
    length(p) == length(q) || throw(DimensionMismatch("Inconsistent array length."))
    s = 0.
    z = zero(T)
    for i = 1:length(p)
        @inbounds pi = p[i]
        @inbounds qi = q[i]
        if pi > z
            s += pi * log(pi / qi)
        end
    end
    return s
end

#############################
#
#   summary
#
#############################

immutable SummaryStats{T<:FloatingPoint}
    mean::T
    min::T
    q25::T
    median::T
    q75::T
    max::T
end

function summarystats{T<:Real}(a::AbstractArray{T})
    m = mean(a)
    qs = quantile(a, [0.00, 0.25, 0.50, 0.75, 1.00])
    R = typeof(convert(FloatingPoint, zero(T)))
    SummaryStats{R}(
        convert(R, m),
        convert(R, qs[1]),
        convert(R, qs[2]),
        convert(R, qs[3]),
        convert(R, qs[4]),
        convert(R, qs[5]))
end

function Base.show(io::IO, ss::SummaryStats)
    println(io, "Summary Stats:")
    @printf(io, "Mean:         %.6f\n", ss.mean)
    @printf(io, "Minimum:      %.6f\n", ss.min)
    @printf(io, "1st Quartile: %.6f\n", ss.q25)
    @printf(io, "Median:       %.6f\n", ss.median)
    @printf(io, "3rd Quartile: %.6f\n", ss.q75)
    @printf(io, "Maximum:      %.6f\n", ss.max)
end

describe{T<:Real}(a::AbstractArray{T}) = show(summarystats(a))


function division(a, b)
	k=0
    while a*k<b
        k = k+1
    end
    return k
end

function syra(n::Int64)
	if n==1
        return 0
    elseif n%2 == 0
        return 1 + syra(division(2,n))
    else
        return 1 + syra(3*n+1)
    end
end

println(syra(42))


test_str = "35
20
15
25
47
40
62
55
65
95
102
117
150
182
127
219
299
277
309
576"

test_list = split(test_str, '\n')

list = readlines("day9_input.txt")

function check(sublist)
    sublen = length(sublist)
    last = sublist[end]
    for j in 1:sublen-2
        for k in j+1:sublen-1
            if sublist[j] + sublist[k] == last
                return true
            end
        end
    end
    return false
end

function invalid(preamble, list)
    int_list = map(x-> parse(Int, x), list)
    for i in preamble+1:length(int_list)
        if !check(int_list[i-preamble:i])
            return i, int_list[i]
        end
    end
    return nothing, nothing
end

@show invalid(5, test_list)
@show invalid(25, list)

function contiguous(preamble, list)
    i_inv, num_inv = invalid(preamble, list)
    int_list = map(x-> parse(Int, x), list)
    len = length(list)
    for i in 1:len
        j = i
        sublistsum = int_list[i]
        while sublistsum <= num_inv
            if sublistsum == num_inv
                return minimum(int_list[i:j]) + maximum(int_list[i:j])
            end
            j += 1
            sublistsum += int_list[j]
        end
    end
end

@show contiguous(5, test_list)
@show contiguous(25, list)

nothing
#=
```julia
julia> @btime invalid(25, list);
  166.235 μs (526 allocations: 155.34 KiB)

julia> @btime contiguous(25, list);
  503.429 μs (531 allocations: 163.75 KiB)
```
 =#
# I restart counting with the second counter j every time I step up the
# first counter i, but that is indeed a waste. I could just subtract the
# first value in the sublist


# doubly periodic dipole
# https://dx.doi.org/10.1103/PhysRevLett.112.145301
# should be eigenstate of Hamiltonian evolution

using Test, Plots, Revise, VortexDistributions

# test dipole
xp = 20
yp = 3
vp = PointVortex(xp,yp,1)

xn = 1.1
yn = -3
vn = PointVortex(xn,yn,-1)

Thetad(.12,.11,xp,yp,xn,yn)

x = LinRange(-π,π,300)
y = x

#TODO add missing method for Thetad(x,y,dipole::Array{PointVortex})
# test double periodicity
testpsi = exp.(im*Thetad(x,y,xp,yp,xn,yn))
@test testpsi[end,:] ≈ testpsi[1,:]
@test testpsi[:,1] ≈ testpsi[:,end]
heatmap(x,y,angle.(testpsi),transpose=true)

# test domain
x = LinRange(-3π,3π,300)
y = LinRange(-4π,4π,300)

# test dipole
xp = 2.1*pi
yp = 2*pi

xn = -2*pi
yn = -2.4*pi

testphase3 = Thetad(x,y,xp,yp,xn,yn)
heatmap(x,y,testphase3,transpose=true)

psivort = Torus(exp.(im*testphase3),x,y)
vortices = findvortices(psivort)

vraw = rawData(vortices)

testpsi = exp.(im*testphase3)
@test testpsi[:,1] ≈ testpsi[:,end]
@test testpsi[1,:] ≈ testpsi[end,:]

#--- add new method for making periodic dipoles
using Test, Plots, Revise, VortexDistributions

x = LinRange(-50,50,300)
y = x

# test dipole
xp = 20
yp = 3
vp = PointVortex(xp,yp,1)

xn = 1.1
yn = -3
vn = PointVortex(xn,yn,-1)

dip = ScalarVortex([vp;vn])

function periodic_dipole!(psi::F,dip::Array{ScalarVortex{T},1}) where {T <: CoreShape, F<:Field}
    @assert length(dip) == 2
    @assert dip[1].vort.qv + dip[2].vort.qv == 0
    @assert hypot(dip[1].vort.xv-dip[2].vort.xv,dip[1].vort.yv-dip[2].vort.yv) >= 2*pi
    @unpack ψ,x,y = psi
    (dip[1].vort.qv > 0) ? (jp = 1;jn = 2) : (jp = 2;jn = 1)
    vp = rawData(dip[jp].vort)[1:2]
    vn = rawData(dip[jn].vort)[1:2]
    @. ψ *= abs(dip[jn](x,y')*dip[jp](x,y'))
    ψ .*= exp.(im*Thetad(x,y,vp...,vn...))
    @pack! psi = ψ
end

psivort = Torus(one.(x.*y'),x,y)

heatmap(abs2.(psivort.ψ))

periodic_dipole!(psivort,dip)

heatmap(x,y,abs2.(psivort.ψ))

heatmap(x,y,angle.(psivort.ψ))

#check that phase is periodic (note we don't enforce periodic density)

testpsi = angle.(psivort.ψ)
@test testpsi[:,1] ≈ testpsi[:,end]
@test testpsi[1,:] ≈ testpsi[end,:]


# https://adventofcode.com/2020/day/25

"""
Single transformation step
"""
function transform(value::Integer, subject_number::Integer)
    return rem(value * subject_number, 20201227)
end

"""
Using the `subject_number` and `loop_size`, calculate the `key`
"""
function get_key(subject_number::Integer, loop_size::Integer)
    value = 1
    for _ in 1:loop_size
        value = transform(value, subject_number)
    end
    return value
end

# example: subject number -> public key
@assert get_key(7, 8) == 5764801
@assert get_key(7, 11) == 17807724
# example: public key -> encryption key
@assert get_key(17807724, 8) == 14897079
@assert get_key(5764801, 11) == 14897079

"""
Using the `subject_number` and `public_key`, guess the `loop_size`
"""
function get_loop_size(subject_number::Integer, public_key::Integer)
    loop_size = 0
    value = 1
    while value != public_key
        loop_size += 1
        value = transform(value, subject_number)
    end
    return loop_size
end

# example: (subject number, public key) -> loop size
@assert get_loop_size(7, 5764801) == 8
@assert get_loop_size(7, 17807724) == 11
# example: (public key, encryption key) -> loop size
@assert get_loop_size(17807724, 14897079) == 8
@assert get_loop_size(5764801, 14897079) == 11

function part1(card_key, door_key)
    card_loop_size = get_loop_size(7, card_key)
    door_loop_size = get_loop_size(7, door_key)

    card_encryption_key = get_key(card_key, door_loop_size)
    door_encryption_key = get_key(door_key, card_loop_size)

    @assert card_encryption_key == door_encryption_key

    return card_encryption_key
end

@assert part1(5764801, 17807724) == 14897079

test = (14205034, 18047856)
println("Part 1: $(result1 = part1(test...))")
# println("Part 2: $(result2 = part2(test...))")

@assert result1 == 297257
# @assert result2 == 418819514477


using ViewerGL, LinearAlgebraicRepresentation
GL = ViewerGL
Lar = LinearAlgebraicRepresentation

V = [0.3307458 0.2735878 0.2527472 0.1955892 0.8735746 0.9604089 1.3600696 1.4469038 0.8952041 0.4087921 0.7697594 0.2833474 0.4465003 0.3177087 0.1293396 0.000548 1.1021824 0.6979647 1.0492281 0.6450104; 1.80716725 1.73722735 1.87091125 1.80097145 1.10328515 1.58712345 1.01597415 1.49981235 1.15705855 1.04345955 1.69419185 1.58059285 2.30620035 1.78225795 2.38416245 1.86022005 1.24012995 1.20346525 1.82393505 1.78727025]
EV = Array{Int64,1}[[1, 2], [3, 4], [1, 3], [2, 4], [5, 6], [7, 8], [5, 7], [6, 8], [9, 10], [11, 12], [9, 11], [10, 12], [13, 14], [15, 16], [13, 15], [14, 16], [17, 18], [19, 20], [17, 19], [18, 20]]
VV = [[k] for k=1:size(V,2)]

GL.VIEW( GL.numbering(.05)((V,[VV, EV]),GL.COLORS[1]) );

# arrangement of input data
W = convert(Lar.Points, V')
cop_EV = Lar.coboundary_0(EV)
W, copEV, copFE = Lar.planar_arrangement(W, cop_EV)

# compute containment graph of components
bicon_comps = Lar.Arrangement.biconnected_components(copEV)

# compute euler characteristic
χ = Lar.euler_characteristic(W, copEV, copFE)
println("χ = $χ ; bicon_comps = $(length(bicon_comps))");

# triangulation
triangulated_faces = Lar.triangulate2D(W, [copEV, copFE]);
FVs = convert(Array{Lar.Cells}, triangulated_faces);
V = convert(Lar.Points, W');
GL.VIEW( GL.GLExplode(V,FVs, 1.5,1.5,1.5,99,0.6) );

EVs = Lar.FV2EVs(copEV, copFE); # polygonal face fragments
GL.VIEW( GL.GLExplode(V,EVs,1.5,1.5,1.5,99,1) );

# visualization of component graphs
# EW = Lar.cop2lar(copEV)
# w = convert(Lar.Points, W')
# comps = [ GL.GLLines(w,EW[comp],GL.COLORS[(k-1)%12+1]) for (k,comp) in enumerate(bicon_comps) ]
# GL.VIEW(comps);


using Bridge, Distributions, StaticArrays
using Plots
using LinearAlgebra

#
t = 2.
n = 100
dt = t / n
x0 = 1.

# define a Float64 Wiener process
P = Wiener() #returns object representing standard Brownian motion

typeof(P)
#prints Bridge.Wiener{Float64}

supertype(typeof(P))
#prints Bridge.ContinuousTimeProcess{Float64}

# sample Brownian motion on an equidistant grid
W = sample(range(0., stop=t, length=n), P)
W = sample(0:dt:t, P) # similar way

println(W.tt)
# prints [0.0,0.03,0.06,..., 0.93.96,0.99]
plot(W.tt, W.yy)
# plots path

plot(W)
# Bridge defines a plot @recipe (via RecipesBase)

# sample complex Brownian motion on a nonequidistant grid

X = sample(sort(rand(1000)), Wiener{Complex{Float64}}())
plot(real(X.yy), imag(X.yy))

# sample a standard Brownian bridge ending in v at time s
s = 1.
v = 0.

B = sample(0:dt:s, WienerBridge(s, v))
plot(W.tt, W.yy, color="blue")
plot!(B.tt, B.yy, color="red")
# displays X and  Brownian bridge B in red

# Define a diffusion process
struct OrnsteinUhlenbeck  <: ContinuousTimeProcess{Float64}
    β::Float64 # drift parameter (also known as inverse relaxation time)
    σ::Float64 # diffusion parameter
    function OrnsteinUhlenbeck(β::Float64, σ::Float64)
        isnan(β) || β > 0. || error("Parameter λ must be positive.")
        isnan(σ) || σ > 0. || error("Parameter σ must be positive.")
        new(β, σ)
    end
end

# define drift and sigma of OrnsteinUhlenbeck
import Bridge: b, σ, a, transitionprob
Bridge.b(t,x, P::OrnsteinUhlenbeck) = -P.β*x
Bridge.σ(t, x, P::OrnsteinUhlenbeck) = P.σ
Bridge.a(t, x, P::OrnsteinUhlenbeck) = P.σ^2

# simulate OrnsteinUhlenbeck using Euler scheme
W = sample(0:0.01:10, Wiener{Float64}())
X = solve(EulerMaruyama(), 0.1, W, OrnsteinUhlenbeck(20., 1.))

plot(X.tt, X.yy)

# define transition density
transitionprob(s, x, t, P::OrnsteinUhlenbeck) = Normal(x*exp(-P.β*(t-s)), sqrt((0.5P.σ^2/P.β) *(1-exp(-2*P.β*(t-s)))))

# plot likelihood of β
LL = [(β, llikelihood(X, OrnsteinUhlenbeck(β, 1.))) for β in 1.:30.]
for (β, ll) in LL
    println("β $β loglikelihood ", ll )
end
plot(Float64[β for (β, ll) in LL], Float64[ll for (β, ll) in LL])

# sample OrnsteinUhlenbeck exactly. compare with euler scheme which degrates as dt = 0.07
X = solve(EulerMaruyama(), 0.1, sample(0:0.07:10, Wiener{Float64}()), OrnsteinUhlenbeck(20., 1.))
X2 = sample(0:0.07:10, OrnsteinUhlenbeck(20., 1.), 0.1)
plot(X.tt, 1 .+ X.yy)
plot!(X2.tt, X2.yy)

# sample vector Brownian motion
W2 = sample(0:0.1:10, Wiener{SVector{4,Float64}}())
llikelihood(W2,Wiener{SVector{4,Float64}}())

P = BridgeProp(OrnsteinUhlenbeck(3., 1.), 0:0.01:1,  (0., 1.), 1.)
X = solve(EulerMaruyama(), 0.1, sample(0:0.01:1, Wiener{Float64}()),P)

P2 = PBridgeProp(OrnsteinUhlenbeck(3., 1.), 0., 0., 1., 2., 2., 0., 1., 0.3, 1.)
Y = solve(EulerMaruyama(), 0.1, sample(0:0.01:2, Wiener{Float64}()),P2)
p = plot(Y.tt, Y.yy, xlim=(0, 2), ylim=(-3,3), legend=false)

for i in 1:100
    global Y = solve(EulerMaruyama(), 0.1, sample(0:0.01:2, Wiener{Float64}()),P2)
    plot!(p, Y.tt, Y.yy, xlim=(0, 2), ylim=(-3,3), linewidth=0.2)
end
display(p)

# Define a diffusion process
struct VOrnsteinUhlenbeck{d}  <: ContinuousTimeProcess{SVector{d,Float64}}
    β # drift parameter (also known as inverse relaxation time)
    σ # diffusion parameter
    function VOrnsteinUhlenbeck{d}(β, σ) where d
           new(β, σ)
    end
end

# define drift and sigma of VOrnsteinUhlenbeck

Bridge.b(t, x, P::VOrnsteinUhlenbeck) = -P.β*x
Bridge.σ(t, x, P::VOrnsteinUhlenbeck) = P.σ*I
Bridge.a(t, x, P::VOrnsteinUhlenbeck) = P.σ*P.σ'*I

# Simulate
tt = 0.:0.003:10.
X = solve(EulerMaruyama(), SVector(0., 0.), sample(tt, Wiener{SVector{2,Float64}}()), VOrnsteinUhlenbeck{2}(3., 1.))

yy = Bridge.mat(X.yy)
plot(yy[1,:], yy[2,:], xlim=(-2, 2), ylim=(-2,2), linewidth=0.5)


using DataArrays
using DataFrames
using Gadfly

function θestimation_()
    # 1970 batting averages for 18 major league players. first column = batting average for first 45 at bats 2nd column = batting everage for remainder of season
    data = [.400 .346
            .378 .298
            .356 .276
            .333 .222
            .311 .273
            .311 .270
            .289 .263
            .267 .210
            .244 .269
            .244 .230
            .222 .264
            .222 .256
            .222 .303
            .222 .264
            .222 .226
            .200 .285
            .178 .316
            .156 .200]

    # data transformation
    y = data[:,1]
    ytest = data[:,2]
    n = 45
    # arcsin transform
    x = sqrt(n).*asin(2.*y-1)

    # shrinkage estimate
    μBar = mean(x)
    V    = sumabs2(x - μBar)
    d    = length(x)
    s2   = V/d
    # by construction of the arcsin transform
    σ2   = 1
    B    = σ2/(σ2 + max(0, s2-σ2))
    μShrunk = μBar + (1-B).*(x-μBar)

    # back transform
    θShrunk = 0.5*(sin(μShrunk/sqrt(n))+1)
    θMLE    = y
    (ytest, θShrunk, θMLE)
end

"""
Baseball Batting Average Shrinkage Estimates
Reproduce example from "Data Analysis Using Stein's Estimator and its Generalizations" Bradley Efron; Carl Morris JASA Vol. 70, No. 350. (Jun., 1975), pp. 311-319.
"""
function shrinkageDemoBaseball()
    θtrue, θShrunk, θMLE = θestimation_()
    # plot Shrinkage Estimates
    p1 = plot([layer(x=[θMLE[i] θShrunk[i]], y=[1 0], Geom.line) for i=1:d]...,
              layer(x=θMLE, y=ones(1, d), Geom.point),
              layer(x=θShrunk, y=zeros(1, d), Geom.point),
              Theme(default_color=colorant"blue"))

    # histograms
    df = DataFrame(PN=[1:d, 1:d, 1:d], Theta=[ytest, θShrunk, θMLE], Type=[ones(d), 2*ones(d), 3*ones(d)])
    p2 = plot(df, y="Theta", x="PN", color="Type", Geom.bar(position=:dodge))

    [p1, p2]
end

function shrinkageMSEBaseball()
    θtrue, θShrunk, θMLE = θestimation_()
    mseMLE = mean((θtrue-θMLE).^2)
    mseShrink = mean((θtrue-θShrunk).^2)
    [mseMLE, mseShrink, mseMLE/mseShrink]
end