"""
```
Model(y, H, A, Q, R, x, P)
```

The type Model contains the measurements matrix as well as the kalman setup:
Measurement:  y(t)     = H * x(t) + e(t) ~ N(0, R)
State:        x(t + 1) = A * x(t) + n(t) ~ N(0, Q)

where both y(t) and x(t+1) are vectors of length p and m, respectively.
The state transition matrix, A, is (m x m) while the measurement model matrix, H, is (p x m).

Time-varying H and A are not currently implemented. Similarly, the covariances
R and Q are constant across time.
"""

mutable struct Model
    y::Matrix{<:Real}
    H::Matrix{<:Real}
    A::Matrix{<:Real}
    Q::Matrix{<:Real}
    R::Matrix{<:Real}
    x::Vector{<:Real}
    P::Matrix{<:Real}
    predicted::Predicted
    filtered::Filtered
    smoothed::Smoothed
    function Model(y, H, A, Q, R, x, P)
        str = "y is $(size(y, 1))x$(size(y, 2)), " *
              "H is $(size(H, 1))x$(size(H, 2)), " *
              "A is $(size(A, 1))x$(size(A, 2)), " *
              "Q is $(size(Q, 1))x$(size(Q, 2)), " *
              "R is $(size(R, 1))x$(size(R, 2)), " *
              "x is $(length(x)), " *
              "P is $(size(P, 1))x$(size(P, 2))"
        !(size(H, 2) == size(A, 1) == size(A, 2) == size(Q, 1) == size(Q, 2) == length(x) == size(P, 1) == size(P, 2)) && throw(DimensionMismatch(str))
        !(size(y, 2) == size(H, 1) == size(R, 1) == size(R, 2)) && throw(DimensionMismatch(str))

        @assert all(diag(R) .> 0) "All diagonal elements of R must be strictly positive."
        @assert all(diag(Q) .> 0) "All diagonal elements of Q must be strictly positive."
        @assert all(diag(P) .> 0) "All diagonal elements of P must be strictly positive."

        predicted = Predicted(size(y, 1), size(y, 2), length(x))
        filtered  = Filtered( size(y, 1), size(y, 2), length(x))
        smoothed  = Smoothed( size(y, 1), size(y, 2), length(x))

        return new(y, H, A, Q, R, x, P, predicted, filtered, smoothed)
    end
end

# Outer constructors for univariate measurement
Model(y::Vector{<:Real}, H::Vector{<:Real}, A::Matrix{<:Real}, Q::Matrix{<:Real}, R::Real, x::Vector{<:Real}, P::Matrix{<:Real}) = Model(repeat(y, 1, 1), reshape(H, 1, length(H)), A, Q, fill(R, 1, 1), x, P)
Model(y::Vector{<:Real}, H::Real, A::Real, Q::Real, R::Real, x::Real, P::Real) = Model(repeat(y, 1, 1), fill(H, 1, 1), fill(A, 1, 1), fill(Q, 1, 1), fill(R, 1, 1), [x], fill(P, 1, 1))

# Univariate state
Model(y::Matrix{<:Real}, H::Matrix{<:Real}, A::Real, Q::Real, R::Matrix{<:Real}, x::Real, P::Real) = Model(y, H, fill(A, 1, 1), fill(Q, 1, 1), R, [x], fill(P, 1, 1))

# Special constructors
LocalLevel(y::Vector{<:Real}, Q::Real, R::Real, x::Real, P::Real) = Model(repeat(y, 1, 1), ones(1, 1), ones(1, 1), fill(Q, 1, 1), fill(R, 1, 1), [x], fill(P, 1, 1))
LocalLevel(y::Matrix{<:Real}, Q::Real, R::Matrix, x::Real, P::Real) = Model(y, ones(size(y, 2), 1), ones(1, 1), fill(Q, 1, 1), R, [x], fill(P, 1, 1))

LocalLevel(y::Vector{<:Real}, Q::Real, R::Real) = Model(repeat(y, 1, 1), ones(1, 1), ones(1, 1), fill(Q, 1, 1), fill(R, 1, 1), [y[1]], fill(1000.0, 1, 1))
LocalLevel(y::Matrix{<:Real}, Q::Real, R::Matrix) = Model(repeat(y, 1, 1), ones(1, 1), ones(1, 1), fill(Q, 1, 1), R, [0.0], fill(1000.0, 1, 1))

"""
```
Output
```

The type Output contains the means and covariances for the measurements
and states, respectively, at each time step, t = 1,...,T:
x is a (T x m) matrix of state means
P is a (T x m x m) array of state covariances
μ is a (T x p) matrix of measurement means
Σ is a (T x p x p) matrix of measurement covariances
"""

abstract type Output end

mutable struct Predicted <: Output
    x::Matrix{AbstractFloat}
    P::Array{AbstractFloat, 3}
    μ::Matrix{AbstractFloat}
    Σ::Array{AbstractFloat, 3}
end

mutable struct Filtered <: Output
    x::Matrix{AbstractFloat}
    P::Array{AbstractFloat, 3}
    μ::Matrix{AbstractFloat}
    Σ::Array{AbstractFloat, 3}
end

mutable struct Smoothed <: Output
    x::Matrix{AbstractFloat}
    P::Array{AbstractFloat, 3}
    μ::Matrix{AbstractFloat}
    Σ::Array{AbstractFloat, 3}
end

Predicted(T::Int, p::Int, m::Int) = Predicted(zeros(T, m), zeros(T, m, m), zeros(T, p), zeros(T, p, p))
Filtered(T::Int, p::Int, m::Int)  = Filtered(zeros(T, m), zeros(T, m, m), zeros(T, p), zeros(T, p, p))
Smoothed(T::Int, p::Int, m::Int)  = Smoothed(zeros(T, m), zeros(T, m, m), zeros(T, p), zeros(T, p, p))


using JuMP
using GLPK

model = Model(GLPK.Optimizer)
@variable(model, x >= 0)
@variable(model, y >= 0)
@constraint(model, 6x + 8y >= 100)
@constraint(model, 7x + 12y >= 120)
@objective(model, Min, 12x + 20y)

optimize!(model)

@show value(x);
@show value(y);
@show objective_value(model);

using JuMP

using GLPK

model = Model(GLPK.Optimizer);

@variable(model, x >= 0)
@variable(model, y >= 0);

@constraint(model, 6x + 8y >= 100)
@constraint(model, 7x + 12y >= 120);

@objective(model, Min, 12x + 20y);

optimize!(model)

@show value(x);
@show value(y);
@show objective_value(model);


# Using the BBP formula
#
# The Bailey–Borwein–Plouffe formula (BBP formula) is a formula for π

#=
Computing Bailey–Borwein–Plouffe formula for Pi
to ascertain the performance of FastRational{BigInt}
relative to Rational{BigInt} by BPP iterations

FastRational{BigInt}s perform better below n=328
Rational{BigInt}s     perform better above n=330

As BPP iterations increase, numerator and denominator grow.
After some digit count in numerator aor denominator, ~10_000,
The extra time spent in divgcd reductions becomes less than
the time spent performing arithmetic on very large integers;
so reducing the numerator and denominator begins to win.
=#

using FastRationals
using BenchmarkTools

const BT=BenchmarkTools.DEFAULT_PARAMETERS;
BT.overhead=BenchmarkTools.estimate_overhead();
BT.evals=1; ; BT.time_tolerance = 5.0e-7; BT.samples = 100;

const big1 = BigInt(1)
const big2 = BigInt(2)
const big4 = BigInt(4)
const big5 = BigInt(5)
const big6 = BigInt(6)
const big8 = BigInt(8)
const big16 = BigInt(16)

function bpp(::Type{T}, n) where {T}
    result = zero(T)
    for k = 0:n
       eightk = big8 * k
       cur = T(big4,eightk+1) -
             T(big2,eightk+4) -
             T(big1,eightk+5) -
             T(big1,eightk+6)
       cur = T(big1, big16^k) * cur
       result = result + cur
    end
    return result
end

systemqtime = @belapsed bpp(Rational{BigInt},   15);
fastqtime = @belapsed bpp(FastRational{BigInt}, 15);
bpp15 = round(systemqtime/fastqtime, digits=1)

systemqtime = @belapsed bpp(Rational{BigInt},   125);
fastqtime = @belapsed bpp(FastRational{BigInt}, 125);
bpp125 = round(systemqtime/fastqtime, digits=1)

systemqtime = @belapsed bpp(Rational{BigInt},   250);
fastqtime = @belapsed bpp(FastRational{BigInt}, 250);
bpp250 = round(systemqtime/fastqtime, digits=1)

# relspeeds meet at n=328

systemqtime = @belapsed bpp(Rational{BigInt},   500);
fastqtime = @belapsed bpp(FastRational{BigInt}, 500);
bpp500 = round(systemqtime/fastqtime, digits=1)

systemqtime = @belapsed bpp(Rational{BigInt},   1000);
fastqtime = @belapsed bpp(FastRational{BigInt}, 1000);
bpp1000 = round(systemqtime/fastqtime, digits=1)

systemqtime = @belapsed bpp(Rational{BigInt},   2000);
fastqtime = @belapsed bpp(FastRational{BigInt}, 2000);
bpp2000 = round(systemqtime/fastqtime, digits=1)

systemqtime = @belapsed bpp(Rational{BigInt},   3000);
fastqtime = @belapsed bpp(FastRational{BigInt}, 3000);
bpp3000 = round(systemqtime/fastqtime, digits=1)

relspeeds = (bpp15=bpp15, bpp125=bpp125, bpp250=bpp250,
             bpp500=bpp500, bpp1000=bpp1000, bpp2000=bpp2000,
             bpp3000=bpp3000)

relspeeds = (bpp15 = 0.3, bpp125 = 0.3, bpp250 = 0.7, bpp500 = 2.4, bpp1000 = 8.1, bpp2000 = 20.9, bpp3000 = 36.0)

xs = [15,125,250,500,1000,2000,3000];
ys = [relspeeds...,];

using GR
plot(xs, ys, size=(500,500))


using Test
using AutoDiff.Backward

@testset "f(x,y,z) = x + 2*y/z" begin
    f(x, y, z) = x + 2.0*y/z
    g = gradient(f)

    @test all(isapprox.(g(1.0, 2.0, 3.0), [1.0, 2.0/3.0, -4.0/9.0]))
end

@testset "one-arg functions" begin
    f(x) = 2.0*x + 3.0
    g = gradient(f)

    @test isapprox(g(1.0), 2.0)
end

@testset "vector-mode functions" begin
    f(xs) = sqrt(xs[1]*xs[1] + xs[2]*xs[2] + xs[3]*xs[3])
    g = gradient(f)
    xs = randn(3)
    @test all(isapprox.(g(xs), [xs[1]/f(xs), xs[2]/f(xs), xs[3]/f(xs)]))
end

@testset "array-mode functions" begin
    f(xs) = sqrt(sum(xs.*xs))
    g = gradient(f)
    xs = randn(3, 5, 7)
    r = f(xs)
    @test all(isapprox.(g(xs), xs./r))
    @test size(g(xs))==(3,5,7)
end

@testset "reuse variable" begin
    f(x, y, z) = x - 3.0*x/(y*z) + z*y
    g = gradient(f)

    x = randn()
    y = randn()
    z = randn()

    @test all(isapprox.(g(x, y, z), [1.0 - 3.0/(y*z), 3.0*x/(y*y*z) + z, 3.0*x/(y*z*z) + y]))
end

@testset "unary minus" begin
    f(x,y,z) = -x + (-y)*z
    g = gradient(f)
    x,y,z = randn(3)
    @test all(isapprox.(g(x,y,z), [-1.0, -z, -y]))
end

@testset "exp" begin
    f(x,y,z) = x*exp(y*z)
    g = gradient(f)
    x,y,z = randn(3)
    ff = f(x,y,z)
    @test all(isapprox.(g(x,y,z), [ff/x, z*ff, y*ff]))
end

@testset "sqrt" begin
    f(x,y,z) = sqrt(x*x + y*y + z*z)
    f_and_g = value_and_gradient(f)
    x,y,z = randn(3)

    ff, gg = f_and_g(x,y,z)

    @test all(isapprox.(gg, [x/ff, y/ff, z/ff]))
end

@testset "trig" begin
    @testset "sin" begin
        f(x,y,z) = x*sin(y*z)
        g = gradient(f)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), [sin(y*z), x*z*cos(y*z), x*y*cos(y*z)]))
    end

    @testset "cos" begin
        f(x,y,z) = x*cos(y*z)
        g = gradient(f)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), [cos(y*z), -z*x*sin(y*z), -x*y*sin(y*z)]))
    end

    @testset "tan" begin
        f(x,y,z) = x*tan(y*z)
        ff(x,y,z) = x*sin(y*z)/cos(y*z)
        g = gradient(f)
        gg = gradient(ff)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), gg(x,y,z)))
    end

    @testset "sec" begin
        f(x,y,z) = x*sec(y*z)
        ff(x,y,z) = x/cos(y*z)
        g = gradient(f)
        gg = gradient(ff)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), gg(x,y,z)))
    end

    @testset "csc" begin
        f(x,y,z) = x*csc(y*z)
        ff(x,y,z) = x/sin(y*z)
        g = gradient(f)
        gg = gradient(ff)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), gg(x,y,z)))
    end

    @testset "cot" begin
        f(x,y,z) = x*cot(y*z)
        ff(x,y,z) = x/tan(y*z)
        g = gradient(f)
        gg = gradient(ff)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), gg(x,y,z)))
    end

    @testset "atan" begin
        f(x,y,z) = atan(fff(x,y,z))
        ff(x,y,z) = tan(f(x,y,z))
        fff(x,y,z) = sqrt(x*x+y*y+z*z)
        g = gradient(f)
        gg = gradient(ff)
        ggg = gradient(fff)
        x,y,z = randn(3)
        @test all(isapprox.(ggg(x,y,z), gg(x,y,z)))
    end

    @testset "atan2" begin
        f(theta) = atan(sin(theta),cos(theta))
        g = gradient(f)
        theta = 2*pi*randn()
        @test isapprox(g(theta), 1.0)
    end
end

@testset "sum like functions" begin
    @testset "sum" begin
        f(x, y, z) = sum([x,y,z])
        g = gradient(f)
        x,y,z = randn(3)
        @test all(isapprox.(g(x,y,z), ones(3)))
    end
end

@testset "log" begin
    f(x,y,z) = log(x*y*z)
    g = gradient(f)
    x,y,z = abs.(randn(3))
    @test all(isapprox.(g(x,y,z), [1.0/x, 1.0/y, 1.0/z]))
end

@testset "log1p" begin
    f(x,y,z) = log1p(x*y*z)
    g = gradient(f)
    x,y,z = 1e-3.*randn(3)
    @test all(isapprox.(g(x,y,z), [y*z/(1+x*y*z), x*z/(1+x*y*z), x*y/(1+x*y*z)]))
end

@testset "logsumexp" begin
    f(x,y,z) = logsumexp(x, logsumexp(y, z))
    f_exact(x,y,z) = log(exp(x) + exp(y) + exp(z))
    g = gradient(f)
    g_exact = gradient(f_exact)
    gg = gradient(logsumexp) # Array version

    x,y,z = randn(3)

    @test all(isapprox.(g(x,y,z), g_exact(x,y,z)))
    @test all(isapprox.(gg([x,y,z]), g_exact(x,y,z)))
end

@testset "intermediate reuse" begin
    function f(x,y,z)
        q = x*y + cos(z*x)
        x*sin(q) + z*q*q
    end
    g = gradient(f)
    x,y,z = randn(3)

    @test all(isapprox.(g(x,y,z),
                        [(2.0*z*(x*y + cos(x*z)) + x*cos(x*y + cos(x*z)))*(y-z*sin(z*x)) + sin(x*y + cos(x*z)),
                         x*(2.0*z*(x*y + cos(x*z)) + x*cos(x*y + cos(x*z))),
                         (x*y + cos(x*z))^2 - x*(2.0*z*(x*y + cos(x*z)) + x*(cos(x*y + cos(x*z))))*sin(x*z)]))
end


# ---
# title: 494. Target Sum
# id: problem494
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Dynamic Programming, Depth-first Search
# link: <https://leetcode.com/problems/target-sum/description/>
# hidden: true
# ---
#
# You are given a list of non-negative integers, a1, a2, ..., an, and a target,
# S. Now you have 2 symbols `+` and `-`. For each integer, you should choose one
# from `+` and `-` as its new symbol.
#
# Find out how many ways to assign symbols to make sum of integers equal to
# target S.
#
# **Example 1:**
#
#
#
#     Input: nums is [1, 1, 1, 1, 1], S is 3.
#     Output: 5
#     Explanation:
#
#     -1+1+1+1+1 = 3
#     +1-1+1+1+1 = 3
#     +1+1-1+1+1 = 3
#     +1+1+1-1+1 = 3
#     +1+1+1+1-1 = 3
#
#     There are 5 ways to assign symbols to make the sum of nums be target 3.
#
#
#
#
# **Constraints:**
#
#   * The length of the given array is positive and will not exceed 20.
#   * The sum of elements in the given array will not exceed 1000.
#   * Your output answer is guaranteed to be fitted in a 32-bit integer.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.0
#   kernelspec:
#     display_name: Julia 1.4.2
#     language: julia
#     name: julia-1.4
# ---

# # Gradient descent (勾配法)
#
# 勾配法の練習として下記の簡単な設定のもとで勾配法を逐次適用することで目的関数の極値問題を解く際の点の動きを可視化する.
#
# $A$ を 2x2 の実対称行列とし, $f=f(x,y)$ を次で定める二変数関数とする.
#
# $$
# f(x,y) =  \begin{bmatrix}x & y\end{bmatrix} A \begin{bmatrix} x \\ y \end{bmatrix}
# $$
#
# このとき $f$ の勾配ベクトルは $\nabla f$ は
#
# $$
# \nabla f= \begin{bmatrix} \frac{\partial f}{\partial x} \\ \frac{\partial f}{\partial y} \end{bmatrix} = A \begin{bmatrix} x \\ y \end{bmatrix}
# $$
#
# と書ける. 初期位置を $v_0\in\mathbb{R}^2$, 学習率を $\eta$ とすれば $i$ 番目のステップで得られる点は次のようになる.
#
# $$
# v_{i} = v_{i-1} - \eta \nabla f|_{(x,y)=v_{i-1}} \quad (i=1,2,\dots)
# $$

using Colors
using LinearAlgebra
using Plots

# +
A = [
    2.0  0.2
    0.2  1.0
]

f(v) =  dot(v,A,v)
f(x, y) = f([x, y])
∇f(v) = A * v
η = 0.05 # learning rate
# -

# ## 等高線の描画

x = range(-10,stop=10,length=50)
y = range(-10,stop=10,length=50)
p = contour(x, y, f)

# ## 勾配法を試す

# +
x = range(-5, stop=5, length=50)
y = range(-5, stop=5, length=50)
p = contour(x, y, f, levels=range(0, stop=100, step=3))

v = [-5., -5.] # initial point
for i in 1:100
    v .-= η .* ∇f(v)
    scatter!(
        p, [v[1]], [v[2]],
        label=:none, color=:blue, msize=3,
    )
end

plot(p)
# -

# ## ランダムに点を配置しそれを初期値とする勾配法

# +
x = range(-5, stop=5, length=50)
y = range(-5, stop=5, length=50)
p = contour(x, y, f, levels=range(0,stop=100,step=3))
for _ in 1:10
    v = rand(range(-5,stop=5,step=0.01), 2)
    c =　RGB(rand(3)...)
    for _ in 1:30 # num steps
        v .-= η .* ∇f(v)
        scatter!(
            p, [v[1]], [v[2]],
            label=:none, color=c, msize=3,
        )
    end
end

plot(p)


"""
    Ricker(; <keyword arguments>)

Create a Ricker wavelet.

# Keyword arguments
* `dt::Real=0.002`: sampling interval in secs.
* `f0::Real=20.0`: central frequency in Hz.

# Examples
```julia
julia> w = Ricker(); plot(w);
julia> w = Ricker(dt=0.004, f0=20); plot(w);
```

# Reference
Sheriff, Robert, 2002, Encyclopedic Dictionary of Applied Geophysics, fourth
ed.: Society of Exploration Geophysicists. Geophysical Reference Series No. 13.
"""

function Ricker(; dt::Real=0.002, f0::Real=20.0)

    nw = 2.0/(f0*dt)
    nc = floor(Int, nw/2)
    t = dt*collect(-nc:1:nc)
    b = (pi*f0*t).^2
    w = (1.-2.*b).*exp(-b)

end


# This program computes Feigenbaum's alpha constant using
# Feigenbaum's functional equation derived via renormalization
# group methods.
# Oct 2017 -- Stuart Brorson, sdb@cloud9.net

using ForwardDiff

# I define the helper fcns first.

#-------------------------------------------------------
# Define universal fcn g(z, a).  It is expressed as a power
# series, coefficients a.  Only even powers are needed since
# g(z) is even.
function g(z, a::Vector)
  #println("---> Entered g\n")
  s = 0;
  for i=length(a):-1:1
    # println("i = $i, s = $s\n")
    s = z*z*(s + a[i]);
  end
  #println("<--- Leaving g\n")
  return (1+s)
end

#---------------------------------------------------------
# Define relation obeyed by g(z).  Name it f(z, a)
function f(z, a::Vector)
  #println("---> Entered f\n")
  const one = big(1.0)
  alpha = one/g(one, a);
  r = g(z, a) - alpha*g(g(z/alpha, a), a);
  #println("<--- Leaving g\n")
  return r
end

#--------------------------------------------------------
# Define fcn which takes gradient of f w.r.t. a.
# Returns row vector which is gradient of f w.r.t. a,
# evaluated at z, a.
function gradf(z, a::Vector)
  #println("---> Entered gradf\n")
  # Specialize to position z.  Result is fcn of coeffs a only.
  fbeta(a) = f(z, a);

  # Compute gradient w.r.t. beta and return it
  gf1 = ForwardDiff.gradient(fbeta, a)
  #println("<--- Leaving gradf\n")
  return gf1'
end

# ------------------------------------------------------------
# The main event -- a computation of alpha using Newton's method.
function compute_alpha(Numdigs, N, betain)

  # First set up variables, including BigFloat settings
  # Setting precision to 5x Numdigs seems to keep Newton's method
  # from wandering away.  Smaller values don't.  More fidding might
  # improve things here.
  setprecision(5*Numdigs);

  # Set stopping tol for Newton's method so I get the number of digits
  # I want.
  const tol = BigFloat(10.0)^-(Numdigs);

  const zi = big.(linspace(1/N, 1, N));   # Grid of sample points zi
  fn = zeros(BigFloat, N);             # Function f vector
  Jn = zeros(BigFloat, N, N);        # Jacobian

  # Need expansion coeffs beta.  Start with the one passed in, betain.
  # N should always be larger than length(betain), so we are adding to the
  # betas we computed last time.
  betan = zeros(BigFloat, N);
  for i = 1:length(betain)
    betan[i] = betain[i];
  end

  # println("-----------------------------\n")
  sttim = now();  # Keep track of loop timing.
  # Now enter Newton loop.
  for cnt = 1:50
    nowtim = now();  # Keep track of loop timing
    println("Newton iteration = $cnt, time elapsed = $(nowtim-sttim)\n")

    # Compute new f and Jacobian upon each iteration
    for i = 1:N
      fn[i] = f(zi[i], betan);
      Jn[i,:] = gradf(zi[i], betan);
    end

    # Now compute step to take, then take it.
    sn = Jn\fn;
    betanp1 = betan - sn;

    # Check for convergence
    if (norm(sn) < tol)
      println("\nDone!  Converged after $cnt iterations.\n")
      # compute alpha
      myalpha = 1/g(1, betanp1);
      return myalpha, betanp1;
    end

    # Move values back
    betan = betanp1

  end
  println("!!!!!!!  Didn't converge!  !!!!!!!!\n")

end

#--------------------------------------------------
function iterate_alpha()
  # This is a runner -- call this fcn.  This fcn calls
  # compute_alpha asking for different number of digits
  # at different precision/tolerance levels.
  # It calls with the beta coefficients computed in the
  # last iteration as the starting point to help Newton's
  # method from wandering away.

  # Seed betan with 3 values.
  betan = zeros(BigFloat, 3);
  betan[1] = big(-1.52763883147);
  betan[2] = big(1.048327004372e-1);
  betan[3] = big(2.669121419134012e-2);

  # Step up number of pts to compute on.  Change this to get
  # more digits
  for N = 10:20:1700
    println("=================================\n")
    println("N = $N\n")
    Numdigs = Int(floor(2*N));

    # Call the function which computes alpha at this level
    # of precision.
    myalpha, betan = compute_alpha(Numdigs, N, betan)

    # Print out myalpha to N digits of precision.  Must add 3
    # to account for negative sign and "2."
    a = string(myalpha);
    println("myalpha = $(a[1:N+3])\n")

    f = open("alpha.dat", "a");
    write(f, "==========\nN = $N\n$(a[1:N+3])\n")
    close(f)

    # Compare against the version from
    # http://www.plouffe.fr/simon/constants/feigenbaum.txt
    oeisalpha = oeis_alpha(3*Numdigs);
    diff = Float64(abs(myalpha) - abs(oeisalpha));
    println("diff = $diff\n")
  end

end

#------------------------------------------------
function oeis_alpha(Numdigs)
  # Return up to 1018 digits of alpha from
  # http://www.plouffe.fr/simon/constants/feigenbaum.txt

  if (Numdigs <= 1018)
    setprecision(Numdigs);
  else
    setprecision(1018);
  end

  str = "-2."
  str = str*"5029078750958928222839028732182157863812713767271499773361920567"
  str = str*"7923546317959020670329964974643383412959523186999585472394218237"
  str = str*"7785445179272863314993372578112163594879503744781260997380598671"
  str = str*"2397117373289276654044010306698313834600094139322364490657889951"
  str = str*"2205843172507873377463087853424285351988587500042358246918740820"
  str = str*"4281700901714823051821621619413199856066129382742649709844084470"
  str = str*"1008054549677936760888126446406885181552709324007542506497157047"
  str = str*"0475419932831783645332562415378693957125097066387979492654623137"
  str = str*"6745918909813116752434221110130913127837160951158341230841503716"
  str = str*"4997020224681219644081216686527458043026245782561067150138521821"
  str = str*"6449532543349873487413352795815351016583605455763513276501810781"
  str = str*"1948369459574850237398235452625632779475397269902012891516645793"
  str = str*"9420198920248803394051699686551494477396533876979741232354061781"
  str = str*"9896112494095990353128997733611849847377946108428833293833903950"
  str = str*"9008914086351525626803381414669279913310743349705143545201344643"
  str = str*"4264752001621384610729922641994332772918977769053802596851"

  alpha = parse(BigFloat, str)

end


using Revise
using NLS_Fit
using DelimitedFiles
using LinearAlgebra

rootDir = joinpath(dirname(pathof(NLS_Fit)), "..")
dataDir = joinpath(rootDir,"data")
dataFile = joinpath(dataDir,"simple_gaussian.txt")

# Getting started example ****************
#

# Data ================
#
XY=readdlm(dataFile);
X=XY[:,1];
Y=XY[:,2];

# Fitting ================
#
model = Gaussian_Peak()

θ_init = Float64[1,10,5]

nls = NLS_ForwardDiff_From_Model2Fit(model,X,Y);
conf = NLS_Solver.LevenbergMarquardt_Conf()
result = NLS_Solver.solve(nls,θ_init,conf)

# Fit result ================
#
@assert NLS_Solver.converged(result)

θ_true = Float64[2,25,3] # cheating...
θ_solver = NLS_Solver.solution(result)

println("Found $θ_solver, error is $(norm(θ_solver-θ_true,Inf))")
println("(caveat: error is mainly due to noisy data, not due to solver accuracy)")

# Check solver accuracy by using a noise free input ****************
#

# Generate noise free input ================
#
n = 50
model = Gaussian_Peak()
θ = Float64[2,n/2,3]

X=Float64[1:n;];
Y=eval_y(model,X,θ);

# Solve and print new noise free results ================
#
nls = NLS_ForwardDiff_From_Model2Fit(model,X,Y);
result = NLS_Solver.solve(nls,θ_init,conf)
θ_solver = NLS_Solver.solution(result)

println("Found $θ_solver, error is $(norm(θ_solver-θ_true,Inf))")


using NaNStatistics
using Test, StatsBase, Statistics, LinearAlgebra

@testset "ArrayStats" begin include("testArrayStats.jl") end
@testset "Covariance and Correlation" begin include("testCovCor.jl") end
@testset "Sorting" begin include("testSorting.jl") end

@testset "Histograms" begin include("testHistograms.jl") end
@testset "Binning" begin include("testBinning.jl") end


using LinearAlgebra
using DataFrames
using CSV

#This file contains various renditions of  which perform numerical
#quadrature given integration endpoints and a function handle
#More importantly, it provides a composite type which stores nodes and weight.

"""
Data-type which stores quadrature nodes and weights
"""

mutable struct nodesWeights{T1<:Array{Float64}, T2<:Array{Float64}}
    nodes::T1
    weights::T2
end

"""
Apply affine transformation to nodes to integration over range r (linear change of variables)
"""
function affineTransformNodes(nw::nodesWeights{O}, r::Array{<:Real}) where O<:Union{Array{Float64, 2}, Array{Float64, 1}}
    b = r[2]; a = r[1]
    nw.nodes = (b-a)/2 .* nw.nodes .+ (b + a)/2
end

"""
Uses the Golub-Welsch eigenvalue method to compute
Gauss-Legendre quadrature nodes and weights for the domain [-1, 1]

INPUTS: n, the number of desired nodes
OUTPUTS: x and w, the nodes and weights
"""
function gausslegpts(n)
    x = zeros(n-1)
    for i = 1:n-1
        x[i] = i/sqrt(4*i^2-1)
    end
    x = SymTridiagonal(zeros(n), x) #Jacobi matrix
    d, V = eigen(x)
    w = 2*V[1,:].^2 #factor of 2 comes from the fact that integral of measure is 2
    return d, w
end

#precomute various integration nodes/weights
dm0, wm0 = gausslegpts(24)
dm, wm = gausslegpts(12)
dm2, wm2 = gausslegpts(50)
dm3, wm3 = gausslegpts(100)

function getGaussQuadraturedata()
    nodesWeights(dm, wm)
end

"""
INPUTS:
f is a function handle, a and b are integration endpoints (a<b), and n
is the number of quadrature nodes (defaults to 10). Note that f can be
multi-output, in which case int1D integrates each entry individually. This way,
the function lends itself to being composed with Gauss-Turan integration.
"""
function int1D(f, arr, num="1")
    if num=="1"
        x, w= dm, wm
    elseif num=="2"
        x, w= dm2, wm2
    elseif num=="0"
        x, w = dm0, wm0
    else
        x, w = dm3, wm3
    end
    sample = f(arr[1])
    res = zeros(size(sample))
    for i = 1:length(x)
        expr = (arr[2]-arr[1])/2 * x[i] .+ (arr[1]+arr[2])/2
        #println(res)
        #println(f(expr) .* w[i])
        res = res .+ f(expr) .* w[i]
    end
    res = (arr[2]-arr[1])/2 .* res
end

function int1D_print(f, arr, num="0")
    if num=="1"
        x, w= dm, wm
    elseif num=="2"
        x, w= dm2, wm2
    elseif num=="0"
        x, w = dm0, wm0
    else
        x, w = dm3, wm3
    end
    sample = f(arr[1])
    res = zeros(size(sample))
    for i = 1:length(x)
        expr = (arr[2]-arr[1])/2 * x[i] .+ (arr[1]+arr[2])/2
        println(res)
        println(f(expr) .* w[i])
        res = res .+ f(expr) .* w[i]
    end
    res = (arr[2]-arr[1])/2 .* res
end

"""
Integrates function over 2D domain using Gauss-Legendre Quadrature
arr[:, 1] are lower bounds, while arr[:, 2] are upper bounds
"""
function int2D(f, arr)
    d = size(arr, 1) #number of dimensions
    int = 0.0
    x, w = dm, wm
    avg1 = (arr[1, 2] - arr[1, 1])/2
    avg2 = (arr[2, 2] - arr[2, 1])/2
    for i = 1:length(x)
        for j = 1:length(x)
           expr1 = avg1*x[i] + (arr[1, 2]+arr[1, 1])/2
           expr2 = avg2*x[j] + (arr[2, 2]+arr[2, 1])/2
           int = int + w[i]*w[j]*f(expr1, expr2)
        end
    end
    int = int*avg1*avg2
end

data50 = DataFrame(CSV.File("nodes_weights_50.csv", header=0))
nodes50 = convert(Array, data50[:,1]) #integration nodes for Gauss-Turan Quadrature
weights50 = convert(Matrix, data50[:, 2:end]) #integration weights

data20 = DataFrame(CSV.File("nodes_weights_20.csv", header=0))
nodes20  = convert(Array, data20[:,1])
weights20 = convert(Array, data20[:,2:end])

data12 = DataFrame(CSV.File("nodes_weights_12.csv", header=0))
nodes12  = convert(Array, data12[:,1])
weights12 = convert(Array, data12[:,2:end])

function getTuranQuadratureData()
    return nodesWeights(nodes12, weights12)
end

"""
Gauss-Turan integration with 2 derivatives
INPUTS:
arr contains enpoints of integration interval
"""
function Gauss_Turan(F, arr, nodes = nodes12, weights=weights12)
    a = arr[1]
    b = arr[2]
    nodes = (b-a)/2 .* nodes .+ (b+a)/2
    T = zeros(length(nodes), 3)
    time = @elapsed begin
    for i = 1:length(nodes)
        (T[i, 1], T[i, 2], T[i, 3]) = F(nodes[i])
    end
    end
    println(time)
    fn = T[:, 1]
    dfn = T[:, 2]
    d2fn = T[:, 3]
    return (fn'*weights[:, 1] + dfn'*weights[:, 2] + d2fn'*weights[:, 3])*(b-a)/2
end

"""
n-dimensional Gauss-Legendre integration
"""
function intnD(f, arr, n)

end


# Removing t = 0, such that Σ is invertible
t = Vector(0.1:0.1:10)
p = 2

# Creating generators U,V that result in a positive-definite matrix Σ
Ut, Vt = spline_kernel(t', p)
K = SymSemiseparableMatrix(Ut,Vt)
Σ    = Matrix(K)
chol = cholesky(Σ)

# Creating a symmetric extended generator representable semiseperable matrix
# Calculating its Cholesky factorization
L = cholesky(K)
# Creating a test vector
xt = ones(size(K,1),10)

# Testing size
@test size(L,1) == length(t)
@test size(L,2) == length(t)

# Testing inverses (Using Cholesky factorizations)
@test chol.L\xt  ≈ L\xt
@test chol.U\xt  ≈ L'\xt
@test chol.L*xt  ≈ L*xt
@test chol.L'*xt ≈ L'*xt

# Testing logdet and det
@test logdet(L) ≈ logdet(chol.L)
@test det(L) ≈ det(chol.L)

# Testing show
@test L.L ≈ tril(Ut'*L.Wt)
@test L.U ≈ triu(L.Wt'*Ut)
@test Matrix(L) ≈ tril(Ut'*L.Wt)
@test L[3,1] ≈ chol.L[3,1]
@test L[2,2] ≈ chol.L[2,2]
@test L[1,3] ≈ chol.L[1,3]


function chobac!(kb::Matrix{Float64}, loads::Vector{Float64})
  #local l, m
  n = size(kb, 1)
  iw = size(kb, 2) - 1
  loads[1] = loads[1] / kb[1, iw+1]
  for i in 2:n
    x = 0.0
    k = 1
    if i <= (iw + 1)
      k = iw - i + 2
    end
    for j in k:iw
      x += kb[i, j]*loads[i+j-iw-1]
    end
    loads[i] = (loads[i] - x) / kb[i, iw+1]
  end
  loads[n] = loads[n] / kb[n, iw+1]
  for i in n-1:-1:1
    x = 0.0
    l = i + iw
    m = l - 1
    if i > (n - iw)
      l = n
      m = i + 1
    end
    for j in m:l
      x += kb[j, iw+i-j+1] * loads[j]
    end
    loads[i] = (loads[i] - x) / kb[i, iw+1]
  end
end


# [18](https://projecteuler.net/problem=18) / [67](https://projecteuler.net/problem=67)

#In a triangle like this:
#
#      a
#     b c
#    d e f
#
#the best way to find the anwser is not to get the maximum from the summaries of
#all possible branches from top to bottom.
#
#There is such a recursive pattern
#
#a + max( b+max(b,c), c+max(e,f)
#
# tri.txt
#75
#95 64
#17 47 82
#18 35 87 10
#20 04 82 47 65
#19 01 23 75 03 34
#88 02 77 73 07 63 67
#99 65 04 28 06 16 70 92
#41 41 26 56 83 40 80 70 33
#41 48 72 33 47 32 37 16 94 29
#53 71 44 65 25 43 91 52 97 51 14
#70 11 33 28 77 73 17 78 39 68 17 57
#91 71 52 38 17 14 91 43 58 50 27 29 48
#63 66 04 68 89 53 67 30 73 16 69 87 40 31
#04 62 98 27 23 09 70 98 73 93 38 53 60 04 23

import Base.zero
zero(::SubString{String}) = 0 # Julia 0.5

ZeroString(::SubString{String}) = 0
ZeroString(x::Int64) = x

A = readdlm("tri.txt")
A = ZeroString.(A)

function myreduction(m)
	if size(m)[1] == 1
		return m[1,1]
	else
		mprime = m[1:(end-1), :]
		for k in 1:(size(m, 1)-1)
			mprime[size(mprime, 1), k] += max(
									 m[size(m, 1),k], m[size(m, 1),k+1])
		end
		return myreduction(mprime)
	end
end

println(myreduction(A))


include("../../common/numder/src/numder_module.jl")

module nonlineq
using LinearAlgebra
using ..numder

include("../../opti/src/line_search.jl")

struct ToleranceOptions
    max_iter::Int
    step_size_tol::Float64
end

function ToleranceOptions(;max_iter = 100, step_size_tol = 1e-15)
    ToleranceOptions(max_iter, step_size_tol)
end

function broyden(
    fcn,
    x0::Vector{T};
    tol_options::ToleranceOptions = ToleranceOptions(),
    B0 = nothing,
    damping = 1.0) where {T}

    n_dim = length(x0)

    if isnothing(B0)
        B = Matrix( one(T) * I, n_dim, n_dim )
        B = numder.jacobian_fw(fcn, x0)
    else
        B = B0
    end

    x = x0
    f = fcn(x)

    x_sol = x

    # merit_fcn(x) = norm( fcn(x) )
    iter = 0
    for outer iter = 1:tol_options.max_iter

        p = -(B \ f)
        # line search? -> damping
        s = damping * p

        if norm(s) <= tol_options.step_size_tol
            x_sol = x
            break
        end

        x = x + s
        f1 = fcn(x)
        y = f1 - f
        f = f1
        B = B + (y - B * s) * (s ./ dot(s, s))'

    end
    if iter == tol_options.max_iter
        x_sol = x
    end
    return (x_sol, iter)

end

end # nonlineq


## This function computes the Generalized Laguerre polynomials using the Tridiagonal matrix
include("orthopoly_evaluate.jl")
function compute_laguerre(n, a, x, all=true)
    # Evaluate Orthonormalized Laguerre polynomials at x
    #  Laguerre:
    #     w(x)=x^a exp(-x) with integral gamma(a+1)  (a>-1)
    #     Equivalent Mathematica:LaguerreL[n,a,x]/Sqrt[Gamma[n+a+1]/Gamma[n+1]]
    ## Laguerre Bidiagonal matrix
    B = diagm(sqrt((a + 1 +(0:n)))) # diagonal
    B = B - diagm(sqrt((1:n)), 1)   # superdiagonal
    T = B'*B
    c = gamma(a+1) #Integral of weight function
    ## Default to 0 through n computation
    if all
        phi = orthopoly_evaluate_all(T, x)
    else
        phi = orthopoly_evaluate_n(T, x)
    end
    phi/sqrt(c)
end


function sieve2(n :: Int)
    ni = (n - 1) ÷ 2
    isprime = trues(ni)
    for i in 1:ni
        if isprime[i]
            j = 2i * (i + 1)
            if j > ni
                m = findall(isprime)
                map!((i::Int) -> 2i + 1, m, m)
                return pushfirst!(m, 2)
            else
                p = 2i + 1
                while j <= ni
                  isprime[j] = false
                  j += p
                end
            end
        end
    end
end


module Day05

using OffsetArrays

export day05

function parseLine(line)
    raw = split.(split(line, " -> "), ",")
    return complex(parse.(Int64, raw[1])...), complex(parse.(Int64, raw[2])...)
end

function readInput(inputfile="inputs/input05.txt")
    return [parseLine(line) for line in eachline(inputfile)]
end

function day05(ventdata = readInput("inputs/input05.txt"))
    minx = minimum((a) -> min(real(a[1]), real(a[2])), ventdata)
    miny = minimum((a) -> min(imag(a[1]), imag(a[2])), ventdata)
    maxx = maximum((a) -> max(real(a[1]), real(a[2])), ventdata)
    maxy = maximum((a) -> max(imag(a[1]), imag(a[2])), ventdata)

    ## previous version used a Dict here
    ## that's ~25x slower
    grid = OffsetArray(zeros(UInt8, maxx-minx+1, maxy-miny+1), minx:maxx, miny:maxy)

    mask = ishorizontal.(ventdata)
    mapvents!(grid, ventdata[mask])
    part1 = count(>=(2), grid)

    mapvents!(grid, ventdata[.!(mask)])
    part2 = count(>=(2), grid)
    return part1, part2
end

function mapvents!(grid, data)
    for (start, stop) in data
        Δ = stop-start
        dist = distance(Δ)
        δ = intdiv(Δ, dist)
        for l in 0:dist
            pos = start+δ*l
            grid[real(pos), imag(pos)] += 1
        end
    end
    return grid
end

ishorizontal((from, to)) = real(from) == real(to) || imag(from) == imag(to)
distance(x::Complex) = let r = abs(real(x)); r == 0 ? abs(imag(x)) : r end
intdiv(x::Complex, by::Int) = complex(div(real(x), by), div(imag(x), by))

count_crossings(grid) = count(>=(2), values(grid))

end #module


push!(LOAD_PATH,joinpath(@__DIR__, "../src/"))
using LinearAlgebra
using StableApproxEPW

# ## Target of the approximation problem

# We consider the Helmholtz solution in the unit disk, with wavenumber
k = 5;
# We need to define the maximum Fourier mode number `P` in the approximation
# target:
P = 25;
# The target approximation will be of the form
# ```math
# u = \mathbf{x} ↦ \sum_{|p| \leq P} u_p b_{p}(\mathbf{x}).
# ```
# Next we construct the vector of coefficients in the basis `b_p`
# for `p` in `[-P,P]`:
U = zeros(ComplexF64, 2P+1);
U[P+1]     = 0.5;  # This is the constant mode (`p=0`)
U[P+1 + P] = 1im;  # This is mode `p = P`
# Here the only non-zero coefficients are ``u_0 = \frac{1}{2}`` and
# ``u_P = \imath`` so that
# ```math
# u = \frac{1}{2} b_{0} + \imath b_{P}.
# ```
# The target of the approximation problem can then be constructed as:
u = solution_surrogate(U; k=k);
# ``u`` can be evaluated at any `(r,θ)` point.
# Alternatively, the target ``u`` could have been a single mode.
# For instance, to get the circular wave with mode number `p=15`, simply set
# ``u = bp(15; k=k)``.
# Of course, any other function (defined in polar coordinates) can be defined
# by the user as target of the approximation problem.

# ## Reconstruction method

# The approximation will be reconstructed by sampling the target on the
# boundary of the unit disk.
# To do so, we need to know now the dimension `N` of the approximation sets
# that we are going to use:
N = 100;
# We can determine the number of sampling nodes necessary for a successful
# reconstruction, based on `N`, `P` (to avoid aliasing) and the oversampling
# ratio `η`:
S = number_of_boundary_sampling_nodes(N; η=2, P=P);
# The (equispaced) boundary nodes are then constructed as:
X = boundary_sampling_nodes(S);
# The right-hand-side of the linear system can then be readily constructed:
b = samples_from_nodes(u, X);

# ## Approximation with **propagative** plane waves

# We construct the approximation set of PPW:
Φppw = approximation_set(N; k=k);
# The matrix and its (SVD) factorization
Appw = samples_from_nodes(Φppw, X);
iAppw = RegularizedSVDPseudoInverse(Appw; ϵ=1e-14);
# The coefficients of the approximation are computed:
ξppw = solve_via_regularizedSVD(iAppw, b);
ũppw = (r,θ) -> sum([ξi * ϕi(r,θ) for (ξi, ϕi) in zip(ξppw, Φppw)]);
# Absolute error function
eppw = (r,θ) -> ũppw(r,θ) - u(r,θ); eppw(1,π/2)
# Let's compute the relative residual:
resppw = norm(Appw * ξppw - b) / norm(b)
# We did not obtained any accuracy whatsoever!
# The reason is that the coefficients are too large:
nrmppw = norm(ξppw) / norm(U)

# ## Approximation with **evanescent** plane waves

# We construct the approximation set of EPW:
Φepw = approximation_set(N, P, sobol_sampling; k=k);
# It is possible to choose other types of sampling methods, instead of
# `sobol_sampling`, for instance `uniform_sampling` and `random_sampling`.
# The matrix and its (SVD) factorization
Aepw = samples_from_nodes(Φepw, X);
iAepw = RegularizedSVDPseudoInverse(Aepw; ϵ=1e-14);
# The coefficients of the approximation are computed:
ξepw = solve_via_regularizedSVD(iAepw, b);
ũepw = (r,θ) -> sum([ξi * ϕi(r,θ) for (ξi, ϕi) in zip(ξepw, Φepw)]);
# Absolute error function
eepw = (r,θ) -> ũepw(r,θ) - u(r,θ); eepw(1,π/2)
# Let's compute the relative residual:
resepw = norm(Aepw * ξepw - b) / norm(b)
# We get almost 8 digits of accuracy!
# The size of the coefficients remains quite high:
nrmepw = norm(ξepw) / norm(U)

# ### Down to machine precision

# Let's double the number of EPW
N = 200
# The above approximation process can be obtained much more rapidly with the
# following convenience function
_, ξepw, ũepw, resepw, nrmepw, eepw = Dirichlet_sampling(k, U, N; smpl_type=sobol_sampling);
# One can check that we obtain now a relative residual very close to machine
# precision (13 digits of accuracy):
resepw
# The size of the coefficients is also greatly reduced:
nrmepw


using FFTW
using DelimitedFiles

# using the convolutional theorem
function convolve_fft(signal1::Array{T}, signal2::Array{T}) where {T <: Number}
    return ifft(fft(signal1).*fft(signal2))
end

function main()

    # Random distribution in x
    x = rand(100)

    # Gaussian signals
    y = [exp(-((i-50)/100)^2/.01) for i = 1:100]

    # Normalization is not strictly necessary, but good practice
    normalize!(x)
    normalize!(y)

    # cyclic convolution via the convolutional theorem
    fft_output = convolve_fft(x, y)

    # outputting convolutions to different files for plotting in external code
    writedlm("fft.dat", fft_output)

end


_clamp(u, u_min, u_max) = max(min(u, u_max), u_min)
_dead_zone(u, u_min, u_max) = ifelse(u > u_max, u - u_max, ifelse(u < u_min, u - u_min, 0))

"""
Limit the range of a signal.

# Parameters:
- `y_max`: Maximum of output signal
- `y_min`: Minimum of output signal
"""
function Limiter(;name, y_max, y_min=y_max > 0 ? -y_max : -Inf)
    y_max ≥ y_min || throw(ArgumentError("`y_min` must be smaller than `y_max`"))
    @named siso = SISO()
    @unpack u, y = siso
    pars = @parameters y_max=y_max y_min=y_min
    eqs = [
        y ~ _clamp(u, y_min, y_max)
    ]
    extend(ODESystem(eqs, t, [], pars; name=name), siso)
end

"""
    DeadZone(; u_max, u_min=-u_max, name)

The DeadZone block defines a region of zero output.
If the input is within uMin ... uMax, the output is zero. Outside of this zone, the output is a linear function of the input with a slope of 1.
```
       y▲
        │     /
        │    /
  u_min │   /
─────|──┼──|───────► u
    /   │   u_max
   /    │
  /     │
```
"""
function DeadZone(; name, u_max, u_min=-u_max)
    if !ModelingToolkit.isvariable(u_max)
        u_max ≥ u_min || throw(ArgumentError("`u_min` must be smaller than `u_max`"))
    end
    @named siso = SISO()
    @unpack u, y = siso
    pars = @parameters u_max=u_max u_min=u_min
    eqs = [
        y ~ _dead_zone(u, u_min, u_max)
    ]
    extend(ODESystem(eqs, t, [], pars; name=name), siso)
end

"""
    SlewRateLimiter(;name, rising=1, falling=-rising, Td=0.001, y_start=0.0)

Limits the slew rate of a signal.

# Parameters:
- `Rising`: Maximum rising slew rate
- `falling`: Maximum falling slew rate
- `Td`: Derivative time constant
"""
function SlewRateLimiter(;name, rising=1, falling=-rising, Td=0.001, y_start=0.0)
    rising ≥ falling || throw(ArgumentError("`rising` must be smaller than `falling`"))
    Td > 0 || throw(ArgumentError("Time constant `Td` must be strictly positive"))
    @named siso = SISO(y_start=y_start)
    @unpack u, y = siso
    pars = @parameters rising=rising falling=falling
    eqs = [
        D(y) ~ max(min((u-y) / Td, rising), falling)
    ]
    extend(ODESystem(eqs, t, [], pars; name=name), siso)
end


# m5.4.AMs.jl

using Pkg, DrWatson

using MonteCarloMeasurements
using StanSample
using StatisticalRethinking

# ### snippet 5.1

df = CSV.read(sr_datadir("WaffleDivorce.csv"), DataFrame);

df = DataFrame(
    :A => df[:, :MedianAgeMarriage],
    :M => df[:, :Marriage],
    :D => df[:, :Divorce]
 )

scale!(df, [:M, :A, :D])

# Define the Stan language model

stan5_4_AM = "
data {
    int N;
    vector[N] A;
    vector[N] M;
}
parameters {
    real a;
    real bAM;
    real<lower=0> sigma;
}
model {
    vector[N] mu = a + bAM * M;
    a ~ normal( 0 , 0.2 );
    bAM ~ normal( 0 , 0.5 );
    sigma ~ exponential( 1 );
    A ~ normal( mu , sigma );
}
";

# Define the SampleModel
#tmpdir=ProjDir*"/tmp"
m5_4_AMs = SampleModel("m5.4.AM", stan5_4_AM);

# Input data

m5_4_data = Dict(
    "N" => size(df, 1),
    "M" => df[:, :M_s],
    "A" => df[:, :A_s]
);

# Sample using cmdstan

rc5_4_AMs = stan_sample(m5_4_AMs, data=m5_4_data);

if success(rc5_4_AMs)

    part5_4_AMs = read_samples(m5_4_AMs, :particles)
    part5_4_AMs |> display

end

# End of m5.3.AMs.jl


Text provided under a Creative Commons Attribution license, CC-BY.  All code is made available under the FSF-approved BSD-3 license.  (c) Lorena A. Barba, Gilbert F. Forsyth 2017. Thanks to NSF for support via CAREER award #1149784.
[@LorenaABarba](https://twitter.com/LorenaABarba)

12 steps to Navier–Stokes
=====
***

For a moment, recall the Navier–Stokes equations for an incompressible fluid, where $\vec{v}$ represents the velocity field:

$$
\begin{eqnarray*}
\nabla \cdot\vec{v} &=& 0 \\
\frac{\partial \vec{v}}{\partial t}+(\vec{v}\cdot\nabla)\vec{v} &=& -\frac{1}{\rho}\nabla p + \nu \nabla^2\vec{v}
\end{eqnarray*}
$$

The first equation represents mass conservation at constant density. The second equation is the conservation of momentum. But a problem appears: the continuity equation for incompressble flow does not have a dominant variable and there is no obvious way to couple the velocity and the pressure. In the case of compressible flow, in contrast, mass continuity would provide an evolution equation for the density $\rho$, which is coupled with an equation of state relating $\rho$ and $p$.

In incompressible flow, the continuity equation $\nabla \cdot\vec{v}=0$ provides a *kinematic constraint* that requires the pressure field to evolve so that the rate of expansion $\nabla \cdot\vec{v}$ should vanish everywhere. A way out of this difficulty is to *construct* a pressure field that guarantees continuity is satisfied; such a relation can be obtained by taking the divergence of the momentum equation. In that process, a Poisson equation for the pressure shows up!

Step 10: 2D Poisson Equation
----
***

Poisson's equation is obtained from adding a source term to the right-hand-side of Laplace's equation:

$$\frac{\partial ^2 p}{\partial x^2} + \frac{\partial ^2 p}{\partial y^2} = b$$

So, unlinke the Laplace equation, there is some finite value inside the field that affects the solution. Poisson's equation acts to "relax" the initial sources in the field.

In discretized form, this looks almost the same as [Step 9](./12_Step_9.ipynb), except for the source term:

$$\frac{p_{i+1,j}^{n}-2p_{i,j}^{n}+p_{i-1,j}^{n}}{\Delta x^2}+\frac{p_{i,j+1}^{n}-2 p_{i,j}^{n}+p_{i,j-1}^{n}}{\Delta y^2}=b_{i,j}^{n}$$

As before, we rearrange this so that we obtain an equation for $p$ at point $i,j$. Thus, we obtain:

$$p_{i,j}^{n}=\frac{(p_{i+1,j}^{n}+p_{i-1,j}^{n})\Delta y^2+(p_{i,j+1}^{n}+p_{i,j-1}^{n})\Delta x^2-b_{i,j}^{n}\Delta x^2\Delta y^2}{2(\Delta x^2+\Delta y^2)}$$

We will solve this equation by assuming an initial state of $p=0$ everywhere, and applying boundary conditions as follows:

$p=0$ at $x=0, \ 2$ and $y=0, \ 1$

and the source term consists of two initial spikes inside the domain, as follows:

$b_{i,j}=100$ at $i=\frac{1}{4}nx, j=\frac{1}{4}ny$

$b_{i,j}=-100$ at $i=\frac{3}{4}nx, j=\frac{3}{4}ny$

$b_{i,j}=0$ everywhere else.

The iterations will advance in pseudo-time to relax the initial spikes. The relaxation under Poisson's equation gets slower and slower as they progress. *Why?*

Let's look at one possible way to write the code for Poisson's equation. As always, we load our favorite Python libraries. We also want to make some lovely plots in 3D. Let's get our parameters defined and the initialization out of the way. What do you notice of the approach below?

```python
import numpy
from matplotlib import pyplot, cm
from mpl_toolkits.mplot3d import Axes3D
%matplotlib inline
```

```python
# Parameters
nx = 50
ny = 50
nt  = 100
xmin = 0
xmax = 2
ymin = 0
ymax = 1

dx = (xmax - xmin) / (nx - 1)
dy = (ymax - ymin) / (ny - 1)

# Initialization
p  = numpy.zeros((ny, nx))
pd = numpy.zeros((ny, nx))
b  = numpy.zeros((ny, nx))
x  = numpy.linspace(xmin, xmax, nx)
y  = numpy.linspace(xmin, xmax, ny)

# Source
b[int(ny / 4), int(nx / 4)]  = 100
b[int(3 * ny / 4), int(3 * nx / 4)] = -100
```

With that, we are ready to advance the initial guess in pseudo-time. How is the code below different from the function used in [Step 9](./12_Step_9.ipynb) to solve Laplace's equation?

```python
for it in range(nt):

    pd = p.copy()

    p[1:-1,1:-1] = (((pd[1:-1, 2:] + pd[1:-1, :-2]) * dy**2 +
                    (pd[2:, 1:-1] + pd[:-2, 1:-1]) * dx**2 -
                    b[1:-1, 1:-1] * dx**2 * dy**2) /
                    (2 * (dx**2 + dy**2)))

    p[0, :] = 0
    p[ny-1, :] = 0
    p[:, 0] = 0
    p[:, nx-1] = 0
```

Maybe we could reuse our plotting function from [Step 9](./12_Step_9.ipynb), don't you think?

```python
def plot2D(x, y, p):
    fig = pyplot.figure(figsize=(11, 7), dpi=100)
    ax = fig.gca(projection='3d')
    X, Y = numpy.meshgrid(x, y)
    surf = ax.plot_surface(X, Y, p[:], rstride=1, cstride=1, cmap=cm.viridis,
            linewidth=0, antialiased=False)
    ax.view_init(30, 225)
    ax.set_xlabel('$x$')
    ax.set_ylabel('$y$')

```

```python
plot2D(x, y, p)
```

![png](output_14_0.png)

Ah! The wonders of code reuse! Now, you probably think: "Well, if I've written this neat little function that does something so useful, I want to use it over and over again. How can I do this without copying and pasting it each time? —If you are very curious about this, you'll have to learn about *packaging*. But this goes beyond the scope of our CFD lessons. You'll just have to Google it if you really want to know.

***

## Learn More

To learn more about the role of the Poisson equation in CFD, watch **Video Lesson 11** on You Tube:

```python
from IPython.display import YouTubeVideo
YouTubeVideo('ZjfxA3qq2Lg')
```

<iframe
    width="400"
    height="300"
    src="https://www.youtube.com/embed/ZjfxA3qq2Lg"
    frameborder="0"
    allowfullscreen
></iframe>

```python
from IPython.core.display import HTML
def css_styling():
    styles = open("../styles/custom.css", "r").read()
    return HTML(styles)
css_styling()
```

<link href='http://fonts.googleapis.com/css?family=Fenix' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:300,400' rel='stylesheet' type='text/css'>
<style>
    @font-face {
        font-family: "Computer Modern";
        src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
    }
    div.cell{
        width:800px;
        margin-left:16% !important;
        margin-right:auto;
    }
    h1 {
        font-family: 'Alegreya Sans', sans-serif;
    }
    h2 {
        font-family: 'Fenix', serif;
    }
    h3{
		font-family: 'Fenix', serif;
        margin-top:12px;
        margin-bottom: 3px;
       }
	h4{
		font-family: 'Fenix', serif;
       }
    h5 {
        font-family: 'Alegreya Sans', sans-serif;
    }
    div.text_cell_render{
        font-family: 'Alegreya Sans',Computer Modern, "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
        line-height: 135%;
        font-size: 120%;
        width:600px;
        margin-left:auto;
        margin-right:auto;
    }
    .CodeMirror{
            font-family: "Source Code Pro";
			font-size: 90%;
    }
/*    .prompt{
        display: None;
    }*/
    .text_cell_render h1 {
        font-weight: 200;
        font-size: 50pt;
		line-height: 100%;
        color:#CD2305;
        margin-bottom: 0.5em;
        margin-top: 0.5em;
        display: block;
    }
    .text_cell_render h5 {
        font-weight: 300;
        font-size: 16pt;
        color: #CD2305;
        font-style: italic;
        margin-bottom: .5em;
        margin-top: 0.5em;
        display: block;
    }

    .warning{
        color: rgb( 240, 20, 20 )
        }
</style>
<script>
    MathJax.Hub.Config({
                        TeX: {
                           extensions: ["AMSmath.js"]
                           },
                tex2jax: {
                    inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                    displayMath: [ ['$$','$$'], ["\\[","\\]"] ]
                },
                displayAlign: 'center', // Change this to 'center' to center equations.
                "HTML-CSS": {
                    styles: {'.MathJax_Display': {"margin": 4}}
                }
        });
</script>

> (The cell above executes the style for this notebook.)


#-----------------------------#
# see page 136 in Sarkka book #
#-----------------------------#

function smoothingrecursion(y; A = A, H = H, Q = Q, R = R, m = m, P = P)

    N = length(y); @assert(N == length(m) == length(P))

    # Store here state distribution

    μstate, Σstate = Array{Vector{Float64}}(undef, N), Array{Matrix{Float64}}(undef, N)

    # Start recursion with prior mean and prior covariance

    mˢₖ₊₁, Pˢₖ₊₁ = m[end], P[end]

    μstate[N], Σstate[N] = mˢₖ₊₁, Pˢₖ₊₁

    for k in N-1:-1:1

        m⁻ₖ₊₁ = A * m[k]

        P⁻ₖ₊₁ = A * P[k] * A' + Q

        Gₖ = (P[k] * A') / P⁻ₖ₊₁

        mˢₖ = m[k] + Gₖ * (mˢₖ₊₁ - m⁻ₖ₊₁)

        Pˢₖ = P[k] + Gₖ * (Pˢₖ₊₁ - P⁻ₖ₊₁) * (Gₖ)'

        # store mean and covariance of Gaussian distribution N(xₖ|mˢₖ, Pˢₖ)
        Pˢₖ = (Pˢₖ + (Pˢₖ)') / 2

        μstate[k], Σstate[k] = mˢₖ, Pˢₖ

        # previous state

        mˢₖ₊₁, Pˢₖ₊₁ = mˢₖ, Pˢₖ

    end

    return μstate, Σstate

end

function testsmoothing(seed=1)

    yclean, y, x, A, H, Q, R, m₀, P₀ = simulatedata(seed=seed)

    μfilter, Σfilter = filteringrecursion(y; A = A, H = H, Q = Q, R = R, m₀ = m₀, P₀ = P₀)

    μsmooth, Σsmooth = smoothingrecursion(y; A = A, H = H, Q = Q, R = R, m = μfilter, P = Σfilter)

    figure()
    subplot(211)
    plot([yᵢ[1]        for yᵢ in yclean], label = "simulated 1")
    plot([(H*xᵢ)[1]    for xᵢ in x], label = "simulated 1")
    legend()

    subplot(212)
    plot([xᵢ[1]    for xᵢ in x], label = "simulated 1")
    plot([xᵢ[1]    for xᵢ in μfilter], label = "filtered 1")
    plot([xᵢ[1]    for xᵢ in μsmooth], "--", label = "smoothed 1")
    legend()

    acc = accfilter = accsmooth = 0.0

    for i in 1:length(y)

        acc += sum((yclean[i] - H*x[i]).^2) / length(y)

        accfilter += sum((yclean[i] - H*μfilter[i]).^2)/ length(y)

        accsmooth += sum((yclean[i] - H*μsmooth[i]).^2)/ length(y)

    end

    @show acc, accfilter, accsmooth

    accx = accfilterx = accsmoothx = 0.0

    for i in 1:length(y)

        accx += sum((x[i] - x[i]).^2)

        accfilterx += sum((x[i] - μfilter[i]).^2)

        accsmoothx += sum((x[i] - μsmooth[i]).^2)

    end

    @show accx, accfilterx, accsmoothx

    x, μfilter, μsmooth

end


"""
    twoD(X)

Project `X` to two dmensions using PCA
"""
function twoD(X)
    X = X .- mean(X,dims=1)
    X = X ./ std(X,dims=1)
    s = svd(X)
    Y = s.U[:,1:2]#.*s.S[1:2]'
    Y[:,1], Y[:,2]
end

"""
    threeD(X)

Project `X` to three dimensions using PCA
"""
function threeD(X)
    X = X .- mean(X,dims=1)
    X = X ./ std(X,dims=1)
    s = svd(X)
    Y = s.U[:,1:3]#.*s.S[1:2]'
    Y[:,1], Y[:,2], Y[:,3]
end

"""
    s1(x, dims=:)

normalize x sums to 1
"""
function s1(x, dims=:)
    if isderiving()
        m = minimum(real(x), dims=dims)
        if any(<(0), m)
            x = x .- m
        end
        return x ./ sum(x, dims=dims)
    else
        m = minimum(x, dims=dims)
        x = float.(x)
        if any(<(0), m)
            x .-= m
        end
        x ./= sum(x, dims=dims)
    end
end

"""
    n1(x)

normalize x norm 1
"""
n1(x) = x./norm(x)
n1(x::AbstractMatrix, dims) = mapslices(n1, x, dims=dims)
"""
    v1(x, dims=:)

normalize x var 1
"""
function v1(x, dims=:)
    x = x .- mean(x, dims=dims)
    x .= x./std(x, dims=dims)
end

function v1!(x, dims=:)
    x .-= mean(x, dims=dims)
    x .= x./std(x, dims=dims)
end

"""
    median1(x, dims=:)

normalize x median 0, median absolute deviation = 1
"""
function m1(x, dims=:)
    x = x .- median(x, dims=dims)
    # x .= x./mapslices(mad, x, dims=dims)
end
function m1(x::AbstractVector)
    x = x .- median(x)
    x .= x./mad(x, normalize=true)
end

"""
    bp_filter(x, passband)

Band-pass filter, passband is tuple, `fs` assumed = 1
"""
function bp_filter(x, passband)
    responsetype = Bandpass(passband..., fs=1)
    designmethod = Butterworth(2)
    filt(digitalfilter(responsetype, designmethod), x)
end

"""
    lp_filter(x, cutoff)

Low-pass filter, `fs` assumed = 1
"""
function lp_filter(x, cutoff)
    responsetype = Lowpass(cutoff, fs=1)
    designmethod = Butterworth(2)
    filt(digitalfilter(responsetype, designmethod), x)
end


"""
## Convex hull

Calculation f convex hull of a sequence of points represented through an Float64
Array of N rows and 2 columns
"""
function convexhull{T <: Real}(points::Array{T,2})
    n = size(points,1)
    # Sort the points by x-coordinate, resulting in a sequence p 1 , . . . , p n
    p = sortrows(points)
    # Put the points p_1 and p_2 in a list L_upper , with p_1 as the first point
    upper = [p[1,:], p[2,:]]

    for i in 3:n
        while size(upper,1) >= 2 && direction(upper[end-1], upper[end], p[i,:]) <= 0
            pop!(upper)
        end

        push!(upper,p[i,:])
    end

    lower = [p[end,:], p[end-1,:]]

    for i in n-2:-1:1
        while size(lower,1) >= 2 && direction(lower[end-1], lower[end], p[i,:]) <= 0
            pop!(lower)
        end

        push!(lower,p[i,:])
    end

    append!(upper, lower[2:end-1])
end

function convexhull{T <: Real}(points::Array{Point2D{T}, 1})
    n = size(points,1)
    # Sort the points by x-coordinate, resulting in a sequence p 1 , . . . , p n
    p = sort(points)
    # Put the points p_1 and p_2 in a list L_upper , with p_1 as the first point
    upper = [p[1], p[2]]

    for i in 3:n
        while size(upper,1) >= 2 && direction(upper[end-1], upper[end], p[i]) <= 0
            pop!(upper)
        end

        push!(upper,p[i])
    end

    lower = [p[end], p[end-1]]

    for i in n-2:-1:1
        while size(lower,1) >= 2 && direction(lower[end-1], lower[end], p[i]) <= 0
            pop!(lower)
        end

        push!(lower,p[i])
    end

    append!(upper, lower[2:end-1])
end


##################################################
## Classical derivative-based, iterative, root-finding algorithms: Newton and Halley
## Historic, we have derivative free versions of similar order

## Newton
"""

    Roots.Newton()

Implements Newton's [method](http://tinyurl.com/b4d7vls): `x_n1 = xn -
f(xn)/f'(xn)`.  This is a quadratically converging method requiring
one derivative. Two function calls per step.
"""
struct Newton <: AbstractUnivariateZeroMethod
end

function update_state(method::Newton, fs, o, options)
    xn = o.xn1
    fxn = o.fxn1
    fpxn = fs(xn,1)

    if isissue(fpxn)
        o.stopped=true
        return
    end

    xn1 = xn - fxn / fpxn
    fxn1 = fs(xn1)
    incfn(o)

    o.xn0, o.xn1 = xn, xn1
    o.fxn0, o.fxn1 = fxn, fxn1

end

"""
    newton(f, fp, x0; kwargs...)

Implementation of Newton's method: `x_n1 = x_n - f(x_n)/ f'(x_n)`

Arguments:

* `f::Function` -- function to find zero of

* `fp::Function` -- the derivative of `f`.

* `x0::Number` -- initial guess. For Newton's method this may be complex.

With the `FowardDiff` package derivatives may be computed automatically. For example,  defining
`D(f) = x -> ForwardDiff.derivative(f, float(x))` allows `D(f)` to be used for the first derivative.

Keyword arguments are passed to `find_zero` using the `Roots.Newton()` method.

"""
newton(f, fp, x0; kwargs...) = find_zero((f, fp), x0, Newton(); kwargs...)
@deprecate newton(f, x0; kwargs...)  newton(f, fp, x0; kwargs...)

## Halley

"""
    Roots.Halley()

Implements Halley's [method](http://tinyurl.com/yd83eytb),
`x_n1 = xn - (2 f(xn)*f'(xn)) / (2 f'(xn)^2 - f(xn) * f''(xn))`.
This method is cubically converging, but requires more function calls per step (3) than
other methods.
"""
struct Halley <: AbstractUnivariateZeroMethod
end

function update_state(method::Halley, fs, o::UnivariateZeroState{T,S}, options::UnivariateZeroOptions) where {T,S}
    xn = o.xn1
    fxn = o.fxn1
    fpxn = fs(xn,1); incfn(o)
    fppxn = fs(xn,2); incfn(o)

    xn1 = xn - 2fxn*fpxn / (2*fpxn*fpxn - fxn * fppxn)
    fxn1 = fs(xn1); incfn(o)

    o.xn0, o.xn1 = xn, xn1
    o.fxn0, o.fxn1 = fxn, fxn1
end

"""
    halley(f, fp, fpp, x0; kwargs...)

Implementation of Halley's method. `xn1 = xn - 2f(xn)*f'(xn) / (2*f'(xn)^2 - f(xn) * f''(xn))`

Arguments:

* `f::Function` -- function to find zero of

* `fp::Function` -- derivative of `f`.

* `fpp:Function` -- second derivative of `f`.

* `x0::Number` -- initial guess

With the `FowardDiff` package derivatives may be computed automatically. For example,  defining
`D(f) = x -> ForwardDiff.derivative(f, float(x))` allows `D(f)` and `D(D(f))` to be used for the first and second
derivatives, respectively.

Keyword arguments are passed to `find_zero` using the `Roots.Halley()` method.

"""
halley(f, fp, fpp, x0; kwargs...) = find_zero((f, fp, fpp), x0, Halley(); kwargs...)
#halley(f,  x0; kwargs...) = find_zero(f, x0, Halley(); kwargs...) # deprecated
#halley(f, fp, x0; kwargs...) = find_zero((f, fp), x0, Halley(); kwargs...) # deprecated
@deprecate halley(f,  x0; kwargs...)    halley(f, fp, fpp, x0; kwargs...)
@deprecate halley(f, fp, x0; kwargs...) halley(f, fp, fpp, x0; kwargs...)


module MINLP

using JuMP, BARON, Test

m = Model(BARON.Optimizer)
ub = [2, 2, 1]
@variable(m, 0 ≤ x[i=1:3] ≤ ub[i])
@variable(m, y[1:3], Bin)

@NLconstraints(m, begin
    0.8log(x[2]+1) + 0.96log(x[1]-x[2]+1) - 0.8x[3]         ≥  0
       log(x[2]+1) + 1.20log(x[1]-x[2]+1)  -   x[3] - 2y[3] ≥ -2
       x[2] ≤  x[1]
       x[2] ≤ 2y[1]
       x[1] - x[2] ≤ 2y[2]
       y[1] + y[2] ≤ 1
end)

@NLobjective(m, Min, 5y[1] + 6y[2] + 8y[3] + 10x[1] - 7x[3] - 18log(x[2]+1) -
                        19.2log(x[1]-x[2]+1) + 10)

optimize!(m)

@test isapprox(value(x[1]), 1.300975890892825, rtol=1e-6)
@test isapprox(value(x[2]), 0.0, rtol=1e-6)
@test isapprox(value(x[3]), 1.0, rtol=1e-6)
@test isapprox(value(y[1]), 0.0, rtol=1e-6)
@test isapprox(value(y[2]), 1.0, rtol=1e-6)
@test isapprox(value(y[3]), 0.0, rtol=1e-6)
@test isapprox(objective_value(m), 6.00975890893, rtol=1e-6)

end # module


using Test

"
Function shootingmethod() for 2nd order BVP ODEs.

Arguments to shootingmethod(f, bvs, initg, steps, tol, limit):

f::Function           : Function to be estimated (see below).\n
bvs::Vector{Float64}  : Vector of boundary values [xa, ya, xb, yb].\n
initg::Matrix{Float64}: Vector of nitial gradients [a0, a1].\n
steps::Int64          : Number of calculations steps.\n
tol::Float64          : Tolerance for convergence.\n
limit::Int64          : Max number of iterations.

Example of how to define the function f(x, y) for y'' = 3x^3+4y:

Decompose in two first order equations:

dy[1] = y[2]\n
dy[2] = 3x^3 + 4y[1]

```
function f(x::Float64, y::Vector{Float64})
  [y[2], 3x^3 +4y[1]]
end
```
"
function shootingmethod(f::Function, bvs::Vector{Float64}, initg::Vector{Float64},
  steps::Int64, tol::Float64=0.00001, limit::Int64=25)
  #println("\n--- Shooting method for Second Order ODEs ---\n")
  @test size(bvs, 1) == 4
  @test size(initg, 1) == 2
  nsteps = steps + 1
  y0 = zeros(Float64, nsteps, 2)
  ystar = zeros(Float64, nsteps)
  y = zeros(Float64, 2)
  (xa, ya, xb, yb) = bvs[:, 1]
  a0 = initg[:, 1]
  h = (xb - xa)/steps
  res = zeros(Float64, nsteps, 2)
  for j in 1:2
    x = xa
    y[1] = ya
    y[2] = a0[j]
    for i in 1:nsteps
      y0[i, j] = y[1]
      k0 = h * f(x, y)
      k1 = h * f(x+h/2, y+k0/2)
      k2 = h * f(x+h/2, y+k1/2)
      k3 = h * f(x+h, y+k2)
      y += (k0+2k1+2k2+k3)/6
      x = x + h
    end
  end
  if (y0[nsteps,1]-yb)*(y0[nsteps,2]-yb) > 0.0
    println("Try new gradients ... ?")
    converged = abs((ystar[nsteps]-yb)/yb) < tol
    return((0, nsteps, converged, res))
  end
  iter = 0
  while iter <= limit
    iter += 1
    if iter > limit
      println("Max number of iterations exceeded ...")
      println("Iterations performed: $iter")
      for i in 1:nsteps
        res[i,:] = [xa+(i-1)*h ystar[i]]
      end
      println("Try new gradients ... ?")
      converged = abs((ystar[nsteps]-yb)/yb) < tol
      return((iter, nsteps, converged, res))
    end
    astar = a0[1] + (yb-y0[nsteps, 1])*(a0[2]-a0[1])/(y0[nsteps,2])
    x = xa
    y[1] = ya
    y[2] = astar
    for i in 1:nsteps
      ystar[i] = y[1]
      k0 = h * f(x, y)
      k1 = h * f(x+h/2, y+k0/2)
      k2 = h * f(x+h/2, y+k1/2)
      k3 = h * f(x+h, y+k2)
      y += (k0+2k1+2k2+k3)/6
      x = x + h
    end
    if abs((ystar[nsteps]-yb)/yb) < tol
      #println("Iterations to convergence: $iter")
      for i in 1:nsteps
        res[i,:] = [xa+(i-1)*h ystar[i]]
      end
      break
    end
    if (ystar[nsteps]-yb)*(y0[nsteps, 1]-yb) > 0.0
      y0[:, 1] = ystar
      a0[1] = astar
    else
      y0[:, 2] = ystar
      a0[2] = astar
    end
  end
  converged = abs((ystar[nsteps]-yb)/yb) < tol
  (iter, nsteps, converged, res)
end


function plot_country_posterior(chains, grouped_dat, country; tspan = [0.0, 365.0], subsample_size=100)

    country_pop = grouped_dat[country].pop[1]
    country_name = grouped_dat[country].Country_Region[1]

    post_dat = DataFrame(chains);

    init = [(country_pop - 100.0) / country_pop, 100.0 / country_pop, 0.0]
    country_param_cols = [Symbol("β[" * string(country) * "]"), Symbol("γ[" * string(country) * "]")]

    posterior_mean_params = mean.(eachcol(post_dat[:, country_param_cols]))
    posterior_mean_prob = ODEProblem(sir_ode, init, tspan, posterior_mean_params)
    posterior_mean_sol = solve(posterior_mean_prob);

    # plt = plot(posterior_mean_sol;
    #     title = country_name * "\n",
    #     title_location = :left,
    #     top_margin = 5mm,
    #     vars = [1],
    #     xguidefontsize = 10, yguidefontsize = 10,
    #     xlabel = "Days since 100 confirmed cases",
    #     ylabel = "Proportion of population",
    #     label = "S",
    #     color = "blue",
    #     linewidth = 2,
    #     legend = :outerright)
    plt = plot(posterior_mean_sol;
        title = country_name * "\n",
        title_location = :left,
        top_margin = 5mm,
        vars = [2],
        xguidefontsize = 10, yguidefontsize = 10,
        xlabel = "Days since 100 confirmed cases",
        ylabel = "Proportion of population",
        label = "I",
        color = "red",
        linewidth = 2,
        legend = :outerright)
    # plot!(posterior_mean_sol; label="I", vars = [2], color = "red", linewidth = 2)
    plot!(posterior_mean_sol; label="R", vars = [3], color = "green", linewidth = 2)

    # scatter!(grouped_dat[country].susc_prop, color = "blue", label = "Obs. S", markerstrokewidth = 0)
    scatter!(grouped_dat[country].case_prop, color = "red", label = "Obs. I", markerstrokewidth = 0)
    scatter!(grouped_dat[country].recov_prop, color = "green", label = "Obs. R", markerstrokewidth = 0)

    # subsample to avoid a crowded plot
    sub_idx = sample(axes(post_dat, 1), subsample_size; replace = false);
    for i in sub_idx
        prob = ODEProblem(sir_ode, init, tspan, post_dat[i, country_param_cols])
        sol = solve(prob);
        # plot!(sol; vars = [1], label="", color = "blue", alpha = 0.1, xlabel = "Days since 100 confirmed cases")
        plot!(sol; vars = [2], label="", color = "red", alpha = 0.1, xlabel = "Days since 100 confirmed cases")
        plot!(sol; vars = [3], label="", color = "green", alpha = 0.1, xlabel = "Days since 100 confirmed cases")
    end

    return plt
end


# The normal fit api in Distributions.jl is:
#   fit(Distr, values)
# so we want to do a more general
#   fitall(values)
# that will try all distributions that can be fitted and return back the "best" one
# for relevant criteria of "best", typically BIC or AIC.

using Distributions

AllContinuousDistributions = [
    Arcsine, Beta, BetaPrime, Cauchy, Chi, Chisq, Erlang, Exponential,
    FDist, Frechet, Gamma, GeneralizedExtremeValue, GeneralizedPareto,
    Gumbel, InverseGamma, InverseGaussian, Laplace, Levy, LogNormal,
    Logistic, Normal, NormalInverseGaussian, Pareto, Rayleigh,
    SymTriangularDist, TDist, TriangularDist, Uniform, VonMises, Weibull
]

samples1 = rand(Normal(3.4, 1.2), 100)

ContinuousDistributionsThatCanFit = filter(AllContinuousDistributions) do D
    try
        fit_mle(D, samples1)
        true
    catch _err
        false
    end
end

# Start for continuous distributions
function fitalldistr{T<:Real}(x::AbstractArray{T}, sortby = :BIC)
    n = length(x)
    allfits = map(ContinuousDistributionsThatCanFit) do D
        d = fit_mle(D, x)
        loglik = loglikelihood(d, x)
        k = length(params(d))
        if sortby == :BIC
            criteria = -2 * loglik + k * log(n)
        elseif sortby == :AIC
            criteria = -2 * loglik + 2 * k
        elseif sortby == :AICc
            criteria = -2 * loglik + 2 * k + ((2*k*(k+1))/(n-k-1))
        elseif sortby == :NLogL
            criteria = -loglik
        end
        (d, criteria)
    end
    sort(allfits, by = t -> t[2])
end

function fitall{T<:Real}(x::AbstractArray{T}, sortby = :BIC)
    fitalldistr(x, sortby)[1][1]
end

samples1 = rand(Normal(3.4, 1.2), 100)
d1 = fitall(samples1)

samples2 = rand(Exponential(0.3), 100)
d2 = fitall(samples2)

samples3 = rand(LogNormal(-5.0, 2.0), 100)
d3s = fitalldistr(samples3)
d3 = fitall(samples3)

samples4 = rand(Gamma(6.20, 1.3), 200)
d4s = fitalldistr(samples4)
d4 = fitall(samples4)

# Should we maybe use a non-parametric test to select between them?


function mirror(A,d::Int=3)
    w = div(d-1,2) #width of window, d is an ODD integer
    mA = [A[(w+1):-1:2];A;A[lastindex(A)-1:-1:(length(A)-w)]]
end

function find_extrema(A::Array,d::Int=3)
    mA = mirror(A)
    count = 0
    maxs = Int[]
    mins = Int[]
    exts = Int[]
    for i in 1:lastindex(mA)-d+1
        win = view(mA,i:i+d-1)
        if (win[2] > win[1] && win[2] > win[3])
                count+=1
                push!(maxs,i)
                push!(exts,i)
        elseif (win[2] < win[1] && win[2] < win[3])
                count+=1
                push!(mins,i)
                push!(exts,i)
        end
    end
    maxs, mins,exts
end

function find_extrema_count(A::Array,d::Int=3)
    dd = diff(sign.(diff(A)))
    return length(dd[dd .!= 0]) + 2
end

function find_extrema_minmax(A::Array,d::Int=3)
    dd = diff(sign.(diff(A)))
    return findall(dd .< 0), findall(dd .> 0)
end

function stream_minmax(env1, env2, A, d::Int)
    a = mirror(A, d)
    N = length(a)
    upper = UInt32[] #buffer for indices
    lower = UInt32[]
    push!(upper,1)
    push!(lower,1)
    w = d
    for i in 2:lastindex(a)
        if i >= w
            env1[i-w+1] = a[upper[1]]
            env2[i-w+1] = a[lower[1]]
        end

        if a[i] > a[i-1]
            pop!(upper)
            # remove maxima from buffer that are less than our new one
            while isempty(upper) != true
                if a[i] <= a[upper[end]]
                    break
                end
                pop!(upper)
            end
        else
            pop!(lower)
            # remove minima from buffer that are greater than our new one
            while isempty(lower) != true
                if a[i] >= a[lower[end]]
                    break
                end
                pop!(lower)
            end
        end
        push!(upper,i)
        push!(lower,i)
        if i == w + upper[1]
            popfirst!(upper)
        elseif i == w + lower[1]
            popfirst!(lower)
        end
        env1[end-w] = a[upper[1]]
        env2[end-w] = a[lower[1]]
    end
    return nothing
end

function moving_average(A, d::Int=3)
    A = mirror(A,d)
    T = typeof(one(eltype(A))/1)
    ret = Vector{T}(undef, length(A) - d + 1)
    id = 1 / d
    s = sum(view(A, 1:d))
    ret[1] = s * id
    @inbounds for n = 1:length(ret)-1
        s += A[n+d] - A[n]
        ret[n+1] = s * id
    end
    return ret
end


function ^(a::STaylor1{N,T}, n::Integer) where {N,T<:Real}
    n == 0 && return one(a)
    n == 1 && return a
    n == 2 && return square(a)
    n < 0 && return a^float(n)
    return power_by_squaring(a, n)
end

^(a::STaylor1{N,T}, b::STaylor1{N,T}) where {N,T<:Number} = exp(b*log(a))

function power_by_squaring(x::STaylor1{N,T}, p::Integer) where {N,T<:Number}
    p == 1 && return x
    p == 0 && return one(x)
    p == 2 && return square(x)
    t = trailing_zeros(p) + 1
    p >>= t

    while (t -= 1) > 0
        x = square(x)
    end

    y = x
    while p > 0
        t = trailing_zeros(p) + 1
        p >>= t
        while (t -= 1) ≥ 0
            x = square(x)
        end
        y *= x
    end

    return y
end

@generated function ^(a::STaylor1{N,T}, r::S) where {N, T<:Number, S<:Real}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]
    ctuple = Expr(:tuple)
    for i = 1:N
        push!(ctuple.args, syms[i])
    end

    for i = 1:N
        push!(ex_calc.args, :($(syms[i]) = zero(T)))
    end

    expr_quote = quote
        iszero(r) && return one(a)
        r == 1 && return a
        r == 2 && return square(a)
        r == 1/2 && return sqrt(a)
        $ex_calc
    end

    c = STaylor1(zero(T), Val{N}())
    for k = 0:(N - 1)
        symk = syms[k + 1]
        temp_quote = quote
            # First non-zero coefficient
            l0 = findfirst(a)
            if l0 < 0
                $symk = zero(T)
            else
                # The first non-zero coefficient of the result; must be integer
                !isinteger(r*l0) && throw(ArgumentError(
                    """The 0th order Taylor1 coefficient must be non-zero
                    to raise the Taylor1 polynomial to a non-integer exponent."""))
                lnull = trunc(Int, r*l0)
                kprime = $k - lnull
                if (kprime < 0) || (lnull > N-1)
                    $symk = zero(T)
                else
                    # Relevant for positive integer r, to avoid round-off errors
                    if isinteger(r) && ($k > r*findlast(a))
                        $symk = zero(T)
                    else
                        if $k == lnull
                            $symk = a[l0]^r
                        else
                            # The recursion formula
                            if l0 + kprime ≤ (N - 1)
                                tup_in = $ctuple
                                $symk = r*kprime*tup_sel(lnull, tup_in)*a[l0 + kprime]
                            else
                                $symk = zero(T)
                            end
                            for i = 1:($k - lnull - 1)
                                if !((i + lnull) > (N - 1) || (l0 + kprime - i > (N - 1)))
                                    aux = r*(kprime - i) - i
                                    tup_in = $ctuple
                                    $symk += aux*tup_sel(i + lnull, tup_in)*a[l0 + kprime - i]
                                end
                            end
                            $symk /= kprime*a[l0]
                        end
                    end
                end
            end
        end
        one_tup = ntuple(i -> i == 1 ? one(T) : zero(T), Val{N}())
        expr_quote = quote
            $expr_quote
            if r == 0
                $ctuple = $one_tup
            elseif r == 1 # DO NOTHING
            elseif r == 2
                temp_st1 = square(STaylor1{N,T}($ctuple))
            elseif r == 0.5
                temp_st2 = sqrt(STaylor1{N,T}($ctuple))
            else
                $temp_quote
            end
        end
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end

    return quote
               Base.@_inline_meta
               $expr_quote
               return STaylor1{N,T}($exout)
            end
end

@generated function square(a::STaylor1{N,T}) where {N, T<:Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    sym = syms[1]
    ex_line = :($(syms[1]) = a[0]^2)
    ex_calc.args[1] = ex_line

    for k in 1:(N-1)
        kodd = k%2
        kend = div(k - 2 + kodd, 2)
        ex_line = :(a[0] * a[$k])
        @inbounds for i = 1:kend
            ex_line = :($ex_line + a[$i] * a[$(k-i)])
        end
        ex_line = :(2.0*($ex_line)) # float(2)* TODO: ADD BACK IN
        if kodd !== 1
            ex_line = :($ex_line +a[$(div(k,2))]^2)
        end
        ex_line = :($(syms[k+1]) = $ex_line)
        ex_calc.args[k+1] = ex_line
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end

@generated function inverse(a::STaylor1{N,T}) where {N,T<:Real}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:(2*N)])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end

    count = 1
    for n = 1:(N - 1)
        ex_calc.args[2*count - 1] = :(syms[n + 1] = zdivfpown[n - 1]/n)
        ex_calc.args[2*count] = :(zdivfpown *= zdivf)
        count += 1
    end

    return quote
               Base.@_inline_meta
               if a[0] != zero(T)
                   throw(ArgumentError(
                   """
                   Evaluation of Taylor1 series at 0 is non-zero. For high accuracy, revert
                   a Taylor1 series with first coefficient 0 and re-expand about f(0).
                   """))
               end
               z = copy(a)
               zdivf = z/a
               zdivfpown = zdivf
               S = eltype(zdivf)
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end

function tup_sel(i, vargs)
    return vargs[i+1]
end

@generated function sqrt(a::STaylor1{N,T}) where {N,T<:Number}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]
    ctuple = Expr(:tuple)
    for i = 1:N
        push!(ctuple.args, syms[i])
    end

    # First non-zero coefficient
    expr_quote = quote
        l0nz = findfirst(a)
        aux = zero(T)
        if l0nz < 0
            return zero(STaylor1{N,T})
        elseif l0nz%2 == 1 # l0nz must be pair
            throw(ArgumentError(
            """First non-vanishing Taylor1 coefficient must correspond
            to an **even power** in order to expand `sqrt` around 0."""))
        end

        # The last l0nz coefficients are set to zero.
        lnull = div(l0nz, 2)
    end

    for i = 1:N
        push!(ex_calc.args, :($(syms[i]) = zero(T)))
    end

    for i = 1:N
        switch_expr = :((lnull == $(i-1)) && ($(syms[i]) = sqrt(a[l0nz])))
        expr_quote = quote
            $expr_quote
            $switch_expr
        end
    end

    for k = 0:(N - 1)
        symk = syms[k + 1]
        temp_expr = quote
            if $k >= lnull + 1
                if $k == lnull
                    $symk = sqrt(a[2*lnull])
                else
                    kodd = ($k - lnull)%2
                    kend = div($k - lnull - 2 + kodd, 2)
                    imax = min(lnull + kend, N - 1)
                    imin = max(lnull + 1, $k + lnull - (N - 1))
                    if imin ≤ imax
                        tup_in = $ctuple
                        $symk = tup_sel(imin, tup_in)*tup_sel($k + lnull - imin, tup_in)
                    end
                    for i = (imin + 1):imax
                        tup_in = $ctuple
                        $symk += tup_sel(i, tup_in)*tup_sel($k + lnull - i, tup_in)
                    end
                    if $k + lnull ≤ (N - 1)
                        aux = a[$k + lnull] - 2*$symk
                    else
                        aux = -2*$symk
                    end
                    tup_in = $ctuple
                    if kodd == 0
                        aux -= tup_sel(kend + lnull + 1, tup_in)^2
                    end
                    $symk = aux/(2*tup_sel(lnull, tup_in))
                end
            end
        end
        expr_quote = quote
            $expr_quote
            $temp_expr
        end
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               $expr_quote
               return STaylor1{N,T}($exout)
            end
end


# This file includes IFS tools.
using LinearAlgebra
using StatsBase

export IFS, attractor, DetAlg, RandAlg, dimension, contfactor, Sierpinski, Fern, Square, Tree

export w, Transformation, randalg_sequential_for_generator, randalg_sequential_generator

"""
    $(TYPEDEF)
Affine transformation

# Fields
    $(TYPEDFIELDS)
"""
struct Transformation{T1<:AbstractMatrix{<:Real}, T2<:AbstractVector{<:Real}}
    "Transformation matrix"
    A::T1
    "Transformation vector"
    b::T2
end

(w::Transformation)(x) = w.A * x + w.b

"""
    $SIGNATURES

Returns dimension of `w`.
"""
dimension(w::Transformation) = size(w.A,1)

"""
    $SIGNATURES

Returns contraction factor of `w`. Contraction factor is computed as the norm of `w.A`.
"""
contfactor(w::Transformation) = norm(w.A)

"""
    $(TYPEDEF)

Iterated fucntion sytem (IFS)

# Fields

    $(TYPEDFIELDS)
"""
struct IFS{T1<:AbstractVector{<:Transformation}, T2<:AbstractVector{<:Real},T3}
    "Vector of transformations of IFS"
    ws::T1
    "Vector of probabilities of IFS"
    probs::T2
    "generator of the ifs"
    generator::T3
end

function IFS(ws::T1, probs::T2, args...) where {T1, T2}
    # Note: For the floating point numbers, aproximation(≈), instead of exact equal (==), should be considered
    sum(probs) ≈ 1 || throw(ArgumentError("Sum of probabilities must be 1."))
    generator = randalg_sequential_generator(ws, probs, args...)
    IFS{T1, T2, typeof(generator)}(ws, probs, generator)
end
IFS(ws,args...) = (n = length(ws); IFS(ws, 1  / n * ones(n), args...))

"""
    $SIGNATURES

Conctructs an IFS for Sierpinski triangle.
"""
Sierpinski() = IFS([
    Transformation([0.5 0.0; 0. 0.5], [0; 0]),
    Transformation([0.5 0.0; 0. 0.5], [0; 1/2.]),
    Transformation([0.5 0.0; 0. 0.5], [1/2.; 1/2.])
    ], [1/3., 1/3., 1/3.])

"""
    $SIGNATURES

Constructs and IFS for a sqaure.
"""
Square() = IFS([
    Transformation([0.5 0.0; 0. 0.5], [1.; 1.]),
    Transformation([0.5 0.0; 0. 0.5], [50.; 1.]),
    Transformation([0.5 0.0; 0. 0.5], [1.; 50.]),
    Transformation([0.5 0.0; 0. 0.5], [50.; 50.])
    ], [0.25, 0.25, 0.25, 0.25])

"""
    $SIGNATURES

Constructs and IFS for a fern.
"""
Fern() = IFS([
    Transformation([0 0; 0 0.16], [0.; 0.]),
    Transformation([0.85 0.04; -0.04 0.85],[0.; 1.6]),
    Transformation([0.2 -0.26; 0.23 0.22], [0.; 1.6]),
    Transformation([-0.15 0.28; 0.26 0.24], [0.; 0.44])
    ], [0.01, 0.85, 0.07, 0.07])

"""
    $SIGNATURES

Constructs and IFS for a fractal tree.
"""
Tree() = IFS([
    Transformation([0 0; 0 0.5], [0.; 0.]),
    Transformation([0.42 -0.42; 0.42 0.42], [0.; 0.2]),
    Transformation([0.42 0.42; -0.42 0.42], [0.; 0.2]),
    Transformation([0.1 0; 0 0.1], [0.; 0.2])
    ], [0.05, 0.40, 0.40, 0.15])

"""
    $SIGNATURES

Returns dimension of `ifs`.
"""
dimension(ifs::IFS) = dimension(ifs.ws[1])

"""
    $SIGNATURES

Returns the contraction factor of `IFS`.
"""
contfactor(ifs::IFS) = maximum(contfactor.(ifs.ws))

"""
    $TYPEDEF

A type signifying that deterministic algorithm is used when calculating the attractor of and IFS.
"""
struct DetAlg end

"""
    $TYPEDEF

A type signifying that random algorithm is used when calculating the attractor of and IFS.
"""
struct RandAlg end

"""
    $TYPEDEF

Attractor of `IFS` type

# Fields

    $TYPEDFIELDS
"""
struct Attractor{T, S, R1, R2}
    "IFS of Attractor"
    ifs::T
    "Type of algorithm to be used to compute attractor(Options are DetAlg and RandAlg"
    alg::S
    "Initial set of attractor"
    initset::R1
    "Set of the attractor"
    set::R2
    "Number of iterations"
    numiter::Int
    "Sequential or parallel"
    parallel::Bool
end

"""
    $SIGNATURES

Computes the attractor of `ifs`. If `alg` is of type `DetAlg`, the deterministic algorithm is used. If `alg` is of type
`RandAlg`, random algorithm is used. `kwargs` may include

* `numiter::Int` : Number of iterations to used to calcuate the attractor (defaults to 10)

* `numtransient::Int` : Number of transient iterations to used to calcuate a transient set. When the transient set is
  constructed, the computation of attractor is continued with distributed computation if `alg` is `RandAlg` and `parallel` is
  `true`. (defaults to 10)

* `parallel::Bool`: If  `true`, the attractor is computed using distrbuted computation. (defaults to false)

* `placedependent::Bool` : If `true`, place dependent attractor is computed if α and β is given accordingly. (default to
  false)

* `α::AbstractVector` : Place-dependent probility coefficient(defaults to nothing)

* `β::AbstractVector` : Place-dependent probility coefficient. (default to nothing)
"""
attractor(ifs, initset; alg=DetAlg(), kwargs...) = typeof(alg) == DetAlg ?
                                                   detalg(ifs,initset; kwargs...) :
                                                   randalg(ifs,initset; kwargs...)

"""
    $SIGNATURES

Computes the attractor of `ifs` with deterministic algorithm.`numiter` is number of iterations. (Defaults to 10). If
`parallel` is true, attractor is computed via parallel computation.
"""
function detalg(ifs, initset; numiter=10, parallel=false)
    copiedset = copy(initset)
    set = parallel ?
          detalg_parallel(ifs.ws, copiedset, numiter) :
          detalg_sequential(ifs.ws, copiedset, numiter)
    Attractor(ifs, DetAlg(), initset, set, numiter, parallel)
end

# Computes the attractor of an ifs via deterministic algorithm sequentially.
function detalg_sequential(ws, set, numiter)
    for i in 1 : numiter
        set = vcat(map(w -> w.(set), ws)...)
    end
    set
end

# Computes the attractor of an ifs via deterministic algorithm in parallel.
function detalg_parallel(ws, set, numiter)
    loadprocs()
    for i in 1 : numiter
        set = vcat(map(w -> pmap(w, set), ws)...)
    end
    set
end

"""
    $SIGNATURES

Computes the attractor of `ifs` with random algorithm.`numiter` is number of iterations. (Defaults to 100). `numtransient` is
the number of transient iterations. If `parallel` is true, attractor is computed via parallel computation. If
`placedependent` is true, the probabilties of the ifs are dependent on the coordinates `x`. This dependency `p(x)` is given
via the parameters `α` and `β` where p(x) = α x + β.
"""
function randalg(ifs, initset; numiter=100, numtransient=10, parallel=false, placedependent=false, α=nothing, β=nothing, allocated::Bool=false)
    ws = ifs.ws
    probs = ifs.probs
    if parallel
        if placedependent
            transient = randalg_sequential_pd(ws, copy(initset), numtransient, probs, α, β, allocated)
            set = randalg_parallel_pd(ws, transient, numiter, probs, α, β, allocated)
        else
            transient = randalg_sequential(ws, copy(initset), numtransient, probs, allocated)
            set = randalg_parallel(ws, transient, numiter, probs, allocated)
        end
    else
        if placedependent
            set = randalg_sequential_pd(ws, copy(initset), numiter, probs, α, β, allocated)
        else
            set = randalg_sequential(ws, copy(initset), numiter, probs, allocated)
        end
    end
    Attractor(ifs, RandAlg(), initset, set, numiter, parallel)
end

function _randalg_sequential(set::AbstractVector, ws, numiter, probs)
    weights = Weights(probs)
    xi = set[end]
    for i = 1 : numiter
        trfmi = sample(ws, weights)
        xi = trfmi(xi)
        push!(set, xi)
    end
    return set
end

# Computes the attractor of an ifs via random algorithm sequentially.
function _randalg_sequential(ch::AbstractChannel, xinit, ws, numiter, probs)
    weights = Weights(probs)
    #TODO: Cancel the numiter, determine the post-error, K(contraction)
    for i = 1 : numiter
        trfmi = sample(ws, weights)
        xnew = trfmi(xinit)
        put!(ch, xnew)
        xinit = xnew
    end
    ch
end

function randalg_sequential(ws, set, numiter, probs, allocated::Bool=false)
    if allocated
        _randalg_sequential(set, ws, numiter, probs)
    else
        # NOTE: The set is assumed to have just a single initial point. If the set consists of more element, then we need a fix.
        ch = Channel(0)
        task = @async _randalg_sequential(ch, only(set), ws, numiter, probs)
        # task = @async _randalg_sequential(ch, ws, probs)
        bind(ch, task)
        ch
    end
end

function _randalg_sequential_generator(ch::AbstractChannel, ws, probs, xinit=nothing; num_iter=nothing, chunk_size=10, ϵ = 1e-8)
    # Compute initial set with a single point.
    if xinit === nothing
        n = size(ws[1].b)[1]
        xinit = rand(n)
    else
        n = size(xinit)[1]
    end

    # Compute number of iterations
    σ, index = findmax(contfactor.(ws))
    if num_iter === nothing
        # Compute num_iter with respect to ϵ
        x1 = ws[index](xinit)
        _k = (log(ϵ) - log(norm(x1 - xinit))) / log(σ) + 1
        k = Int(ceil(_k))
    else
        # Assign num_iter directly
        k = num_iter
    end

    # Compute transients
    weights = Weights(probs)
    xnew = xinit
    for i = 1 : k
        trfmi = sample(ws, weights)
        xnew = trfmi(xnew)
    end

    # Compute attractor
    chunk = zeros(n, chunk_size)
    while true
        for i = 1 : chunk_size
            trfmi = sample(ws, weights)
            xnew = trfmi(xnew)
            chunk[:,i] = xnew
        end
        put!(ch, chunk)
    end
end

function randalg_sequential_generator(ws, probs, args...)
    ch = Channel(0)
    task = @async _randalg_sequential_generator(ch, ws, probs, args...)
    bind(ch, task)
    ch
end

#TODO: Cancel the numiter, determine the post-error, K(contraction), choose num_iter from consraction factor

# function estimate_contraction_factor(ws)
#     σ = zeros(lenght(ws))
#     for i, transformation in ws
#         σ[i] = norm(transformation.A, inf)
#     end
#     # TODO : Returns nothing
#     σ
# end

# Computes the attractor of an ifs via random algorithm sequentially with placedependent probabilties.
function randalg_sequential_pd(ws, set, numiter, probs, α, β, allocated::Bool=false)
    # TODO: Impletement no allocation method
    xi = set[end]
    for i = 1 : numiter
        trfmi = sample(ws, Weights(probs))
        xi = trfmi(xi)
        probs = α * xi + β
    end
    set
end

# Computes the attractor of an ifs via random algorithm in parallel.
function randalg_parallel(ws, set, numiter, probs, allocated::Bool=false)
    # TODO: Impletement no allocation method
    weights = Weights(probs)
    loadprocs()
    vcat(pmap(process_chunk, [(ws, set, floor(Int, numiter / nworkers()), weights) for i =  1 : nworkers()])...)
end

# Computes the attractor of an ifs via random algorithm in parallel with placedependent probabilties.
function randalg_parallel_pd(ws, set, numiter, probs, α, β)
    loadprocs()
    vcat(pmap(process_chunk_pd, [(ws, set, floor(Int, numiter / nworkers()), probs, α, β) for i =  1 : nworkers()])...)
end

# `process_chunk` is the worker function that is used in all processes(both in master and worker process). when calculating
# the attractor if alg is `RandAlg` and `parallel` is true.
function process_chunk(ws_set_niter_weights)
    ws, set, niter, weights = ws_set_niter_weights
    xi = set[end]
    for i = 1 : niter
        wi = sample(ws, weights)
        xi = wi(xi)
        push!(set, xi)
    end
    set
end

# `process_chunk` is the worker function that is used in all processes(both in master and worker process). when calculating
# the attractor if alg is `RandAlg` and `parallel` is true with placedependent probabilties.
function process_chunk_pd(ws_set_niter_probs_alpha_beta)
    ws, set, niter, probs, α, β = ws_set_niter_probs_alpha_beta
    xi = set[end]
    for i = 1 : niter
        wi = sample(ws, Weights(probs))
        xi = wi(xi)
        probs = α * xi + β
        push!(set, xi)
    end
    set
end

# Load worker processes and load FractalTools to those worker processes.
function loadprocs(numprocs=Base.Sys.CPU_THREADS - 1 - nprocs())
    addprocs(numprocs)
    @everywhere @eval using FractalTools
end