# bayesian inference for gauss-gamma distribution

module GaussGamma
  using DataFrames, CSV, Plots, FreqTables, NamedArrays, Distributions
  using StatsBase, Statistics
  pyplot()

  function main(;test=false)
    # step1: generate prior distribution
    # initial parameters
    mu_0 = 200
    zeta_0 = 1
    alpha_0 = 1
    beta_0 = 2
    # prior gamma distribution of lambda
    lambda_dist = gen_lambda_dist(alpha_0, beta_0)
    draw_dist_pdf(lambda_dist, 0, 5, 0.01,
                  "src/bayesian_inference/prior_lambda_dist.png",
                  test)
    # prior normal distribution of mu
    lambda = rand(lambda_dist)
    mu_dist = gen_mu_dist(mu_0, zeta_0, lambda)
    draw_dist_pdf(mu_dist, 180, 220, 0.1,
                  "src/bayesian_inference/prior_mu_dist.png",
                  test)

    # step2: calculate posterior distribution with not many observation
    data_path = joinpath(split(@__FILE__, "src")[1], "data/sensor_data_200.txt")
    df_200_mm = CSV.read(data_path, DataFrame,
                         header=["date", "time", "ir", "lidar"],
                         delim=' ')
    lidar = df_200_mm.lidar
    samples = sample(lidar, 5)
    println("Samples(N=5): $(samples)")
    println("Mean(N=5): $(sum(samples)/length(samples))")
    println("Std dev(N=5): $(Statistics.std(samples, corrected=false))")
    # calculate parameters
    N = length(samples)
    mu_N = 1.0/(N+beta_0)*sum(samples) + beta_0/(N+beta_0)*mu_0
    zeta_N = N + zeta_0
    alpha_N = N/2 + alpha_0
    beta_N = 0.5*(sum([z^2 for z in samples]) + zeta_0*(mu_0^2) - zeta_N*(mu_N^2)) + beta_0
    println("$(mu_N) $(zeta_N) $(alpha_N) $(beta_N)")
    # posterior gamma distribution of lambda
    lambda_dist = gen_lambda_dist(alpha_N, beta_N)
    draw_dist_pdf(lambda_dist, 0, 5, 0.01,
                  "src/bayesian_inference/posterior_lambda_dist_N5.png",
                  test)
    draw_dist_pdf(lambda_dist, 0, 0.01, 0.00001,
                  "src/bayesian_inference/posterior_lambda_dist_N5_zoom.png",
                  test)
    # posterior normal distribution of mu
    lambda = rand(lambda_dist)
    mu_dist = gen_mu_dist(mu_N, zeta_N, lambda)
    draw_dist_pdf(mu_dist, 180, 220, 0.1,
                  "src/bayesian_inference/posterior_mu_dist_N5.png",
                  test)
    println("Mean(N=5): $(mu_dist.μ)")
    println("Std dev(N=5): $(sqrt(1/lambda))")

    # step3: calculate posterior distribution with a lot of observation
    samples = lidar # all data
    println("Samples(all): $(samples)")
    println("Mean(all): $(sum(samples)/length(samples))")
    println("Std dev(all): $(Statistics.std(samples, corrected=false))")
    # calculate parameters
    N = length(samples)
    mu_N = 1.0/(N+beta_0)*sum(samples) + beta_0/(N+beta_0)*mu_0
    zeta_N = N + zeta_0
    alpha_N = N/2 + alpha_0
    beta_N = 0.5*(sum([z^2 for z in samples]) + zeta_0*(mu_0^2) - zeta_N*(mu_N^2)) + beta_0
    println("$(mu_N) $(zeta_N) $(alpha_N) $(beta_N)")
    # posterior gamma distribution of lambda
    lambda_dist = gen_lambda_dist(alpha_N, beta_N)
    draw_dist_pdf(lambda_dist, 0, 5, 0.01,
                  "src/bayesian_inference/posterior_lambda_dist_all.png",
                  test)
    draw_dist_pdf(lambda_dist, 0.035, 0.05, 0.0001,
                  "src/bayesian_inference/posterior_lambda_dist_all_zoom.png",
                  test)
    # posterior normal distribution of mu
    lambda = rand(lambda_dist)
    mu_dist = gen_mu_dist(mu_N, zeta_N, lambda)
    draw_dist_pdf(mu_dist, 180, 220, 0.1,
                  "src/bayesian_inference/posterior_mu_dist_all.png",
                  test)
    println("Mean(all): $(mu_dist.μ)")
    println("Std dev(all): $(sqrt(1/lambda))")
    # compare with histogram
    # draw histogram
    bin_min_max = maximum(df_200_mm.lidar) - minimum(df_200_mm.lidar)
    histogram(df_200_mm.lidar, bins=bin_min_max, color=:orange,
              label="histogram")
    if test == false
      save_path = joinpath(split(@__FILE__, "src")[1], "src/bayesian_inference/lidar_200mm_histogram.png")
      savefig(save_path)
    end
    # calculated distribution
    z_dist = Normal(mu_dist.μ, sqrt(1/(lambda)))
    xs = range(190, 230, length=Int64(floor(40/0.1)))
    ys = [pdf(z_dist, x) for x in xs]
    plot(xs, ys, label="PDF", color=:blue)
    if test == false
      save_path = joinpath(split(@__FILE__, "src")[1], "src/bayesian_inference/lidar_200mm_calc_dist.png")
      savefig(save_path)
    end
  end

  function gen_lambda_dist(alpha, beta)
    return Gamma(alpha, 1/beta)
  end

  function gen_mu_dist(mu_mean, zeta, lambda)
    return Normal(mu_mean, sqrt(1/(zeta*lambda)))
  end

  function draw_dist_pdf(dist, range_min, range_max, step, save_name, test)
    xs = range(range_min, range_max, length=Int64(floor((range_max-range_min)/step)))
    ys = [pdf(dist, x) for x in xs]
    plot(xs, ys, label="PDF")

    if test == false
      save_path = joinpath(split(@__FILE__, "src")[1], save_name)
      savefig(save_path)
    end
  end
end


### A Pluto.jl notebook ###
# v0.16.1

using Markdown
using InteractiveUtils

# ╔═╡ ffed97e2-dc07-48bb-870a-737a115e69dc
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
    using NativeSVG
	using Plots
	using LaTeXStrings
	using GeneralQP
	using NLopt
end

# ╔═╡ 9f5f8730-0188-11eb-209f-e771579d35ce
md"# Sequential Quadratic Programming"

# ╔═╡ c30dbe90-0188-11eb-0c2b-ed5e93a86b0b
md"""## Quadratic Programming

An optimization problem with a quadratic objective function and linear
constraints is called a _quadratic program_. Problems of this
type are important in their own right, and they also arise a subproblems
in methods for general constrained optimization such as sequential
quadratic programming (this chapter) and interior-point methods (lecture 10).

The general quadratic program can be stated as

```math
\begin{aligned}
\min_{\vec{x}}\, & f\left(\vec{x}\right)\overset{\vartriangle}{=}\frac{1}{2}\vec{x}^\mathsf{T}Q\vec{x}-\vec{c}^\mathsf{T}\vec{x}\\
\textrm{subject to}\, & \begin{cases}
A_{\textrm{eq}}\vec{x}=\vec{b}_{\textrm{eq}}\,,\\
A_{\textrm{in}}\vec{x}\leq\vec{b}_{\textrm{in}}\,,
\end{cases}
\end{aligned}
```

where $Q$ is a symmetric $n\times n$ matrix, $\vec{c}\in\mathbb R^{n}$,
$A_{\textrm{eq}}$ is a $m\times n$ matrix, $\vec{b}_{\textrm{eq}}\in\mathbb R^{m}$,
$A_{\textrm{in}}$ is a $p\times n$ matrix and $\vec{b}_{\textrm{in}}\in\mathbb R^{p}$.
If the Hessian matrix $Q$ is positive semidefinite, we have a convex
quadratic program. Non-convex quadratic programs, in which $Q$ is
an indefinite matrix, can be more challenging because they have several
stationary points and local minima."""

# ╔═╡ e0be4cbe-0188-11eb-1991-5b001cf2434a
md"""## Equality Constraints

We begin our discussion of algorithms for quadratic programming by considering
the case in which only equality constraints are present. We consider
to following equality-constrained quadratic problem
```math
\begin{aligned}
\min_{\vec{x}}\, & f\left(\vec{x}\right)\overset{\vartriangle}{=}\frac{1}{2} \vec{x}^\mathsf{T}Q\vec{x}- \vec{c}^\mathsf{T}\vec{x}\\
\textrm{subject to}\, & A\vec{x}=\vec{b}\,,\nonumber
\end{aligned}
```
where $A$ is the $m\times n$ Jacobian of constraints and $\vec{b}\in\mathbb R^{m}$.
We assume that $A$ has rank $m$ so that the constraints are consistent.

The First Order Necessary Condition (FONC) uses the Langrangian function
```math
\mathcal L\left(\vec x, \vec \lambda\right)=\vec{x}^\mathsf{T}Q\vec{x}- \vec{c}^\mathsf{T}\vec{x} + \vec\lambda^\mathsf{T} \left(A\vec{x}-\vec{b}\right)
```
and gives the following condition
```math
\begin{aligned}
\nabla_{\vec x}\mathcal L\left(\vec x ^\star, \vec \lambda ^\star\right)&=\vec 0\,,\\
\nabla_{\vec \lambda}\mathcal L\left(\vec x ^\star, \vec \lambda ^\star\right)&=\vec 0\,,
\end{aligned}
```
So the FONC for $\vec{x}^{\star}$ to be a solution of the quadratic
problem yields a vector $\vec{\lambda}^{\star}$ such
that the following system of equations is satisfied:
```math
\begin{pmatrix}Q & A^\mathsf{T}\\
A & 0
\end{pmatrix}\begin{pmatrix}\vec{x}^{\star}\\
\vec{\lambda}^{\star}
\end{pmatrix}=\begin{pmatrix}\vec{c}\\
\vec{b}
\end{pmatrix}\,.
```
This system can be solved directly by factorization. An alternative
is to use an iterative method.

Consider the quadratic programming problem
```math
\begin{aligned}
\min_{\vec{x}}\, & \frac{1}{2} \vec{x}^\mathsf{T}\begin{pmatrix}6 & 2 & 1\\
2 & 5 & 2\\
1 & 2 & 4
\end{pmatrix}\vec{x}- \begin{pmatrix}8\\
3\\
3
\end{pmatrix}^\mathsf{T}\vec{x}\\
\textrm{subject to}\, & \begin{pmatrix}1 & 0 & 1\\
0 & 1 & 1
\end{pmatrix}\vec{x}=\begin{pmatrix}3\\
0
\end{pmatrix}\,.
\end{aligned}
```"""

# ╔═╡ 56fd4bc0-0189-11eb-04c2-0d000ded6e97
let
	Q = [6 2 1
     	 2 5 2
         1 2 4]
	c = [8, 3, 3]
	A = [1 0 1
		 0 1 1]
	b = [3, 0]
	sol = [Q transpose(A)
		   A zeros(2,2)] \ [c; b]
end

# ╔═╡ 80edfd80-0189-11eb-1687-01a14b31ad39
md"""The solution $\vec{x}^{\star}$ and optimal Lagrange multiplier vector $\vec \lambda^\star$ are given by
```math
\begin{aligned}
\vec{x}^{\star} & = \begin{pmatrix}2 & -1 & 1\end{pmatrix}^\mathsf{T}\,,\\
\vec{\lambda}^{\star} & = \begin{pmatrix}-3 & 2\end{pmatrix}^\mathsf{T}\,.
\end{aligned}
```"""

# ╔═╡ 8f340330-0189-11eb-27c8-7ba7c0954ac7
md"""## Active Set Method

We now describe active-set methods for solving quadratic programs
containing both equality and inequality constraints. We consider only
the convex case, in which the matrix ``Q`` is positive semidefinite.

If the contents of the optimal active set ``J\left(\vec x^\star\right)`` were known in advance, we could find the solution ``\vec{x}^{\star}``
by applying the technique for equality constrained quadratic programs
to the problem
```math
\begin{aligned}
\min_{\vec{x}}\, & f\left(\vec{x}\right)\overset{\vartriangle}{=}\frac{1}{2} \vec{x}^\mathsf{T}Q\vec{x}- \vec{c}^\mathsf{T}\vec{x}\\
\textrm{subject to}\, & \begin{cases}
A_{\textrm{eq}}\vec{x}=\vec{b}_{\textrm{eq}}\,,\\
 \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}=b_{\textrm{in},j}\,, & \forall j\in J\left(\vec{x}^{\star}\right)\,,
\end{cases}
\end{aligned}
```
where ``\vec{a}_{\textrm{in},j}`` is the ``j``th row in the matrix ``A_{\textrm{in}}``
and ``b_{\textrm{in},j}`` is the ``j``th element of the vector ``\vec{b}_{\textrm{in}}``.
Of course, we usually do not have prior knowledge of ``J\left(\vec{x}^{\star}\right)``
and determination of this set is the main challenge facing algorithms
for inequality-constrained quadratic programs.

Active-set methods find a step from one iterate to the next by solving
a quadratic subproblem in which some of the inequality constraints,
and all the equality constraints are imposed as equalities. This subset
is referred to as the _working set_ and is denoted at the ``k``th
iterate by ``W_{k}``. An important requirement we impose on ``W_{k}``
is that the gradients ``\vec{a}_{\textrm{eq},i}``, ``i=1,\dots,m`` and
``\vec{a}_{\textrm{in},j}``, ``j\in W_{k}`` are linearly independent,
even when the full set of active constraints at that point has linearly
dependent gradients.

Given an iterate ``\vec{x}^{\left(k\right)}`` and the working set ``W_{k}``,
we first check whether ``\vec{x}^{\left(k\right)}`` minimizes the quadratic
function ``f`` in the subspace defined by the working set. If not,
we compute a step ``\vec{d}^{\left(k\right)}`` by solving an equality-constrained
quadratic subproblem in which the inequality constraints corresponding
to the working set ``W_{k}`` are regarded as equalities and the other
inequality constraints are temporarily disregarded. To express this
subproblem in terms of the step ``\vec{d}^{\left(k\right)}``, we define
```math
\vec{d}^{\left(k\right)}=\vec{x}^{\left(k+1\right)}-\vec{x}^{\left(k\right)}\,,\quad\vec{g}_{k}=Q\vec{x}^{\left(k\right)}-\vec{c}\,.
```
By substituting for ``\vec{x}^{\left(k+1\right)}`` into the objective
function, we find that
```math
f\left(\vec{x}^{\left(k+1\right)}\right)=f\left(\vec{x}^{\left(k\right)}+\vec{d}^{\left(k\right)}\right)=\frac{1}{2}\left(\vec{d}^{\left(k\right)}\right)^\mathsf{T}Q\vec{d}^{\left(k\right)}+\vec{g}_{k}^\mathsf{T}\vec{d}^{\left(k\right)}+\rho_{k}\,,
```
where ``\rho_{k}=\frac{1}{2} \left(\vec{x}^{\left(k\right)}\right)^\mathsf{T}Q\vec{x}^{\left(k\right)}- \vec{c}^\mathsf{T}\vec{x}^{\left(k\right)}``
is independent of ``\vec{d}^{\left(k\right)}``. Since we can drop ``\rho_{k}``
from the objective function without changing the solution of the problem,
we can write the subproblem to be solved at the ``k``th iteration as
follows
```math
\begin{aligned}
\min_{\vec{d}^{\left(k)\right)}}\, & \frac{1}{2} \left(\vec{d}^{\left(k\right)}\right)^\mathsf{T}Q\vec{d}^{\left(k\right)}+ \vec{g}_{k}^\mathsf{T}\vec{d}^{\left(k\right)}\label{eq:quadratic_subproblem}\\
\textrm{subject to}\, & \begin{cases}
A_{\textrm{eq}}\vec{d}^{\left(k\right)}=\vec{0}\,,\\
 \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}=0\,, & \forall j\in W_{k}\,.
\end{cases}
\end{aligned}
```
Note that for each ``j\in W_{k}``, the value of `` \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}^{\left(k\right)}``
does not change as we move along ``\vec{d}^{\left(k\right)}``, since
we have `` \vec{a}_{\textrm{in},j}^\mathsf{T}\left(\vec{x}^{\left(k\right)}+\alpha\vec{d}^{\left(k\right)}\right)= \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}^{\left(k\right)}=b_{\textrm{in},j}``
for all ``\alpha``. Since the constraints in ``W_{k}`` were satisfied
at ``\vec{x}^{\left(k\right)}``, they are also satisfied at ``\vec{x}^{\left(k\right)}+\alpha\vec{d}^{\left(k\right)}``,
for any value of ``\alpha``.

Supposing for the moment that the optimal ``\vec{d}^{\left(k\right)}``
is nonzero, we need to decide how far to move along this direction.
If ``\vec{x}^{\left(k\right)}+\vec{d}^{\left(k\right)}`` is feasible
with respect to all the constraints, we set ``\vec{x}^{\left(k+1\right)}=\vec{x}^{\left(k\right)}+\vec{d}^{\left(k\right)}``.
Otherwise, we set
```math
\vec{x}^{\left(k+1\right)}=\vec{x}^{\left(k\right)}+\alpha_{k}\vec{d}^{\left(k\right)}\,,
```
where the step-length parameter ``\alpha_{k}`` is chosen to be the
largest value in the range ``\left[0,1\right]`` for which all constraints
are satisfied. We can derive an explicit definition of ``\alpha_{k}``
by considering what happens to the constraints ``j\notin W_{k}``, since
the constraints ``j\in W_{k}`` will certainly be satisfied regardless
of the choice of ``\alpha_{k}``. If `` \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}\leq0``
for some ``j\notin W_{k}``, then for all ``\alpha_{k}\geq0``, we have
`` \vec{a}_{\textrm{in},j}^\mathsf{T}\left(\vec{x}^{\left(k\right)}+\alpha_{k}\vec{d}^{\left(k\right)}\right)\leq \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}^{\left(k\right)}\leq b_{\textrm{in},j}``.
Hence, constraint ``j`` will be satisfied for all nonnegative choices
of the step-length parameter. Whenever `` \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}>0``
for some ``j\notin W_{k}``, however, we have that `` \vec{a}_{\textrm{in},j}^\mathsf{T}\left(\vec{x}^{\left(k\right)}+\alpha_{k}\vec{d}^{\left(k\right)}\right)\leq b_{\textrm{in},j}``
only if
```math
\alpha_{k}\leq\frac{b_{\textrm{in},j}- \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}^{\left(k\right)}}{ \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}}\,.
```
To maximize the decrease in ``f``, we want ``\alpha_{k}`` to be as large
as possible in ``\left[0,1\right]`` subject to retaining feasibility,
so we obtain the following definition
```math
\alpha_{k}\overset{\textrm{def}}{=}\min\left\{ 1,\min_{j\notin W_{k}, \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}>0}\frac{b_{\textrm{in},j}- \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{x}^{\left(k\right)}}{ \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}}\right\} \,.
```
We call the constraints ``j`` for which this minimum is achieved the
_blocking constraints_. Note that it is quite possible for ``\alpha_{k}``
to be zero, because we could have `` \vec{a}_{\textrm{in},j}^\mathsf{T}\vec{d}^{\left(k\right)}>0``
for some constraint ``j`` that is active at ``\vec{x}^{\left(k\right)}``
but not a member of the current working set ``W_{k}``.

If ``\alpha_{k}<1``, that is, the step along ``\vec{d}_{k}`` was blocked
by some constraint not in ``W_{k}``, a new working set ``W_{k+1}`` is
constructed by adding one of the blocking constraints to ``W_{k}``.
We continue to iterate in this manner, adding constraints to the working
set until the subproblem has solution ``\vec{d}^{\circ}=\vec{0}``.
Since ``\vec{d}^{\circ}=\vec{0}`` satisfy the optimality condition, we have that
```math
\sum_{i=1}^{m}\lambda_{i}^{\circ}\vec{a}_{\textrm{eq},i}+\sum_{j\in W^{\circ}}\mu_{j}^{\circ}\vec{a}_{\textrm{in},j}=-\vec{g}^{\circ}=-Q\vec{x}^{\circ}+\vec{c}\,,
```
for some Lagrange multipliers ``\lambda_{i}^{\circ}``, ``i=1,\dots,m``
and ``\mu_{j}^{\circ}``, ``j\in W^{\circ}``. It follows that ``\vec{x}^{\circ}``,
``\vec{\lambda}^{\circ}`` and ``\vec{\mu}^{\circ}`` satisfy the second
KKT condition, if we define the multipliers corresponding to the inequality
constraints not in the working set to be zero. Because of the control
imposed on the step length, ``\vec{x}^{\circ}`` is also feasible with
respect to all the constraints, so the third, fourth and fifth KKT
conditions are satisfied at this point.

We now examine the signs of the KKT multipliers in the working set,
that is, the indices ``j\in W^{\circ}``. If these conditions are all
nonnegative, the first KKT condition is also satisfied, so we conclude
that ``\vec{x}^{\circ}`` is a KKT point for the original problem. In
fact, since ``Q`` is positive semidefinite, we have that ``\vec{x}^{\circ}``
is a global minimum.

If, on the other hand, on or more of the multipliers ``\mu_{j}^{\circ}``,
``j\in W^{\circ}``, are negative, the first KKT condition is not satisfied
and the objective function ``f`` may be decreased by dropping one of
these constraints. Thus, we remove an index ``j`` corresponding to
one of the negative multipliers from the working set and solve a new
subproblem for the next step. While any index ``j`` for which ``\mu_{j}^{\circ}<0``
usually will yield in a direction ``\vec{d}`` along which the algorithm
can make progress, the most negative multiplier is often chosen in
practice. This choice is motived by a sensitivity analysis, which
shows that the rate of decrease in the objective function when one
constraint is removed, is proportional to the magnitude of the Lagrange
multiplier for that constraint."""

# ╔═╡ db088c90-0189-11eb-338d-81af9469d556
md"""## Example

Apply the active-set method to the following problem:
```math
\begin{aligned}
\min_{\vec{x}}\, & f\left(\vec{x}\right)=\left(x_{1}-1\right)^{2}+\left(x_{2}-2.5\right)^{2}\\
\textrm{subject to}\, & \begin{cases}
-x_{1}+2x_{2}-2\leq0\,,\\
x_{1}+2x_{2}-6\leq0\,.\\
x_{1}-2x_{2}-2\leq0\,,\\
-x_{1}\leq0\,,\\
-x_{2}\leq0\,.
\end{cases}
\end{aligned}
```"""

# ╔═╡ 6bdfb630-018a-11eb-1010-b3c332121581
begin
	Q = Float64[2 0
                0 2]
	c = Float64[2, 5]
	A = Float64[-1  2
				 1  2
				 1 -2
				-1  0
				 0 -1]
	b = Float64[2, 6, 2, 0, 0]

	x = -1:0.05:5
	y = -1:0.05:5
	z = Surface((x,y)->(0.5 .* [x y]*Q*[x;y] .- transpose(c)*[x;y] .+ 0.5 .* transpose(c)*Q*c)[1], x, y)
	contour(x, y, z, levels=35)
	plot!(x, (2 .+ x) ./ 2, linestyle=:dash, label=L"-x_1+2x_2-2\le 0")
	plot!(x, (6 .- x) ./ 2, linestyle=:dash, label=L"x_1+2x_2-6\le 0")
	plot!(x, (2 .- x) ./ -2, linestyle=:dash, label=L"x_1-2x_2-2\le 0")
	plot!([0,2,4,2,0,0],[0,0,1,2,1,0], linewidth=2, label="domain")
end

# ╔═╡ 88c16630-018b-11eb-3ea1-a555d69b2e26
md"""We refer the constraints, in order, by indices $1$ through $5$.
For this problem it is easy to determine a feasible initial point;
say $\vec{x}^{\left(0\right)}=\begin{pmatrix}2 & 0\end{pmatrix}\mathsf{T}$.
Constraints $3$ and $5$ are active at this point, and we set $W_{0}=\left\{ 3,5\right\} $.
Note that we could just as validly have chosen $W_{0}=\left\{ 5\right\} $
or $W_{0}=\left\{ 3\right\} $ or even $W_{0}=\emptyset$; each choice
would lead the algorithm to perform somewhat differently."""

# ╔═╡ 4a460a20-018e-11eb-3da1-4d37893ead18
x₀ = [2, 0]

# ╔═╡ 8df4e190-018b-11eb-2eae-1d8f6c4e6248
let x = x₀
	g = Q*x - c
	ain = [reshape(A[3,:], 1, 2); reshape(A[5,:], 1, 2)]
	sol = [Q transpose(ain)
		   ain zeros(2,2)] \ [-g; 0; 0]
end

# ╔═╡ a8042d20-018b-11eb-0f74-3b70e587f20d
md"""Since $\vec{x}^{\left(0\right)}$ lies on a vertex of the feasible
region, it is obviously a minimizer of the objective function $f$
with respect to the working set $W_{0}$; that is, the solution of
the subproblem with $k=0$ is $\vec{d}^{\left(0\right)}=\vec{0}$.
We can then find the multipliers $\mu_{3}^{\circ}$ and $\mu_{5}^{\circ}$
associated with the active constraints. Substitution of the data from
our problem yields
```math
\begin{pmatrix}-1\\
2
\end{pmatrix}\lambda_{3}^{\circ}+\begin{pmatrix}0\\
1
\end{pmatrix}\lambda_{5}^{\circ}=\begin{pmatrix}2\\
-5
\end{pmatrix}\,,
```
which has solution $\lambda_{3}^{\circ}=-2$ and $\lambda_{5}^{\circ}=-1$.

We now remove constraint $3$ from the working set, because it has
the most negative multiplier, and set $W_{1}=\{5\}$."""

# ╔═╡ b9b7e250-018b-11eb-1eca-51dc8250a3eb
sol₁ = let x = x₀
	g = Q*x - c
	ain = reshape(A[5,:], 1, 2)
	sol = [Q transpose(ain)
		   ain zeros(1,1)] \ [-g; 0]
end

# ╔═╡ e14bb850-018b-11eb-0bb0-15ebdb2f7fcf
md"""We begin iteration $1$ by finding the solution of the subproblem for $k=1$, which is
$\vec{d}^{\left(1\right)}= \begin{pmatrix}-1 & 0\end{pmatrix}^\mathsf{T}$."""

# ╔═╡ ea6595a0-018b-11eb-07dc-f394137f59f4
α₁, d₁ = let x = x₀
	d = sol₁[1:2]
	α = min(1.0, [(reshape(A[j,:], 1, 2) * d)[1,1] ≤ 0 ? 1.0 : ((b[j] .- reshape(A[j,:], 1, 2) * x) / (reshape(A[j,:], 1, 2) * d))[1,1] for j in (1,2,3,4)]...)
	α, d
end

# ╔═╡ 0d2eeb40-018c-11eb-0fdc-1b6668c5941f
md"""The step-length formula yields $\alpha_{1}=1$, and the new iterate
is $\vec{x}^{\left(2\right)}=\begin{pmatrix}1 & 0\end{pmatrix}^\mathsf{T}$. There are no blocking constraints, so that ``W_{2}=W_{1}=\left\{ 5\right\}``"""

# ╔═╡ ee702bf0-018c-11eb-1ec9-6b481c119add
x₁ = x₀ + α₁ .* d₁

# ╔═╡ 2cb44400-018d-11eb-0138-c78f6a18b8f8
sol₂ = let x = x₁
	g = Q*x - c
	ain = reshape(A[5,:], 1, 2)
	sol = [Q transpose(ain)
		   ain zeros(1,1)] \ [-g; 0]
end

# ╔═╡ 456d9c82-018d-11eb-0e81-bf7346539a32
md"""We find at the start of iteration $2$ that the solution of the
subproblem is $\vec{d}^{\left(2\right)}=\vec{0}$. We deduce that
the Lagrange multiplier for the lone working constraint is $\lambda_{5}^{\circ}=-5$,
so we drop the working set to obtain $W_{3}=\emptyset$."""

# ╔═╡ 84712210-018f-11eb-0bef-53cdfb93e0e2
x₂ = x₁

# ╔═╡ 96ecfeb2-018e-11eb-2dda-994575ee6011
sol₃ = let x = x₂
	g = Q*x - c
	sol = Q \ (-g)
end

# ╔═╡ 2ddf0860-018d-11eb-2884-7dacdda18597
md"""Iteration $3$ starts by solving the unconstrained problem, to obtain
the solution $\vec{d}^{\left(3\right)}=\begin{pmatrix}0 & 2.5\end{pmatrix}^\mathsf{T}$."""

# ╔═╡ c71e2f00-018e-11eb-1c30-a54d6d4d04ff
α₃, d₃ = let x = x₂
	d = sol₃[1:2]
	α = min(1.0, [(reshape(A[j,:], 1, 2) * d)[1,1] ≤ 0 ? 1.0 : ((b[j] .- reshape(A[j,:], 1, 2) * x) / (reshape(A[j,:], 1, 2) * d))[1,1] for j in (1,2,3,4,5)]...)
	α, d
end

# ╔═╡ e470e5c0-018e-11eb-229b-e521909f84e3
md"""The step-length formula yields a step length of $\alpha_{3}=0.6$
and a new iterate $\vec{x}^{\left(4\right)}=\begin{pmatrix}1 & 1.5\end{pmatrix}^\mathsf{T}$. There
is a single blocking constraint (constraint $1$), so we obtain ``W_{4}=\left\{ 1\right\}``."""

# ╔═╡ 596ba4f0-018f-11eb-321e-03ec8a6d18db
x₃ = x₂ + α₃ .* d₃

# ╔═╡ 91ada700-018f-11eb-37d0-757a263bae99
sol₄ = let x=x₃
	g = Q*x - c
	ain = reshape(A[1,:], 1, 2)
	sol = [Q transpose(ain)
		   ain zeros(1,1)] \ [-g; 0]
end

# ╔═╡ abfbf760-018f-11eb-18dc-bd573cc8be06
md"""The solution of the subproblem for $k=4$ is then $\vec{d}^{\left(4\right)}=\begin{pmatrix}0.4 & 0.2\end{pmatrix}^\mathsf{T}$."""

# ╔═╡ ba0f5450-018f-11eb-2188-4327aa9c02cc
α₄, d₄ = let x = x₃
	d = sol₄[1:2]
	α = min(1.0, [(reshape(A[j,:], 1, 2) * d)[1,1] ≤ 0 ? 1.0 : ((b[j] .- reshape(A[j,:], 1, 2) * x) / (reshape(A[j,:], 1, 2) * d))[1,1] for j in (2,3,4,5)]...)
	α, d
end

# ╔═╡ d8c62ef0-018f-11eb-21ce-ebe037f9d91d
md"""The new step-length is $1$. There are no blocking constraints
on this step, so the next working set in unchanged: $W_{5}=\left\{ 1\right\} $. The new iterate is $\vec{x}^{\left(5\right)}=\begin{pmatrix}1.4 & 1.7\end{pmatrix}^\mathsf{T}$."""

# ╔═╡ e59a7c30-018f-11eb-2b94-7b8dff2a6383
x₄ = x₃ + α₄ .* d₄

# ╔═╡ f5f705d0-018f-11eb-2462-e7e2265b700c
sol₅ = let x = x₄
	g = Q*x - c
	ain = reshape(A[1,:], 1, 2)
	sol = [Q transpose(ain)
		   ain zeros(1,1)] \ [-g; 0]
end

# ╔═╡ 1254d540-0190-11eb-3066-b5a853eef0c0
md"""Finally, we solve the subproblem for $k=5$ to obtain a solution $\vec{d}^{\left(5\right)}=\vec{0}$.
We find a multiplier $\mu_{1}^{\circ}=0.8$, so we have found the
solution. Wet set $\vec{x}^{\star}=\begin{pmatrix}1.4 & 1.7\end{pmatrix}^\mathsf{T}$
and terminate."""

# ╔═╡ 02787820-0190-11eb-24ed-c5296c91df6b
let
	x = -1:0.05:5
	y = -1:0.05:5
	z = Surface((x,y)->(0.5 .* [x y]*Q*[x;y] .- transpose(c)*[x;y] .+ 0.5 .* transpose(c)*Q*c)[1], x, y)
	contour(x, y, z, levels=35)
	plot!(x, (2 .+ x) ./ 2, linestyle=:dash, label=L"-x_1+2x_2-2\le 0")
	plot!(x, (6 .- x) ./ 2, linestyle=:dash, label=L"x_1+2x_2-6\le 0")
	plot!(x, (2 .- x) ./ -2, linestyle=:dash, label=L"x_1-2x_2-2\le 0")
	plot!([0,2,4,2,0,0],[0,0,1,2,1,0], linewidth=2, label="domain")
	plot!([2,2,1,1,1,1.4], [0,0,0,0,1.5,1.7], linewidth=2, label="iterates", markershape=:circle)
end

# ╔═╡ aa606210-018a-11eb-29bd-5d4f9ae0966e
md"""## Julia

The package `GeneralQP` solves the quadratic programming problems using the function `solve(Q, -c, A, b, x₀)`."""

# ╔═╡ f3007460-018a-11eb-3fa2-8753aceaf89a
sol = GeneralQP.solve(Q, -c, A, b, Float64[2,0])

# ╔═╡ 24c67580-018b-11eb-3eb7-47a2e2ae5251
md"""## Sequential Quadratic Programming

We consider the general constrained problem
```math
\begin{aligned}
\min\, & f\left(\vec{x}\right)\\
\textrm{subject to} & \begin{cases}
\vec{h}\left(\vec{x}\right)=\vec{0}\\
\vec{g}\left(\vec{x}\right)\leq\vec{0}
\end{cases}
\end{aligned}
```
where $f:\mathbb{R}^{n}\rightarrow\mathbb{R}$, $\vec{h}:\mathbb{R}^{n}\rightarrow\mathbb{R}^{m}$,
$m\leq n$, and $\vec{g}:\mathbb{R}^{n}\rightarrow\mathbb{R}^{p}$. The idea
behind the _sequential quadratic programming_ (SQP) approach
is to model the general problem at the current iterate $\vec{x}^{\left(k\right)}$
by a quadratic programming subproblem, then use the minimizer of this
subproblem to define a new iterate $\vec{x}^{\left(k+1\right)}$.
The challenge is to design the quadratic subproblem so that it yields
a good step for the general optimization problem.

We know that the extended Lagrangian function for this problem is

```math
\mathcal{L}\left(\vec{x},\vec{\lambda},\vec{\mu}\right)=f\left(\vec{x}\right)+\vec{\lambda}^\mathsf{T}\vec{h}\left(\vec{x}\right)+\vec{\mu}^\mathsf{T}\vec g\left(\vec{x}\right)\,.
```

Applying Newton's method to the Lagrangian function and linearizing
both the equality and the inequality constraints yields the following
subproblem
```math
\begin{aligned}
\min_{\vec{d}^{\left(k\right)}}\, & \frac{1}{2}\left(\vec{d}^{\left(k\right)}\right)^\mathsf{T} \mathsf{H} \mathcal{L}\left(\vec{x}^{\left(k\right)},\vec{\lambda}^{\left(k\right)},\vec{\mu}^{\left(k\right)}\right)\vec{d}^{\left(k\right)}+ \mathsf{D} \mathcal{L}\left(\vec{x}^{\left(k\right)},\vec{\lambda}^{\left(k\right)},\vec{\mu}^{\left(k\right)}\right)\vec{d}^{\left(k\right)}\\
\textrm{subject to}\, & \begin{cases}
 \mathsf{J}\vec{h}\left(\vec{x}^{\left(k\right)}\right)\vec{d}^{\left(k\right)}=-\vec{h}\left(\vec{x}^{\left(k\right)}\right)\\
 \mathsf{D} g_{j}\left(\vec{x}^{\left(k\right)}\right)\vec{d}^{\left(k\right)}=-g_{j}\left(\vec{x}^{\left(k\right)}\right)\,, & j\in W_{k}\,,
\end{cases}
\end{aligned}
```

where $\mu_{j}^{\left(k\right)}=0$, for all
$j\notin W_{k}$. We can use the active-set method for quadratic programming
to solve this subproblem. The new iterate is given by $\vec{x}^{\left(k+1\right)}$,
$\vec{\lambda}^{\left(k+1\right)}$, $\vec{\mu}^{\left(k+1\right)}$
and $W_{k+1}$.

If the SQP method is able to identify the optimal active set then
it will act like a Newton method for equality-constrained optimization
and will converge rapidly.

It is also remarkable that, far from the solution, the SQP approach
is usually able to improve the estimate of the active set and guide
the iterates towards a solution.

Non-quadratic objective functions, however, can impede progress of
the SQP algorithm, a phenomenon known as the Maratos effect. Steps
that make good progress toward a solution are rejected and the algorithm
fails to converge rapidly. These difficulties can be overcome by means
of a _second-order correction_."""

# ╔═╡ a98c8890-0190-11eb-25a4-bfd18c42b749
function myfunc(x::Vector, grad::Vector)
    if length(grad) > 0
        grad[1] = 0
        grad[2] = 0.5/sqrt(x[2])
    end
    return sqrt(x[2])
end

# ╔═╡ acfea4e0-0190-11eb-07a9-a3f38d97ff80
function myconstraint(x::Vector, grad::Vector, a, b)
    if length(grad) > 0
        grad[1] = 3a * (a*x[1] + b)^2
        grad[2] = -1
    end
    (a*x[1] + b)^3 - x[2]
end

# ╔═╡ a4824010-0190-11eb-0d97-a55353709552
let
	opt = Opt(:LD_SLSQP, 2)
	opt.lower_bounds = [-Inf, 0.]
	opt.xtol_rel = 1e-4

	opt.min_objective = myfunc
	inequality_constraint!(opt, (x,g) -> myconstraint(x,g,2,0), 1e-8)
	inequality_constraint!(opt, (x,g) -> myconstraint(x,g,-1,1), 1e-8)

	(minf, minx, ret) = optimize(opt, [1.234, 5.678])
	numevals = opt.numevals # the number of function evaluations
	minf, minx, numevals, ret
end

# ╔═╡ Cell order:
# ╟─ffed97e2-dc07-48bb-870a-737a115e69dc
# ╟─9f5f8730-0188-11eb-209f-e771579d35ce
# ╟─c30dbe90-0188-11eb-0c2b-ed5e93a86b0b
# ╟─e0be4cbe-0188-11eb-1991-5b001cf2434a
# ╠═56fd4bc0-0189-11eb-04c2-0d000ded6e97
# ╟─80edfd80-0189-11eb-1687-01a14b31ad39
# ╟─8f340330-0189-11eb-27c8-7ba7c0954ac7
# ╟─db088c90-0189-11eb-338d-81af9469d556
# ╠═6bdfb630-018a-11eb-1010-b3c332121581
# ╟─88c16630-018b-11eb-3ea1-a555d69b2e26
# ╠═4a460a20-018e-11eb-3da1-4d37893ead18
# ╠═8df4e190-018b-11eb-2eae-1d8f6c4e6248
# ╟─a8042d20-018b-11eb-0f74-3b70e587f20d
# ╠═b9b7e250-018b-11eb-1eca-51dc8250a3eb
# ╟─e14bb850-018b-11eb-0bb0-15ebdb2f7fcf
# ╠═ea6595a0-018b-11eb-07dc-f394137f59f4
# ╟─0d2eeb40-018c-11eb-0fdc-1b6668c5941f
# ╠═ee702bf0-018c-11eb-1ec9-6b481c119add
# ╠═2cb44400-018d-11eb-0138-c78f6a18b8f8
# ╟─456d9c82-018d-11eb-0e81-bf7346539a32
# ╠═84712210-018f-11eb-0bef-53cdfb93e0e2
# ╠═96ecfeb2-018e-11eb-2dda-994575ee6011
# ╟─2ddf0860-018d-11eb-2884-7dacdda18597
# ╠═c71e2f00-018e-11eb-1c30-a54d6d4d04ff
# ╟─e470e5c0-018e-11eb-229b-e521909f84e3
# ╠═596ba4f0-018f-11eb-321e-03ec8a6d18db
# ╠═91ada700-018f-11eb-37d0-757a263bae99
# ╟─abfbf760-018f-11eb-18dc-bd573cc8be06
# ╠═ba0f5450-018f-11eb-2188-4327aa9c02cc
# ╟─d8c62ef0-018f-11eb-21ce-ebe037f9d91d
# ╠═e59a7c30-018f-11eb-2b94-7b8dff2a6383
# ╠═f5f705d0-018f-11eb-2462-e7e2265b700c
# ╟─1254d540-0190-11eb-3066-b5a853eef0c0
# ╠═02787820-0190-11eb-24ed-c5296c91df6b
# ╟─aa606210-018a-11eb-29bd-5d4f9ae0966e
# ╠═f3007460-018a-11eb-3fa2-8753aceaf89a
# ╟─24c67580-018b-11eb-3eb7-47a2e2ae5251
# ╠═a98c8890-0190-11eb-25a4-bfd18c42b749
# ╠═acfea4e0-0190-11eb-07a9-a3f38d97ff80
# ╠═a4824010-0190-11eb-0d97-a55353709552


module problem104

description = """
Pandigital Fibonacci ends
Problem 104

The Fibonacci sequence is defined by the recurrence relation:

    Fn = Fn−1 + Fn−2, where F1 = 1 and F2 = 1.

It turns out that F541, which contains 113 digits, is the first Fibonacci number for which the last nine digits are 1-9 pandigital (contain all the digits 1 to 9, but not necessarily in order). And F2749, which contains 575 digits, is the first Fibonacci number for which the first nine digits are 1-9 pandigital.

Given that Fk is the first Fibonacci number for which the first nine digits AND the last nine digits are 1-9 pandigital, find k.
"""

# test if the string representation of a number starts with digits 1-9
is_start_pandigital(x :: String) = (length(x) >= 9) && all([in(c, x[1:9]) for c = '1':'9'])

# test if an integer ends with digits 1-9
# This has been optimized to avoid memory allocations as it is the inner loop
function is_end_pandigital(x :: Int, digits=zeros(Bool, 10))
  digits[:] = false
  for i = 1:9
    digits[x % 10  + 1] = true
    x = div(x, 10)
  end
  for i = 2:10
    if !digits[i]
      return false
    end
  end
  return true
end

# Find the first fibonacci that starts with and ends with digits 1-9
# Calculate fibonacci sequence in BigInts, and in Int64s (modulo 10^10)
# Test for end-pandigital on the native ints, and when found, test the BigInts for start-pandigital
function find_start_end_pandigital()
  a, b = 1, 1
  A, B = BigInt(a), BigInt(b)
  buf = zeros(Bool, 10)
  for k = 3:1000000
    a, b = b, (a+b) % 10000000000
    A, B = B, (A+B)
    if is_end_pandigital(b, buf)
      if is_start_pandigital(string(B))
        return k
      end
    end
  end
end

using Base.Test

@test is_start_pandigital("3219485766456")
@test !is_start_pandigital("10201049586723")
@test is_end_pandigital(958928534791826)

end


# # Optimal advertising

# This example is taken from <https://web.stanford.edu/~boyd/papers/pdf/cvx_applications.pdf>.

# Setup:
#
# * We have $m$ adverts and $n$ timeslots
# * The total traffic in time slot $t$ is $T_t$
# * The number of ad $i$ displayed in period $t$ is $D_{it} \geq 0$
# * We require $\sum_{i=1}^m D_{it} \leq T_t$ since we cannot show more than $T_t$ ads during time slot $t$.
# * We require $\sum_{t=1}^n D_{it} \geq c_i$ to fulfill a contract to show advertisement $i$ at least $c_i$ times.
#
# Goal: Choose $D_{it}$.
#
# For some empirical $P_{it}$ with $0 \leq P_{it} \leq 1$, we obtain $C_{it} = P_{it}D_{it}$ clicks for ad $i$, which pays us some number $R_i > 0$ up to a budget $B_i$.
# The ad revenue for ad $i$ is $S_i = \min( R_i \sum_t C_{it}, B_i )$ which is concave in $D$.
# We aim to maximize $\sum_i S_i$.

using Random
using Distributions: LogNormal
Random.seed!(1);

m = 5; # number of adverts
n = 24; # number of timeslots
SCALE = 10000;
B = rand(LogNormal(8), m) .+ 10000;
B = round.(B, digits=3); # Budget

P_ad = rand(m);
P_time = rand(1,n);
P = P_ad * P_time;

T = sin.(range(-2*pi/2, stop=2*pi-2*pi/2, length=n)) * SCALE;
T .+= -minimum(T) + SCALE; # traffic
c = rand(m); # contractual minimum
c *= 0.6*sum(T)/sum(c);
c = round.(c, digits=3);
R = [rand(LogNormal(minimum(c)/c[i]), 1) for i=1:m]; # revenue

#-

## Form and solve the optimal advertising problem.
using Convex, SCS;
D = Variable(m, n);
Si = [min(R[i]*dot(P[i,:], D[i,:]'), B[i]) for i=1:m];
problem = maximize(sum(Si),
               [D >= 0, sum(D, dims=1)' <= T, sum(D, dims=2) >= c]);
solve!(problem, () -> SCS.Optimizer(verbose=0));

#-

# Plot traffic.
using Plots
plot(1:length(T), T, xlabel="hour", ylabel="Traffic")

#-

# Plot P.
heatmap(P)

#-

# Plot optimal D.
heatmap(evaluate(D))


"""
    logsumexp(x::Vector{Complex{T}})::Complex{T}

Log-sum-exp with choosing the shift to be the maximum absolute value for
the real and imaginary parts of x's components.

# Example:
```
a = randn(3) + randn(3) .* im
u = exp.(a)
RHS = logsumexp(u)
LHS = log(sum(exp.(u)))
println("RHS = ", RHS)
println("LHS = ", LHS)
```
"""
function logsumexp(x::Vector{Complex{T}})::Complex{T} where T <: Real

    # use the maximum magnitude for real and imaginary parts to get the shift, a.
    val_unused, 𝑖_real = findmax( real.(x) )
    val_unused, 𝑖_imag = findmax( imag.(x) )

    a::Complex{T} = real(x[𝑖_real]) + im*imag(𝑖_imag)

    # log-sum-exp.
    running_sum::Complex{T} = zero(Complex{T})
    for i = 1:length(x)
        running_sum += exp( x[i] - a )
    end

    return a + log(running_sum)
end


function basis(elem::Quad, N, r, s)
    Np = convert(Int, (N + 1) * (N + 1))
    sk = 1
    V, Vr, Vs = ntuple(x->zeros(length(r), Np), 3)
    for j = 0:N
        P_j = jacobiP(s, 0, 0, j)
        for i = 0:N
            P_i = jacobiP(r, 0, 0, i)
            V[:, sk]  = P_i .* P_j
            Vr[:, sk] = grad_jacobiP(r, 0, 0, i) .* P_j
            Vs[:, sk] = P_i .* grad_jacobiP(s, 0, 0, j)
            sk += 1
        end
    end
    return V,Vr,Vs
end

function nodes(elem::Quad, N)
    r1D, w1D = gauss_lobatto_quad(0, 0, N)
    s, r = meshgrid(r1D)
    return r[:], s[:]
end

function equi_nodes(elem::Quad, N)
    r1D = LinRange(-1, 1, N + 1)
    s, r = meshgrid(r1D)
    return r[:], s[:]
end

function quad_nodes(elem::Quad,N)
    r1D, w1D = gauss_quad(0, 0, N)
    s, r = meshgrid(r1D)
    ws, wr = meshgrid(w1D)
    w = @. wr * ws
    return r[:], s[:], w[:]
end


# """
# Source Term Function
# ```julia
# source_term = SourceTerm(λ, μ)
# source_term(x, p)
# ```
# where x is a discritzed price point, source_params is a vector of parameters for
# the source term and p is the last "observed" mid price point
#
# The function returns a single scalar output
#
# """

mutable struct SourceTerm
    λ::Float64
    μ::Float64
end

function (st::SourceTerm)(x::Float64,p::Float64)
    return st.λ*(p-x)*exp(-st.μ*(p-x)^2)
end

# function (st::SourceTerm)(x::Float64,p::Float64)
#     return st.λ*tanh(st.μ*(p-x))
# end


# THIS FILE ADAPTED FROM GraphLayout.jl
# see https://github.com/IainNZ/GraphLayout.jl

"""
    Use the spring/repulsion model of Fruchterman and Reingold (1991):
        Attractive force:  f_a(d) =  d^2 / k
        Repulsive force:  f_r(d) = -k^2 / d
    where d is distance between two vertices and the optimal distance
    between vertices k is defined as C * sqrt( area / num_vertices )
    where C is a parameter we can adjust

    Arguments:
    adj_matrix Adjacency matrix of some type. Non-zero of the eltype
               of the matrix is used to determine if a link exists,
               but currently no sense of magnitude
    C          Constant to fiddle with density of resulting layout
    MAXITER    Number of iterations we apply the forces
    INITTEMP   Initial "temperature", controls movement per iteration
"""
function layout_spring_adj(
    adj_matrix::Array{T,2};
    C = 2.0,
    MAXITER = 100,
    INITTEMP = 2.0,
) where {T}

    size(adj_matrix, 1) != size(adj_matrix, 2) && error("Adj. matrix must be square.")
    N = size(adj_matrix, 1)

    # Initial layout is random on the square [-1,+1]^2
    locs_x = 2 * rand(N) .- 1.0
    locs_y = 2 * rand(N) .- 1.0

    # The optimal distance bewteen vertices
    K = C * sqrt(4.0 / N)

    # Store forces and apply at end of iteration all at once
    force_x = zeros(N)
    force_y = zeros(N)

    # Iterate MAXITER times
    @inbounds for iter = 1:MAXITER
        # Calculate forces
        for i = 1:N
            force_vec_x = 0.0
            force_vec_y = 0.0
            for j = 1:N
                i == j && continue
                d_x = locs_x[j] - locs_x[i]
                d_y = locs_y[j] - locs_y[i]
                d = sqrt(d_x^2 + d_y^2)
                if adj_matrix[i, j] != zero(eltype(adj_matrix)) ||
                   adj_matrix[j, i] != zero(eltype(adj_matrix))
                    # F = d^2 / K - K^2 / d
                    F_d = d / K - K^2 / d^2
                else
                    # Just repulsive
                    # F = -K^2 / d^
                    F_d = -K^2 / d^2
                end
                # d  /          sin θ = d_y/d = fy/F
                # F /| dy fy    -> fy = F*d_y/d
                #  / |          cos θ = d_x/d = fx/F
                # /---          -> fx = F*d_x/d
                # dx fx
                force_vec_x += F_d * d_x
                force_vec_y += F_d * d_y
            end
            force_x[i] = force_vec_x
            force_y[i] = force_vec_y
        end
        # Cool down
        TEMP = INITTEMP / iter
        # Now apply them, but limit to temperature
        for i = 1:N
            force_mag = sqrt(force_x[i]^2 + force_y[i]^2)
            scale = min(force_mag, TEMP) / force_mag
            locs_x[i] += force_x[i] * scale
            #locs_x[i]  = max(-1.0, min(locs_x[i], +1.0))
            locs_y[i] += force_y[i] * scale
            #locs_y[i]  = max(-1.0, min(locs_y[i], +1.0))
        end
    end

    # Scale to unit square
    min_x, max_x = minimum(locs_x), maximum(locs_x)
    min_y, max_y = minimum(locs_y), maximum(locs_y)
    function scaler(z, a, b)
        2.0 * ((z - a) / (b - a)) - 1.0
    end
    locs_x = map(z -> scaler(z, min_x, max_x), locs_x)
    locs_y = map(z -> scaler(z, min_y, max_y), locs_y)

    return locs_x, locs_y
end

function _spring(G::SimpleGraph{T}, nits::Int = 1000) where {T}
    n = NV(G)
    A, vv = private_adj(G)

    d = Dict{T,Vector{Float64}}()

    if n == 0
        return d
    end

    if n == 1
        v = first(vv)
        d[v] = [0.0, 0.0]
        return d
    end

    x, y = layout_spring_adj(A, MAXITER = nits)

    for i = 1:n
        v = vv[i]
        d[v] = [x[i], y[i]]
    end
    return d
end


println("Test metrics")

import Distributions.SpecialFunctions.logabsbeta
import Distributions.StatsBase.harmmean
lbeta(a, b) = logabsbeta(a, b)[1]

@testset "metrics" begin
  @testset "dic" begin
    x = randn(10000)
    ll = normlogpdf.(x)
    @test isfinite(dic(ll))
  end

  @testset "logharmmean" begin
    Random.seed!(0)
    x = randn(100)
    @test log(harmmean(exp.(x))) ≈ MCMC.logharmmeanexp(x) atol=1e-6
  end

  @testset "log_bayes_factor" begin
    Random.seed!(0)
    n = 100
    y = rand(Bernoulli(.7), n)
    sumy = sum(y)

    m0 = Beta(70, 100)
    m1 = Beta(100, 70)

    compute_logc_true(m) = lbeta(m.α + sumy, m.β + n - sumy) - lbeta(m.α, m.β)
    c0_true = compute_logc_true(m0)
    c1_true = compute_logc_true(m1)

    true_log_bf = c1_true - c0_true

    post_samps_m0 = rand(Beta(m0.α + sumy, m0.β + n - sumy), 100_000)
    post_samps_m1 = rand(Beta(m1.α + sumy, m1.β + n - sumy), 100_000)
    ll0 = map(p -> sum(logpdf.(Bernoulli(p), y)), post_samps_m0)
    ll1 = map(p -> sum(logpdf.(Bernoulli(p), y)), post_samps_m1)

    approx_log_bf = log_bayes_factor(ll0, ll1)

    @test c0_true ≈ MCMC.logharmmeanexp(ll0)  atol=0.2
    @test c1_true ≈ MCMC.logharmmeanexp(ll1)  atol=0.2

    @test true_log_bf ≈ approx_log_bf atol=0.2
  end

  @testset "logmeanexp" begin
    Random.seed!(0)
    x = randn(10000)
    @test logmeanexp(x) ≈ log(mean(exp.(x)))
  end

  @testset "hellinger" begin
    Random.seed!(0)
    d1 = Normal(0, 1)
    d2 = Normal(1, 1)
    @test hellinger(d1, d2, Int(1e6)) ≈ hellinger(d2, d1, Int(1e6))  atol=0.02
    @test hellinger(d1, d2, Int(1e6)) ≈ 0.34 atol=0.01
  end
end


struct PolynomialFeature
    _degree::Int64
end

function transform(feature::PolynomialFeature, x::AbstractArray{Float64,1})
    # return the design matrix [phi(x_1), phi(x_2), ,,, phi(x_N)]^T of size (n_samples, n_features)
    n_samples = size(x)[1]
    degree = feature._degree
    Phi = zeros(Float64, n_samples, degree + 1)
    for i = 1:n_samples
        Phi[i, :] = [x[i]^j for j = 0:degree]
    end

    return Phi
end


gyroid(v) = cos(v[1])*sin(v[2])+cos(v[2])*sin(v[3])+cos(v[3])*sin(v[1])
gyroid_shell(v) = max(gyroid(v)-0.4,-gyroid(v)-0.4)
xr,yr,zr = ntuple(_->LinRange(0,pi*4,50),3)
A = [gyroid_shell((x,y,z)) for x in xr, y in yr, z in zr]
A[1,:,:] .= 1e10
A[:,1,:] .= 1e10
A[:,:,1] .= 1e10
A[end,:,:] .= 1e10
A[:,end,:] .= 1e10
A[:,:,end] .= 1e10

volume(A, algorithm=:iso, isovalue=0.5, isorange=0.05)

using FFTW
using LinearAlgebra
using Random
# Fourier synthesis of an isosurface using 1/f^α noise
function one_on_f_noise(T, α, f)
    # 2α needed to convert powers to amplitudes here
    ϵ = sqrt(eps(real(T)))
    A = randn(T)
    return convert(T, A / (ϵ + norm(f))^2α)
end

N = 200
α = 1.2f0
Random.seed!(2)
# symmetric frequency space
fx = range(-1,1,length=N)
fy = reshape(fx,1,:)
fz = reshape(fx,1,1,:)
spectrum = one_on_f_noise.(ComplexF32, α, Vec3f0.(fx, fy, fz))
iso = real.(fft(fftshift(spectrum)))
mini, maxi = extrema(iso)
iso_norm = (iso .- mini) ./ (maxi - mini)
volume(iso_norm, algorithm=:iso, isovalue=0.5, isorange=0.05)


using Econometrics

# sample size
n = 30

# draw n values of x between 0 and 4
x = 0:4/(n-1):4

# true function (unknown to researcher)
function f(x)
    10 + 3*x - x^2
end

# derivative of the function
function fprime(x)
    3 - 2x
end

# elasticity of the function: marginal function divided by average function
elasticity = x -> fprime(x)/(f(x)/x)
e = elasticity.(x) # elasticities at the x values

# noisy observation of true function
y = f.(x) + randn(n)
# researcher only observes y and x

# linear fit and elasticity according to linear fit
X = [ones(n) x]
Xprime = [zeros(n) ones(n)]
b, fit, junk = lsfit(y,X)
elinear = Xprime*b./(fit./x)

# Fourier fit and elasticity
#X = [ones(n) x cos.(x) sin.(x) cos.(2x) sin.(2x)]
#Xprime = [zeros(n) ones(n) -sin.(x) cos.(x) -2sin.(2x) 2cos.(2x)]
X = [ones(n) x cos.(x) sin.(x)]
Xprime = [zeros(n) ones(n) -sin.(x) cos.(x)]
b, fourierfit, junk = lsfit(y,X)
efourier = Xprime*b./(fourierfit./x)

# plot function and fits
p1 = plot(x,[f.(x) fit fourierfit], title = "Function and fits", legend=:bottomleft, label=["true" "linear" "fourier"])

# plot true elasticities and fits
p2 = plot(x, [e elinear efourier], title = "Elasticity and fitted elasticities", legend=:bottomleft, label=["true" "linear" "fourier"])

plot(p1, p2)


"""
    Sigmoid(x)

Sigmoid activation function.

    Usage:
```julia-repl
Sigmoid(x)
```
Arguments:
- `x` : Input value.
"""
function Sigmoid(x::Float64)
    x₁ = 1.0 / (1.0 + exp(-x))
    return x₁
end


#=
   Urn puzzle in Turing.jl

   From Berlin Bayesians
   https://app.slack.com/client/TFPMSKW3F/CFQHMRD6K/thread/CFQHMRD6K-1623812230.000500
   """
   Three urns, first blindly take one from the first and put in the second,
   then blindly take one from the second and put in the third and then blindly pick
   one from the third. How likely is it to pick a black one in the last step?

   [
     Urns

     1: white black black
     2: white white black
     3: white black

   ]
   """

   Probability of drawing a black in the last step:
   Distributions of variable (num:0)
   0.00000 =>    5273  (0.527300)
   1.00000 =>    4727  (0.472700)

   (Cf the WebPPL model's exact solution: 0.47222222222222227)

   Cf ~/webppl/urn_puzzle
=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model urn_puzzle() = begin
    white = 1
    black = 2

    urn1 = [white,black,black]
    urn2 = [white,white,black]
    urn3 = [white,black]

    # From urn 1 to urn 2
    # Note: We have to do this in two steps, i.e.
    # first pick some integer and then fetch that p1'th ball
    # from the urn
    p1 ~ DiscreteUniform(1,length(urn1))
    urn2b = push!(urn2,urn1[p1])

    # From urn 2 to urn 3
    p2 ~ DiscreteUniform(1,length(urn2b))
    urn3b = push!(urn3,urn2b[p2])

    # Pick a ball from urn 3
    p3 ~ DiscreteUniform(1,length(urn3b))
    ball = urn3b[p3]

    # Is it black?
    pblack ~ Dirac(ball==black)

end

model = urn_puzzle()

num_chns = 4

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(1000), 10_000)
# chns = sample(model, IS(), 10_000)
#

display(chns)
# display(plot(chns))

show_var_dist_pct(chns,:pblack)
show_var_dist_pct(chns,:ball)


"""
Information Gain.

The uncertainty of the starting node, minus the weighted impurity of
two child nodes.
"""
function info_gain(left, right, current_uncertainty::Float64)::Float64
    p = Float64(length(left)) / (length(left) + length(right))
    current_uncertainty - p * gini(left) - (1 - p) * gini(right)
end


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# ╔═╡ 9947a66a-7fe5-11eb-0955-87881a67fbea
begin
	using PlutoUI
	using Plots

	# plotly()
end

# ╔═╡ 9872b1a8-7fe7-11eb-0b3b-cb6181515182
using SymPy

# ╔═╡ 84fc02d6-7fef-11eb-2e21-cb577c3974f0
# Pkg.add("ForwardDiff)
using ForwardDiff

# ╔═╡ e529f550-7fe3-11eb-06f8-6beaa1506e66
md"""
## AutoDiff - Exploration

Status:
 - ref. 2019-10-26 - ref. [Automatic Differentiation in 10 minutes with Julia](https://www.youtube.com/watch?v=vAp6nUMrKYg)
 - 2021-03-08 moved to `Pluto.jl`

"""

# ╔═╡ 28aa4e9c-7fe4-11eb-3d87-c5a10540c2c1
md"""
Let's start with a simple example, the computation of $\sqrt(x)$ where how autodiff works comes as both a mathematical surprise, and a computing wonder.
The example is the Babylonian algorithm, known to mankind for millenia, to compute $\sqrt(x)$:

repeat

   $~~~~t \leftarrow \frac{(t + \frac{x}{t})}{2}$

until t converges to $\sqrt(x)$

Each iteration has one add and two divides.
For illustration purposes, 10 iterations suffice
"""

# ╔═╡ 1ecd40bc-7fe6-11eb-0a2f-cb9964fc5dbf
md"""
### And now the derivative, almost by magic

In a few lines of code. No mention of $\frac{1}{2}$ over  $\sqrt(x)$. We will use the "dual number" denoted as $D$ in what follows, those where invented by the famous algebraist Clifford in 1873.
"""

# ╔═╡ 4ac7200c-7fe6-11eb-259a-8d881d2f7e3e
struct D <: Number # D is a <function, derivative> pair (a pair of floats)
    f::Tuple{Float64, Float64}
end

# ╔═╡ 54f29be2-7fe6-11eb-38c5-d163b159730a
md"""
- Sum Rule: $(x + y)' = x' + y'$
- Quotient Rule: $(\frac{x}{y})' = \frac{yx' - xy'}{y^2}$
"""

# ╔═╡ 6e7efe02-7fe6-11eb-19a7-8b72c0836c8b
begin
	import Base: +, /, -, *, convert, promote_rule

	## overload:
	+(x::D, y::D) = D(x.f .+ y.f)
	/(x::D, y::D) = D((x.f[1] / y.f[1],
			(y.f[1] * x.f[2] - x.f[1] * y.f[2]) / y.f[1]^2))
	-(x::D, y::D) = D(x.f .- y.f)
	*(x::D, y::D) = D((x.f[1] * y.f[1], (y.f[1] * x.f[2] + x.f[1] * y.f[2])))

	## convert ordinary number to Dual number, intro. 0 for derivative
	convert(::Type{D}, x::Real) = D((x, zero(x)))

	## then promote...
	promote_rule(::Type{D}, ::Type{<:Number}) = D
end

# ╔═╡ 7a9389a2-7fe5-11eb-06b8-c52b073c876a
function babylonian(x; n = 10)
    t = (1. + x) / 2.
    for i ∈ 2:n
        t = (t + x/t) / 2.
    end
    t
end

# ╔═╡ 9091553e-7fe5-11eb-2131-9b1b1097c932
begin
	α = π
	babylonian(α), √α
end

# ╔═╡ b9ccd308-7fe5-11eb-0b0c-efa652660cd5
with_terminal() do
	for α ∈ 2:3
   		println(babylonian(α), " ", √α)
	end
end

# ╔═╡ ff570916-7fe5-11eb-35ab-e915308e7bfc
## WARN: first plots require to load package which takes time

begin
	ix = 0:.01:49

	plot([x -> babylonian(x, n=i) for i ∈ 1:5],
    	ix,
    	label=["Iteration $jx" for _ ∈ 1:1, jx ∈ 1:5])

	plot!(sqrt,
    	ix,
    	c="darkslateblue",
    	label="sqrt",
    	title = "Those Babylonian really knew about √")
end

# ╔═╡ 732679da-7ff0-11eb-1145-3750276a9601
begin
	using LinearAlgebra

	n = 4
	Strang = SymTridiagonal(2 * ones(n), - ones(n-1))
end

# ╔═╡ 6e64c064-7fe6-11eb-1e6d-3dfeebace21b
md"""
The same algorithm with no rewrite at all computes properly the derivative as the check shows:
"""

# ╔═╡ 6e487e0e-7fe6-11eb-23a2-35a3a2ed7015
x₁ = 49; babylonian(D((x₁, 1))), (√x₁, .5 / √x₁)

# ╔═╡ a362fc86-7fe6-11eb-1b7a-03cd0549e790
x₂ = π; babylonian(D((x₂, 1))), (√x₂, .5 / √x₂)

# ╔═╡ a348cb0e-7fe6-11eb-1c9d-4b4278a367aa
md"""
#### It just works!

How does this work?
We will explain in a moment. Right now marvel that it does. Note we did not import any autodiff package. Everything is just basic Julia.

#### The assembler

Most folks don't read assembler, but one can see that it is short (minus comments). The shortness is a clue that suggests speed!

"""

# ╔═╡ a32df540-7fe6-11eb-073f-1b34489aa4a0
with_terminal() do
	@show @inline function babylonian_(x; n = 10)
    	 t = (1 + x) / 2.
     	for i = 2:n; t = (t + x/t) / 2. end
     	t
 	end
end

# ╔═╡ a312dfda-7fe6-11eb-0bae-37587f221888
with_terminal() do
	@code_native babylonian(D((2, 1)))
end

# ╔═╡ 37574936-7fe7-11eb-132a-2df44bf8d6bb
md"""
### Symbolically

We haven't yet explained how it works, but it may be of some value to understand that the below is mathematically equivalent, though not what the computation is doing.
Notice in the below that babylonian works on SymPy Symbols.

Note: Python and Julia are very good friends.It's not a competition! Watch how nicely we can use the same code now in SymPy.

"""

# ╔═╡ 98544e52-7fe7-11eb-1094-9382f7126ce4
with_terminal()do
	x = symbols("x")

	println("Iterations as a function of x")
	for k ∈ 1:5
    	println(simplify(babylonian(x, n=k)))
	end
end

# ╔═╡ 98399af8-7fe7-11eb-1276-13c05fa36ed9
with_terminal() do
	x = symbols("x")

	println("Derivatives as a function of x")

	for k ∈ 1:5
    	println(simplify(diff(simplify(babylonian(x, n=k)), x)))
	end
end

# ╔═╡ 981ea264-7fe7-11eb-06ef-97ee66d5619b
md"""
Let's by hand take the "derivative" of the babylonian iteration with respect to x.

Specifically $t' = \frac{dt}{dx}$. This is the old fashioned way of a human writing code.
"""

# ╔═╡ d05af69e-7feb-11eb-3874-d1b05fb6434a
function dbabylonian(x; n = 10)
    t = (1. + x) / 2.
    dt = 1. / 2.

    for i ∈ 2:n
        t = (t + x/t) / 2.
        dt = (dt + (t - x * dt) / t^2) / 2.
    end
    dt
end

# ╔═╡ d0310bf4-7feb-11eb-2d05-c9531128eb4a
md"""

Note:
  -  $t = \frac{1}{2} \times (t + \frac{x}{t})$

  - then $(\frac{dt}{dx})' = \frac{1}{2} \times (t' + \frac{x' \times t - x \times t'}{t^2})$, as $x' = 1$, we get:

$(\frac{dt}{dx})' = \frac{1}{2} \times (\frac{t' + (t - x \times t')}{t^2})$
"""

# ╔═╡ d01a58be-7feb-11eb-1f29-6d23db7914da
x= π; dbabylonian(x), .5 / √x

# ╔═╡ cffee818-7feb-11eb-079b-a7dbfdd8cdd6
md"""
What just happened?

Answer: we created an iteration by hand for t' given our iteration for t, Then we ran the iteration alongside the iteration for t.
"""

# ╔═╡ d0ca45ac-7fec-11eb-202b-87ae22b66616
babylonian(D((x, 1)))

# ╔═╡ d0addd04-7fec-11eb-0213-53ef5d729159
md"""

How did this work?

It created the same derivative iteration that we did by hand, using very general rules that are set once and need not be written by hand (and multiple dispatch).

Important: the derivative is substituted before the JIT compiler, and thus efficient compiled code is executed.
"""

# ╔═╡ d092a21e-7fec-11eb-0cc1-d1515675f521
md"""
### Dual Number Notation

Instead of $D(a, b)$ we can write: $a + b \epsilon$ where $\epsilon$ satisfies $\epsilon^2 = 0$. Some people like to recall imaginary numbers where an $i$ is introduced with $i^2 = -1$. Others like to think of how engineers just fdrop the $O(\epsilon^2)$ terms.

The four rules are:
  - 1 & 2. $(a + b\epsilon) \pm (c + d\epsilon) = (a + c) \pm (b + d)\epsilon$
  - 3.$(a + b\epsilon) \times (c + d\epsilon) = (ac) + (bc + ad)\epsilon$
  - 4.$\frac{(a + b\epsilon)}{(c + d\epsilon)} = (\frac{a}{c}) + \frac{(bc - ad)}{d^2}\epsilon$

"""

# ╔═╡ 1818fb9c-7fed-11eb-0bd3-21411f08fcdf
Base.show(io::IO, x::D) = print(io, x.f[1], " + ", x.f[2], " ϵ")

# ╔═╡ 14125e9c-7fef-11eb-3dc4-a75f84383d5d
D((1, 0))

# ╔═╡ 13e452cc-7fef-11eb-2e1d-a5e3b9936a4c
D((0, 2))^2 # should be zero!

# ╔═╡ 13ab59ea-7fef-11eb-2078-6d8ee11d1c90
D((2, 1))^2

# ╔═╡ 2886e6c4-7fef-11eb-0fff-9b3033e4f943
begin
	ϵ = D((0, 1))
	with_terminal() do
		@code_native(ϵ^2)
	end
end

# ╔═╡ 3f1f4bba-7fef-11eb-2f39-cf906aec10ae
ϵ * ϵ

# ╔═╡ 3f037186-7fef-11eb-200f-3579d0ab8e1f
ϵ^2

# ╔═╡ 3eea0d36-7fef-11eb-3ede-c7d54243194c
(1 + ϵ)^5 # note it just works (we did not train powers)

# ╔═╡ 3ece19f8-7fef-11eb-3b91-ff25fd3ca8f4
md"""
### Generalization to arbitrary roots
"""

# ╔═╡ 3eb0843a-7fef-11eb-2c67-9ba2e4d1eefc
function nth_root(x, n=2; t=1, p=10)
    for i = 1:p
        t += (x / t^(n-1) - t) / n
    end
    t
end

# ╔═╡ 5c9fbbc6-7fef-11eb-32ad-b986ca56aa9b
nth_root(2, 3), ∛2  # copied from https://www.alt-codes.net/root-symbols

# ╔═╡ 5c84a84c-7fef-11eb-3117-01f1842890f5
nth_root(2 + ϵ, 3)

# ╔═╡ 5c665cd4-7fef-11eb-2c89-e153a973b30e
nth_root(7, 12), 7^(1/12)

# ╔═╡ 5c4f64ac-7fef-11eb-1864-13f293e85fc4
let x = 2.0
	nth_root( x + ϵ), ∛x, 1/x*(2/3)/3
end

# ╔═╡ 5c33d6ce-7fef-11eb-12c1-29396395358d
md"""
### Forward Diff

Now that you understand it, you can use the official package.
"""

# ╔═╡ 84ddd280-7fef-11eb-29cb-3fc8c8547134
ForwardDiff.derivative(sqrt, 2)

# ╔═╡ 916346b8-7fef-11eb-0d53-29bd11c34510
ForwardDiff.derivative(babylonian, 2)

# ╔═╡ 99672fe6-7fef-11eb-305e-d98ce4bc8dc1
with_terminal() do
	println(@which ForwardDiff.derivative(√, 2))
end

# ╔═╡ be076e8a-7fef-11eb-0741-79beea838677
md"""
### Close Look at Convergence with big floats

the $-log10$ gives the number of correct digits. Watch the quadratic convergence right before your eyes.
"""

# ╔═╡ d5b96cac-7fef-11eb-0772-551e4d96efb2
with_terminal() do
	setprecision(3000)

	# round.(Float64.(log10.([babylonian(BigFloat(2), n=k) for k=1:10] - √BigFloat(2))), 3)
	println(round.(Float64.(log10.([babylonian(BigFloat(2), n=k) - √BigFloat(2.) for k=1:10])), sigdigits=6))
end

# ╔═╡ d59d0508-7fef-11eb-20a3-3f108b859670
struct D1{T} <: Number # D is a <function, derivative> pair
    f::Tuple{T, T}
end

# ╔═╡ d57e75ac-7fef-11eb-1cc7-db506a7e4e2f
begin
	z = D((2., 1.))
	z₁ = D1((BigFloat(2.), BigFloat(1.)))
end

# ╔═╡ fbb04502-7fef-11eb-28fd-414c2f982855
# begin
# 	#import Base: +, /, convert, promote_rule

# 	## overload:
# 	+(x::D1, y::D1) = D1(x.f .+ y.f)
# 	/(x::D1, y::D1) = D1((x.f[1] / y.f[1], (y.f[1] * x.f[2] - x.f[1] * y.f[2]) / y.f[1]^2))

# 	convert(::Type{D1{T}}, x::Real) where {T} = D1((convert(T, x), zero(T)))
# 	promote_rule(::Type{D1{T}}, ::Type{S}) where {T, S <: Number} = D1{promote_type(T, S)}
# end

# ╔═╡ 3e2da118-7ff0-11eb-36ca-e32af03b2a0c
A = randn(3, 3)

# ╔═╡ 3e0a484e-7ff0-11eb-17be-e9e7aa66fb2c
x₃= randn(3)

# ╔═╡ 3dd5c57e-7ff0-11eb-3330-cf7d02b5c632
ForwardDiff.gradient(x -> x'A*x, x₃)

# ╔═╡ Cell order:
# ╟─e529f550-7fe3-11eb-06f8-6beaa1506e66
# ╠═9947a66a-7fe5-11eb-0955-87881a67fbea
# ╟─28aa4e9c-7fe4-11eb-3d87-c5a10540c2c1
# ╠═7a9389a2-7fe5-11eb-06b8-c52b073c876a
# ╠═9091553e-7fe5-11eb-2131-9b1b1097c932
# ╠═b9ccd308-7fe5-11eb-0b0c-efa652660cd5
# ╠═ff570916-7fe5-11eb-35ab-e915308e7bfc
# ╟─1ecd40bc-7fe6-11eb-0a2f-cb9964fc5dbf
# ╠═4ac7200c-7fe6-11eb-259a-8d881d2f7e3e
# ╟─54f29be2-7fe6-11eb-38c5-d163b159730a
# ╠═6e7efe02-7fe6-11eb-19a7-8b72c0836c8b
# ╟─6e64c064-7fe6-11eb-1e6d-3dfeebace21b
# ╠═6e487e0e-7fe6-11eb-23a2-35a3a2ed7015
# ╠═a362fc86-7fe6-11eb-1b7a-03cd0549e790
# ╟─a348cb0e-7fe6-11eb-1c9d-4b4278a367aa
# ╠═a32df540-7fe6-11eb-073f-1b34489aa4a0
# ╠═a312dfda-7fe6-11eb-0bae-37587f221888
# ╟─37574936-7fe7-11eb-132a-2df44bf8d6bb
# ╠═9872b1a8-7fe7-11eb-0b3b-cb6181515182
# ╠═98544e52-7fe7-11eb-1094-9382f7126ce4
# ╠═98399af8-7fe7-11eb-1276-13c05fa36ed9
# ╟─981ea264-7fe7-11eb-06ef-97ee66d5619b
# ╠═d05af69e-7feb-11eb-3874-d1b05fb6434a
# ╟─d0310bf4-7feb-11eb-2d05-c9531128eb4a
# ╠═d01a58be-7feb-11eb-1f29-6d23db7914da
# ╟─cffee818-7feb-11eb-079b-a7dbfdd8cdd6
# ╠═d0ca45ac-7fec-11eb-202b-87ae22b66616
# ╟─d0addd04-7fec-11eb-0213-53ef5d729159
# ╟─d092a21e-7fec-11eb-0cc1-d1515675f521
# ╠═1818fb9c-7fed-11eb-0bd3-21411f08fcdf
# ╠═14125e9c-7fef-11eb-3dc4-a75f84383d5d
# ╠═13e452cc-7fef-11eb-2e1d-a5e3b9936a4c
# ╠═13ab59ea-7fef-11eb-2078-6d8ee11d1c90
# ╠═2886e6c4-7fef-11eb-0fff-9b3033e4f943
# ╠═3f1f4bba-7fef-11eb-2f39-cf906aec10ae
# ╠═3f037186-7fef-11eb-200f-3579d0ab8e1f
# ╠═3eea0d36-7fef-11eb-3ede-c7d54243194c
# ╟─3ece19f8-7fef-11eb-3b91-ff25fd3ca8f4
# ╠═3eb0843a-7fef-11eb-2c67-9ba2e4d1eefc
# ╠═5c9fbbc6-7fef-11eb-32ad-b986ca56aa9b
# ╠═5c84a84c-7fef-11eb-3117-01f1842890f5
# ╠═5c665cd4-7fef-11eb-2c89-e153a973b30e
# ╠═5c4f64ac-7fef-11eb-1864-13f293e85fc4
# ╟─5c33d6ce-7fef-11eb-12c1-29396395358d
# ╠═84fc02d6-7fef-11eb-2e21-cb577c3974f0
# ╠═84ddd280-7fef-11eb-29cb-3fc8c8547134
# ╠═916346b8-7fef-11eb-0d53-29bd11c34510
# ╠═99672fe6-7fef-11eb-305e-d98ce4bc8dc1
# ╟─be076e8a-7fef-11eb-0741-79beea838677
# ╠═d5b96cac-7fef-11eb-0772-551e4d96efb2
# ╠═d59d0508-7fef-11eb-20a3-3f108b859670
# ╠═d57e75ac-7fef-11eb-1cc7-db506a7e4e2f
# ╠═fbb04502-7fef-11eb-28fd-414c2f982855
# ╠═3e2da118-7ff0-11eb-36ca-e32af03b2a0c
# ╠═3e0a484e-7ff0-11eb-17be-e9e7aa66fb2c
# ╠═3dd5c57e-7ff0-11eb-3330-cf7d02b5c632
# ╠═732679da-7ff0-11eb-1145-3750276a9601


using IntervalArithmetic
if VERSION < v"0.7.0-DEV.2004"
    using Base.Test
else
    using Test
end

@testset "Interval loop tests" begin
    i = 1

    @test Interval(i,i).lo == 1
    @test @interval(i).lo == 1

    for i in 1:10
        a = @interval(i)
        @test a.lo == i
    end

end

## Calculate pi by summing 1/i^2 to give pi^2/6:

setprecision(Interval, 53)

function calc_pi1(N)
    S1 = @interval(0)

    for i in 1:N
        S1 += @interval(1/i^2)
    end
    S1 += @interval(1/(N+1), 1/N)

    sqrt(6*S1)
end

function calc_pi2(N)
    S2 = @interval(0)

    for i in 1:N
        S2 += 1/i^2
    end
    S2 += @interval(1/(N+1), 1/N)

    sqrt(6*S2)
end

function calc_pi3(N)
    S3 = @floatinterval(0)

    for i in 1:N
        S3 += 1/i^2
    end
    S3 += @floatinterval(1/(N+1), 1/N)

    sqrt(6*S3)
end

function calc_pi4(N)
    S4 = @floatinterval(0)
    II = @floatinterval(1)

    for i in N:-1:1
        S4 += II / (i^2)
    end
    S4 += II / @floatinterval(N, N+1)

    sqrt(6*S4)
end

function calc_pi5(N)
    S5 = @floatinterval(0)
    II = @floatinterval(1)

    for i in N:-1:1
        S5 += 1 // (i^2)
    end
    S5 += 1 / @floatinterval(N, N+1)

    sqrt(6*S5)
end

@testset "Pi tests" begin

    big_pi = setprecision(256) do
        big(pi)
    end

    N = 10000
    pi1 = calc_pi1(N)
    pi2 = calc_pi2(N)
    pi3 = calc_pi3(N)
    pi4 = calc_pi4(N)
    pi5 = calc_pi5(N)

    @test big_pi ∈ pi1
    @test big_pi ∈ pi2
    @test big_pi ∈ pi3
    @test big_pi ∈ pi4
    @test big_pi ∈ pi5

    @test pi1 == pi2
    @test pi2 == pi3

end

setprecision(Interval, Float64)


### A Pluto.jl notebook ###
# v0.18.1

using Markdown
using InteractiveUtils

# ╔═╡ 321b951c-9bea-11ec-0d10-31f76d44cc9e
md"""
# HW 7

Eric Nguyen
"""

# ╔═╡ 710b2cc8-45e7-4eee-9066-3fd4080aebc9
md"""
### Problem 1

First we start with the Cartesian coordinates ``x``, ``y``, and ``z`` in terms of spherical coordinates,

$\begin{align*}
x &= r \sin{\theta} \cos{\phi} \\
y &= r \sin{\theta} \sin{\phi} \\
z &= r \cos{\theta}
\end{align*}$

Imagine that we change the position ``\vec{r}`` by a tiny ``d\vec{r} = dx \, \hat{i} + dy \, \hat{j} + dz \, \hat{k}`` such that ``x``, ``y``, and ``z`` change by infinitesimal amounts,

$\begin{align*}
dx &= \sin{\theta} \cos{\phi} \,dr + r \cos{\theta} \cos{\phi} \,d\theta - r \sin{\theta} \sin{\phi} \,d\phi \\
dy &= \sin{\theta} \sin{\phi} \,dr + r \cos{\theta} \sin{\phi} \,d\theta + r \sin{\theta} \cos{\phi} \,d\phi \\
dz &= \cos{\theta} \,dr - r \sin{\theta} \,d\theta
\end{align*}$

Consider the change ``d\vec{r}`` in the ``r,\theta,\phi`` directions found by setting the infinitesimals equal to zero accordingly (e.g., for ``\hat{r}`` set ``d\theta = d\phi = 0``, etc.),

$\begin{align*}
d\vec{r}_r &= \hat{i} \sin{\theta} \cos{\phi} \,dr + \hat{j} \sin{\theta} \sin{\phi} \,dr + \hat{k} \cos{\theta} \,dr \\
&= (\hat{i} \sin{\theta} \cos{\phi} + \hat{j} \sin{\theta} \sin{\phi} + \hat{k} \cos{\theta}) \,dr \\
d\vec{r}_{\theta} &= \hat{i} r \cos{\theta} \cos{\phi} \,d\theta + \hat{j} r \cos{\theta} \sin{\phi} \,d\theta - \hat{k} r \cos{\theta} \,d\theta \\
&= (\hat{i} \cos{\theta} \cos{\phi} + \hat{j} \cos{\theta} \sin{\phi} - \hat{k} \sin{\theta}) r \,d\theta \\
d\vec{r}_{\phi} &= \hat{i} r \sin{\theta} \sin{\phi} \,d\phi + \hat{j} r \sin{\theta} \cos{\phi} \,d\phi \\
&= (\hat{i} \sin{\phi} + \hat{j} \cos{\phi}) r \sin{\theta} \,d\phi
\end{align*}$

It follows that the unit vectors in the spherical polar coordinates are

$\begin{align*}
\hat{r} &= \hat{i} \sin{\theta} \cos{\phi} + \hat{j} \sin{\theta} \sin{\phi} + \hat{k} \cos{\theta} \\
\hat{\theta} &= \hat{i} \cos{\theta} \cos{\phi} + \hat{j} \cos{\theta} \sin{\phi} - \hat{k} \sin{\theta} \\
\hat{\phi} &= \hat{i} \sin{\phi} + \hat{j} \cos{\phi}
\end{align*}$
"""

# ╔═╡ 370b6a47-79b6-41e3-ad41-01544affa97d
md"""
### Problem 2

Using ``\vec{v} = \frac{d\vec{r}}{dt}``, rewrite the expression for kinetic energy as

$K = \frac{1}{2} m \left(\vec{v} ⋅ \vec{v}\right) = \frac{1}{2} m \left(\sqrt{{v_x}^2 + {v_y}^2 + {v_z}^2}\right)^2 = \frac{1}{2} m ({v_x}^2 + {v_y}^2 + {v_z}^2)$

We start with the transformation equations for the Cartesian coordinates,

$\begin{align*}
x &= r \sin{\theta} \cos{\phi} \\
y &= r \sin{\theta} \sin{\phi} \\
z &= r \cos{\theta}
\end{align*}$

Take the time derivative for each coordinate (i.e., find expressions for the components of ``\vec{v}``):

$\begin{align*}
v_x &= ẋ = \sin{\theta} \cos{\phi} \, ṙ + r \cos{\theta} \cos{\phi} \,\dot{\theta} - r \sin{\theta} \sin{\phi} \,\dot{\phi} \\
v_y &= ẏ = \sin{\theta} \sin{\phi} \, ṙ + r \cos{\theta} \sin{\phi} \,\dot{\theta} + r \sin{\theta} \cos{\phi} \, \dot{\phi} \\
v_z &= ż = \cos{\theta} \,ṙ - r \sin{\theta} \,\dot{\theta}
\end{align*}$

Going back to the expression for kinetic energy, we can write the velocity expression in terms of ``r``, ``\theta``, and ``\phi`` and simplify.
First, we square each velocity component:

$\begin{align*}
{v_x}^2 &= (\sin{\theta} \cos{\phi} \, ṙ + r \cos{\theta} \cos{\phi} \,\dot{\theta} - r \sin{\theta} \sin{\phi} \,\dot{\phi})^2 \\
&= \sin{\theta}^2 \cos^2{\phi} \, ṙ^2 + r^2 \cos^2{\theta} \cos^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \sin^2{\phi} \,\dot{\phi}^2 \\
&\quad + 2r \sin{\theta} \cos{\theta} \cos^2{\phi} \,ṙ \,\dot{\theta} - 2r \sin^2{\theta} \sin{\phi} \cos{\phi} \,ṙ \,\dot{\phi} \\
&\quad - 2r \sin{\theta} \cos{\theta} \sin{\phi} \cos{\phi} \,\dot{\theta} \,\dot{\phi} \\
\\
{v_y}^2 &= (\sin{\theta} \sin{\phi} \, ṙ + r \cos{\theta} \sin{\phi} \,\dot{\theta} + r \sin{\theta} \cos{\phi} \, \dot{\phi})^2 \\
&= \sin^2{\theta} \sin^2{\phi} \,ṙ^2 + r^2 \cos^2{\theta} \sin^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \cos^2{\phi} \,\dot{\phi}^2 \\
&\quad + 2r \sin{\theta} \cos{\theta} \sin^2{\phi} \,ṙ \,\dot{\theta} + 2r \sin^2{\theta} \sin{\phi} \cos{\phi} \,ṙ \,\dot{\phi} \\
&\quad + 2r \sin{\theta} \cos{\theta} \sin{\phi} \cos{\phi} \,\dot{\theta} \,\dot{\phi} \\
\\
{v_z}^2 &= (\cos{\theta} \,ṙ - r \sin{\theta} \,\dot{\theta})^2 \\
&= \cos^2{\theta} \,ṙ^2 + r^2 \sin^2{\theta} \,\dot{\theta}^2 - 2r \sin{\theta} \cos{\theta} \,ṙ \,\dot{\theta}
\end{align*}$

Adding the terms together, we get:

$\begin{align*}
{v_x}^2 + {v_y}^2 + {v_z}^2 &= \sin{\theta}^2 \cos^2{\phi} \, ṙ^2 + r^2 \cos^2{\theta} \cos^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \sin^2{\phi} \,\dot{\phi}^2 \\
&\quad + 2r \sin{\theta} \cos{\theta} \cos^2{\phi} \,ṙ \,\dot{\theta} - 2r \sin^2{\theta} \sin{\phi} \cos{\phi} \,ṙ \,\dot{\phi} \\
&\quad - 2r \sin{\theta} \cos{\theta} \sin{\phi} \cos{\phi} \,\dot{\theta} \,\dot{\phi} \\
&\quad + \sin^2{\theta} \sin^2{\phi} \,ṙ^2 + r^2 \cos^2{\theta} \sin^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \cos^2{\phi} \,\dot{\phi}^2 \\
&\quad + 2r \sin{\theta} \cos{\theta} \sin^2{\phi} \,ṙ \,\dot{\theta} + 2r \sin^2{\theta} \sin{\phi} \cos{\phi} \,ṙ \,\dot{\phi} \\
&\quad + 2r \sin{\theta} \cos{\theta} \sin{\phi} \cos{\phi} \,\dot{\theta} \,\dot{\phi} \\
&\quad + \cos^2{\theta} \,ṙ^2 + r^2 \sin^2{\theta} \,\dot{\theta}^2 - 2r \sin{\theta} \cos{\theta} \,ṙ \,\dot{\theta} \\

&= \sin{\theta}^2 \cos^2{\phi} \, ṙ^2 + r^2 \cos^2{\theta} \cos^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \sin^2{\phi} \,\dot{\phi}^2 \\
&\quad + \sin^2{\theta} \sin^2{\phi} \,ṙ^2 + r^2 \cos^2{\theta} \sin^2{\phi} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \cos^2{\phi} \,\dot{\phi}^2 \\
&\quad + \cos^2{\theta} \,ṙ^2 + r^2 \sin^2{\theta} \,\dot{\theta}^2 \\

&= \sin^2{\theta} \,ṙ^2 + r^2 \cos^2{\theta} \,\dot{\theta}^2 + r^2 \sin^2{\theta} \,\dot{\phi}^2 \\
&\quad + \cos^2{\theta} \,ṙ^2 + r^2 \sin^2{\theta} \,\dot{\theta}^2 \\

&= ṙ^2 + r^2 \, \dot{\theta}^2 + r^2 \sin^2{\theta} \,\dot{\phi}^2
\end{align*}$

Substitute the result into the expression for kinetic energy:

$K = \frac{1}{2} m (ṙ^2 + r^2 \, \dot{\theta}^2 + r^2 \sin^2{\theta} \,\dot{\phi}^2)$
"""

# ╔═╡ a1799513-6a61-4443-8ec7-428c0696bc5f
md"""
### Problem 3

**(a).**
The square of the magnitude of the cross product ``|\vec{A} × \vec{B}|^2`` is

$\begin{align*}
|\vec{A} × \vec{B}|^2 &= (A_y B_z - A_z B_y)^2 |\hat{i}| + (A_z B_x - A_x B_z)^2 |\hat{j}| + (A_x B_y - A_y B_x)^2 |\hat{k}| \\
&= {A_y}^2 {B_z}^2 + {A_z}^2 {B_y}^2 - 2 A_y A_z B_y B_z \\
&\quad + {A_z}^2 {B_x}^2 + {A_x}^2 {B_z}^2 - 2 A_x A_z B_x B_z \\
&\quad + {A_x}^2 {B_y}^2 + {A_y}^2 {B_x}^2 - 2 A_x A_y B_x B_y \\
\end{align*}$

The square of the magnitude of ``|\vec{A}|^2`` and ``|\vec{B}|^2`` are

$\begin{align*}
|\vec{A}|^2 &= {A_x}^2 + {A_y}^2 + {A_z}^2 \\
|\vec{B}|^2 &= {B_x}^2 + {B_y}^2 + {B_z}^2
\end{align*}$

The product ``|\vec{A}|^2 |\vec{B}|^2`` is then

$\begin{align*}
|\vec{A}|^2 |\vec{B}|^2 &= ({A_x}^2 + {A_y}^2 + {A_z}^2)({B_x}^2 + {B_y}^2 + {B_z}^2) \\
&= {A_x}^2 {B_x}^2 + {A_x}^2 {B_y}^2 + {A_x}^2 {B_z}^2 \\
&\quad + {A_y}^2 {B_x}^2 + {A_y}^2 {B_y}^2 + {A_y}^2 {B_z}^2 \\
&\quad + {A_z}^2 {B_x}^2 + {A_z}^2 {B_y}^2 + {A_z}^2 {B_z}^2 \\
\end{align*}$

The square of the magnitude of ``(\vec{A} ⋅ \vec{B})^2`` is

$\begin{align*}
(\vec{A} ⋅ \vec{B})^2 &= (A_x B_x + A_y B_y + A_z B_z)^2 \\
&= {A_x}^2 {B_x}^2 + {A_y}^2 {B_y}^2 + {A_z}^2 {B_z}^2 \\
&\quad + 2 A_x A_y B_x B_y + 2 A_x A_z B_x B_z + 2 A_y A_z B_y B_z
\end{align*}$

Notice that

$|\vec{A}|^2 |\vec{B}|^2 - (\vec{A} ⋅ \vec{B})^2 = |\vec{A} × \vec{B}|^2$

Thus,

$|\vec{A} × \vec{B}|^2 = |\vec{A}|^2 |\vec{B}|^2 - (\vec{A} ⋅ \vec{B})^2$
"""

# ╔═╡ 08ef5ce6-9d82-4758-ade2-4ea62b514451
md"""
**(b).**
Using components, summation ceonvention, and the relationship between the totally antisymmetric symbol and the Kronecker delta it's possible to arrive at the same expression.

$\begin{align*}
|\vec{A} × \vec{B}|^2 &= (\epsilon_{ijk} \hat{e}_i A_j B_k) ⋅ (\epsilon_{ℓmn} \hat{e}_ℓ A_m B_n) \\
&= A_j B_k A_m B_n \epsilon_{ijk} \epsilon_{ℓmn} \delta_{iℓ} \\
&= A_j B_k A_m B_n \epsilon_{ijk} \epsilon_{imn} \\
&= A_j B_k A_m B_n (\delta_{jm} \delta_{kn} - \delta_{jn} \delta_{km}) \\
&= \sum_{jkmn} A_j B_k A_m B_n \delta_{jm} \delta_{kn} - \sum_{jkmn} A_j B_k A_m B_n \delta_{jn} \delta_{km} \\
&= \sum_{jm} A_j A_m \delta_{jm} \sum_{kn} B_k B_n \delta_{kn} - \sum_{jn} A_j B_m \delta_{jn} \sum_{km} A_m B_k \delta_{km} \\
&= (\vec{A} ⋅ \vec{A}) (\vec{B} ⋅ \vec{B}) - (\vec{A} ⋅ \vec{B}) (\vec{A} ⋅ \vec{B}) \\
&= |\vec{A}|^2 |\vec{B}|^2 - (\vec{A} ⋅ \vec{B})^2
\end{align*}$
"""

# ╔═╡ c0971a8a-7d86-4cb6-aae9-5423beeb1da4
md"""
### Problem 4

From Problem (1) I showed that

$\begin{align*}
d\vec{r}_{r} &= (\hat{i} \sin{\theta} \cos{\phi} + \hat{j} \sin{\theta} \sin{\phi} + \hat{k} \cos{\theta}) \,dr = \,dr \,\hat{r} \\
d\vec{r}_{\theta} &= (\hat{i} \cos{\theta} \cos{\phi} + \hat{j} \cos{\theta} \sin{\phi} - \hat{k} \sin{\theta}) r \,d\theta = r \,d\theta \,\hat{\theta}  \\
d\vec{r}_{\phi} &= (\hat{i} \sin{\phi} + \hat{j} \cos{\phi}) r \sin{\theta} \,d\phi = r \sin{\theta} \,d\phi \,\hat{\phi}
\end{align*}$

This shows that when moving in the ``r``-direction only, the derivative is ``\frac{∂}{∂r}``, in the ``\theta``-direction, the derivative is ``\frac{1}{r} \frac{∂}{∂\theta}``, and in the ``\phi``-direction, the derivative is ``\frac{1}{r \sin{\theta}} \frac{∂}{∂\phi}``.
The gradient in spherical coordinates is then

$\vec{∇} = \hat{r} \frac{∂}{∂r} + \hat{\theta} \frac{1}{r} \frac{∂}{∂\theta} + \hat{\phi} \frac{1}{r \sin{\theta}} \frac{∂}{∂\phi}$
"""

# ╔═╡ 74bea6fd-90e1-4aad-bb58-50b141eb0255
md"""
### Problem 5

From the notes, it is given

$\begin{align*}
\hat{r} &= \hat{i} \cos{\phi} + \hat{j} \sin{\phi} \\
\hat{\phi} &= -\hat{i} \sin{\phi} + \hat{j} \cos{\phi}
\end{align*}$

The partial derivatives of each unit vector with respect to ``\phi`` are

$\begin{align*}
\frac{∂\hat{r}}{∂\phi} &= -\hat{i} \sin{\phi} + \hat{j} \cos{\phi} = \hat{\phi} \\
\frac{∂\hat{\phi}}{∂\phi} &= -\hat{i} \cos{\phi} - \hat{j} \sin{\phi} = \hat{r}
\end{align*}$

Proceeding with the derivation, the result is

$\begin{align*}
\vec{∇}^2 f(r,\phi) &= \vec{∇} ⋅ \vec{∇} f(r,\phi) \\
&= \left(\hat{r} \frac{∂}{∂r} + \hat{\phi} \frac{1}{r} \frac{∂}{∂\phi}\right) ⋅ \left(\hat{r} \frac{∂f}{∂r} + \hat{\phi} \frac{1}{r} \frac{∂f}{∂\phi}\right) \\
&= \hat{r} \frac{∂}{∂r} \left(\hat{r} \frac{∂f}{∂r} + \hat{\phi} \frac{1}{r} \frac{∂f}{∂\phi}\right) + \hat{\phi} \frac{1}{r} \frac{∂}{∂\phi} \left(\hat{r} \frac{∂f}{∂r} + \hat{\phi} \frac{1}{r} \frac{∂f}{∂\phi}\right) \\
&= \hat{r} \left(\hat{r} \frac{∂^2f}{∂r^2} - \hat{\phi} \frac{1}{r^2} \frac{∂f}{∂\phi} + \hat{\phi} \frac{1}{r} \frac{∂f}{∂\phi ∂r}\right) \\
&\quad + \hat{\phi} \frac{1}{r} \left(\hat{\phi} \frac{∂f}{∂r} + \hat{r} \frac{∂f}{∂r ∂\phi} - \hat{r} \frac{1}{r} \frac{∂f}{∂\phi} + \hat{\phi} \frac{1}{r} \frac{∂^2 f}{∂\phi^2}\right) \\
&= \frac{∂^2f}{∂r^2} + \frac{1}{r} \frac{∂f}{∂r} + \frac{1}{r^2} \frac{∂^2 f}{∂\phi^2} \\
&= \frac{1}{r} \frac{∂}{∂r} \left(r \frac{∂f}{∂r}\right) + \frac{1}{r^2} \frac{∂^2 f}{∂\phi^2}
\end{align*}$
"""

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

julia_version = "1.7.1"
manifest_format = "2.0"

[deps]
"""

# ╔═╡ Cell order:
# ╟─321b951c-9bea-11ec-0d10-31f76d44cc9e
# ╟─710b2cc8-45e7-4eee-9066-3fd4080aebc9
# ╟─370b6a47-79b6-41e3-ad41-01544affa97d
# ╟─a1799513-6a61-4443-8ec7-428c0696bc5f
# ╟─08ef5ce6-9d82-4758-ade2-4ea62b514451
# ╟─c0971a8a-7d86-4cb6-aae9-5423beeb1da4
# ╟─74bea6fd-90e1-4aad-bb58-50b141eb0255
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


using SequenceTransformations, Polynomials # use my Polynomials fork Pkg.clone("https://github.com/MikaelSlevinsky/Polynomials.jl.git")

x = RatPoly(Poly([0;big(1)]),one(Poly{BigInt}))

# In this example, we use nonlinear sequence transformations to obtain rational approximants to power series.
# Consider the power series for the inverse tangent function.

a = Sequence(i->x*(-x^2)^(i-1)/(2i-1))

# The ϵ-algorithm corresponds to creating a diagonal sequence in the Padé table.

Epsilon = ϵ(cumsum(a))

Epsilon(1:20)

# Now we can see how the Padé aproximants compare to the function inverse tangent.

myatan(x,n) = convert(typeof(x),Epsilon(n)(x))

myatan(1.0,20)-atan(1.0) # Double precision. Summing 10 million terms can only deliver a little over 8 digits.

myatan(2.0,20)-atan(2.0) # Eight digits beyond region of convergence. Not bad!

# More powerful sequence transformations create useful approximants to factorially divergent power series.
# This series appears when computing the asymptotic expansion of the exponential integral.

a = Sequence(i->(-x)^(i)*factorial(big(i-1)))

D,L,W = Drummond(cumsum(a)),Levin(cumsum(a)),Weniger(cumsum(a))

W(1:20)

mydivergentseries(x,n) = convert(typeof(x),W(n)(x))

Gompertz = -.596347362323194074341078499369279376074

mydivergentseries(1.0,20)-Gompertz # It does quite well!

# The ratio of these sequences is used in Stirling's asymptotic approximation to the Gamma function.
A001163 = big([1,1,1,-139,-571,163879,5246819,-534703531,-4483131259,432261921612371,6232523202521089,-25834629665134204969,-1579029138854919086429,746590869962651602203151,1511513601028097903631961,-8849272268392873147705987190261,-142801712490607530608130701097701])
A001164 = big([1,12,288,51840,2488320,209018880,75246796800,902961561600,86684309913600,514904800886784000,86504006548979712000,13494625021640835072000,9716130015581401251840000,116593560186976815022080000,2798245444487443560529920000,299692087104605205332754432000000,57540880724084199423888850944000000])

# Here, we use the ϵ-algorithm to generate Padé approximants instead of a Poincaré-type asymptotic expansion.

x = RatPoly(Poly([0;big(1)]),one(Poly{BigInt}))

a = Sequence(i->A001163[i]//A001164[i]*x^(i-1))

ϵ(cumsum(a))(1:2:17)

# We can create the Levin d-transformations to the asymptotic expansion for different rational approximants.

Levin(cumsum(a),:d)(1:10)


##
import LinearAlgebra as linalg
import Plots as plt

include("KalmanFilter.jl")
import .KalmanFilter as kf

##
function get_data()
    # Observations
    x = [4000.0 4260.0 4550.0 4860.0 5110.0] # position [m]
    v = [ 280.0  282.0  285.0  286.0  290.0] # velocity [m/s]
    # x = [4000.0 4260.0 4550.0 4860.0 5110.0 5220.0 5115.0 4995.0 4871.0] # position [m]
    # v = [ 280.0  282.0  285.0  286.0  290.0  295.0  292.0  287.0  284.0] # velocity [m/s]
    X = [x; v]

    # Initial values
    a = 2.0 # acceleration
    Δt = 1.0 # Time step
    ΔP = [20.0; 5.0] # Process errors in process covariance matrix
    Δ = [25.0; 6.0] # Observation errors
    return X, a, Δ, ΔP, Δt
end

## Define the state space model

function define_model(X, a, Δ, ΔP, Δt)
    m, n = size(X)
    Q = zeros(m, m) # Process noise covariance matrix
    F = [1.0 Δt; 0.0 1.0] # Transition matrix
    B = [Δt^2/2.0 Δt]' # Control-input matrix
    u = 2.0 # # Control vector, acceleration
    R = linalg.Diagonal([Δ[1]^2 0.0; 0.0 Δ[2]^2]) # Sensor noise covariance matrix
    H = kf.eye(m) # Measurement matrix

    # Initial predicted process covariance matrix
    P = linalg.Diagonal([ΔP[1]^2 ΔP[1]*ΔP[2]; ΔP[2]*ΔP[1] ΔP[2]^2])
    return P, Q, R, F, H, B, u, m, n
end

## Data input
function data_input(X, n)
    X = [ [X[1,i], X[2,i]] for i in 1:n ]
    Xp = [ X[1] ] # Initialize predicted state
    Xk = copy(Xp) # Initialized new state matrix
    return Xp, Xk, X
end

## Smoothers
function run_smoothers_bts(Xk, F, Σ)
    k = length(Xk)
    sm = Vector{kf.RTSSmoother}(undef, k)
    sm[k] = kf.RTSSmoother(Xk[k], Σ[k])
    ksmoother = kf.RauchTungStriebel()
    while k > 1
        s = kf.kalman_smooth(ksmoother, sm[k].xk, Xk[k-1], Xk[k], Σ[k-1], Σ[k], F)
        k -= 1
        sm[k] = s
    end
    return sm
end

## UKF
function run_ukf(X, f, g, P, R, Q, α; filter=kf.CholeskySqrt())
    m, n = size(X)
    _, _, Xx = data_input(X, n)
    k = 1
    K = kf._kalman_gain(P, kf.eye(m), R)
    sukf = [ kf.UnscentedKF(
                Xx[k], Xx[k],
                P, K,
                zeros(2*m+1), zeros(2*m+1), zeros(m, 2*m+1),
                zeros((2*m)*2+1), zeros((2*m)*2+1), zeros(m, (2*m)*2+1),
                P, P,
                filter,
            )
    ]
    while k < n
        s = kf.unscented_kalman(
            Xx[k+1], sukf[k].xk, f, g, sukf[k].P, R, Q; sqrt_filter=filter, α=α,
        )
        push!(sukf, s)
        k += 1
    end
    return sukf
end

## Plots
function plot_kf(Xp, Xk, Xx, Δ)
    # Xp, Xk, Xx = reduce(hcat, Xp), reduce(hcat, Xk), reduce(hcat, Xx)
    p1 = plt.plot(Xp[1,:], m=:o, label="Predicted", legend=:topleft, ylabel="Position [m]", xlabel="Time [s]")
    plt.plot!(p1, Xx[1,:], m=:s, label="Measured", ribbon=Δ[1])
    plt.plot!(p1, Xk[1,:], m=:p, label="KF")
    p2 = plt.plot(Xp[2,:], m=:o, label="Predicted", legend=:topleft, ylabel="Velocity [m/s]", xlabel="Time [s]")
    plt.plot!(p2, Xx[2,:], m=:s, label="Measured", ribbon=Δ[2])
    plt.plot!(p2, Xk[2,:], m=:p, label="KF")
    p = plt.plot(p1, p2, size=(700,350))
    return p
end

function plot_kf_smooth(Xp, Xk, Xx, Xks, Δ)
    # Xp, Xk = reduce(hcat, Xp), reduce(hcat, Xk)
    # Xx, Xks = reduce(hcat, Xx), reduce(hcat, Xks)
    p1 = plt.plot(Xp[1,:], m=:o, label="Predicted", legend=:topleft, ylabel="Position [m]", xlabel="Time [s]")
    plt.plot!(p1, Xx[1,:], m=:s, label="Measured", ribbon=Δ[1])
    plt.plot!(p1, Xk[1,:], m=:p, label="KF")
    plt.plot!(p1, Xks[1,:], m=:h, label="Smooth")
    p2 = plt.plot(Xp[2,:], m=:o, label="Predicted", legend=:topleft, ylabel="Velocity [m/s]", xlabel="Time [s]")
    plt.plot!(p2, Xx[2,:], m=:s, label="Measured", ribbon=Δ[2])
    plt.plot!(p2, Xk[2,:], m=:p, label="KF")
    plt.plot!(p2, Xks[2,:], m=:h, label="Smooth")
    p = plt.plot(p1, p2, size=(700,350))
    return p
end

## Run KFs
X, a, Δ, ΔP, Δt = get_data()
P, Q, R, F, H, B, u, m, n = define_model(X, a, Δ, ΔP, Δt)

## Unscented Kalman Filter
f(x) = F*x .+ B .* u
g(x) = H*x
skf = run_ukf(X, f, g, P, R, Q, 1e-3; filter=kf.SVDSqrt())
Xp, Xk = kf.unpack(skf, [:xp, :xk])
p1 = plot_kf(Xp, Xk, X, Δ)

## Rauch Tung Striebel Smoother
Σ, Xkm = kf.unpack_matrix(skf, [:P, :xk])
skm = run_smoothers_bts(Xkm, F, Σ)
Xkm = kf.unpack(skf, :xk)
p1m = plot_kf_smooth(Xp, Xk, X, Xkm, Δ)


###
  Name     : c7_17_optim_example.jl
  Book     : Hands-on Data Science with Anaconda )
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan and James Yan
  Date     : 3/15/2018
  email    : yany@canisius.edu
             paulyxy@hotmail.com
###

using Optim
f(x) = (1.0 - x[1])^2 + 200.0 * (x[2] - x[1]^2)^2
optimize(f, [0.0, 0.0])


#=
  Biased coin

  From cplint: http://cplint.eu/example/inference/coin.pl
  """
  Throwing a coin with uncertainty on its fairness, from
  J. Vennekens, S. Verbaeten, and M. Bruynooghe. Logic programs with annotated
  disjunctions. In International Conference on Logic Programming,
  volume 3131 of LNCS, pages 195-209. Springer, 2004.
  """

  Model 1: Probability of tossing a head (no observation)
  Distributions of variable coinHead (num:0)
  1.00000 =>    5049  (0.504900)
  0.00000 =>    4951  (0.495100)

  Model 2: Probability of tossing a head given that we know it's biased
  Distributions of variable coinHead (num:0)
  1.00000 =>    5712  (0.571200)
  0.00000 =>    4288  (0.428800)

  Cf ~/cplint/coin.pl
     ~/blog/biased_coin.blog
     ~/webppl/biased_coin.wppl

=#

using Turing
include("jl_utils.jl")

@model function biased_coin(experiment="observe biased")
    fair = 1
    biased = 2

    coinType ~ Categorical([0.9,0.1]) # fair,biased
    head = 1
    tail = 2
    coinResult ~ coinType == fair ? Categorical([0.5,0.5]) :
            Categorical([0.6,0.4])

    if experiment == "observe biased"
        true ~ Dirac(coinType == biased)
    end

    coinHead ~ Dirac(coinResult == head)
    coinTail ~ Dirac(coinResult == tail)

end

println("Model 1: Probability of tossing a head (no observation)")
model = biased_coin("no observation")

num_chns = 4
# chs = sample(model, Prior(), 1000)
# chs = sample(model, MH(), 10_000)
# chs = sample(model, PG(15), 10_000)
# chs = sample(model, IS(), 10_000)
chs = sample(model, SMC(), 10_000)
# chs = sample(model, SMC(), MCMCThreads(), 10_000, num_chs)

# chs = sample(model, SGLD(), 10_000)
# chs = sample(model,NUTS(), 10_000)
# chs = sample(model,HMC(0.01,5), 10_000)

# display(chs)
# display(plot(chs))

show_var_dist_pct(chs,:coinHead)

println("\nModel 2: Probability of tossing a head given that we know it's biased")
model = biased_coin("observe biased")

num_chns = 4
# chs = sample(model, Prior(), 1000)
# chs = sample(model, MH(), 10_000)
# chs = sample(model, PG(15), 10_000)
# chs = sample(model, IS(), 10_000)
chs = sample(model, SMC(), 10_000)
# chs = sample(model, SMC(), MCMCThreads(), 10_000, num_chs)

# chs = sample(model, SGLD(), 10_000)
# chs = sample(model,NUTS(), 10_000)
# chs = sample(model,HMC(0.01,5), 10_000)

# display(chs)
# display(plot(chs))

show_var_dist_pct(chs,:coinHead)


using Polylogarithms
using SpecialFunctions
using Test
using DataFrames, CSV
import Base.MathConstants: π, pi, ℯ, e, γ, eulergamma, catalan, φ, golden
include("test_defs.jl")

@testset "Harmonic numbers" begin

    @testset "    throws errors" begin
        @test_throws DomainError harmonic(-1)
        @test_throws MethodError harmonic(Float32(1.0))
    end

    @testset "    types" begin
        @test typeof(harmonic(1)) == Float64
        @test typeof(harmonic(1.0)) == Float64
        @test typeof(harmonic(complex(1.0))) == Complex{Float64}
    end

    @testset "    basics" begin
        @test harmonic(1) ≅ 1.0
        @test harmonic(2) ≅ 1.5
        @test harmonic(3) ≅ 11.0/6.0
        @test harmonic(4) ≅ 25.0/12.0
        @test harmonic(5) ≅ 137.0/60.0
    end

    @testset "    identities" begin
        for n=4:10
            @test harmonic(n) ≅ 1.0/n + harmonic(n-1)
        end

        for n=20:30
            @test harmonic(n) ≅ sum( 1.0./(1:n) )
        end
    end

    @testset "    dataset 1 (integer arguments)" begin
        data1 = CSV.read("../data/harmonic_test_data_1.csv", DataFrame)
        m = size(data1,1)
        H = Symbol("real(H_n)")
        for i=1:m
            @test harmonic( data1[i,:n] ) ≅  data1[i,H]
        end
    end

    @testset "    dataset 2 (real arguments)" begin
        data2 = CSV.read("../data/harmonic_test_data_2.csv", DataFrame)
        m = size(data2,1)
        H = Symbol("real(H_n)")
        for i=1:m
            @test harmonic( data2[i,:x] ) ≈  data2[i,H] # note the lower accuracy requirement here
        end
    end

    @testset "    dataset 3 (complex arguments)" begin
        data3 = CSV.read("../data/harmonic_test_data_3.csv", DataFrame)
        m = size(data3,1)
        zr = Symbol("Re(z)")
        zi = Symbol("Im(z)")
        Hr = Symbol("Re(H(z))")
        Hi = Symbol("Im(H(z))")
        for i=1:m
            z = data3[i,zr] + im*data3[i,zi]
            Hz = data3[i,Hr] + im*data3[i,Hi]
            @test harmonic( z ) ≈  Hz # note the lower accuracy requirement here
        end
    end
end

@testset "Generalised Harmonic numbers" begin

    @testset "    throws errors" begin
        @test_throws DomainError harmonic(-1, 1.2)
        @test_throws MethodError harmonic(1.0, 1)
        @test_throws MethodError harmonic(1.0, 1.0)
        @test_throws MethodError harmonic(complex(1.0), 1)
    end

    @testset "    output types" begin
        @test typeof(harmonic(1, 1)) == Float64
        @test typeof(harmonic(1, 1.0)) == Float64
    end

    @testset "     identities" begin
        for n=1:10
            @test harmonic(n,1.0) ≅ harmonic(n)
            @test harmonic(n,0.0) ≅ n

            for r = 3:2:9
                @test harmonic(n,r) ≅ Float64(n)^(-r) + polygamma(r-1,n)/gamma(r) + zeta(r)
            end
        end
    end

    @testset "    dataset 4 (integer arguments)" begin
        H = Symbol("real(H_{n,r})")
        data4 = CSV.read("../data/harmonic_test_data_4.csv", DataFrame)
        m = size(data4,1)
        for i=1:m
            @test harmonic( data4[i,:n], data4[i,:r] ) ≅ data4[i,H]
        end
    end

    @testset "    dataset 5 (integer n, real r)" begin
        H = Symbol("real(H_{n,r})")
        data5 = CSV.read("../data/harmonic_test_data_5.csv", DataFrame)
        m = size(data5,1)
        for i=1:m
            @test harmonic( data5[i,:n], data5[i,:r] )  ≅ data5[i,H]
        end
    end
end


using LinearAlgebra
using SparseArrays
using Plots

"This routine solves the periodic advection-diffusion equation using Forward Euler
in time and 2nd order finite differences in space."

m = 100 # number of points

# define spatial grid
x = LinRange(-1,1,m+2) # x_0, x_1, ..., x_m, x_{m+1} = x_0
xint = x[1:end-1]
h = x[2]-x[1]

a = 1.0 # advection speed
ϵ = h/2 # diffusion coefficient
dt = h # timestep
T = 2.0 # final time

# initial condition and forcing
u0(x) = sin(pi*x)
f(x,t) = 0.0

# approximation of second derivative: largest eig = O(1/h^2)
A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
A[1,end] = -1/h^2
A[end,1] = -1/h^2

# approximation of first derivative: largest eig = O(1/h)
Q = (1/(2*h)) * spdiagm(1=>ones(m),-1=>-ones(m))
Q[1,end] = -1/(2*h)
Q[end,1] = 1/(2*h)

u = u0.(xint)
Nsteps = ceil(Int,T/dt)
dt = T / Nsteps

interval = 10
@gif for k = 1:Nsteps
    tk = k*dt
    u .= u + dt * (f.(xint,tk) - (a*Q + ϵ*A)*u)
    if k % interval==0
        plot(xint,u,linewidth=2,label="Solution",ylims=(-1.5,1.5))
        println("on timestep $k out of $Nsteps.")
    end
end every interval

maxλ = Float64[]
ϵvec = .0:.0001:.025
for ϵ = ϵvec
    append!(maxλ,maximum(abs.(eigvals(Matrix(I - dt*(Q + ϵ*A))))))
end
scatter(ϵvec,maxλ,leg=false)
plot!(title="Largest eigenvalue magnitude with ϵ")


#
# Derivatives of f: R -> R
#

@assert norm(Calculus.finite_difference(x -> x^2, 1.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference(x -> x^2, 1.0, :central) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference(x -> x^2, 1.0) - 2.0) < 10e-4

@assert norm(Calculus.finite_difference(x -> sin(x), 1.0, :forward) - cos(1.0)) < 10e-4
@assert norm(Calculus.finite_difference(x -> sin(x), 1.0, :central) - cos(1.0)) < 10e-4
@assert norm(Calculus.finite_difference(x -> sin(x), 1.0) - cos(1.0)) < 10e-4

@assert norm(Calculus.finite_difference(x -> exp(-x), 1.0, :forward) - (-exp(-1.0))) < 10e-4
@assert norm(Calculus.finite_difference(x -> exp(-x), 1.0, :central) - (-exp(-1.0))) < 10e-4
@assert norm(Calculus.finite_difference(x -> exp(-x), 1.0) - (-exp(-1.0))) < 10e-4

#
# Gradients of f: R^n -> R
#

@assert norm(Calculus.finite_difference(x -> x[1]^2, [1.0], :forward) - [2.0]) < 10e-4
@assert norm(Calculus.finite_difference(x -> x[1]^2, [1.0], :central) - [2.0]) < 10e-4
@assert norm(Calculus.finite_difference(x -> x[1]^2, [1.0]) - [2.0]) < 10e-4

@assert norm(Calculus.finite_difference(x -> sin(x[1]), [1.0], :forward) - [cos(1.0)]) < 10e-4
@assert norm(Calculus.finite_difference(x -> sin(x[1]), [1.0], :central) - [cos(1.0)]) < 10e-4
@assert norm(Calculus.finite_difference(x -> sin(x[1]), [1.0]) - [cos(1.0)]) < 10e-4

@assert norm(Calculus.finite_difference(x -> exp(-x[1]), [1.0], :forward) - [-exp(-1.0)]) < 10e-4
@assert norm(Calculus.finite_difference(x -> exp(-x[1]), [1.0], :central) - [-exp(-1.0)]) < 10e-4
@assert norm(Calculus.finite_difference(x -> exp(-x[1]), [1.0]) - [-exp(-1.0)]) < 10e-4

#
# Second derivatives of f: R -> R
#

@assert norm(Calculus.finite_difference_hessian(x -> x^2, x -> 2 * x, 1.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference_hessian(x -> x^2, x -> 2 * x, 10.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference_hessian(x -> x^2, x -> 2 * x, 100.0, :forward) - 2.0) < 10e-4

@assert norm(Calculus.finite_difference_hessian(x -> x^2, 1.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference_hessian(x -> x^2, 10.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.finite_difference_hessian(x -> x^2, 100.0, :forward) - 2.0) < 10e-4

#
# Hessians of f: R^n -> R
#

fx(x) = sin(x[1]) + cos(x[2])
gx = gradient(fx)
@assert norm(gx([0.0, 0.0]) - [cos(0.0), -sin(0.0)]) < 10e-4
@assert norm(Calculus.finite_difference_hessian(fx, gx, [0.0, 0.0], :central) - [-sin(0.0) 0.0; 0.0 -cos(0.0)]) < 10e-4

#
# Taylor Series first derivatives
#

@assert norm(Calculus.taylor_finite_difference(x -> x^2, 1.0, :forward) - 2.0) < 10e-4
@assert norm(Calculus.taylor_finite_difference(x -> x^2, 1.0, :central) - 2.0) < 10e-4

#
# Taylor Series second derivatives
#

# TODO: Fill this in

#
# Speed comparisons
#

@elapsed Calculus.finite_difference(x -> x^2, 1.0, :forward)
@elapsed Calculus.finite_difference(x -> x^2, 1.0, :central)
@elapsed Calculus.finite_difference(x -> x^2, 1.0)
@elapsed Calculus.taylor_finite_difference(x -> x^2, 1.0, 10e-4, :forward)
@elapsed Calculus.taylor_finite_difference(x -> x^2, 1.0, 10e-4, :central)


"""
    Matern(lnℓ::AbstractFloat, ln𝓋::AbstractFloat) <: RadialKernel{SqEuclidean}

The matern kernel with parameters
``\\exp(\\ln\\ell) = \\ell > 0`` and
``\\exp(\\ln\\mathscr{v}) = \\mathscr{v} > 0``.

```math
k_{\\ell,\\mathscr{v}}(x, y)
= k_{\\ell,\\mathscr{v}}(\\lVert x - y\\rVert_2) =
\\frac{2^{1 - \\mathscr{v}}}{\\Gamma(\\mathscr{v})}
\\bigg(\\sqrt{2\\mathscr{v}}\\frac{\\lVert x - y\\rVert_2}{\\ell}
\\bigg)^\\mathscr{v}K_\\mathscr{v}
\\bigg(\\sqrt{2\\mathscr{v}}\\frac{\\lVert x - y\\rVert_2}{\\ell}\\bigg)
```

External links
* [Matérn covariance function on Wikipedia](https://en.wikipedia.org/wiki/Matérn_covariance_function)

"""
mutable struct Matern{
    F<:AbstractFloat
} <: RadialKernel{Euclidean}
    dist::Euclidean
    lnℓ::F
    ln𝓋::F

    𝓋::F
    c::F # 2^(1 - 𝓋) / Γ(𝓋)
    sqrttwo𝓋_ℓ::F

    function Matern(
        dist::Euclidean,
        lnℓ::AbstractFloat,
        ln𝓋::AbstractFloat
    )
        lnℓ, ln𝓋 = promote(lnℓ, ln𝓋)
        𝓋 = exp(ln𝓋)
        new{typeof(lnℓ)}(
            dist,
            lnℓ,
            ln𝓋,
            𝓋,
            exp2(1 - 𝓋) / gamma(𝓋),
            exp((float(logtwo) + ln𝓋) / 2),
        )
    end
end

Matern(lnℓ::AbstractFloat, ln𝓋::AbstractFloat) =
    Matern(Euclidean(), lnℓ, ln𝓋)

@inline function (k::Matern)(τ::AbstractFloat)
    if τ == 0
        return oneunit(promote(τ, k.𝓋)[1])
    end
    t = k.sqrttwo𝓋_ℓ * τ
    return k.c * (t^k.𝓋) * besselk(k.𝓋, t)
end

numparams(::Matern) = (1, 1)
paramtypes(::Matern{F}) where F = (F, F)
params(k::Matern) = (lnℓ = k.lnℓ, ln𝓋 = k.ln𝓋)
function setparams!(k::Matern{F}, lnℓ::F, ln𝓋::F) where F
    𝓋 = exp(ln𝓋)
    k.lnℓ = lnℓ
    k.ln𝓋 = ln𝓋
    k.𝓋 = 𝓋
    k.c = exp2(1 - 𝓋) / gamma(𝓋)
    k.sqrttwo𝓋_ℓ = exp((float(logtwo) + ln𝓋) / 2 - lnℓ)
end


using Compat.Test
import Roots.newton, Roots.halley

@test norm(newton(sin, cos, 0.5) - 0.0) <= 100*eps(1.0)
@test newton(cos, x -> -sin(x), 1.0)  ≈ pi/2
@test newton(x -> x^2 - 2x - 1, x -> 2x - 2, 3.0)  ≈ 2.414213562373095
@test norm(newton(x -> exp(x) - cos(x), x -> exp(x) + sin(x), 3.0) - 0.0) <= 1e-14
@test halley(x -> x^2 - 2x - 1,x -> 2x - 2,x -> 2, 3.0)  ≈ 2.414213562373095
a = halley(x -> exp(x) - cos(x),
           x -> exp(x) + sin(x),
           x -> exp(x) + cos(x), 3.0)
@test norm(a - 0.0) <= 1e-14

## tests with auto derivatives
@test newton(sin, 3) ≈ pi
@test halley(sin, 3) ≈ pi

## More tests with autoderivaitves. Derivative of D operation:
isdefined(ForwardDiff, :derivative) && @test newton(D(sin), 1.5) ≈ pi/2

## test with Complex input

@test real(newton(x ->  x^3 - 1, x ->  3x^2, 1+im)) ≈ 1.0
@test real(newton(x ->  x^3 - 1, x ->  3x^2, 1+10im)) ≈ (-1/2)


using Printf, Combinatorics

derangements(n::Int) = (perm for perm in permutations(1:n)
                        if all(indx != p for (indx, p) in enumerate(perm)))

function subfact(n::Integer)::Integer
    if n in (0, 2)
        return 1
    elseif n == 1
        return 0
    elseif 1 ≤ n ≤ 18
        return round(Int, factorial(n) / e)
    elseif n > 0
        return (n - 1) * ( subfact(n - 1) + subfact(n - 2) )
    else
        error()
    end
end

println("Derangements of [1, 2, 3, 4]")
for perm in derangements(4)
    println(perm)
end

@printf("\n%5s%13s%13s\n", "n", "derangements", "!n")
for n in 1:10
    ders = derangements(n)
    subf = subfact(n)
    @printf("%5i%13i%13i\n", n, length(collect(ders)), subf)
end

println("\n!20 = ", subfact(20))


using LinearAlgebra
using Plots

function flower(x, a=1, b=1, c=4)
    return a * norm(x) + b * sin(c * atan(x[2], x[1]));
end

contour(-3:0.1:3, -3:0.1:3, (x, y)->flower([x, y]))