# p16.jl - Poisson eq. on [-1,1]x[-1,1] with u=0 on boundary

# Set up grids and tensor product Laplacian and solve for u:
N = 24; (D,x) = cheb(N); y = x;
xx = x[2:N]; yy = y[2:N];
f = 10*sin.(8*xx'.*(yy-1));
D2 = D^2; D2 = D2[2:N,2:N]; I = eye(N-1);
L = kron(I,D2) + kron(D2,I);                       # Laplacian
figure(1); clf(); spy(L);
tic(); u = L\f[:]; toc();          # solve problem and watch the clock

# Reshape long 1D results onto 2D grid (flipping orientation):
uu = zeros(N+1,N+1); uu[N:-1:2,N:-1:2] = reshape(u,N-1,N-1);
value = uu[Int(3N/4+1),Int(3N/4+1)];

# Interpolate to finer grid and plot:
xxx = yyy = -1:.04:1;
s = interpolate((x[end:-1:1],y[end:-1:1]),uu,Gridded(Linear()));
uuu = s[xxx,yyy];
figure(2); clf(); surf(xxx,yyy,uuu,rstride=1,cstride=1);
xlabel("x"); ylabel("y"); zlabel("u"); view(-37.5,30);
text3D(.4,-.3,-.3,"\$u(2^{-1/2},2^{-1/2})\$ = $(signif(value,11))",fontsize=9);


#! /usr/bin/julia

# Rosetta Code, Find largest left truncatable prime in a given base

function addmsdigit{T<:Integer}(p::T, b::T, s::T)
    a = T[]
    q = p
    for i in 1:(b-1)
        q += s
        isprime(q) || continue
        push!(a, q)
    end
    return a
end

function lefttruncprime{T<:Integer}(pbase::T)
    b = convert(BigInt, pbase)
    a = BigInt[]
    append!(a, primes(b-1))
    mlt = zero(BigInt)
    s = one(BigInt)
    while !isempty(a)
        mlt = maximum(a)
        s *= b
        for i in 1:length(a)
            p = shift!(a)
            append!(a, addmsdigit(p, b, s))
        end
    end
    return mlt
end

lo, hi = 3, 17
print("The largest left truncatable primes for bases")
println(@sprintf " %d to %d." lo hi)
for i in lo:hi
    mlt = lefttruncprime(i)
    println(@sprintf "   %3d %d (%s)" i mlt base(i, mlt))
end


"""
    Tcmb(z::Real)

Computes the temperature of the CMB at redshift `z`.
"""
Tcmb(z::T) where T = ( 2.728 * ( 1.0 + z ) )

"""
    kSzPrefac(ν::Real, z::Real, DI_over_I::Bool)

Prefactor for the kinetic Sunyaev-Zel'dovich effect.
"""
function kSzPrefac(ν::Real, z::Real, DI_over_I::Bool)

    kSzPrefac  = -1.0 * σ_T / c_light

    if DI_over_I
        x   = h_planck * ν / ( k_B * Tcmb(z) )

        kSzPrefac *= exp(x) - 1 / (x * exp(x))
    end

    return kSzPrefac
end

"""
    kinetic_SZ(n_cm3::Real, vel_y_cgs::Real,
                    ν::Real=1.e9, z::Real=0.0;
                    DI_over_I::Bool=false)

Computes the kinetic Sunyaev-Zel'dovich effect from electron density `n_cm3` and velocity in y-direction to the projection plane in cgs units `vel_y_cgs`.
If `DI_over_I` is set to `true` you also need to provide an observation frequency `ν` and redshift `z`.
"""
function kinetic_SZ(n_cm3::Real, vel_y_cgs::Real,
                    ν::Real=1.e9, z::Real=0.0;
                    DI_over_I::Bool=false)
    return kSzPrefac(ν, z, DI_over_I) * n_cm3 * vel_y_cgs
end

"""
    comptonY(n_cm3::Real, T_K::Real, z::Real)

Computes the Compton-Y parameter from electron density `n_cm3` and temperature `T` in Kelvin at redshift `z`.
"""
function comptonY(n_cm3::Real, T_K::Real, z::Real)
    return yPrefac * n_cm3 * ( T_K - Tcmb(z) )
end

"""
    tSzPrefac(ν::Real, z::Real)

Computes the prefactor for the thermal Sunyaev-Zel'dovich effect.
"""
function tSzPrefac(ν::Real, z::Real, DI_over_I::Bool)

    x   = h_planck * ν / ( k_B * Tcmb(z) )
    tSzPrefac = (x * (exp(x) + 1.0) / (exp(x) - 1.0) - 4.0)

    if DI_over_I
        tSzPrefac *= exp(x) - 1 / (x * exp(x))
    end

    return tSzPrefac
end

"""
    thermal_SZ(n_cm3::Real, T_K::Real, z::Real=0.0, ν::Real=1.44e9)

Computes the thermal Sunyaev-Zel'dovich effect for electron density `n_cm3` and temperature `T_K` in Kelvin at redshift `z` and observer frequency `ν`.
"""
function thermal_SZ(n_cm3::Real, T_K::Real,
                    z::Real=0.0, ν::Real=1.44e9;
                    DI_over_I::Bool=false)

    return tSzPrefac(ν, z, DI_over_I) * comptonY(n_cm3, T_K, z)
end


using StaticArrays

"""
    butcher_tableau_radau_2stages()

Returns (A,b,c) corresponding to the Butcher tableau for the 2 stage Radau IIA scheme.
"""
function butcher_tableau_radau_2stages()
	A = @SMatrix [5/12   -1/12
	              3/4    1/4];
	b = @SVector [3/4, 1/4];
	c = @SVector [1/3, 1.0];
	return (A, b, c);
end

"""
    butcher_tableau_radau_3stages()

Returns (A,b,c) corresponding to the Butcher tableau for the 3 stage Radau IIA scheme.
"""
function butcher_tableau_radau_3stages()
	A = @SMatrix [(88.0-7.0*sqrt(6.0))/360.0       (296.0-169.0*sqrt(6.0))/1800.0   (-2.0+3.0*sqrt(6.0))/225.0
	              (296.0+169.0*sqrt(6.0))/1800.0   (88.0+7.0*sqrt(6.0))/360.0       (-2.0-3.0*sqrt(6.0))/225.0
	              (16.0-sqrt(6.0))/36.0            (16.0+sqrt(6.0))/36.0             1/9];
	b = @SVector [(16.0-sqrt(6.0))/36.0,    (16.0+sqrt(6.0))/36.0,    1/9];
	c = @SVector [(4.0-sqrt(6.0))/10.0,     (4.0+sqrt(6.0))/10.0,     1.0];
	return (A, b, c);
end


using Optim

mutable struct HLT
	α::Float64
	β::Float64
	l₀::Float64
	b₀::Float64

	HLT() = new(0, 0, 0, 0)
	HLT(α::Number, β::Number, l₀::Number, b₀::Number) = new(Float64(α), Float64(β),
		Float64(l₀), Float64(b₀))
end

function loss(model::HLT, time_series)
	α, β, l₀, b₀ = model.α, model.β, model.l₀, model.b₀

	N = length(time_series)

	l_t, l_t_, b_t = 0, 0, 0 # l_t_ is the variable to save l(t-1)
	loss = 0

	for t in 1:N
		if t == 1
			l_t = l₀
			b_t = b₀
		else
			l_t = time_series[t - 1] * α + (l_t + b_t) * (1 - α) #b_t is taking b(t-1) value
		end

		l_t_ = l_t
		y_pred = l_t + b_t
		loss += (time_series[t] - y_pred)^2
	end

	return loss
end

function fit(model::HLT, y)
	lower = [-Inf, -0.001, -Inf, -Inf]
	upper = [1., 1., Inf, Inf]
	initial = [model.α, model.β, model.l₀, model.b₀]

	function loss_(parameters::Array{Float64, 1})
		α, β, l₀, b₀ = parameters
		return loss(HLT(α, β, l₀, b₀), y)
	end
	res = Optim.optimize(loss_, lower, upper, initial)
	optimal = Optim.minimizer(res)
	return HLT(optimal[1], optimal[2], optimal[3], optimal[4])
end

function forecast(model::HLT, time_series, forecast_length)
	N = length(time_series)
	α, β, l₀, b₀ = model.α, model.β, model.l₀, model.b₀
	l_t, l_t_, b_t = 0, 0, 0
	pred = Array{Float64, 1}(undef, forecast_length)

    #go through the whole time series making the point by point estimate
	for t in 1:N
		if t == 1
			l_t = l₀
			b_t = b₀
		else
			l_t = time_series[t - 1] * α + (l_t + b_t) * (1 - α) #b_t "is" b(t-1)
			b_t = β * (l_t - l_t_) + (1 - β) * b_t
		end
		l_t_ = l_t
	end

    #The parameter´s values to make the forecast are those estimated in the last step of the time series
	l_t = time_series[end] * α + (l_t + b_t) * (1 - α)
	b_t = β * (l_t - l_t_) + (1 - β) * b_t

	for i in 1:forecast_length
		#y_pred = l_t + b_t * i
		#push!(pred, y_pred)
		pred[i] = l_t + b_t * i
	end

	return pred
end


using Interpolations
using Plots
pyplot()

f(x) = 2 .* cos(6x) .+ sin(14x) .+ 2.5
c_grid = 0:.2:1
n = length(c_grid)

Af = interpolate((c_grid, ), f(c_grid), Gridded(Linear()))

f_grid = linspace(0, 1, 150)

plot(f_grid, f, color=:blue, linewidth=2, alpha=0.8, label="true function")
plot!(f_grid, Af[f_grid], color=:green, linewidth=2, alpha=0.8,
      label="linear approximation", legend=:top, grid=false)
N = repmat(c_grid, 1, 2)'
heights = [zeros(1,n); f(c_grid)']
plot!(N, heights, color=:black, linestyle=:dash, alpha=0.5, label="")


s = string(
"73167176531330624919225119674426574742355349194934",
"96983520312774506326239578318016984801869478851843",
"85861560789112949495459501737958331952853208805511",
"12540698747158523863050715693290963295227443043557",
"66896648950445244523161731856403098711121722383113",
"62229893423380308135336276614282806444486645238749",
"30358907296290491560440772390713810515859307960866",
"70172427121883998797908792274921901699720888093776",
"65727333001053367881220235421809751254540594752243",
"52584907711670556013604839586446706324415722155397",
"53697817977846174064955149290862569321978468622482",
"83972241375657056057490261407972968652414535100474",
"82166370484403199890008895243450658541227588666881",
"16427171479924442928230863465674813919123162824586",
"17866458359124566529476545682848912883142607690042",
"24219022671055626321111109370544217506941658960408",
"07198403850962455444362981230987879927244284909188",
"84580156166097919133875499200524063689912560717606",
"05886116467109405077541002256983155200055935729725",
"71636269561882670428252483600823257530420752963450")

a = [Int32(s[i]-48) for i in 1:length(s)]
println(maximum([reduce(*, a[i:i+12]) for i in 1:length(a)-13]))

008 - Largest Product in a Series/008 - Largest_Product_in_a_Series.jl


# # Fitting a polynomial with kernels
#
# First, let us load the relevant packages and define our problem.
# We want to interpolate a simple polynomial: `(2x-1)² + 2y + xy - 3`.

using KernelMachines, Statistics, Plots

f(x, y) = (2x-1)^2 + 2y + x * y - 3
f((x, y)) = f(x, y)
rg = 0:0.01:1
N = length(rg)
flat = hcat(repeat(rg, inner=N), repeat(rg, outer=N))
truth = map(f, eachrow(flat))
theme(:wong)
surface(rg, rg, reshape(truth, N, N), clims=(-3, 1))

# Let us generate a `6 x 6` trainig grid.

N_train = 6
rg_train = range(0, 1, length=N_train)
X = hcat(repeat(rg_train, inner=N_train), repeat(rg_train, outer=N_train))
Y = map(f, eachrow(X));

# Now, let us train a Kernel Machine on the problem.

krm = KernelMachineRegression(X, Y;
    dims=(3, 3, 3), kernel=multiplicativegaussiankernel)
fit!(krm)

pred_krm = predict(krm, flat)
surface(rg, rg, reshape(pred_krm, N, N), clims=(-3, 1))

# In this problem, a simple Kernel Ridge regression also performs very well.

kr = KernelRegression(X, Y)
fit!(kr)
pred_kr = predict(kr, flat)
surface(rg, rg, reshape(pred_kr, N, N), clims=(-3, 1))


using Winston

## We create spline bases on for the function space [5,15] -> Real with evenly-spaced nodes, and
## apply them to the grid x
x = linspace(5,15,101)
X1 = linearSplineBFE(x, (5,15), 11)
X2 = quadraticSplineBFE(x, (5,15), 11)
X3 = cubicSplineBFE(x, (5,15), 11)

p = FramedPlot()
map(i -> add(p, Curve(x, X1[:,i], "color", "black")), 1:3)
Winston.display(p)

p = FramedPlot()
map(i -> add(p, Curve(x, X2[:,i], "color", "red")), 1:5)
Winston.display(p)

p = FramedPlot()
map(i -> add(p, Curve(x, X3[:,i], "color", "blue")), 1:7)
Winston.display(p)

## Now we use a pre-specified set of nodes
u = [8.05,11.34,12.31,14.53, 14.75]
X1 = splineBFE(1, x, u)
X2 = splineBFE(2, x, u)

p = FramedPlot()
map(i -> add(p, Curve(x, X1[:,i], "color", "black")), 1:3)
Winston.display(p)

p = FramedPlot()
map(i -> add(p, Curve(x, X2[:,i], "color", "black")), 1:2)
Winston.display(p)


#   Unit "tools.jl" of the Diagonalization.jl Package for Julia language
#
#   MIT License
#   Copyright (c) 2019-2021,
#   Marco Congedo, CNRS, Grenoble, France:
#   https://sites.google.com/site/marcocongedo/home

# ? CONTENTS :
#   This unit implements general tools and internal functions.

"""
```julia
function eig(A)

function eig(A, B)
```
Call Julia function [eigen](https://docs.julialang.org/en/v1/stdlib/LinearAlgebra/#LinearAlgebra.eigen)
and return its output sorted by descending
order of eigenvalues.
"""
function eig(A)
   λ, U=eigen(A)
   return (reverse(λ), reverse(U, dims=2))
end

function eig(A, B)
   λ, U=eigen(A, B)
   return (reverse(λ), reverse(U, dims=2))
end

"""
```julia
function nonDiagonality(C::Union{Matrix, Diagonal, SorH})
```

Measure of deviancy from diagonality of ``n⋅n`` square matrix `C`, defined as
(Congedo et al., 2008)[🎓](@ref).

``\\frac{\\sum_{i≠j}|c_{ij}|^2}{(n-1)\\sum_{i}|c_{ii}|^2}``

It is equal to ``0`` if ``C`` is diagonal, equal to ``1`` if
``C`` is perfectly uniform.

**Examples:**
```julia
using Diagonalizations
C=ones(10, 10)                   # uniform matrix
nd=nonDiagonality(C)             # must be 1
D=Diagonal(abs.(randn(10, 10)))  # diagonal matrix
nd=nonDiagonality(D)             # must be 0
```
"""
function nonDiagonality(C::Union{Matrix, Diagonal, SorH})
   n = size(C, 1)
   n ≠ size(C, 2) && throw(ArgumentError("📌, nonDiagonality function: input matrix must be square"))
   ssDiag=sumOfSqrDiag(C)
   return ((sumOfSqr(C)-ssDiag)/ssDiag)/(n-1)
end
nonD=nonDiagonality

"""
```julia
function spForm(P::Union{Mat, Real, Complex})
```
Measure of deviancy from scaled permutation form of ``n⋅n`` square matrix
`P`, defined as

``\\frac{1}{2(n-1)}\\bigg(\\sum_{row}1-\\beta(row)+\\sum_{col}1-\\beta(col)\\bigg)``,

where for each *row* and *column* of `P`, β is the maximum of the
absolute values divided by the sum of the absolute values.

This index is equal to ``0`` if in each row and column ``P``
has only one non-zero element, that is, if ``P`` is a scaled permutation matrix.
The larger the index, the farther away ``P`` is from this form.

This measure and several existing variants are well-known in the blind source
separation / independent component analysis community,
where it is used to compare approximate joint diagonalization
algorithms on simulated data. In fact, if ``A`` is the inverse of the
approximate joint diagonalizer that is used to generate the data
and ``B`` the approximate joint diagonalizer estimated by an algorithm,
``P=BA`` must be as close as possible to a scaled permutation matrix
(see [scale and permutation](@ref)).

Return 0.0 (zero) if `P` is a real of complex number.

**Examples:**

```julia
using Diagonalizations, PosDefManifold
# create 20 random commuting matrices
# they all have the same eigenvectors
Cset=randP(3, 20; eigvalsSNR=Inf, commuting=true)
# estimate the approximate joint diagonalizer (ajd)
a=ajd(Cset)
# the ajd must be equivalent to the eigenvector matrix of
# any of the matrices in Cset
spForm(a.F'*eigvecs(Cset[1]))+1.0≈1.0 ? println(" ⭐ ") : println(" ⛔ ")
```

"""
function spForm(P::Union{Mat, Real, Complex})
   if P isa Number return 0.0 end
   r, c=size(P)
   r ≠ c && throw(ArgumentError("📌, spForm function: input matrix must be square"))
   mos(v::AbstractArray)=1.0-(maximum(v)/sum(v)) # 1- max over sum of a vector
   (sum(mos(abs.(p)) for p∈eachcol(P)) + sum(mos(abs.(p)) for p∈eachrow(P)))/(2*(r-1))
end

"""
```julia
(1)
function genDataMatrix(t::Int, n::Int, A=nothing)

(2)
function genDataMatrix(::Type{Complex{T}},
                       t::Int, n::Int, A=○) where {T<:AbstractFloat}
```

(1)
Generate a ``t⋅n`` random data matrix as ``XA``,
where ``X`` is a ``t⋅n`` matrix with entries randomly drawn
from a Gaussian distribution and ``A`` a ``n⋅n`` symmetric
matrix, which, if not provided as argument `A`,
will be generated with entries randomly drawn from a uniform
distribution ∈[-1, 1].

(2)
as (1), but ``X`` is generated randomly from a complex
Gaussian distribution and ``A``is Hermitian (complex)
which, if not provided as argument `A`,
will be generated with entries randomly drawn from a uniform
distribution ∈[-1-i1, 1+i1].

**Examples:**
```julia
A=genDataMatrix(100, 20) # (1), real
A=genDataMatrix(ComplexF64, 100, 20) # (2), complex
```

"""
function genDataMatrix(t::Int, n::Int, A=○)
   if A===○ A=Symmetric((rand(n, n).-0.5).*2) end
   return randn(t, n)*A
end

function genDataMatrix(::Type{Complex{T}},
                       t::Int, n::Int, A=○) where {T<:AbstractFloat}
  if A===○ A=Hermitian([(rand(T).-(0.5+0.5im)).*2. for i=1:n, j=1:n]) end
  return randn(T, t, n)*A
end

# -------------------------------------------------------- #
# INTERNAL FUNCTIONS #
# -------------------------------------------------------- #

# EigenDecomposition with a covariance matrix as input
function _getEVD(C :: Union{Hermitian, Symmetric, Mat}, eVar::TeVaro,
                 eVarMeth::Function, simple::Bool)

   λ, U = eig(C)
   λ=_checkλ(λ) # make sure no imaginary noise is present (for complex data)
   simple ? (U, Matrix(U'), Diagonal(λ), ○, ○, ○) :
   begin
     eVar===○ ? eVar=0.999 : ○
     eVar, D, U, p, arev=_ssd!(eVar, λ, U, _minDim(C), eVarMeth)
     (U, Matrix(U'), D, eVar, λ, arev)
   end
end

# EigenDecomposition with a data matrix as input
_getEVD(X::Mat, covEst::StatsBase.CovarianceEstimator, dims::Int64,
        mean::Tmean, w::Tw, eVar::TeVaro, eVarMeth::Function, simple::Bool) =
  _getEVD(_cov(X, covEst, dims, mean, w), eVar, eVarMeth, simple)

# Whitening with a covariance matrix as input
function _getWhi(C :: Union{Hermitian, Symmetric, Mat}, eVar::TeVaro,
                 eVarMeth::Function, simple::Bool)

  U, Uⁱ, D, eVar, λ, arev=_getEVD(C, eVar, eVarMeth, simple)
  if simple
     if eltype(C)<:Real
        ispos(diag(D); tol=eps(eltype(C)), rev=true, 🔔=true,
        msg="negative or almost zero eigenvalue") || throw(ArgumentError("A `simple` linear filter cannot be created. See the warning that has been printed in Julia's REPL"))
     end
     (U*D^-0.5, D^0.5*Uⁱ, D, ○, ○, ○)
  else
     (U*D^-0.5, D^0.5*Uⁱ, D, eVar, λ, arev)
  end
end

# Whitening with a data matrix as input
_getWhi(X::Mat, covEst::StatsBase.CovarianceEstimator, dims::Int64,
        mean::Tmean, w::Tw, eVar::TeVaro, eVarMeth::Function, simple::Bool) =
   _getWhi(_cov(X, covEst, dims, mean, w), eVar, eVarMeth, simple)

# convert mean vector for compatibility with StatsBase.jl
function _convert_mean(mean::Tmean, dims::Int, argName::String)
  length(mean)≠n && throw(ArgumentError(📌*", _convert_mean internal function: vector "*argName*" must have length $n"))
  return dims==1 ? Matrix(mean') : mean
end

# return `X` or `X` with the mean subtracted, depending on `meanX`
function _deMean(X::Mat, dims::Int, meanX::Tmean)
   if       meanX isa Int
            return X
   elseif   meanX===○
            meanX_=mean(X; dims=dims)
   elseif   meanX isa AbstractVector
            meanX_=_convert_mean(meanX, dims, "meanX")
   end
   #println("dims ", dims, "  sizemeanX_ ", size(meanX_), " sizeX ", size(X))
   if       dims==1
            s=(1, size(X, 2))
   elseif   dims==2
            s=(size(X, 1), 1)
   end
   size(meanX_)≠s && throw(ArgumentError(📌*", _deMean internal function: The size of `meanX_` does not fit input matrix `X` with `dims`=$dims"))
   return X.-meanX_
end

# check arguments for one data matrix input
function _check_data(X::Mat, dims::Int64, covEst::StatsBase.CovarianceEstimator, meanX::Tmean, wX::Tw)
   dims ∈ (1, 2) || throw(ArgumentError(📌*", _check-data internal function: Argument `dims` may be 1 or 2. dims=$dims"))
   wX≠○ && lenght(wX)≠size(X, dims) && throw(ArgumentError(📌*", _check-data internal function: The size of `wX` does not fit input matrix `X` with `dims`=$dims"))
   eltype(X)<:Complex && covEst≠SCM && throw(ArgumentError(📌*", _check-data internal function: Only the `SCM` (sample covariance matrix) `covEst` estimator can be used for complex data"))
   # TODO add check for meanX
   ishermitian(X) && throw(ArgumentError(📌*", _check-data internal function: it looks like
   you want to call a filter constuctor that takes covariance matrices as input,
   but you are actually calling the constructor that takes data matrices as input.
   Solution: flag your covariance matrix(ces) argument(s) as Symmetric or Hermitian,
   for example, `Hermitian(C)`. To do so, you will need to be using LinearAlgebra."))
   return true
end

# check arguments for two data matrices input
function _check_data(X::Mat, Y::Mat, dims::Int64, covEst::StatsBase.CovarianceEstimator, meanX::Tmean, meanY::Tmean, wXY::Tw)
   dims ∈ (1, 2) || throw(ArgumentError(📌*", _check-data internal function: Argument `dims` may be 1 or 2. dims=$dims"))
   size(X, dims)==size(Y, dims) || throw(ArgumentError(📌*", _check-data internal function: The `dims` dimension of argument `X` and `Y` must be the same"))
   wXY≠○ && lenght(wXY)≠size(X, dims) && throw(ArgumentError(📌*", _check-data internal function: The size of `wXY` does not fit input matrix `X` with `dims`=$dims"))
   (eltype(X)<:Complex || eltype(Y)<:Complex) && covEst≠SCM && throw(ArgumentError(📌*", _check-data internal function: Only the `SCM` (sample covariance matrix) `covEst` estimator can be used for complex data"))
   # TODO add check for meanX and meanY
   return true
end

# check arguments for one vector of data matrix input
function _check_data(𝐗::VecMat, dims::Int64, covEst::StatsBase.CovarianceEstimator, meanX::Into, w::Twf)
   dims ∈ (1, 2) || throw(ArgumentError(📌*", _check-data internal function: Argument `dims` may be 1 or 2. dims=$dims"))
   x=collect(size(X, _flip12(dims)) for X ∈ 𝐗)
   all(y->y==x[1], x) ||  throw(ArgumentError(📌*", _check-data internal function: Given `dims`=$dims, all matrices in `𝐗` must have the same dimension $(_flip12(dims))"))
   w≠○ && !(w isa Function) && lenght(w)≠length(𝐗) && throw(ArgumentError(📌*", _check-data internal function: The size of `w` must be equal to the number of matrices in `𝐗`"))
   !isempty(findall(x->eltype(x)<:Complex, 𝐗)) && covEst≠SCM && throw(ArgumentError(📌*", _check-data internal function: Only the `SCM` (sample covariance matrix) `covEst` estimator can be used for complex data"))
   # TODO add check for meanX
   return true
end

# check arguments for two vectors of data matrix input
function _check_data(𝐗::VecMat, 𝐘::VecMat, dims::Int64, covEst::StatsBase.CovarianceEstimator, meanX::Into, meanY::Into, w::Twf)
   dims ∈ (1, 2) || throw(ArgumentError(📌*", _check-data internal function: Argument `dims` may be 1 or 2. dims=$dims"))
   length(𝐗)==length(𝐘) || throw(ArgumentError(📌*", _check-data internal function: The number of matrices in `𝐗` and `𝐘` must be the same"))
   x=collect(size(X, dims)-size(Y, dims) for (X, Y) ∈ (𝐗, 𝐘))
   norm(x)==0 || throw(ArgumentError(📌*", _check-data internal function: The `dims` dimension of all pairs of matrices in `𝐗` and `𝐘` must be the same"))
   w≠○ && !(w isa Function) && lenght(w)≠length(𝐗) && throw(ArgumentError(📌*", _check-data internal function: The size of `w` must be equal to the number of matrices in `𝐗` and `𝐘`"))
   (!isempty(findall(x->eltype(x)<:Complex, 𝐗)) || !isempty(findall(y->eltype(y)<:Complex, 𝐘))) && covEst≠SCM && throw(ArgumentError(📌*", _check-data internal function: Only the `SCM` (sample covariance matrix) `covEst` estimator can be used for complex data"))
   # TODO add check for meanX and meanY
   return true
end

#TODO _check_data(𝑿::VecVecMat, dims::Int64, covEst::StatsBase.CovarianceEstimator, meanX::Into, ○)===○ && return

# call StatsBase.cov within one line with or without weights
# Also, flag the covariance as Symmetric if is real, Hermitian if is complex
# The mean is subtracted separatedly for consistence with the other _cov method
# NB!!! covarianceestimations.jl does not work for complex data input!
function _cov(X::Matrix{R},
              covEst   :: StatsBase.CovarianceEstimator = SCM,
              dims     :: Int64 = 1,
              meanX    :: Tmean = 0,
              wX       :: Tw = ○) where R<:Real
   #T = R isa Real ? Symmetric : Hermitian
   X_=_deMean(X, dims, meanX)
   return wX===○ ? Symmetric(cov(covEst, X_; dims=dims, mean=0)) : # do NOT remove `mean`=0
                   Symmetric(cov(covEst, X_, wX; dims=dims, mean=0)) # "
end

# `covest` is not used but left in for code homogeneity
function _cov(X::Matrix{R},
              covEst   :: StatsBase.CovarianceEstimator = SCM,
              dims     :: Int64 = 1,
              meanX    :: Tmean = 0,
              wX       :: Tw = ○) where R<:Complex
   X_=_deMean(X, dims, meanX)
   wX≠○ ? ( dims==1 ? X__=wX.*X_ : X__=wX'.*X_ ) : X__=X_
   return dims==1 ? Hermitian((X__'*X__)/size(X, 1)) : Hermitian((X__*X__')/size(X, 2))
end

# as before for a vector of data matrices at once
# NB!!! does not work for complex data input as uses the above method!
function _cov(𝐗::VecMat;
              covEst   :: StatsBase.CovarianceEstimator = SCM,
              dims     :: Int64 = 1,
              meanX    :: Into = 0)
   # once PosDefManifold supports vectors of Symmetric matrices
   # T = R===Real ? Symmetric : Hermitian
   # remove `Hermitian` here below and use T instead
   # _cov will automatically flag its output
   𝐂=Vector{Hermitian}(undef, length(𝐗))
   @threads for i=1:length(𝐗)
               𝐂[i]=Hermitian(_cov(𝐗[i], covEst, dims, meanX, ○))
   end
   return 𝐂
end

# cross-covariance within one line with or without weights
# The mean is subtracted separately since there is no crosscov method in StatsBase
function _cov(X::Matrix{R}, Y::Matrix{R},
              dims     :: Int64 = 1,
              meanX    :: Tmean = 0,
              meanY    :: Tmean = 0,
              wXY      :: Tw = ○) where R<:Union{Real, Complex}
   (size(X, dims) ≠ size(Y, dims)) && throw(ArgumentError(📌*", _cov internal function: the size of matrices `X` and `Y` are not conform for computing cross-covariance with $dims as value of `dims`"))
   X_=_deMean(X, dims, meanX)
   Y_=_deMean(Y, dims, meanY)
   return wXY===○ ? ( dims==1 ? (X_'*Y_)/size(X, 1) : (X_*Y_')/size(X_, 2) ) :
                    ( dims==1 ? ((wXY'.*X_')*Y_)/wXY.sum : ((wXY'.*X_)*Y_')/wXY.sum )
end

# The same as above, for 2 vectors of data matrices at once
# the cross-covariance if computed for all corresponding pairs
function _cov(𝐗::VecMat, 𝐘::VecMat;
              dims     :: Int64 = 1,
              meanX    :: Into = 0,
              meanY    :: Into = 0)
   (length(𝐗)≠length(𝐘)) && throw(ArgumentError(📌*", _cov internal function: vectors 𝐗 and 𝐘 must hold the same number of data matrices"))
   𝐂=Vector{Matrix}(undef, length(𝐗))
   @threads for i=1:length(𝐗)
               𝐂[i]=_cov(𝐗[i], 𝐘[i], dims, meanX, meanY, ○)
            end
   return 𝐂
end

# if     m=1 𝐗 is a vector of k data matrices.
#           Return a kx1x1 array of their covariance matrices in the k dimension
# elseif k=1 𝐗 is a vector of m data matrices.
#           Return a 1xmxm array of all cross-covariances of 𝐗[i] and 𝐗[j], for i,j=1:m
# elseif 𝐗 is a k-vector of m data matrices.
#           Return a kxmxm array of all cross-covariances of 𝐗[l][i] and 𝐗[l][j], for l=1:k, i,j=1:m
function _crossCov(𝐗, m, k;
                   covEst  :: StatsBase.CovarianceEstimator=SCM,
                   dims    :: Int64 = 1,
                   meanX   :: Into = 0,
                   trace1  :: Bool = false,
                   w       :: Union{Tw, Function}=○)
    𝒞=Array{Matrix}(undef, k, m, m)
    if      m==1
      @inbounds for κ=1:k 𝒞[κ, 1, 1] = _cov(𝐗[κ], covEst, dims, meanX, ○) end
    elseif  k==1
      @inbounds for i=1:m-1, j=i+1:m
                        𝒞[1, i, j] = _cov(𝐗[i], 𝐗[j], dims, meanX, meanX, ○)
                        𝒞[1, j, i] = 𝒞[1, i, j]'
                end
      @inbounds for i=1:m 𝒞[1, i, i] = _cov(𝐗[i], covEst, dims, meanX, ○) end # This is needed for scaling in any case
    else
      @inbounds for κ=1:k, i=1:m-1, j=i+1:m
                        𝒞[κ, i, j] = _cov(𝐗[κ][i], 𝐗[κ][j], dims, meanX, meanX, ○)
                        𝒞[κ, j, i]=𝒞[κ, i, j]'
                end
      @inbounds for κ=1:k, i=1:m 𝒞[κ, i, i] = _cov(𝐗[κ][i], covEst, dims, meanX, ○) end # This is needed for scaling in any case
    end

    # trace normalize
    if trace1 || w ≠ ○ _Normalize!(𝒞, m, k, trace1, w) end

    return 𝒞
end

# get index and value of the
# first value in 𝜆 greater than or equal to eVar (eVarMeth=searchsortedfirst) or
# last value in 𝜆 less than or equal to eVar (eVarMeth=searchsortedlast),
# where 𝜆 is the vector with accumulated regularized (sum-normalized) eigenvalues
# INPUT:
# the desired explained variance (real) of subspace dimension (int) (evar),
# the eigenvalues in descending order (λ),
# the corresponding eigenvectors (U),
# the maximum theoretical rank of the input matrix (r),
# the method (eVarMeth function) for determining the subspace dimension.
# OUTPUT:
# the actual explained variance (evar!),
# the first p eigenvalues (λ!),
# the corresponding first p eigenvectors (U!),
# the subspace dimension (p),
# the vector with the accumulated regularized eigenvalues (arev)
function _getssd!(eVar::TeVaro, λ::Vec, r::Int64, eVarMeth::Function)
   eltype(λ)<:Complex && @warn "📌, internal function `_getssd!`: the `λ` vector is complex, subspace dimension is based on its absolute values."
   eVar===○ ? eVar=0.999 : ○
   λ_=abs.(λ)
   arev = accumulate(+, λ_./sum(λ_))
   return (eVar isa Int64 ? clamp(eVar, 1, r) : clamp(eVarMeth(arev, eVar), 1, r), arev)
end

#see PCA and Whitening
function _ssd!(eVar::TeVaro, λ::Vec, U::Mat, r::Int64, eVarMeth::Function)
   eltype(λ)<:Complex && @warn "📌, internal function `_ssd!`: the `λ` vector is complex, subspace dimension is based on its absolute values."
   p, arev = _getssd!(eVar, λ, r, eVarMeth)
   return p==r ? 1. : arev[p], Diagonal(λ[1:p]), U[:, 1:p], p, arev
end

#see PMCA and CCA
function _ssdxy!(eVar::TeVaro, λ::Vec, U1::Mat, U2::Mat, r::Int64, eVarMeth::Function)
   eltype(λ)<:Complex && @warn "📌, internal function `_ssdxy!`: the `λ` vector is complex, subspace dimension  is based on its absolute values."
   p, arev = _getssd!(eVar, λ, r, eVarMeth)
   return p==r ? 1. : arev[p], Diagonal(λ[1:p]), U1[:, 1:p], U2[:, 1:p], p, arev
end

# see CSP
function _ssdcsp!(eVar::TeVaro, λ::Vec, U::Mat, r::Int64, eVarMeth::Function, selMeth::Symbol)
   eltype(λ)<:Complex && @warn "📌, internal function `_ssdcsp!`: the `λ` vector is complex, subspace dimension  is based on its absolute values."
   λ_=abs.(λ)
   ratio = λ_./(1.0.-λ_)
   d = (log.(ratio)).^2
   h = selMeth==:extremal ? sortperm(d; rev=true) : [i for i=1:length(λ)]
   arev = accumulate(+, d[h]./sum(d))
   if     eVar isa Int
      p = clamp(eVar, 1, r)
   elseif eVar isa Real
      p = clamp(eVarMeth(arev, eVar), 1, r)
   else    #eVar isa nothing, the default
      if selMeth==:extremal
         g=exp(sum(log, d)/length(d))
         p = clamp(searchsortedlast(d[h], g; rev=true), 1, r)
      else
         p = clamp(searchsortedlast(ratio, 1; rev=true), 1, clamp(argmin(d), 1, r))
      end
   end
   return p==r ? 1. : arev[p], Diagonal(λ[h[1:p]]), U[:, h[1:p]], p, arev
end

# see CSTP
function _ssdcstp!(eVar::TeVaro, λ::Vec, U::Mat, V::Mat, r::Int64, eVarMeth::Function)
   eltype(λ)<:Complex && @warn "📌, internal function `_ssdcstp!`: the `λ` vector is complex, subspace dimension is based on its absolute values."
   λ_=abs.(λ)
   arev = accumulate(+, λ_./sum(λ_))
   if     eVar isa Int
      p = clamp(eVar, 1, r)
   elseif eVar isa Real
      p = clamp(eVarMeth(arev, eVar), 1, r)
   else    #eVar isa nothing, the default
      p = clamp(eVarMeth(arev, 0.999), 1, r)
   end
   return p==r ? 1. : arev[p], Diagonal(λ[1:p]), U[:, 1:p], V[:, 1:p], p, arev
end

_flip12(i::Int) =
   if      i==1 return 2
   elseif  i==2 return 1
   else throw(ArgumentError, 📌*", _flip12 internal function: the `dims` argument must be 1 or 2")
   end

_set_dims(X::Mat)=argmax(collect(size(X)))
_set_dims(X::Mat, Y::Mat)=argmax(collect(size(X))+collect(size(Y)))
_set_dims(𝐗::VecMat)=argmax(sum(collect(size(X)) for X ∈ 𝐗))
_set_dims(𝐗::VecMat, 𝐘::VecMat)=
    argmax(sum(collect(size(X)) for X ∈ 𝐗)+sum(collect(size(Y)) for Y ∈ 𝐘))
_set_dims(𝑿::VecVecMat)=argmax(sum(collect(size(𝑿[i][j])) for i=1:length(𝑿) for j=1:length(𝑿[i])) )

_minDim(X::Matrix) = minimum(size(X))
_minDim(X::Matrix, Y::Matrix) = min(minimum(size(X)), minimum(size(Y)))
_minDim(C::SorH) = size(C, 1)
_minDim(𝐂::ℍVector) = minimum(size(C, 1) for C ∈ 𝐂)
_minDim(C1::SorH, C2::SorH) = min(size(C1, 1), size(C2, 1))
_minDim(𝐗::VecMat) = minimum(minimum(size(X)) for X ∈ 𝐗)
_minDim(𝐗::VecMat, 𝐘::VecMat) = min(_minDim(𝐗), _minDim(𝐘))
_minDim(𝑿::VecVecMat) = minimum((minimum(minimum(size(X)) for X ∈ 𝑿[i]) for i=1:length(𝑿)))


"""
        mapr(t, a, b, c, d) -> Real

Maps the value `t` from the interval `[a, b]` to `[c, d]`
"""
function mapr(t::Real, a::Real, b::Real, c::Real, d::Real)
    return c + ((d - c) / (b - a)) * (t - a)
end
"""
        mapr(t, a, b) -> Real

Maps the value `t` from the interval `[0, 1]` to `[a, b]`
"""
mapr(t::Real, a::Real, b::Real) = mapr(t, 0, 1, a, b)
mapr(t::Real, (a, b)::Tuple, (c, d)::Tuple) = mapr(t, a, b, c, d)
mapr(t::Real, (a, b)::Tuple) = mapr(t, a, b)
mapr(t::Real, A::AbstractRange, B::AbstractRange) = mapr(t, extrema(A), extrema(B))
mapr(t::Real, A::AbstractRange) = mapr(t, extrema(A))


using TuringModels

Turing.setadbackend(:reversediff);

d = CSV.read(joinpath(@__DIR__, "..", "..", "data", "Kline.csv"), delim=';');
size(d) # Should be 10x5

# New col log_pop, set log() for population data
d[!, :log_pop] = map((x) -> log(x), d[:, :population]);

# New col contact_high, set binary values 1/0 if high/low contact
d[!, :contact_high] = map((x) -> ifelse(x=="high", 1, 0), d[:, :contact]);

# New col where we center(!) the log_pop values
mean_log_pop = mean(d[:, :log_pop]);
d[!, :log_pop_c] = map((x) -> x - mean_log_pop, d[:, :log_pop]);

@model m10_10stan_c(total_tools, log_pop_c, contact_high) = begin
    α ~ Normal(0, 100)
    βp ~ Normal(0, 1)
    βc ~ Normal(0, 1)
    βpc ~ Normal(0, 1)

    for i ∈ 1:length(total_tools)
        λ = exp(α + βp*log_pop_c[i] + βc*contact_high[i] +
            βpc*contact_high[i]*log_pop_c[i])
        total_tools[i] ~ Poisson(λ)
    end
end;

chns = sample(m10_10stan_c(d[:, :total_tools], d[:, :log_pop_c],
  d[:, :contact_high]), Turing.NUTS(0.65), 1000);

# Rethinking result

m_10_10t_c_result = "
    mean   sd  5.5% 94.5% n_eff Rhat
 a   3.31 0.09  3.17  3.45  3671    1
 bp  0.26 0.03  0.21  0.32  5052    1
 bc  0.28 0.12  0.10  0.47  3383    1
 bcp 0.07 0.17 -0.20  0.34  4683    1
";

# Describe the draws

describe(chns)

# End of m10.10t2.jl


"""
`AllenCahnProblem1D` - structure to solve the one-dimensional Allen-Cahn equation, that is, solves

uₜ = Auₓₓ + f(u)

on (a, b), where A = Δ and f(u) = -u³ + u.
"""
struct AllenCahnProblem1D{TLBC, TRBC, TA}
    a::Real # Left endpoint
    b::Real # Right endpoint
    ϵ::Real # Model parameter
    nₓ::Int # Total number of interior spatial mesh points
    Δx::Real # Spatial mesh spacing
    x::Vector # Spatial mesh points including boundary
    nₜ::Int # Number of timesteps
    t_max::Real # Final time for integration
    Δt::Real # Length of timestep
    left_bc::TLBC # Boundary condition on the left endpoint
    right_bc::TRBC # Boundary condition on the right endpoint
    A::TA # Matrix approximation of differential operator for the problem
    rhs::Vector # Right hand side of the end system
    f::Function # Nonlinear part of semilinear ODE system (u - u³ for this problem)
    u::Vector # Solution of the problem
end

"""
`BoundaryCondition` - abstract type to include boundary conditions
"""
abstract type BoundaryCondition end

"""
`NeumannBC` - structure to house a Neumann boundary condition, i.e., a boundary condition for which

uₓ(a) = vₗ  or  uₓ(b) = vᵣ
"""
struct NeumannBC <: BoundaryCondition
    boundary_condition::Real
end

"""
`PeriodicBC` - structure to house a Periodic boundary condition, i.e., a boundary condition for which

u(a) = u(b), uₓ(a) = uₓ(b)
"""
struct PeriodicBC <: BoundaryCondition
end

"""
`TimeSteppingMethod` - abstract type to include time stepping methods
"""
abstract type TimeSteppingMethod end

"""
`BackwardEulerMethod` - structure for Backward Euler method
"""
struct BackwardEulerMethod <: TimeSteppingMethod
end

"""
`CrankNicolsonMethod` - structure for Crank-Nicolson method
"""
struct CrankNicolsonMethod <: TimeSteppingMethod
end


# g + g^2 + ... + g^(T-1)
pvfactor(g, T) = (g ^ T - 1.0) / (g - 1.0);

# Present value. First entry not discounted.
present_value(xV, R) = sum(xV ./ (R .^ (0 : (length(xV)-1))));

# -----------


#=
  Euler #20 in Julia.

  Problem 20:
  """
  n! means n (n 1) ... 3 2 1

  Find the sum of the digits in the number 100!")
  """

  This Julia program was created by Hakan Kjellerstrand, hakank@gmail.com
  See also my Julia page: http://www.hakank.org/julia/

=#

include("Euler.jl")

# 0.00029684s
function euler20a()
    return sum(split(factorial(BigInt(100))|>string,"").|>j->parse(Int,j))
end

# 0.00031793s
function euler20b()
    return sum(digits(factorial(BigInt(100))))
end

# 0.00032192s
function euler20c()
    return factorial(BigInt(100))|>digits|>sum
end

run_euler(euler20a)
# run_euler(euler20b)
# run_euler(euler20c)


matrixdict = Dict("hilb" => hilb, "hadamard" => hadamard,
                  "cauchy" => cauchy, "circul" => circul,
                  "dingdong" => dingdong, "frank" => frank,
                  "invhilb" => invhilb, "forsythe" => forsythe,
                  "magic" => magic, "grcar" => grcar,
                  "triw" => triw, "moler" => moler,
                  "pascal" => pascal, "kahan" => kahan,
                  "pei" => pei, "vand" => vand,
                  "invol" => invol, "chebspec" => chebspec,
                  "lotkin" => lotkin, "clement" => clement,
                  "fiedler" => fiedler, "minij" => minij,
                  "binomial" => binomialm, "tridiag" => tridiag,
                  "lehmer" => lehmer, "parter" => parter,
                  "chow" => chow, "randcorr" => randcorr,
                  "poisson" => poisson, "neumann" => neumann,
                  "rosser" => rosser, "sampling" => sampling,
                  "wilkinson" => wilkinson, "rando" => rando,
                  "randsvd" => randsvd, "rohess" => rohess,
                  "kms" => kms, "wathen" => wathen,
                  "oscillate" => oscillate, "toeplitz" => toeplitz,
                  "hankel" => hankel, "golub" => golub,
                  "companion" => companion,
                  "prolate" => prolate, "deriv2" => deriv2,
                  "shaw" => shaw, "wing" => wing,
                  "foxgood" => foxgood, "heat" => heat,
                  "baart" => baart, "phillips" => phillips,
                  "gravity" => gravity, "blur" => blur,
                  "spikes" => spikes, "ursell" => ursell,
                  "parallax" => parallax, "erdrey" => erdrey,
                  "gilbert" => gilbert, "smallworld" => smallworld
                  );

matrixclass = Dict("symmetric" => ["hilb", "cauchy", "circul", "dingdong",
                                   "invhilb", "moler", "pascal", "pei",
                                   "clement", "fiedler", "minij", "tridiag",
                                   "lehmer", "randcorr", "poisson", "wilkinson",
                                   "kms", "wathen", "oscillate", "prolate",
                                   "hankel"],

             "inverse" => ["hilb", "hadamard", "cauchy", "invhilb",
                           "forsythe", "magic", "triw", "moler", "pascal",
                           "kahan", "pei", "vand", "invol", "lotkin",
                           "clement", "fiedler", "minij", "tridiag",
                           "lehmer", "poisson", "kms" ],

             "ill-cond" => ["hilb", "cauchy", "frank", "invhilb",
                            "forsythe", "triw", "moler", "pascal",
                            "kahan","pei", "vand", "invol", "lotkin",
                            "tridiag", "rosser", "randsvd", "kms",
                            "oscillate", "prolate", "golub"],

             "pos-def" => ["hilb", "cauchy", "circul", "invhilb",
                           "moler", "pascal", "pei", "minij", "tridiag",
                           "lehmer", "poisson", "kms", "wathen", "oscillate"],

             "eigen" =>   ["hadamard", "circul", "dingdong", "frank",
                           "forsythe", "grcar", "pascal", "invol","chebspec",
                           "lotkin", "clement", "fiedler", "minij",
                           "tridiag", "parter", "chow", "poisson", "neumann",
                           "rosser", "sampling", "wilkinson","wathen",
                           "oscillate"],

             "sparse" => ["poisson", "neumann", "wathen", "blur", "erdrey", "gilbert",
                          "smallworld"],

             "random" => ["rosser", "rando", "randcorr", "randsvd", "rohess",
                          "wathen", "oscillate", "golub", "erdrey", "gilbert", "smallworld"],

             "regprob" => ["deriv2", "shaw", "wing", "foxgood", "heat",
                           "baart", "phillips", "gravity", "blur",
                           "spikes", "ursell", "parallax"],
              "graph" => ["erdrey", "gilbert", "smallworld"]
               );


__precompile__(true)

module IntroLinearAlgebra

using Pkg, LinearAlgebra, Requires

export rowswitch,
       rowscale,
       rowadd,
       rref,
       eigenvalues,
       eigenspaces,
       charpoly,
       zerooutabove!,
       zerooutbelow!,
       rownormalize!,
       setdigits,
       setalignment,
       transformation_movie,
       svdviz

import Base: show, eps, inv

#-------------------------------------------------------
# MATRIX TOOLS
#-------------------------------------------------------

DIGITS = 4
ALIGNMENT = true

RatOrInt = Union{Rational,Integer}

"""
    setdigits!(n::Integer)

Set the number of digits (after the decimal point) to
display (for matrices with floating-point entries).
"""
function setdigits!(n::Integer)
    global DIGITS
    DIGITS = n
end

"""
    setalignment!(b::Bool)

Determine whether to align displayed matrix entries
disregarding the sign (true) or including the sign (false)
"""
function setalignment!(b::Bool)
    global ALIGNMENT
    ALIGNMENT = b
end

"""
    texstring(M)

Return a string for processing by LaTeX to pretty print
the matrix `M`
"""
function texstring(M::AbstractArray{T,2}) where T
    ϵ = T<:RatOrInt ? 0 : eps(norm(M,Inf))
    function neg(x::Real) # test whether a number is
                          # genuinely negative (-0.0 isn't)
        return x < -ϵ
    end
    prettystring(x::Bool;padding=false) = "\\mathrm{"*(x ? "true" : "false")*"}"
    function prettystring(x::RatOrInt;padding=false)
        if isa(x,Integer)
            if padding
                return neg(x) ? string(x) : "\\hphantom{-}"*string(x)
            else
                return string(x)
            end
        elseif x.den == 1
            return prettystring(x.num,padding=padding)
        else
            sgn = neg(x.num) ? "-" : (padding ? "\\hphantom{-}" : "")
            return sgn*"\\frac{"*string(abs(x.num))*"}{"*string(x.den)*"}"
        end
    end
    function prettystring(x::Real;padding=false)
        global DIGITS
        return (neg(x) ? "-" : (padding ? "\\hphantom{-}" : "")) *
            string(round(abs(x);digits=DIGITS))
    end
    function prettystring(z::Complex;padding=false)
        if ~_isnonzero(z.im,[one(typeof(z.im))])
            return prettystring(z.re)
        elseif ~_isnonzero(z.re,[one(typeof(z.re))])
            return (abs(z.im) == 1 ? "" : prettystring(z.im))*"i"
        else
            return prettystring(z.re)*(neg(z.im) ? "-" : "+")*prettystring(abs(z.im)*im)
        end
    end

    s = "\$\\left[\\begin{array}{" * repeat("c",size(M,2)) * "}"
    global ALIGNMENT
    columnpadding = T <: Real ? any(M .< -ϵ, dims=1) : repeat([false],size(M,2))

    for i=1:size(M,1)
        for j=1:size(M,2)
            thiscolumnpadding = T <: Real ? any(M[:,j] .< -ϵ) : false
            s *= prettystring(M[i,j],padding=ALIGNMENT&&columnpadding[j])
            if j < size(M,2)
                s *= " & "
            end
        end
        s *= " \\\\ "
    end
    s *= "\\end{array}\\right]\$"
    return s
end

texstring(M::AbstractArray{T,1}) where T<:Real = texstring(reshape(M,(length(M),1)))
texstring(M::RowVector{T}) where T<:Real = texstring(convert(Array{T,2},M))

show(io::IO,
     ::MIME"text/latex",
     s::AbstractArray{T,2}) where T <: Real = write(io, texstring(s))

show(io::IO,
     ::MIME"text/latex",
     s::AbstractArray{T,1}) where T <: Real = write(io, texstring(s))

show(io::IO,
     ::MIME"text/latex",
     s::AbstractArray{T,2}) where T <: Complex = write(io, texstring(s))

show(io::IO,
     ::MIME"text/latex",
     s::AbstractArray{T,1}) where T <: Complex = write(io, texstring(s))

"""
    rowswitch(M,i,j)

Return matrix obtained by switching rows `i` and `j` in matrix `M`

```julia
julia> M = [1//1 2 3; 4 5 6]
julia> rowswitch(M,1,2)
2×3 Array{Rational{Int64},2}:
 4//1  5//1  6//1
 1//1  2//1  3//1
```
"""
function rowswitch(M::AbstractArray{T,2},i::Integer,j::Integer) where T
    A = copy(M)
    A[i,:], A[j,:] = A[j,:], A[i,:]
    return A
end

"""
    rowscale(M,i,x)

Return matrix obtained by scaling row `i` in matrix `M` by
a factor of `x`

```julia
julia> M = [1 2 3; 4 5 6]
julia> rowscale(M,1,16)
2×3 Array{Int64,2}:
 16  32  48
  4   5   6
```
"""
function rowscale(M::AbstractArray{T,2},i::Integer,x) where T
    A = copy(convert(Array{promote_type(T,typeof(x)),2},M))
    A[i,:] *= x
    return A
end

"""
    rowadd(M,i,j,x)

Return matrix obtained by adding `x` times row `j` to
row `i` in matrix `M`

```julia
julia> M = [1 2 3; 4 5 6]
julia> rowadd(M,1,2,-1)
2×3 Array{Int64,2}:
 -3  -3  -3
  4   5   6
```
"""
function rowadd(M::AbstractArray{T,2},
                i::Integer,
                j::Integer,
                x) where T
    A = copy(convert(Array{promote_type(T,typeof(x)),2},M))
    A[i,:] += x*A[j,:]
    return A
end

"""
    rowswitch!(M,i,j)

Version of `rowswitch` that modifies the argument rather than
returning a new matrix
"""
function rowswitch!(M::AbstractArray{T,2},i::Integer,j::Integer) where T
    if i == j
        return nothing
    else
        M[i,:], M[j,:] = M[j,:], M[i,:]
        return nothing
    end
end

"""
    rowscale!(M,i,j)

Version of `rowscale` that modifies the argument rather than
returning a new matrix
"""
function rowscale!(M::AbstractArray{T,2},i::Integer,x) where T
    M[i,:] *= x
    return nothing
end

"""
    rowadd!(M,i,j)

Version of `rowadd` that modifies the argument rather than
returning a new matrix
"""
function rowadd!(M::AbstractArray{T,2},i::Integer,j::Integer,x) where T
    M[i,:] += x*M[j,:]
    return nothing
end

"""
    _isnonzero(x,M)

Check whether x is nonzero. Symbols count as nonzero,
and floats count as zero whenever their absolute value
is less than roundoff error for floats the size of the
norm of M.
"""
function _isnonzero(x,M::AbstractArray{T}) where T
    if isa(x,AbstractFloat) || (isa(x,Complex) && isa(x.re,AbstractFloat))
        ϵ = T <: AbstractFloat ? eps(norm(M,Inf)) : eps(typeof(x))
        return abs(x) > ϵ
    else
        return x != 0
    end
end
eps(::Type{Complex{T}}) where T = 3*eps(T)
eps(z::Complex{T}) where T <: AbstractFloat = 3*(eps(z.re) + eps(z.im))

simplify(x) = isa(x,Real) ? x :
    (string(typeof(x)) == "SymPy.Sym" ? symplify(x) : x)

"""
    rref(M;showsteps=false)

Compute the reduced row echelon form of M and return
the RREF if `showsteps` is `false`, and the sequence of
steps performed to compute the RREF if `showsteps` is
`true`

```julia
julia> M = [1//1 2 3; 4 5 6]
julia> rref(M)
2×3 Array{Rational{Int64},2}:
 1//1  0//1  -1//1
 0//1  1//1   2//1
julia> rref(M;showsteps=true)
4-element Array{Array{Rational{Int64},2},1}:
 Rational{Int64}[1//1 2//1 3//1; 4//1 5//1 6//1]
 Rational{Int64}[1//1 2//1 3//1; 0//1 -3//1 -6//1]
 Rational{Int64}[1//1 2//1 3//1; 0//1 1//1 2//1]
 Rational{Int64}[1//1 0//1 -1//1; 0//1 1//1 2//1]
```
"""
function rref(M::AbstractArray{T,2};showsteps=false) where T
    isnonzero(x) = _isnonzero(x,M)
    S = T <: Rational ? T.parameters[1] : T
    if T <: RatOrInt
        newtype = Rational{S}
    elseif T <: Complex && T.parameters[1] <: Integer
        newtype = Complex{Rational{T.parameters[1]}}
    elseif T <: Complex && T.parameters[1] <: Rational
        newtype = Complex{T.parameters[1]}
    else
        newtype = promote_type(typeof(inv(one(T))),T)
    end
    A = copy(convert(Array{newtype,2},M))
    steps = typeof(A)[]
    current_row = 1
    for j=1:size(A,2)
        if ~any(isnonzero.(A[current_row:end,j]))
            if issubtype(T,AbstractFloat)
                A[current_row:end,j] = 0
            end
        else
            i = current_row-1 + findfirst(isnonzero.(A[current_row:end,j]))
            rowswitch!(A,current_row,i)
            if showsteps push!(steps,copy(A)) end
            zerooutbelow!(A,current_row,j)
            if showsteps push!(steps,copy(A)) end
            rownormalize!(A,current_row)
            if showsteps push!(steps,copy(A)) end
            current_row += 1
            if current_row > size(A,1)
                break
            end
        end
        map!(simplify,A,A)
    end
    for i=size(A,1):-1:1
        if ~any(isnonzero.(A[i,:]))
            continue
        else
            j = findfirst(isnonzero.(A[i,:]))
            zerooutabove!(A,i,j)
            if showsteps push!(steps,copy(A)) end
        end
    end
    if showsteps
        uniquesteps = typeof(A)[]
        for i=1:length(steps)
            if i == 1 || steps[i] != uniquesteps[end]
                push!(uniquesteps,steps[i])
            end
        end
        return uniquesteps
    else
        return A
    end
end

"""
    zerooutbelow!(M,i,j)

Use row operations to zero out entries in column `j` below row `i`

```julia
julia> M = [1//1 2 3; 4 5 6]
julia> zerooutbelow!(M,1,2)
julia> M
2×3 Array{Rational{Int64},2}:
 1//1  2//1   3//1
 3//2  0//1  -3//2
```
"""
function zerooutbelow!(M::Array{T,2},i::Integer,j::Integer)::Nothing where T
    for k=i+1:size(M,1)
        rowadd!(M,k,i,T<:RatOrInt ? -M[k,j]//M[i,j] : -M[k,j]/M[i,j])
    end
    return nothing
end

#----------------------------------------------------------------
"""
    zerooutabove!(M,i,j)

Use row operations to zero out entries in column `j` above row `i`

```julia
julia> M = [1//1 2 3; 4 5 6]
julia> zerooutabove!(M,2,2)
julia> M
julia> M
2×3 Array{Rational{Int64},2}:
 -3//5  0//1  3//5
  4//1  5//1  6//1
```
"""
function zerooutabove!(M::Array{T,2},i::Integer,j::Integer)::Nothing where T
    for k=1:i-1
        rowadd!(M,k,i,T<:RatOrInt ? -M[k,j]//M[i,j] : -M[k,j]/M[i,j])
    end
    return nothing
end

#----------------------------------------------------------------
"""
    rownormalize!(M,i)

Scale row `i` in matrix `M` to make the leading entry 1

```julia
julia> M = [1//1 2 3; 4 5 6]
julia> rownormalize!(M,2)
julia> M
2×3 Array{Rational{Int64},2}:
 1//1  2//1  3//1
 1//1  5//4  3//2
```
"""
function rownormalize!(M::Array{T,2},i::Integer)::Nothing where T
    isnonzero(x) = _isnonzero(x,M)
    j = findfirst(isnonzero.(M[i,:]))
    rowscale!(M,i, T<:RatOrInt ? 1//M[i,j] : 1/M[i,j])
    return nothing
end

function inv(A::Union{Array{Rational{T},2},Array{T,2}}) where T <: Integer
    if size(A,1) != size(A,2)
        error("Matrix not square")
    end
    n = size(A,1)
    E = rref([A I])
    C,D = E[:,1:n], E[:,n+1:end]
    if C != I
        error("Not invertible")
    else
        return D
    end
end

#----------------------------------------------------------------------------
# GRAPHICS TOOLS
#----------------------------------------------------------------------------

function __init__()

    Requires.@require SymPy="24249f21-da20-56a4-8eb1-6a02cf4ae2e6" begin

        symplify(x) = SymPy.simplify(x)

        texstring(A::Array{SymPy.Sym}) = SymPy.sympy[:latex](A)

        struct EigenSpace
            λ
            mult
            eigenmatrix
        end

        function texstring(E::EigenSpace)
            s = "\\mathrm{eigenspace}\\left("
            s *= "λ = "
            s *= SymPy.latex(E.λ,itex=true)*", "
            s *= "\\mathrm{mult} = "*string(E.mult)*", "
            s *= "\\mathrm{Col}\\left("
            io = IOBuffer()
            show(io,(MIME"text/latex").instance,E.eigenmatrix)
            s *= String(take!(io))
            s *= "\\right)\\right)"
        end

        function show(io::IO,M::MIME"text/latex",E::EigenSpace)
            write(io,"\$"*texstring(E)*"\$")
        end

        function show(io::IO,E::EigenSpace)
            s = "eigenspace(\n"
            s *= "λ = "
            _io = IOBuffer()
            show(_io,(MIME"text/plain").instance,E.λ)
            s *= String(take!(_io))
            s *= ", mult = "*string(E.mult)*", "
            show(_io,E.eigenmatrix)
            s *= String(take!(_io))
            s *= "\n)"
            print(io,s)
        end

        function show(io::IO,::MIME"text/plain",A::Array{EigenSpace,1})
            for (k,E) in enumerate(A)
                show(io,E)
                if k < length(A)
                    print(io,"\n\n")
                end
            end
        end

        function show(io::IO,M::MIME"text/latex",A::Array{EigenSpace,1})
            print(io,"\$\$\\begin{bmatrix}")
            for E in A
                write(io,texstring(E))
                print(io,"\\\\")
            end
            print(io,"\\end{bmatrix}\$\$")
        end

        """
        eigenvalues(M)

        Uses SymPy to find the exact eigenvalues of M

        ```julia
        julia> M = [1 2; 4 5]
        julia> eigenvalues(M)
        2-element Array{SymPy.Sym,1}
        ⎡3 + 2⋅√3 ⎤
        ⎢         ⎥
        ⎣-2⋅√3 + 3⎦
        ```
        """
        function eigenvalues(A::Array{T,2}) where T
            _loadsympy()
            return SymPy.solve(charpoly(A))
        end

        """
        eigenvalues(M)

        Use SymPy to find the exact eigenvalues of M

        ```julia
        julia> M = [1 2; 4 5]
        julia> eigenspaces(M)
        eigenspace(
        λ = 3 + 2 \\sqrt{3}, mult = 1,
        ⎡  1   √3⎤
        ⎢- ─ + ──⎥
        ⎢  2   2 ⎥
        ⎢        ⎥
        ⎣   1    ⎦
        )

        eigenspace(
        λ = - 2 \\sqrt{3} + 3, mult = 1,
        ⎡  √3   1⎤
        ⎢- ── - ─⎥
        ⎢  2    2⎥
        ⎢        ⎥
        ⎣   1    ⎦
        )
        ```
        """
        function eigenspaces(A::Array{T,2}) where T
            B = map(x->convert(SymPy.Sym,x),A)
            return [EigenSpace(a,b,map(SymPy.simplify,hcat(c...))) for
                    (a,b,c) in SymPy.getindex(B,:eigenvects)()]
        end

        """
        charpoly(M)

        Uses SymPy to find the characteristic polynomial of M

        ```julia
        julia> M = [1 2; 4 5]
        julia> charpoly(M)
        ```
        """
        function charpoly(A::Array{T,2}) where T
            λ = SymPy.symbols("lambda")
            B = map(x->convert(SymPy.Sym,x),A)
            return SymPy.getindex(B,:charpoly)()(λ)
        end

    end

    Requires.@require AsyPlots="77e5a97a-5ef9-58df-9d21-21957d92d960" begin

        """
        transformation_movie(A,n;frames=20)

        Return an array of Graphics2D arrays which give a dynamic
        picture of how A transforms integer grid lines in [-n,n]^2

        ```julia
        julia> A = [0 2; -1 0]
        julia> movie = transformation_movie(A,10,20);
        julia> using Interact
        julia> @manipulate for i=1:length(movie)
                   movie[i]
               end
        ```
        """
        transformation_movie(A::Array{T,2};kwargs...) where T =
            transformation_movie((x,y)->A*[x;y];kwargs...)

        function transformation_movie(f::Function;
                                      gridlines::Integer=8,
                                      frames::Integer=20)

            n = gridlines

            Path = AsyPlots.Path
            Point = AsyPlots.Point
            NamedColor = AsyPlots.NamedColor

            mypath(A::Array{<:Real,2};kwargs...) = Path([tuple(A[k,:]...) for k=1:size(A,1)];kwargs...)
            mypath(A::Array{T,1};kwargs...) where T = Path([tuple(v...) for v in A];kwargs...)
            sc(r::Real,s::String) = r*NamedColor(s)

            m = 2.05*max(norm(f(n,n),Inf),norm(f(-n,n),Inf))
            box = mypath([-m -m; m -m; m m; -m m; -m -m];color="white")
            return [AsyPlots.Plot(vcat([[Path([0 0; m 0]),Path([0 0; m 0])];
                                        [mypath([(1-t)*[j,k] + t*f(j,k) for j=-n:0.1:n];
                                                linewidth=1.5,color=sc(0.9,"DarkRed")) for k=-n:n];
                                        [mypath([(1-t)*[j,k] + t*f(j,k) for k=-n:0.1:n];
                                                linewidth=1.5,color=sc(0.9,"MidnightBlue")) for j=-n:n];
                                        [Point(0,0;linewidth=1e-2)];
                                        [box]]...))
                    for t=range(0,stop=1,length=frames)]
        end

        function svdviz(A::Array{T,2},
                        B::Array{T,2}=A;
                        n=5,
                        frames=20,
                        gridlinewidth=1,
                        graylinewidth=1,
                        pointsize=1,
                        kwargs...) where T <: Real

            Path = AsyPlots.Path
            Point = AsyPlots.Point
            Arrow = AsyPlots.Arrow
            Plot = AsyPlots.Plot

            U, Σ, V = svd(A)
            m = 1.05*max(norm(A*V*[n,n],Inf),norm(A*V*[-n,n],Inf),norm(A*V*[-n,-n],Inf))
            mybox = Path([-m -m; m -m; m m; -m m; -m -m];color="white")
            loc(j,k,t) = tuple((j*B*V[:,1] + k*B*V[:,2])*t + (j*V[:,1] + k*V[:,2])*(1-t)...)
            pts(t) = vcat([Point(loc(j,k,t);linewidth=pointsize) for j=-n:n,k=-n:n]...)
            bluelines(t) = [Path([loc(j,k,t) for j=-n:n];linewidth=gridlinewidth,color="MidnightBlue") for k=-n:n]
            redlines(t) = [Path([loc(j,k,t) for k=-n:n];linewidth=gridlinewidth,color="DarkRed") for j=-n:n]
            graylines = [[Path([(j,k) for j=-n:n];color="Gray",linewidth=graylinewidth,opacity=0.3) for k=-n:n];
                         [Path([(j,k) for k=-n:n];color="Gray",linewidth=graylinewidth,opacity=0.3) for j=-n:n]]
            axes = [Path([0 0; 0 m]),Path([0 0; m 0])]
            basisvectors(t) = [Path([(0,0),tuple([1,0]*(1-t) + A*[1,0]*(t)...)];arrow=Arrow(),linewidth=1.5,color="SeaGreen"),
                               Path([(0,0),tuple([0,1]*(1-t) + A*[0,1]*(t)...)];arrow=Arrow(),linewidth=1.5,color="DarkRed")]
            [Plot([graylines;
                   basisvectors(t);
                   bluelines(t);
                   redlines(t);
                   pts(t);
                   [mybox]];kwargs...) for t = range(0,stop=1,length=frames)]
        end

    end

end

end # module


""" Julia module to check data for a Simpson's statistical paradox """
module Simpsons

export has_simpsons_paradox, make_paradox, plot_clusters, plot_kmeans_by_factor, find_clustering_elbow, simpsons_analysis

using DataFrames, Distributions, Polynomials, Clustering, Plots

"""
    has_simpsons_paradox(df, cause, effect, factor;
        continuous_threshold = 5, cmax = 5, verbose = true)

Returns true if the data aggregated by `factor` exhibits Simpson's paradox.
Note that the `cause` and `effect` columns will be converted to Int columns if
they are not already numeric in type. A continuous data `factor` column (one
with `continuous_threshold` or more discrete levels) will be grouped into a
at most cmax clusters so as to avoid too many clusters. Prints the regression
slope directions for overall data and groups if verbose is true.
Example:
    df = DataFrame(
        treatment = [1, 2, 1, 1, 2, 2],
        recovery = [1, 0, 1, 1, 0, 0],
        kidney_stone_size = ["small", "small", "large", "small", "large", "large"])
    has_simpsons_paradox(df, :treatment, :recovery, :kidney_stone_size)
"""
function has_simpsons_paradox(df, cause, effect, factor; continuous_threshold = 5, cmax = 5, verbose = true)
    df1 = DataFrame()

    # Convert cause and effect column data types to numeric if needed
    df1[:, cause] = (df[1, cause] isa Number ? df[!, cause] : _makenumeric(df[!, cause]))
    df1[:, effect] = (df[1, effect] isa Number ? df[!, effect] : _makenumeric(df[!, effect]))
    if df[1, factor] isa Number || length(unique(df[!, factor])) < continuous_threshold
        df1[:, factor] = df[!, factor]
    else
        df1[:, factor] = _makenumeric(df[!, factor])
    end

    # Do linear regression on the cause versus effect columns.
    m = Polynomials.fit(df1[!, effect], df1[!, cause], 1)
    overallslope = m.coeffs[2]

    # Group by the factor and do a similar linear regression on each group when possible
    # first check for continous factor type, if number of unique values > continuous_threshold
    fac = df1[!, factor]
    uni = unique(fac)
    if length(uni) >= continuous_threshold && uni[1] isa Number
        # continuous factor, so find best cluster number up to cmax
        groupmat = zeros(eltype(uni), (2, length(fac)))
        groupmat[1, :] .= fac
        _, kmclust = find_clustering_elbow(groupmat, 2, cmax)
        grou = Symbol("grouped" * string(factor))
        insertcols!(df1, grou => kmclust.assignments)
        grouped = groupby(df1, grou)
    else
        grouped = groupby(df1, factor)
    end
    subgroupslopes = Float64[]
    for gdf in grouped
        length(gdf[!, effect]) < 2 && continue
        gm = Polynomials.fit(gdf[!, effect], gdf[!, cause], 1)
        length(gm.coeffs) < 2 && continue
        push!(subgroupslopes, gm.coeffs[2])
    end
    if verbose
        println("For cause $cause, effect $effect, and factor $factor:")
        println("Overall linear trend from cause to effect is ",
            overallslope > 0 ? "positive." : "negative.")
    end
    differentslopesigns = false
    for (i, slp) in enumerate(subgroupslopes)
        verbose && println("    Subgroup $i trend is ", slp > 0 ? "positive." : "negative.")
        if sign(slp) != sign(overallslope)
            verbose && println("        This shows a Simpson paradox type reversal.")
            differentslopesigns = true
        end
    end
    verbose && println()
    return differentslopesigns
end

"""
    make_paradox(nsubgroups = 3 , N = 1024)

Return a dataframe containing `N` rows of random data in 3 columns `:x` (cause),
`:y` (effect), and `:z` (cofactor) which displays the Simpson's paradox.
"""
function make_paradox(nsubgroups = 3 , N = 1024)
    rw = rand(nsubgroups)
    w = rw ./ sum(rw)
    m = rand(MvNormal([0, 0], 3 .* [1 0.7; 0.7 1]), nsubgroups)
    cv = [[1 -c; -c 1] for c in rand(Uniform(0.1, 0.9), nsubgroups)]

    dfs = DataFrame(:x => Float64[], :y => Float64[], :z => Int[])
    for subgroup in 1:nsubgroups
        subN = Int(round(N .* w[subgroup]))
        xarr, yarr = Float64[], Float64[]
        for _ in 1:subN
            x, y = rand(MvNormal(m[:, subgroup], cv[subgroup]), 2)
            push!(xarr, x)
            push!(yarr, y)
        end
        samp = DataFrame(:x => xarr, :y => yarr, :z => fill(subgroup, subN))
        append!(dfs, samp)
    end
    return has_simpsons_paradox(dfs, :x, :y, :z, verbose=false) ? dfs : make_paradox(nsubgroups, N)
end

"""
    plot_clusters(df, cause, effect)

Plot, with subplots, clustering of the dataframe using `cause` (X axis) and `effect` (Y axis)
plotted and color coded by clusterings. Use kmeans clustering analysis on all fields of
dataframe. Use 2 to 5 as cluster number. Converts non-numeric columns to numeric for processing.
"""
function plot_clusters(df, cause, effect)
    # convert non-numeric columns to numeric ones
    df1 = DataFrame()
    for nam in names(df)
        df1[:, nam] = df[1, nam] isa Number ? df[!, nam] : _makenumeric(df[!, nam])
    end
    factors = collect(Matrix(df1)')
    subplots = Plots.Plot[]
    for n in 2:5
        push!(subplots, scatter(df1[!, cause], df1[!, effect],
            marker_z = kmeans(factors, n).assignments, color = :lightrainbow,
            title = "$cause -> $effect with $n clusters", legend = false,
            xlabel = cause, ylabel = effect, smooths = true))
    end
    plt = scatter(subplots..., layout = (2, 2), smooths=true)
    display(plt)
end

"""
    plot_kmeans_by_factor(df, cause_column, effect_column, factor_column)

Plot clustering of the dataframe using cause plotted as X, effect as Y, with the `factor_column`
used for kmeans clustering into between 2 and 5 clusters on the plot.
"""
function plot_kmeans_by_factor(df, cause_column, effect_column, factor_column)
    df1 = DataFrame(cause_column => df[!, cause_column], effect_column => df[!, effect_column],
        factor_column => df[1, factor_column] isa Number ? df[!, factor_column] : _makenumeric(df[!, factor_column]))
    fac = df1[!, factor_column]
    uni = unique(fac)
    groupmat = zeros(eltype(uni), (2, length(fac)))
    groupmat[1, :] .= fac
    karray = [kmeans(groupmat, i) for i in 1:6]
    x1, y1 = 1, karray[1].totalcost
    x2, y2 = 6, karray[6].totalcost
    (_, idx) = findmax(map(i -> distance(x1, y1, x2, y2, i, karray[i].totalcost), 2:5))
    grou = Symbol("grouped" * string(factor_column))
    df1[:, grou] = karray[idx + 1].assignments
    groups = groupby(df1, grou)
    plt = scatter(df1[!, cause_column], df[!, effect_column], color = :black, smooth = true,
        title = "$cause_column -> $effect_column with cofactor $factor_column",
        xlabel = cause_column, ylabel = effect_column, legend = false)
    for (i, gf) in enumerate(groups)
        scatter!(plt, gf[!, cause_column], gf[!, effect_column], color = _pcolor(i), smooth = true, legend = false)
    end
    display(plt)
end

"""
    simpsons_analysis(df, cause_column, effect_column; verbose = true, show_plots = true)

Analyze the dataframe `df` assuming a cause is in `cause_column` and an effect in
`effect_column` of the dataframe. Output data including any Simpson's paradox type
reversals in subgroups found. Plots shown if show_plots is true (default).
"""
function simpsons_analysis(df, cause_column, effect_column; verbose=true, show_plots = true)
    # Plot cluster analysis for clustering numbers 2 through 5
    show_plots && plot_clusters(df, cause_column, effect_column)
    # plot clusterings by factor
    for factor in Symbol.(names(df))
        factor in [cause_column, effect_column] && continue
        if show_plots && df[1, factor] isa Number
            plot_kmeans_by_factor(df, cause_column, effect_column, factor)
        end
        has_simpsons_paradox(df, cause_column, effect_column, factor, verbose=verbose)
    end
end

"""
    find_clustering_elbow(dataarray::AbstractMatrix{<:Real}, cmin = 1, cmax = 5; fclust = kmeans, kwargs...)

Find the "elbow" of the totalcost versus cluster number curve, where
cmin <= elbow <= cmax. Note that in pathological cases where the actual
minimum of the totalcosts occurs at a cluster count less than that of the
curve "elbow", the function will return either cmin or the actual cluster
count at which the totalcost is at minimum, whichever is larger. The default
clustering function is kmeans, but other functions which take two required
aguments, a matrix of data plus a desired cluster number and return a
ClusteringResult as defined in the Clustering package may be used.
<br>
Returns a tuple: the cluster count and the ClusteringResult at the "elbow" optimum.
"""
function find_clustering_elbow(dataarray::AbstractMatrix{<:Real}, cmin = 1, cmax = 5; fclust = kmeans, kwargs...)
    allkmeans = [fclust(dataarray, i, kwargs...) for i in 1:cmax+1]
    alltotals = map(x -> x.totalcost, allkmeans)
    _, cidx = findmin(alltotals)
    x1, y1 = 1, alltotals[1]
    x2, y2 = cmax + 1, alltotals[cmax + 1]
    _, idx = findmax(map(i -> distance(x1, y1, x2, y2, i, alltotals[i]), 2:cmax))
    nclust = cidx < idx + 1 ? max(cmin, cidx) : idx + 1
    return nclust, allkmeans[nclust]
end

# internal helper functions

"""
    _makenumeric(a)

Returns an array of counting number integers, `1 <= values <= length(a)`,
where the returned array has its shape and sort order the same as `a`.
"""
function _makenumeric(a)
    d = Dict{eltype(a), Int}()
    for (i, s) in enumerate(sort(unique(a)))
        d[s] = i
    end
    return map(x -> d[x], a)
end

"""
    distance(x1, y1, x2, y2, x0, y0)

Return the Euclidean distance from point at `[x0, y0]` to line from `[x1, y1]` to `[x2, y2]`)
"""
function distance(x1, y1, x2, y2, x0, y0)
    return abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1)) / sqrt((x2 - x1)^2 + (y2 - y1)^2)
end

""" get one of 6 colors by an index """
_pcolor(i) = collect(palette(:lightrainbow))[mod1(i, 6)]

end  # module Simpsons


## Scalar functions on intervals (no directed rounding used)

mag(a::Interval) = max( abs(a.lo), abs(a.hi) )
mag(::EmptyInterval{T}) where T = T(NaN)

function mig(a::Interval{T}) where T
    zero_in(a) && return zero(T)
    r1, r2 = (@round_down(abs(a.lo)), @round_down(abs(a.hi)))
    min( r1, r2 )
end
mig(::EmptyInterval{T}) where T = T(NaN)

# Infimum and supremum of an interval
inf(a::Interval{T}) where T =
    ifelse(iszero(a.lo), -zero(T), a.lo) # IEEE 1788-2015 requires negative sign
sup(a::Interval{T}) where T =
    ifelse(iszero(a.hi), zero(T), a.hi) # IEEE 1788-2015 requires positive sign

# by convention, empty intervals return values which imply emptiness
inf(a::EmptyInterval{T}) where T = T(Inf)
sup(a::EmptyInterval{T}) where T = T(-Inf)

# mid, diam, radius

#= Fallback in case of overflow: a.hi + a.lo == +Inf or a.hi + a.lo == -Inf.
    This case can not be the default one as it does not pass several
    IEEE1788-2015 tests for small floats.
=#
midpoint_bounded_safe(a::Interval) = 0.5a.lo + 0.5a.hi
midpoint_bounded_accurate(a::Interval{T}) where T = 0.5(a.lo + a.hi) |>
    x -> ifelse(iszero(x), zero(T), x) # ensure positive sign for zero

# NOTE: mid could be renamed to mean? or median?
# Compare pg. 64 of the IEEE 1788-2015 standard:
"""
    mid(a::Interval)

Find the midpoint of interval `a`.

For intervals of the form `[-Inf, x]` or `[x, +Inf]` where `x` is finite, return
respectively `nextfloat(-Inf)` and `prevfloat(+Inf)`. Note that it differs from the
behavior of `mid(a, α=0.5)`.
"""
function mid(a::Interval{T}) where T
    isentire(a) && return zero(T)
    a.lo == -Inf && return nextfloat(a.lo)
    a.hi == +Inf && return prevfloat(a.hi)

    accurate_midpoint = midpoint_bounded_accurate(a)
    if isfinite(accurate_midpoint)
        return accurate_midpoint
    else
        return midpoint_bounded_safe(a)
    end
end

mid(::EmptyInterval{T}) where T = T(NaN)

"""
    diam(a::Interval)

Return the diameter (length) of the `Interval` `a`.
"""
diam(a::Interval{T}) where T = @round_up(a.hi - a.lo) # cf page 64 of IEEE1788
diam(::EmptyInterval{T}) where T = T(NaN)

# Should `radius` this yield diam(a)/2? This affects other functions!
"""
    radius(a::Interval)

Return the radius of the `Interval` `a`, such that
`a ⊆ m ± radius`, where `m = mid(a)` is the midpoint.
"""
function radius(a::Interval{T}) where T
    m = mid(a)
    return max(m - a.lo, a.hi - m)
end
radius(::EmptyInterval{T}) where T = T(NaN)

# midpoint-radius forms
midpoint_radius(a::ValidInterval) = (mid(a), radius(a))


using Parameters

"""
Contains a distribution over a triangulated state space. `dist::Vector{Float64}`
is the distribution, and `nonzero_inds::Vector{Int}` are the indices of the
simplices with non-zero measure.
"""
@with_kw struct InvDist
    dist::Vector{Float64} = Vector{Float64}() # Distribution over the simplices
    nonzero_inds::Vector{Int} = Vector{Int}() # indices of nonzero entries
end

"""
Compute the invariant probability distribution from a square Markov matrix `M`.
This is done by repeated application of `M` on an initially random distribution
until the distribution converges.
"""
function estimate_invdist(
        M::AbstractArray{Float64, 2};
        N::Int = 100,
        tolerance::Float64 = 1/10^5,
        delta::Float64 = 1/10^5
        )

    #=
    # Start with a random distribution `Ρ` (big rho). Normalise it so that it
    # sums to 1 and forms a true probability distribution over the simplices.
    =#
    Ρ = rand(Float64, 1, size(M, 1))
    Ρ = Ρ ./ sum(Ρ, 2)

    #=
    # Start estimating the invariant distribution. We could either do this by
    # finding the left-eigenvector of M, or by repeated application of M on Ρ
    # until the distribution converges. Here, we use the latter approach,
    # meaning that we iterate until Ρ doesn't change substantially between
    # iterations.
    =#
    distribution = Ρ * M

    distance = norm(distribution - Ρ) / norm(Ρ)

    check = floor(Int, 1 / delta)
    check_pts = floor.(Int, collect(1:N).' ./ check) .* collect(1:N).'
    check_pts = check_pts[check_pts .> 0]
    num_checkpts = size(check_pts, 1)
    check_pts_counter = 1

    counter = 1
    while counter <= N && distance >= tolerance
        counter += 1
        Ρ = distribution

        # Apply the Markov matrix to the current state of the distribution
        distribution = Ρ * M

        if (check_pts_counter <= num_checkpts &&
           counter == check_pts[check_pts_counter])

            check_pts_counter += 1
            colsum_distribution = sum(distribution, 2)[1]
            if abs(colsum_distribution - 1) > delta
                distribution = distribution ./ colsum_distribution
            end
        end

        distance = norm(distribution - Ρ) / norm(Ρ)
    end

    # Do the last normalisation and check
    colsum_distribution = sum(distribution, 2)[1]

    if abs(colsum_distribution - 1) > delta
        distribution = distribution ./ colsum_distribution
    end
    # Find simplices with strictly positive measure.
    simplex_inds_nonzero = heaviside(distribution) .* collect(1:size(M, 1)).'
    simplex_inds_nonzero = round(Int, simplex_inds_nonzero)
    simplex_inds_nonzero = simplex_inds_nonzero[simplex_inds_nonzero .> 0]

    # Extract the elements of the invariant measure corresponding to these indices
    return InvDist(dist = vec(distribution), nonzero_inds = simplex_inds_nonzero)
end


"""
horner(c,x)

Evaluate a polynomial whose coefficients are given in ascending
order in `c`, at the point `x`, using Horner's rule.
"""
function horner(c,x)

n = length(c)
y = c[n]
for k in n-1:-1:1
    y = x*y + c[k]
end

return y
end


# m5.4.MAs.jl

using Pkg, DrWatson

using MonteCarloMeasurements
using StanSample
using StatisticalRethinking

# ### snippet 5.1

df = CSV.read(sr_datadir("WaffleDivorce.csv"), DataFrame);

df = DataFrame(
    :A => df[:, :MedianAgeMarriage],
    :M => df[:, :Marriage],
    :D => df[:, :Divorce]
 )

scale!(df, [:M, :A, :D])

# Define the Stan language model

stan5_4_MA = "
data {
    int N;
    vector[N] A;
    vector[N] M;
}
parameters {
    real a;
    real bMA;
    real<lower=0> sigma;
}
model {
    vector[N] mu = a + bMA * A;
    a ~ normal( 0 , 0.2 );
    bMA ~ normal( 0 , 0.5 );
    sigma ~ exponential( 1 );
    M ~ normal( mu , sigma );
}
";

# Define the SampleModel
m5_4_MAs = SampleModel("m5.4", stan5_4_MA);

# Input data

m5_4_data = Dict(
    "N" => size(df, 1),
    "M" => df[:, :M_s],
    "A" => df[:, :A_s]
);

# Sample using cmdstan

rc5_4_MAs = stan_sample(m5_4_MAs, data=m5_4_data);

if success(rc5_4_MAs)

    # Rethinking results

    rethinking_results = "
           mean   sd  5.5% 94.5%
    a      0.00 0.09 -0.14  0.14
    bMA   -0.69 0.10 -0.85 -0.54
    sigma  0.68 0.07  0.57  0.79
    ";

    part5_4_MAs = read_samples(m5_4_MAs, :particles)
    part5_4_MAs |> display

end

# End of m5.4.MAs.jl


module KuboOscillator

    export kubo_oscillator_sde_1, kubo_oscillator_psde_1, kubo_oscillator_spsde_1
    export kubo_oscillator_sde_2, kubo_oscillator_psde_2, kubo_oscillator_spsde_2
    export kubo_oscillator_sde_3, kubo_oscillator_psde_3, kubo_oscillator_spsde_3

    using GeometricIntegrators.Equations

    q_init_A=[0.5, 0.0]
    q_init_B=[0.5 0.0 -0.5; 0.0 0.5 0.0]

    noise_intensity = 0.1

    Δt  = 0.01
    nt  = 10

    function kubo_oscillator_sde_v(t,q, v_out)
        v_out[1]=  q[2]
        v_out[2]= -q[1]
    end

    function kubo_oscillator_sde_B(t,q, B_out; col=0)
        noise_intensity = 0.1

        if col==0
            B_out[1,1]=  noise_intensity*q[2]
            B_out[2,1]= -noise_intensity*q[1]
        elseif col==1
            B_out[1]=  noise_intensity*q[2]
            B_out[2]= -noise_intensity*q[1]
        end
    end

    function kubo_oscillator_sde_1()
        # q_init_A - single deterministic initial condition
        # Generating 3 sample paths
        # 1-dimensional noise
        SDE(1, 3, kubo_oscillator_sde_v, kubo_oscillator_sde_B, q_init_A)
    end

    function kubo_oscillator_sde_2()
        # q_init_B - interpreted as a single random initial condition
        # The 3 columns correspond to 3 sample paths
        # 1-dimensional noise
        SDE(1, kubo_oscillator_sde_v, kubo_oscillator_sde_B, q_init_B)
    end

    function kubo_oscillator_sde_3()
        # q_init_B - interpreted as multiple deterministic initial conditions
        # The 3 columns correspond to 3 different deterministic initial conditions
        # Generating 3 sample paths for each initial condition
        # 1-dimensional noise
        SDE(1, 3, kubo_oscillator_sde_v, kubo_oscillator_sde_B, q_init_B)
    end

    # PSDE

    q_init_C=[0.5]
    p_init_C=[0.0]

    q_init_D=[0.5 0.0 -0.5]
    p_init_D=[0.0 0.5 0.0]

    function kubo_oscillator_psde_v(t, q, p, v_out)
        v_out[1]=  p[1]
    end

    function kubo_oscillator_psde_f(t, q, p, f_out)
        f_out[1]=  -q[1]
    end

    function kubo_oscillator_psde_B(t, q, p, B_out)
        B_out[1,1]= noise_intensity*p[1]
    end

    function kubo_oscillator_psde_G(t, q, p, G_out)
        G_out[1,1]= -noise_intensity*q[1]
    end

    function kubo_oscillator_psde_1()
        # q_init_C - single deterministic initial condition
        # Generating 3 sample paths
        # 1-dimensional noise
        PSDE(1, 3, kubo_oscillator_psde_v, kubo_oscillator_psde_f, kubo_oscillator_psde_B, kubo_oscillator_psde_G, q_init_C, p_init_C)
    end

    function kubo_oscillator_psde_2()
        # q_init_D - interpreted as a single random initial condition
        # The 3 columns correspond to 3 sample paths
        # 1-dimensional noise
        PSDE(1, kubo_oscillator_psde_v, kubo_oscillator_psde_f, kubo_oscillator_psde_B, kubo_oscillator_psde_G, q_init_D, p_init_D)
    end

    function kubo_oscillator_psde_3()
        # q_init_D - interpreted as multiple deterministic initial conditions
        # The 3 columns correspond to 3 different deterministic initial conditions
        # Generating 3 sample paths for each initial condition
        # 1-dimensional noise
        PSDE(1, 3, kubo_oscillator_psde_v, kubo_oscillator_psde_f, kubo_oscillator_psde_B, kubo_oscillator_psde_G, q_init_D, p_init_D)
    end

    # SPSDE

    function kubo_oscillator_spsde_v(t, q, p, v_out)
        v_out[1]=  p[1]
    end

    function kubo_oscillator_spsde_f1(t, q, p, f_out)
        f_out[1]=  -q[1]
    end

    function kubo_oscillator_spsde_f2(t, q, p, f_out)
        f_out[1]=  0.0
    end

    function kubo_oscillator_spsde_B(t, q, p, B_out)
        B_out[1,1]= noise_intensity*p[1]
    end

    function kubo_oscillator_spsde_G1(t, q, p, G_out)
        G_out[1,1]= -noise_intensity*q[1]
    end

    function kubo_oscillator_spsde_G2(t, q, p, G_out)
        G_out[1,1]= 0.0
    end

    function kubo_oscillator_spsde_1()
        # q_init_C - single deterministic initial condition
        # Generating 3 sample paths
        # 1-dimensional noise
        SPSDE(1, 3, kubo_oscillator_spsde_v, kubo_oscillator_spsde_f1, kubo_oscillator_spsde_f2, kubo_oscillator_spsde_B, kubo_oscillator_spsde_G1, kubo_oscillator_spsde_G2, q_init_C, p_init_C)
    end

    function kubo_oscillator_spsde_2()
        # q_init_D - interpreted as a single random initial condition
        # The 3 columns correspond to 3 sample paths
        # 1-dimensional noise
        SPSDE(1, kubo_oscillator_spsde_v, kubo_oscillator_spsde_f1, kubo_oscillator_spsde_f2, kubo_oscillator_spsde_B, kubo_oscillator_spsde_G1, kubo_oscillator_spsde_G2, q_init_D, p_init_D)
    end

    function kubo_oscillator_spsde_3()
        # q_init_D - interpreted as multiple deterministic initial conditions
        # The 3 columns correspond to 3 different deterministic initial conditions
        # Generating 3 sample paths for each initial condition
        # 1-dimensional noise
        SPSDE(1, 3, kubo_oscillator_spsde_v, kubo_oscillator_spsde_f1, kubo_oscillator_spsde_f2, kubo_oscillator_spsde_B, kubo_oscillator_spsde_G1, kubo_oscillator_spsde_G2, q_init_D, p_init_D)
    end

end


##
using Random
using Plots
using LinearAlgebra
theme(:dark)
##
Random.seed!(7)
X = randn(2,100)
A = randn(2,2)
Y = A*X
scatter(Y[1,:], Y[2,:], label="", size=(300,300),color=3)
xlabel!("x")
ylabel!("y")
##
savefig("pca-2d-fig.pdf")

##
using Interact
using Statistics
using Printf
settheme!(:nativehtml)
mp = @manipulate for θ in 0.0:2π/100:2π
    v = [cos(θ),sin(θ)]
    z = vec(v'*Y)
    p = plot(
      scatter(Y[1,:], Y[2,:], label="",color=3, aspect_ratio=:equal),
      histogram(z,label=""),
      size=(900,450),tickfont=16,titlefont=20
      )
    plot!(p[1],2*[0,v[1]],2*[0,v[2]],label="",linewidth=2)
    scatter!(p[1], v[1].*z, v[2].*z, color=2, label="",
      markerstrokewidth=0,alpha=0.3, markersize=4)
    xlims!(p[2],(-5,5))
    ylims!(p[2],(0,30))
    xlims!(p[1],(-4,4))
    ylims!(p[1],(-4.5,4.5))
    title!(p[1],@sprintf("Angle = %.2f", θ))
    title!(p[2],@sprintf("Variance = %.2f", var(z)))
end
## If the figure doesn't show, run
using Interact
ui = button()
display(ui)
##
θs = 0.0:π/1000:π
plot(collect(θs), map(x -> var(vec([cos(x),sin(x)]'*Y)), θs),
  label="", size=(500,250))
xlabel!("Angle")
ylabel!("Variance")
##
savefig("pca-2d-angle-variance.pdf")
## PCA is a way to get this angle automatically!
function mypca(X::AbstractMatrix, donormalize::Bool = false)
  means = vec(mean(X, dims=2)) # take the average of each coordinate
  C = X .- means # center each coordinate
  if donormalize
    for i=1:size(C,1)
      normalize!(@view C[i,:])
    end
  end
  U,S,V = svd(C)
end
U,S,V = mypca(Y, false)
U[:,1]
##
θpca = atan(-U[2,1],-U[1,1])
##
plot(collect(θs), map(x -> var(vec([cos(x),sin(x)]'*Y)), θs),
label="", size=(500,250))
xlabel!("Angle")
ylabel!("Variance")
annotate!(θpca, var(vec([cos(θpca),sin(θpca)]'*Y)), text("*", 24, :red))
##
savefig("pca-2d-angle-variance-opt.pdf")
##
scatter(Y[1,:], Y[2,:], label="", size=(300,300),color=3)
plot!(2*[0,-U[1,1]],2*[0,-U[2,1]],label="",linewidth=2, color=5)
##
savefig("pca-2d-fig-PCA.pdf")
##
S
## Save a set of points
pyplot()
theme(:default)
scatter(Y[1,:], Y[2,:], label="", size=(300,300),color=3,
  framestyle=:none, background=:transparent)
savefig("pca-2d-data.pdf")


function calc_pi(n::Integer)
   x = rand(n)
   y = rand(n)
   num_in_unit_circle = sum(x.*x.+y.*y .< 1)
   num_in_unit_circle * 4/n
end

n = 10_000;
calc_pi(1)  # Force compile
@time pi_estimate = calc_pi(n)
println("# After ", n, " itterations, estimated pi to be...")
println(pi_estimate)


# ------------------------------------------------------------------------------------------
# ## What is learning?
#
# Computers read data, as we saw in notebooks 1 and 2. We can then build functions that
# model that data to make decisions, as we saw in notebooks 3 and 5.
#
# But how do you make sure that the model actually fits the data well? In the last notebook,
# we saw that we can fiddle with the parameters of our function defining the model to reduce
# the loss function. However, we don't want to have to pick the model parameters ourselves.
# Choosing parameters ourselves works *well enough* when we have a simple model and only a
# few data points, but can quickly become extremely complex for more detailed models and
# larger data sets.
#
# Instead, we want our machine to *learn* the parameters that fit the model to our data,
# without needing us to fiddle with the parameters ourselves. In this notebook, we'll talk
# about the "learning" in machine learning.
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ### Motivation: Fitting parameters by hand
#
# Let's go back to our example of fitting parameters from notebook 3. Recall that we looked
# at whether the amount of green in the pictures could distinguish between an apple and a
# banana, and used a sigmoid function to model our choice of "apple or banana" using the
# amount of green in an image.
# ------------------------------------------------------------------------------------------

using Plots; gr()
using Images; using Interact

σ(x,w,b) = 1 / (1 + exp(-w*x+b))

apple =  load("data/10_100.jpg")
banana = load("data/104_100.jpg")
apple_green_amount =  mean(Float64.(green.(apple)))
banana_green_amount = mean(Float64.(green.(banana)));

@manipulate for w in -10:0.01:30, b in 0:0.1:30

    plot(x->σ(x,w,b), 0, 1, label="Model", legend = :topleft, lw=3)
    scatter!([apple_green_amount],  [0.0], label="Apple")
    scatter!([banana_green_amount], [1.0], label="Banana")

end

# ------------------------------------------------------------------------------------------
# Intuitively, how did you tweak the sliders so that way the model sends apples to 0 and
# bananas to 1? Most likely, you did the following:
#
# #### Move the sliders a bit, see whether the curve moves in the right direction, and if it
# did, keep doing it.
#
# For a machine, "learning" is that same process, translated into math!
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ## "Learning by nudging": The process of descent
#
# Let's start to formalize this idea. In order to push the curve in the "right direction",
# we need some measurement of "how right" and "how wrong" the model is. When we translate
# the idea of a "right direction" into math, we end up with a **loss function**, `L(w, b)`,
# as we saw in notebook 5. We say that the loss function is lowest when the model `σ(x, w,
# b)` performs the best.
#
# Now we want to create a loss function that is the lowest when the apple is at `0` and the
# banana is at `1`. If the data (the amount of green) for our apple is $x_1$, then our model
# will output $σ(x_1,w, b)$ for our apple. So, we want the difference $0 - σ(x_1, w, b)$ to
# be small. Similarly, if our data for our banana (the banana's amount of green) is $x_2$,
# we want the difference $1 - σ(x_2, w, b)$ to be small.
#
# To create our loss function, let's add together the squares of the difference of the
# model's output from the desired output for the apple and the banana. We get
#
# $$ L(w,b) = (0 - σ(x_1, w, b))^2 + (1 - σ(x_2, w, b))^2. $$
#
# $L(w, b)$ is lowest when it outputs `0` for the apple and `1` for the banana, and thus the
# cost is lowest when the model "is correct".
#
# We can visualize this function by plotting it in 3D with the `surface` function.
# ------------------------------------------------------------------------------------------

# plotly()
gr()

L(w, b) = (0 - σ(apple_green_amount,w,b))^2 + (1 - σ(banana_green_amount,w,b))^2

w_range = 10:0.1:13
b_range = 0:1:20

L_values = [L(w,b) for b in b_range, w in w_range]

@manipulate for w in w_range, b in b_range
    p1 = surface(w_range, b_range, L_values, xlabel="w", ylabel="b", cam=(70,40), cbar=false, leg=false)
    scatter!(p1, [w], [b], [L(w,b)+1e-2], markersize=5, color = :blue)
    p2 = plot(x->σ(x,w,b), 0, 1, label="Model", legend = :topleft, lw=3)
    scatter!(p2, [apple_green_amount],  [0.0], label="Apple", markersize=10)
    scatter!(p2, [banana_green_amount], [1.0], label="Banana", markersize=10, xlim=(0,1), ylim=(0,1))
    plot(p1, p2, layout=(2,1))
end

# ------------------------------------------------------------------------------------------
# The blue ball on the 3D plot shows the current parameter choices, plotted as `(w,b)`.
# Shown below the 3D plot is a 2D plot of the corresponding model with those parameters.
# Notice that as the blue ball rolls down the hill, the model becomes a better fit. Our loss
# function gives us a mathematical notion of a "hill", and the process of "learning by
# nudging" is simply rolling the ball down that hill.
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# To do this mathematically, we need to know which direction is "downhill". Recall from
# calculus that the derivative of `L` with respect to `b` tells you how `L` changes when `b`
# changes. Thus to roll downhill, we should go in the direction where the derivative is
# negative (the function goes down) for each parameter. This direction is the negative of
# what's called the **gradient**, $\nabla L$. This means that the "learn by nudging method"
# can be rephrased in mathematical terms as:
#
# 1. Calculate the gradient
# 2. Move a little bit in the direction of the negative gradient
# 3. Repeat
#
# This process of rolling the ball in the direction of the negative gradient is called
# **gradient descent**; written mathematically, it is
#
# $$p_{n+1} = p_n - \eta \nabla L(p_n).$$
#
# Here, $p_n$ represents the vector of current parameters $(w, b)$; $\nabla L(p_n)$ is the
# gradient of the loss function, given those parameters. We start from $p_n$ and change it
# by $\eta \nabla L(p_n)$, where $\eta$ is a small step size that determines how far we move
# the parameters in the direction of the negative gradient; notice that if you step too far,
# you'll overshoot the minimum!. The result is $p_{n+1}$, the new vector of parameters.
#
# [Picture of Gradient Descent Vectors]
#
# If we repeat this process, then we will end up at parameters where the model correctly
# labels apples as `0` and bananas as `1`. When this happens, the model has learned from the
# data and can then read pictures and tell you whether they are apples or bananas!
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# #### Exercise 1
#
# Use the following terms to fill in the sentences below. Terms may be used more than once
# or not at all:
# > gradient, loss function, derivative, gradient descent, learning.
#
# * We can think of a _(A)_ as a 1D version of a _(B)_.
# * We can visualize a _(C)_ as a hill.
# * In the explanation above, rolling downhill is called _(D)_ and means traveling along the
# _(E)_.
# * To quantify the correctness of a model we use a _(F)_.
# * When our program can minimize a _(G)_ on its own, we say it is _(H)_.
#
#

#
# A)

# B)

# C)

# D)

# E)

# F)

# G)

# H)

# ------------------------------------------------------------------------------------------

.nbexports/introductory-tutorials/broader-topics-and-ecosystem/intro-to-ml/09. ML - What is learning.jl


# squared Euclidean distance from a set

immutable SqrDistL2{R <: Real} <: ProximableFunction
  ind::IndicatorConvex
  lambda::R
  function SqrDistL2(ind::IndicatorConvex, lambda::R)
    if lambda < 0
      error("parameter λ must be nonnegative")
    else
      new(ind, lambda)
    end
  end
end

SqrDistL2{R <: Real}(ind::IndicatorConvex, lambda::R=1.0) = SqrDistL2{R}(ind, lambda)

function (f::SqrDistL2){T <: RealOrComplex}(x::AbstractArray{T})
  p, = prox(f.ind, x)
  return (f.lambda/2)*vecnorm(x-p)^2
end

function prox!{T <: RealOrComplex}(f::SqrDistL2, x::AbstractArray{T}, y::AbstractArray{T}, gamma::Real=1.0)
  p, = prox(f.ind, x)
  sqrd = (f.lambda/2)*vecnorm(x-p)^2
  c1 = 1/(1+f.lambda*gamma)
  c2 = f.lambda*gamma*c1
  for k in eachindex(p)
    y[k] = c1*x[k] + c2*p[k]
  end
  return sqrd*c1^2
end

fun_name(f::SqrDistL2) = "squared Euclidean distance from a convex set"
fun_dom(f::SqrDistL2) = fun_dom(f.ind)
fun_expr(f::SqrDistL2) = "x ↦ (λ/2) inf { ||x-y||^2 : y ∈ S} "
fun_params(f::SqrDistL2) = string("λ = $(f.lambda), S = ", typeof(f.ind))

function prox_naive{T <: RealOrComplex}(f::SqrDistL2, x::AbstractArray{T}, gamma::Real=1.0)
  p, = prox(f.ind, x)
  sqrd = (f.lambda/2)*vecnorm(x-p)^2
  gamlam = f.lambda*gamma
  return 1/(1+gamlam)*x + gamlam/(1+gamlam)*p, sqrd/(1+gamlam)^2
end


export rotate

function rotate(x::Array{Float64,2}; angle=0.0, point=[0.0, 0.0, 0.0], vector_=[1.0, 0.0, 0.0])
    t = angle/180*pi
    x = x .- vec(point)
    vector_ = vec(vector_)
    unit_vector = vector_/(sqrt(sum(vector_.^2)))
    l, m, n = unit_vector
    c, s = cos(t), sin(t)
    rot = [l*l*(1-c)+c     m*l*(1-c)-n*s   n*l*(1-c)+m*s;
           l*m*(1-c)+n*s   m*m*(1-c)+c     n*m*(1-c)-l*s;
           l*n*(1-c)-m*s   m*n*(1-c)+l*s   n*n*(1-c)+c;]
    x = rot * x
    x = x .+ vec(point)
    return x
end

function rotate(obj::T; angle=0.0, point=[0.0, 0.0, 0.0], vector_=[1.0, 0.0, 0.0]) where T <: Union{Shape, PostOpObj}
    # x, v, y, vol, type
    function func(out)
        x = out[:x]
        x = rotate(x; angle=angle, point=point, vector_=vector_)
        return repack!(out, [:x], [x])
    end
    if isa(obj, Shape)
        return PostOpObj(obj, func)
    elseif isa(obj, PostOpObj)
        push!(obj.operations, func)
        return obj
    else
        error("Not allowed.")
    end
end


function main()
	N = 4_000_000
	s = 0
	i1 = 1
	i2 = 1
	while true
		fib = i1 + i2
		i1 = i2
		i2 = fib
		if fib > N
			break
		end
		if fib % 2 == 0
			s += fib
		end
	end
	println(s)
end

main()


using Distributions

xObs     = [18 132; 45 178]
rowSums  = [sum(xObs[i,:]) for i in 1:2]
colSums  = [sum(xObs[:,i]) for i in 1:2]
n        = sum(xObs)

rowProps = rowSums/n
colProps = colSums/n

xExpect  = [colProps[c]*rowProps[r]*n for r in 1:2, c in 1:2]
testStat = sum([(xObs[r,c]-xExpect[r,c])^2 / xExpect[r,c] for r in 1:2,c in 1:2])
pVal = ccdf(Chisq(1),testStat)

println("Chi-squared value: ", testStat)
println("P-value: ", pVal)


"""

```
sphereradiusfromshellvolume(volume,step)
```

Computes the radius that corresponds to a spherical shell of
a given volume.

"""
function sphereradiusfromshellvolume(volume, step)

    fourthirdsofpi = 4 * pi / 3
    if 3 * step * volume - pi * step^4 <= 0.0
        return 0.0
    end
    rmin =
        (sqrt(3 * pi) * sqrt(3 * step * volume - pi * step^4) - 3 * pi * step^2) /
        (6 * pi * step)
    return (0.5 * (volume / fourthirdsofpi + 2 * rmin^3))^(1 / 3)

end


abstract type AbstractUtility end

# Euler equation deviation. Generic!
function euler_dev(
    u :: AbstractUtility,
    cV :: AbstractVector{Float64},
    betaR :: Float64
    )
    muV = marg_utility(u, cV);
    T = length(cV);
    devV = muV[1 : (T-1)] ./ muV[2 : T] .- betaR;
    return devV
end

## ----------  Log

struct UtilityLog <: AbstractUtility end

# Note the broadcasting dot. This now works for scalars and arrays.
utility(u :: UtilityLog, c) = log.(c);
marg_utility(u :: UtilityLog, c) = 1.0 ./ c;
inv_utility(u :: UtilityLog, util) = exp.(util);
inv_marg_utility(u :: UtilityLog, mu) = 1.0 ./ mu;
c_growth(u :: UtilityLog, betaR) = betaR;

## -----------  CRRA

struct UtilityCRRA <: AbstractUtility
    sigma :: Float64
end

utility(u :: UtilityCRRA, c) = (c .^ (1.0 - u.sigma)) ./ (1.0 - u.sigma) .- 1.0;
marg_utility(u :: UtilityCRRA, c) = c .^ (-u.sigma);
inv_utility(u :: UtilityCRRA, util) = ((util .+ 1.0) .* (1.0 - u.sigma)) .^ (1.0 / (1.0 - u.sigma));
inv_marg_utility(u :: UtilityCRRA, mu) = mu .^ (-1.0 / u.sigma);
c_growth(u :: UtilityCRRA, betaR) = betaR .^ (1.0 / u.sigma);

# -------------


# This file includes the MonteCarlo simulation of ASK modulation scheme and compares
# the numerical results with the theoretical results.

using DigitalCommunications
using Plots

# Simulation parameters
theme(:default)
plt = plot(legend=:bottomleft)
for k in 1 : 3
    M = 2^k
    nsymbols = Int(1e6)
    nbits = k * nsymbols
    ebno = collect(0 : 10)
    esno = ebno .+ 10 * log10(k)

    # Communcation system components
    gen = SymbolGenerator(nsymbols, M)
    modulator = Modulator(ASK(M))
    channel = AWGNChannel()
    detector = Detector(modulator(1:M))

    # Monte Carlo simulation
    message = gen.symbols
    symerr = zeros(length(esno))
    for i in 1 : length(symerr)
        channel.esno = esno[i]  # Update channel snr
        mbar = message |> modulator |> channel |> detector  # Extracted message signal
        symerr[i] = sum(mbar .!= message) / length(message)  # Symbol error rate
    end

    plot!(ebno, berask.(ebno, M), yscale=:log10, lw=0.5,
        markershape=:auto, color=:black, gridalpha=0.9, minorgrid=true, minorgridalpha=0.5, label="$M-ASK-theoretical")
    plot!(ebno, symerr,  yscale=:log10, lw=0.5,
        markershape=:auto, color=:black, gridalpha=0.9, minorgrid=true, minorgridalpha=0.5, label="$M-ASK-montecarlo")
end
display(plt)


using ITensors,
      LinearAlgebra,
      QuadGK

βc() = 0.5*log(sqrt(2.0)+1.0)

function ising_free_energy(β::Real,J::Real=1.0)
  k = β*J
  c = cosh(2.0*k)
  s = sinh(2.0*k)
  xmin = 0.0
  xmax = π
  integrand(x) = log(c^2+sqrt(s^4+1-2*s^2*cos(x)))
  integral,err = quadgk(integrand, xmin, xmax)::Tuple{Float64,Float64}
  return -(log(2.0)+integral/π)/(2.0*β)
end

function ising_magnetization(β::Real)
 β>βc && return (1.0-sinh(2.0*β)^(-4))^(1/8)
 return 0.0
end

function ising_mpo(sh::Tuple{Index,Index},sv::Tuple{Index,Index},
                   β::Real,J::Real=1.0;
                   sz::Bool=false)
  d = dim(sh[1])
  T = ITensor(sh[1],sh[2],sv[1],sv[2])
    for i = 1:d
      T[i,i,i,i] = 1.0
    end
    sz && (T[1,1,1,1] = -T[1,1,1,1])
    Q = [exp(β*J) exp(-β*J); exp(-β*J) exp(β*J)]
    D,U = eigen(Symmetric(Q))
    sqrtQ = U*Diagonal(sqrt.(D))*U'
    Xh1 = ITensor(vec(sqrtQ),sh[1],sh[1]')
    Xh2 = ITensor(vec(sqrtQ),sh[2],sh[2]')
    Xv1 = ITensor(vec(sqrtQ),sv[1],sv[1]')
    Xv2 = ITensor(vec(sqrtQ),sv[2],sv[2]')
    T = replacetags(T*Xh1*Xh2*Xv1*Xv2,"1","0")
  return T
end

function ising_partition(sh,sv,β)
  ny,nx = size(sh)
  T = Matrix{ITensor}(undef,ny,nx)
  for iy = 1:ny, ix = 1:nx
    ixp = per(ix+1,nx)
    iyp = per(iy+1,ny)
    T[iy,ix] = ising_mpo((sh[iy,ix],sh[iy,ixp]),(sv[iy,ix],sv[iyp,ix]),β)
  end
  return T
end