struct FractionalFactorial <: GSAMethod
end

"""
Code based on the theory presetned in:
Saltelli, A. (2008). Global sensitivity analysis: The primer. Chichester: Wiley, pp. 71-76.
"""

using LinearAlgebra

function _recursive_hadamard(k::Integer)
     """
     Generate a hadamard matrix via recursion.
     """
     # base case
     if k == 2
          return [1 1; 1 -1]
     else
          h = _recursive_hadamard(k ÷ 2)
     end

     return hcat(vcat(h, h), vcat(h, -h))
end

function _expanding_window_hadamard(k::Integer)
     """
     Generate hadamard matrix of size k using expanding window approach.
     """
     @assert ispow2(k)

     # intialize
     h = ones(Int64, k, k)
     h[2,2] = -1

     let
        bot_row = 2
        right_col = 2
        while bot_row < k

             @inbounds cur = h[1:bot_row, 1:right_col]

             new_bot = bot_row+1:bot_row*2
             new_right = right_col+1:right_col*2
             # update right
             @inbounds h[1:bot_row, new_right] = cur

             # update below
             @inbounds h[new_bot, 1:right_col] = cur

             # update diagonal
             @inbounds h[new_bot, new_right] = - cur

             # update window for each to 'copy' from.
             bot_row *= 2
             right_col *= 2
        end
     end
     return h
end

function _generate_hadamard(k::Integer)
     @assert ispow2(k)
     return _expanding_window_hadamard(k)
end

function generate_ff_design_matrix(num_parameters::Integer)
     """
     param: num_parameters
          The number of parameters to be sampled for.

     return:
          A 2-level fractional factorial design matrix of -1s and 1s with a design of resolution IV.

          The number of rows is 2*number of parameters and the number of columns is the number of parameters.

          See equation (2.31) in Saltelli, A. (2008) for details.
     """

     # If k is not a power of 2, get the next larger number that is
     # so the hadamard matrices can be computed.
     k = Integer(round(2^ceil(log2(num_parameters))))

     s = _generate_hadamard(k)
     design_matrix = vcat(s, -s) # design of resolution IV.

     return design_matrix
end

function generate_ff_sample_matrix(design_matrix::Array{Int64,2},  levels_list=nothing)
      """

      Convert the desgin matrix to a matrix whose rows can
      be used as inputs to a model by replacing low and high values for each parameter
      with the the low and high end of each parameters range, as specified in levels_list.

      param: design_matrix

            The matrix outputted by 'ff_design' i.e a 2-level fractional
            factorial design matrix of -1s and 1s with a design of resolution IV.
            The number of rows is 2*number of parameters and the
            number of columns is the number of parameters.

      param: levels_list (optional)
          A list of length equal to the number of parameters, where the elements of
          the list are 2-tuples.

          The ith tuple corresponds to the the low-value and the high-value of the ith parameter.
          i.e. levels_list[2][1] corresponds to the low-value of the 2nd parameter,
          and levels_list[2][2] corresponds to the high-value of the 2nd parameter.

          If no list is provided, the low value for each parameter
          is assumed to be 0 and the high value is assumed to be 1.

      returns:
          A matrix with the same dimensions as the design matrix, but with the '-1's replaced with
          the low value specified for each parameter in level_list, and similar the '1's are replaced
          with the high values.

          Each row corresponds to a single input vector to be used as input to a model/function.
      """
      sample_matrix = copy(design_matrix)
      if !isnothing(levels_list)
           for (col_index, (low_value, high_value)) in enumerate(levels_list)
                design_col = @view design_matrix[:,col_index]
                sample_col = @view sample_matrix[:,col_index]

                sample_col[design_col .== -1] .= low_value
                sample_col[design_col .== 1] .= high_value
            end
      end
      return sample_matrix
end

function run_model(sample_matrix::AbstractArray, model)
      """

      param: sample_matrix
            A matrix where each row corresponds to a input vector for "model"
      param: model
            The function/model to be executed on each row of the sample_matrix.
            Must return a scalar.
      return:
            A vector containing the result of running each row of the
            sample_matrix through model.
      """

      N = size(sample_matrix)[1]
      y_out = zeros(N)
      Threads.@threads for i in 1:N
          y_out[i] = model(@view sample_matrix[i,:])
      end
      return y_out
end

function ff_main_effects(design_matrix::AbstractArray, response_values::AbstractArray)
      """
           Computes the main effect for each parameter

           param: design_matrix

                The matrix outputted by 'ff_design' i.e a 2-level fractional
                factorial design matrix of -1s and 1s with a design of resolution IV.
                The number of rows is 2*number of parameters and the
                number of columns is the number of parameters.

           param: response_values

                An array of values corresponding to the result of running the
                model on each row of a sample matrix generated by the design_matrix

           return: main_effects

               A vector of the main effects of each parameter on the output.

            See Saltelli, A. equation 2.32 for details.
      """

      num_rows, num_cols = size(design_matrix)
      @assert num_rows == length(response_values) "Number of rows in design matrix must match number of responses"

      main_effects = zeros(num_cols)
      for column in 1:num_cols
            contrast_vector = @view design_matrix[:,column]
            main_effects[column] = dot(response_values, contrast_vector)
      end
      return main_effects ./ num_rows
end

function gsa(f, method::FractionalFactorial; num_params, N, p_range = nothing, kwargs...)
    design_matrix = generate_ff_design_matrix(num_params)
    sample_matrix = generate_ff_sample_matrix(design_matrix, p_range)

    response_vec = run_model(sample_matrix, f)

    main_effects = ff_main_effects(design_matrix, response_vec)
    return main_effects, main_effects.^2
end


using LinearAlgebra
using Base: require_one_based_indexing

"""
    systemSolver(A::AbstractMatrix, b::Vector, x_initial::Vector) -> Vector

    Compute the orthogonal projection of a given point to the
solution set of a system of linear equations. Also a direct
method for solving systems of linear equations. The output
is either the projection or inconsistency of the system.

# Arguments
 - `A::AbstractMatrix{Real}`: coefficients of linear equations
 - `b::AbstractVector{Real}`: intended results of equations
 - `x0::AbstractVector{Real}`: initial point for projection
 - `β::Real`: set in order to obtain comparable summands in norm

# References
> Ján Plesník, Finding the orthogonal projection of a point onto
> an affine subspace, Linear Algebra and its Applications, Volume 422,
> Issues 2–3, 2007, Pages 455-470
"""
function systemSolver(A::AbstractMatrix{<:Real},    # in: equations matrix (m x n) rows = coefficients
                    b::AbstractVector{<:Real},      # in: equations results
                    x0::AbstractVector{<:Real},     # in: initial point
                    β::Real=1.0)                    # in: optional normalizer

    require_one_based_indexing(A, b, x0)
    m, n = size(A)
    if m != length(b) || n != length(x0)
        throw(DimensionMismatch("dimensions of `A`, $m * $n, does not match one of the lengths of `x0`, $(length(x0)), or `b`, $(length(b))"))
    end
    for i in 1:m
        iszero(A[i, :]) && throw(ArgumentError("`A` rows must be nonzero"))
    end
    iszero(β) && throw(ArgumentError("`β` must be nonzero"))

    if m == 0
        return x0
    end

    k = 1
    p = 1
    V = hcat(A[1, :])
    x = x0 + (b[1] - dot(A[1, :], x0))/(dot(A[1, :], V[:, 1])) * V[:, 1]

    denominators = zeros(Real, m-1)

    while k != m
        k = k + 1
        p = p + 1

        denominators[k-1] = dot((A[k, :] - A[1, :]), V[:, k-1])
        if denominators[k-1] == 0
            A[k, :] = - A[k, :]
            b[k] = - b[k]
            denominators[k-1] = dot((A[k, :] - A[1, :]), V[:, k-1])
        end

        w = x0 + β * A[k, :]
        for j in 2:k
            w = w + (b[j]-b[1]-dot(A[j, :]-A[1, :], w))/denominators[j-1] * V[:, j-1]
        end

        u = x + (b[k] - dot(A[k, :], x))/denominators[k-1] * V[:, k-1]
        if u != w
            V = hcat(V, w - u)
            x = u + (b[1] - dot(A[1, :], u))/dot(A[1, :], V[:, k]) * V[:, k]
        elseif dot(A[k, :], x) == b[k]
            if k != m
                A = A[setdiff(1:end, k), :]
                b = b[setdiff(1:end, k)]
                k = k - 1
                m = m - 1
            end
        else
            throw(DomainError("the ($p)th equation is inconsistent"))
        end
    end
    return x
end


"""
     l63!(du,u,p,t)

 Steps forward the Lorenz63 equations for time `t`.
 Prognostic variables x, y, z are defined as
 elements 1, 2 and 3 of vector `u` and with parameters
 σ, ρ and β defined as elements of vector `p`.
 Elements of du are set to dxdt, dydt and dzdt respectively.

```
 julia> using DifferentialEquations, Plots
 julia> u0=[1.,0.,0.]
 julia> p=[10.,28.,8/3]
 julia> tspan=(0.,100.)
 julia> prob=ODEProblem(l63!,u0,tspan,p)
 julia> sol=solve(prob)
 julia> plot(sol,vars=(1,2,3))
```

"""
function l63!(du,u,p,t)
        x, y, z=u
        σ, ρ, β=p

        dxdt=σ*(y-x)
        dydt=ρ*x-x*z-y
        dzdt=x*y-β*z

        du[1], du[2], du[3]=
        dxdt , dydt,  dzdt
        return
end


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ db4a6b48-99a7-11eb-097b-676aed417c48
begin

include("MultiArmBandit.jl")

using .MultiArmBandit
using DataFrames
using StatsBase, Statistics
using Distributions
using Gadfly
import Random

end

# ╔═╡ b65dd189-456b-40ac-88fc-d05b5e29b396
md"

# Multi-Arm Bandits

A Multi-Arm Bandit is a concept from Reinforcement Learning which is useful for decision making under uncertainty. Take a situation where you have limited time or resources to test multiple drugs, perhaps the COVID vaccines for example. While you could use the standard approach of a clinical trial with randomization, a balanced design, and performing AB tests, this results in many participants not recieving the best treatment, as some of them will be allocated to inferior choices. Ideally, we would like to do both: 1) Show a given treatment is the best choice and 2) Have most people end up benefitting from it.

Using a strategy known as $\epsilon$-greedy Q learning, the Multi Arm bandit can help accomplish this task in an online manner where the treatments are allocated on the spot. The arm refers to different treatment groups that the bandit can select from. In the beginning, each treatment arm is assigned the same value of $Q_{tmt}=Q_{0}$. Some small amount of noise can be added to break ties. At each step, $(1-\epsilon)$ of the time the bandit randomly selects from the treatment with the highest $Q_{tmt}$ and updates its corresponding value. $\epsilon$ of the time, the bandit will select a random treatment instead of the one with the highest Q. The update rule is as follows:

$Q_{tmt,t} = Q_{tmt,t-1} + \frac{R-Q_{tmt,t-1}}{N_{tmt,t}}$

where R is the reward, which is usually just the response Y corresponding to the treatment given. $N_{t+1}$ corresponds to the sample size in the given treatment group after the sample from the group is randomly selected. The bandit will be trying to go for the treatment which maximizes the reward, and so care should be taken that the response is transformed or sign-flipped if necessary so that higher is better. The above update rule can be shown to correspond to the calculation of the sample mean in an online manner if $Q_{0} = 0$. However, we can modify  the updates by adding an explicit learning rate parameter $\eta$:

$Q_{tmt,t+1} = Q_{tmt,t} + \eta(R-Q_{tmt,t})$

This can help improve learning. In this notebook, we will simulate data from 5 groups and examine the impact of setting different parameters for $Q_{0}$ and $\eta$.

The source code for the functions used in this demo Pluto notebook can be found in the **MultiArmBandit.jl** module in this repo.

"

# ╔═╡ 38528382-a1ec-4117-88d4-fbcc019c7302
md"

## Simulate Data

Below, we simulate n=10000 points from 5 groups each A, B, C, D, E from a normal distibution with mean $\mu = (-0.3,0.0,0.5,0.8,1.1)$ and $\sigma = (1, 0.5, 0.3, 0.3, 0.5)$ respectively. The bandit will select points from this simulated dataset. Note that in the ideal situation the ground truth in this simulated example is:

$ E > D > C > B > A $

Ideally, the bandit will come to this same conclusion and choose E most of the time. However, the different SDs for the groups can complicate this choice, as the bandit is greedy and thus converging to E in a finite number of iterations (in the real world, this would correspond to resources or number of patients in the trial) can be more complicated. We can track the bandit's history and plot it over time to examine when convergence occurs.

The sample mean/SD of each Arm are shown below. The bandit will be selecting samples from each group in an online manner, and so at each step it does NOT have access to every single data point. Instead it must figure out in the finite number of iterations which group is the best

"

# ╔═╡ 60e50c4e-8120-4306-9c6c-9282fff8eb47
begin

n = 1000
Random.seed!(101)
A= rand(Normal(-0.3,1),n)
B= rand(Normal(0,0.5),n)
C = rand(Normal(0.5,0.3),n)
D = rand(Normal(0.8,0.3),n)
E = rand(Normal(1.1,0.5),n)

simdata = DataFrame(Arm = vcat(fill("A",n),fill("B",n),fill("C",n),fill("D",n),fill("E",n)),
                    Y = vcat(A,B,C,D,E))

summary_simdata = combine(groupby(simdata,:Arm),nrow,:Y=>mean=>:meanY,:Y=>std=>:sdY)
summary_simdata
end

# ╔═╡ 65107f38-7916-43a4-b36f-1ef012c43b51
md"

## Create Bandit: $Q_0=0, \eta=\frac{1}{N_{tmt,t}}, \epsilon = 0.1$

Below we instantiate the first bandit with all default settings, and allow it to explore 10% of the time while 90% of the time it selects the current best group in a greedy manner.

"

# ╔═╡ 8ed00c06-7348-49dc-a9f2-041465beb969
band1=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,track_hist=true)

# ╔═╡ 472fee46-6e48-47a7-9235-ccb7c460f735
begin
Random.seed!(1001)
MultiArmBandit.Train!(band1,999,ϵ = 0.1)
end

# ╔═╡ 1243d0ab-f95c-482d-b81e-3b3eb60907eb
band1_df = MultiArmBandit.GetHistoryDF(band1)

# ╔═╡ 794d15a3-a4b4-4ffa-9afb-bb19d79e79e5
plot(band1_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ 5e8e7408-aa48-4a2d-a427-40e8adf81d3c
band1_summary = summary(band1)

# ╔═╡ 32f83952-c660-4fde-8bd5-ae1a97e7bacd
md"

Based on the above figure, the correct ordering is found after around 250 steps. For the first few steps, the bandit picks Group D which is the 2nd best but eventually finds Group C as a result of the exploration. Lets try to see what happens if we lower the bandit's exploration by setting $\epsilon = 0.03$. That is, the bandit will select a group randomly (instead of the group it deems the best group at the moment) 3% of the time.

## Create Bandit: $Q_0=0, \eta=\frac{1}{N_{tmt,t}}, \epsilon = 0.03$

"

# ╔═╡ 65945e4f-17e9-40ba-870f-88da0c5d13d0
begin
band2=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,track_hist=true)
Random.seed!(1102)
MultiArmBandit.Train!(band2,999,ϵ = 0.03)
band2_df = MultiArmBandit.GetHistoryDF(band2)
band2_summary = summary(band2)
end

# ╔═╡ 3566a579-84dc-4cb6-91ab-833d0d5e9782
plot(band2_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ a15c8f8f-f077-4758-a668-01c8b562ca95
md"

In this situation, it took longer (~100 iterations) for the bandit to arrive at the best group

In the next section, we will change the starting value to be higher at $Q_{0} = 5$. This will tend to encourage a lot more early exploration as this value is much higher than what is possible based on sampling.

## Create Bandit: $Q_0=5, \eta=\frac{1}{N_{tmt,t}}, \epsilon = 0.1$

"

# ╔═╡ 77d6e956-219e-40e9-978d-f7e0906a8d6c
begin
band3=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,Q₀=5,track_hist=true)
Random.seed!(901)
MultiArmBandit.Train!(band3,99,ϵ = 0.1)
band3_df = MultiArmBandit.GetHistoryDF(band3)
band3_summary = summary(band3)
end

# ╔═╡ 043f9d99-e8b5-4b33-82ac-3870a2238f85
plot(band3_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ 50196d6d-8a52-4043-aa9f-e3f43a9ed2a1
md"

In this case, starting at a value much higher than possible leads to earlier exploration, resulting in the Q value for each group falling rapidly. However, the bandit arrives at the best option E much quicker since it is pulled down less and is essentially guaranteed to be selected since the bandit desperately switches options more in the beginning to try to maintain a high Q value.

In this next part, we keep $Q_0 = 5$ but set the learning rate $\eta=0.01$

## Create Bandit: $Q_0=5, \eta=0.01, \epsilon = 0.1$

"

# ╔═╡ 9f3d4773-ac0a-4dd2-992d-b549f58f4021
begin
band4=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,Q₀=5,track_hist=true)
Random.seed!(555)
MultiArmBandit.Train!(band4,2999,ϵ = 0.1,η=0.01)
band4_df = MultiArmBandit.GetHistoryDF(band4)
band4_summary = summary(band4)
end

# ╔═╡ 3789a09f-73fb-4ca8-b267-bbe2d7b32848
plot(band4_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ 7825e432-9f3b-4099-bf37-0b1b3ccf38c1
md"

In this case, setting the learning rate to 0.01 leads the bandit to take longer to reach the optimal solution E. Until around 1500 steps, the Q value for all the groups is even.

What happens if we increase the learning rate to 1 (but keep $Q_0 = 5$) ?

## Create Bandit: $Q_0=5, \eta=1, \epsilon = 0.1$

"

# ╔═╡ 2342907c-75e0-4126-b2df-6ce5a5901bf1
begin
band5=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,Q₀=5,track_hist=true)
Random.seed!(1556)
MultiArmBandit.Train!(band5,2999,ϵ = 0.1,η=1)
band5_df = MultiArmBandit.GetHistoryDF(band5)
band5_summary = summary(band5)
end

# ╔═╡ 7c511272-01f8-4270-bdf7-f4f66e78df07
plot(band5_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ c1deba65-a93f-4992-958e-38a987346691
md"

Now the plot is extremely noisy, and the bandit cannot seem to differentiate between Group D and Group E very well. We can see in the summary that the bandit tended to select D the most but in the end the Q values of both D/E are about the same.

In this final parts, we now set $Q_0=0$ again and examine the impact of the two learning rates.

## Create Bandit: $Q_0=0, \eta=0.01, \epsilon = 0.1$

"

# ╔═╡ 8bbfae0c-6648-4f42-a40c-ada03d17bb3a
begin
band6=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,Q₀=0,track_hist=true)
Random.seed!(121)
MultiArmBandit.Train!(band6,2999,ϵ = 0.1,η=0.01)
band6_df = MultiArmBandit.GetHistoryDF(band6)
band6_summary = summary(band6)
end

# ╔═╡ 613e3215-4402-4ba2-8402-2ab5bf1f1247
plot(band6_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ e7772e4c-b798-4bdd-85c1-6ee291fce282
md"

In this case, starting at $Q_0=0$ and setting the learning rate $\eta = 0.01$ leads to the bandit favoring Group C, a suboptimal group, for a long time. It is only after ~2700 iterations that it picks up on Group E.

What happens if the learning rate is increased? Will we see the same noisy behavior as before when starting from $Q_0=0$?

## Create Bandit: $Q_0=0, \eta=1, \epsilon = 0.1$

"

# ╔═╡ bf545060-f55c-4a91-99f8-c41543c25ebb
begin
band7=MultiArmBandit.CreateBandit(simdata,:Arm,:Y,Q₀=0,track_hist=true)
Random.seed!(13)
MultiArmBandit.Train!(band7,2999,ϵ = 0.1,η=1)
band7_df = MultiArmBandit.GetHistoryDF(band7)
band7_summary = summary(band7)
end

# ╔═╡ 5af03195-5199-48b9-8792-f02da75dc320
plot(band7_df,x=:steps,y=:Q,color=:Group,Geom.line)

# ╔═╡ a94f1ce9-b874-4da2-af2b-ee8c6840a2d2
md"

The higher learning rate again leads to noise, and in the end, the bandit has trouble differentiating Group D and E just as before.

In summary, lowering the learning rate appears to increase the chance of arriving at a suboptimal solution. Increasing the learning rate too much confuses the Bandit and the Q value curves become extremely noisy.

Starting at a higher Q (with no set learning rate) encourages quicker exploration between the groups

"

# ╔═╡ Cell order:
# ╟─b65dd189-456b-40ac-88fc-d05b5e29b396
# ╠═db4a6b48-99a7-11eb-097b-676aed417c48
# ╟─38528382-a1ec-4117-88d4-fbcc019c7302
# ╠═60e50c4e-8120-4306-9c6c-9282fff8eb47
# ╟─65107f38-7916-43a4-b36f-1ef012c43b51
# ╠═8ed00c06-7348-49dc-a9f2-041465beb969
# ╠═472fee46-6e48-47a7-9235-ccb7c460f735
# ╠═1243d0ab-f95c-482d-b81e-3b3eb60907eb
# ╠═794d15a3-a4b4-4ffa-9afb-bb19d79e79e5
# ╠═5e8e7408-aa48-4a2d-a427-40e8adf81d3c
# ╟─32f83952-c660-4fde-8bd5-ae1a97e7bacd
# ╠═65945e4f-17e9-40ba-870f-88da0c5d13d0
# ╠═3566a579-84dc-4cb6-91ab-833d0d5e9782
# ╟─a15c8f8f-f077-4758-a668-01c8b562ca95
# ╠═77d6e956-219e-40e9-978d-f7e0906a8d6c
# ╠═043f9d99-e8b5-4b33-82ac-3870a2238f85
# ╟─50196d6d-8a52-4043-aa9f-e3f43a9ed2a1
# ╠═9f3d4773-ac0a-4dd2-992d-b549f58f4021
# ╠═3789a09f-73fb-4ca8-b267-bbe2d7b32848
# ╟─7825e432-9f3b-4099-bf37-0b1b3ccf38c1
# ╠═2342907c-75e0-4126-b2df-6ce5a5901bf1
# ╠═7c511272-01f8-4270-bdf7-f4f66e78df07
# ╟─c1deba65-a93f-4992-958e-38a987346691
# ╠═8bbfae0c-6648-4f42-a40c-ada03d17bb3a
# ╠═613e3215-4402-4ba2-8402-2ab5bf1f1247
# ╟─e7772e4c-b798-4bdd-85c1-6ee291fce282
# ╠═bf545060-f55c-4a91-99f8-c41543c25ebb
# ╠═5af03195-5199-48b9-8792-f02da75dc320
# ╟─a94f1ce9-b874-4da2-af2b-ee8c6840a2d2


function bernoulli(n)
    A = Vector{Rational{BigInt}}(undef, n + 1)
    for m = 0 : n
        A[m + 1] = 1 // (m + 1)
        for j = m : -1 : 1
            A[j] = j * (A[j] - A[j + 1])
        end
    end
    return A[1]
end

function display(n)
    B = map(bernoulli, 0 : n)
    pad = mapreduce(x -> ndigits(numerator(x)) + Int(x < 0), max, B)
    argdigits = ndigits(n)
    for i = 0 : n
        if numerator(B[i + 1]) & 1 == 1
            println(
                "B(", lpad(i, argdigits), ") = ",
                lpad(numerator(B[i + 1]), pad), " / ", denominator(B[i + 1])
            )
        end
    end
end

display(60)


"""
    black_body(T, K₀, σ)
    black_body(T)

Thermal infrared, *i.e.* longwave radiation emitted from a black body at temperature T.

- `T`: temperature of the object in Celsius degree
- `K₀`: absolute zero (°C)
- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law)

# Note

`K₀` and `σ` are taken from [`Constants`](@ref) if not provided.

"""
function black_body(T, K₀, σ)
  Tₖ = T - K₀
  σ * (Tₖ^4.0)
end

function black_body(T)
    constants = Constants()
    black_body(T, constants.K₀, constants.σ)
end

"""
Thermal infrared, *i.e.* longwave radiation emitted from an object at temperature T.

- `T`: temperature of the object in Celsius degree
- `ε` object [emissivity](https://en.wikipedia.org/wiki/Emissivity) (not to confuse with ε the
ratio of molecular weights from [`Constants`](@ref)). A typical value for a leaf is 0.955.
- `K₀`: absolute zero (°C)
- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law)

# Note

`K₀` and `σ` are taken from [`Constants`](@ref) if not provided.

# Examples

```julia
# Thermal infrared radiation of water at 25 °C:
grey_body(25.0, 0.96)
```
"""
function grey_body(T, ε, K₀, σ)
  ε * black_body(T, K₀, σ)
end

function grey_body(T, ε)
    constants = Constants()
    grey_body(T, ε, constants.K₀, constants.σ)
end

"""
    net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁,K₀,σ)
    net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁)

Net longwave radiation fluxes (*i.e.* thermal radiation, W m-2) between an object and another.
The object of interest is at temperature T₁ and has an emissivity ε₁, and the object with
wich it exchanges energy is at temperature T₂ and has an emissivity ε₂.

If the result is positive, then the object of interest gain energy.

# Arguments

- `T₁` (Celsius degree): temperature of the target object (object 1)
- `T₂` (Celsius degree): temperature of the object with which there is potential exchange (object 2)
- `ε₁`: object 1 emissivity
- `ε₂`: object 2 emissivity
- `F₁`: view factor (0-1), *i.e.* visible fraction of object 2 from object 1 (see note)
- `K₀`: absolute zero (°C)
- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law)

# Note

`F₁`, the view factor (also called shape factor) is a coefficient applied to the semi-hemisphere
field of view of object 1 that "sees" object 2. E.g. a leaf can be viewed as a plane. If one side
of the leaf sees only object 2 in its field of view (e.g. the sky), then `F₁ = 1`.
Then the net longwave radiation flux for this part of the leaf is multiplied by its actual
surface to get the exchange. Note that we apply reciprocity between the two objects for
the view factor (they have the same value), *i.e.*: A₁F₁₂ = A₂F₂₁.

Then, if we take a leaf as object 1, and the sky as object 2, the visible fraction of
sky viewed by the leaf would be:

- `0.5` if the leaf is on top of the canopy, *i.e.* the upper side of the leaf sees the sky,
the side bellow sees other leaves and the soil.
- between 0 and 0.5 if it is within the canopy and partly shaded by other objects.

Note that `A₁` for a leaf is twice its common used leaf area, because `A₁` is the **total**
leaf area of the object that exchange energy.

```julia
# Net thermal radiation fluxes between a leaf and the sky considering the leaf at the top of
# the canopy:
Tₗ = 25.0 ; Tₐ = 20.0
ε₁ = 0.955 ; ε₂ = 1.0
Rₗₗ = net_longwave_radiation(Tₗ,Tₐ,ε₁,ε₂,1.0)
Rₗₗ

# Rₗₗ is the net longwave radiation flux between the leaf and the atmosphere per surface area.
# To get the actual net longwave radiation flux we need to multiply by the surface of the
# leaf, e.g. for a leaf of 2cm²:
leaf_area = 2e-4 # in m²
Rₗₗ * leaf_area

# The leaf lose ~0.0055 W towards the atmosphere.
```

# References

Cengel, Y, et Transfer Mass Heat. 2003. A practical approach. New York, NY, USA: McGraw-Hill.
"""
function net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁,K₀,σ)
    (black_body(T₂,K₀,σ) - black_body(T₁,K₀,σ)) / (1.0 / ε₁ + 1.0 / ε₂ - 1.0) * F₁
end

function net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁)
    constants = Constants()
    net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁,constants.K₀,constants.σ)
end

"""
    atmosphere_emissivity(Tₐ,eₐ)

Emissivity of the atmoshpere at a given temperature and vapor pressure.

# Arguments

- `Tₐ` (°C): air temperature
- `eₐ` (kPa): air vapor pressure
- `K₀` (°C): absolute zero

# Examples

```julia
Tₐ = 20.0
VPD = 1.5
atmosphere_emissivity(Tₐ, vapor_pressure(Tₐ,VPD))
```

# References

Leuning, R., F. M. Kelliher, DGG de Pury, et E.-D. SCHULZE. 1995. Leaf nitrogen,
photosynthesis, conductance and transpiration: scaling from leaves to canopies ». Plant,
Cell & Environment 18 (10): 1183‑1200.
"""
function atmosphere_emissivity(Tₐ,eₐ,K₀)
    0.642 * (eₐ * 100 / (Tₐ - K₀))^(1 / 7)
end

function atmosphere_emissivity(Tₐ,eₐ)
    atmosphere_emissivity(Tₐ,eₐ,Constants().K₀)
end


# This code presents an example to illustrate how SmolyakApprox can be used

using SmolyakApprox

function test_smolyak_approx()

  d  = 5  # Set the number of dimensions
  mu = 3  # Set the level of approximation

  g2, m2 =  smolyak_grid(clenshaw_curtis_equidistant,d,mu)
  grid, multi_ind = smolyak_grid(chebyshev_gauss_lobatto,d,mu)  # Construct the Smolyak grid and the multi index

  # An arbitrary test function

  function test(grid)

    y_value = (grid[:,1].+1).^0.1.*exp.(grid[:,2]).*log.(grid[:,3].+2).^0.2.*(grid[:,4].+2).^0.8.*(grid[:,5].+7).^0.1

    return y_value

  end

  y = test(grid)  # Evaluate the test function on the Smolyak grid
  y_pl = test(g2)

  point = [0.75, 0.45, 0.82, -0.15, -0.95]  # Choose a point to evaluate the approximated function

  # One way of computing the weights and evaluating the approximated function

  weights = smolyak_weights(y,grid,multi_ind)        # Compute the Smolyak weights
  y_hat = smolyak_evaluate(weights,point,multi_ind)  # Evaluate the approximated function

  #= A second way of computing the weights and evaluating the approximated function that
  computes the interpolation matrix just once. =#

  interp_mat = smolyak_inverse_interpolation_matrix(grid,multi_ind)  # Compute the interpolation matrix
  w = smolyak_weights(y,interp_mat)             # Compute the Smolyak weights
  y_hatt = smolyak_evaluate(w,point,multi_ind)  # Evaluate the approximated function

  # Piecewise linear

  w_pl = smolyak_pl_weights(y_pl,g2,m2)
  y_pl_hat = smolyak_pl_evaluate(w_pl,point,g2,m2)

  # Evaluate the exact function at point

  y_actual = test(point')

  # Now consider the ansiotropic case

  mu = [3, 2, 2, 2, 3]
  grid, multi_ind = smolyak_grid(chebyshev_gauss_lobatto,d,mu)  # Construct the Smolyak grid and the multi index
  y = test(grid)
  weights = smolyak_weights(y,grid,multi_ind)                   # Compute the Smolyak weights
  y_hat_ansio = smolyak_evaluate(weights,point,multi_ind)       # Evaluate the approximated function
  weights_th = smolyak_weights_threaded(y,grid,multi_ind)                   # Compute the Smolyak weights

  g3, m3 =  smolyak_grid(clenshaw_curtis_equidistant,d,mu)
  y_pl = test(g3)
  w_pl_ansio = smolyak_pl_weights(y_pl,g3,m3)
  y_pl_hat_ansio = smolyak_pl_evaluate(w_pl,point,g3,m3)
  w_pl_ansio_th = smolyak_pl_weights_threaded(y_pl,g3,m3)

  # Now test the full grid results

  mu = 3
  grid_full, multi_ind_full = smolyak_grid_full(chebyshev_gauss_lobatto,d,mu) # Construct the Smolyak grid and the multi index
  y_full = test(grid_full)
  weights_full = smolyak_weights_full(y_full,grid_full,multi_ind_full)        # Compute the Smolyak weights
  y_hat_full = smolyak_evaluate_full(weights_full,point,multi_ind_full)       # Evaluate the approximated function

  return y_hat, y_hatt, y_pl_hat, y_actual, y_hat_ansio, y_pl_hat_ansio, y_hat_full

end

test_smolyak_approx()


using LinearAlgebra
using PyPlot
using Statistics
using Distributions
using ProgressMeter

σ = 1
τ = sqrt(10)
μ = 5
n = 5

X = [9.37;10.18;9.16;11.60;10.33]
μn = mean(X)*n/σ^2/(n/σ^2 + 1/τ^2) + μ*(1/τ^2)/(n/σ^2 + 1/τ^2)
τn2 = 1/(n/σ^2+1/τ^2)

# Hyperparameters
# proposal step size
δ = 3/100
burnin = 2000
N = 10000
θ0 = 0.0

function logf(θ)
    -sum((X.-θ).^2)/2σ^2 - (θ-μ)^2/2τ^2
end

function proposal(x)
    x + (rand()-0.5)*2 * δ
end

sim = zeros(N)
sim[1] = θ0
@showprogress for i = 1:N-1
    x = sim[i]
    x_star = proposal(x)
    Δ =  logf(x_star) - logf(x)
    if log(rand())<Δ
        sim[i+1] = x_star
    else
        sim[i+1] = x
    end
end
L = logf.(sim)

# sim = sim[burnin+1:end]
figure(figsize=(12,4))
subplot(131)
plot(sim)
plot(1:length(sim), ones(length(sim))*μn, "--")
title("\$\\theta\$ Value")
xlabel("Iteration")
ylabel("\$\\theta\$")
subplot(132)
plot(L)
title("Log likelihood")
xlabel("Iteration")
ylabel("Log likelihood")
subplot(133)
hist(sim[burnin+1:end], density=true, bins=50)
y = pdf.(Normal(μn, sqrt(τn2)), LinRange(8.5,11.5,100))
plot(LinRange(8.5,11.5,100), y, label="Exact")
title("Distribution")
xlabel("\$\\theta\$")
ylabel("Density")
legend()
tight_layout()


include("center.jl")
include("cross_validation.jl")
include("mse.jl")

function ridge( X, y, lambda::Number )

    colmeans_X = center!( X )
    mean_y = center!( y )

    #A = X'*X + diagm( fill( lambda, size(X,2) ) )
    #w = A\(X'*y)

    A = [ X ; fill( sqrt( lambda ), ( 1, size(X,2) ) ) ]
    B = [ y ; fill( 0, ( 1, size(y,2) ) ) ]
    w = A\B

    broadcast!( +, X, X, colmeans_X )
    broadcast!( +, y, y, mean_y )

    bias = mean_y - colmeans_X * w

    return w, bias

end

function ridge( X, y, ls, gen = Kfold( size(X,1), 5 ) )
    if Base.length( ls ) == 1
        return ridge( X, y, ls[1] )
    end
    l_errs = Array( Float64, size(ls,1) )
    for i in 1:size(ls,1)
        l = ls[i]
        train_f(is) = ridge( X[is,:], y[is,:], l )
        error_f(model,is) = mse( predict( model, X[is,:] ), y[is,:] )
        l_errs[i] = mean( cross_validate( train_f, error_f, gen ) )
    end
    l = ls[ indmin( l_errs ) ]
    return ridge( X, y, l ), l, l_errs
end

function predict( w, bias, X )
    return broadcast( +, X*w, bias )
end

function ridgeKernel( K, y, lambda )
    m = size( K, 1 )

    colmeans_K = center!( K )
    mean_y = center!( y )

    for i = 1:m
        K[i,i] += lambda
    end
    w = K \ y
    for i = 1:m
        K[i,i] -= lambda
    end

    broadcast!( +, y, y, mean_y )
    broadcast!( +, K, K, colmeans_K )

    bias = mean_y - colmeans_K * w

    return w, bias
end

function ridgeKernel!( K, y, lambda )
    m = size( K, 1 )

    colmeans_K = center!( K )
    mean_y = center!( y )

    for i = 1:m
        K[i,i] += lambda
    end
    w = K \ y

    broadcast!( +, y, y, mean_y )

    bias = mean_y - colmeans_K * w

    return w, bias
end

function predictKernel( w, bias, K )
    return broadcast( +, K*w, bias )
end

function ridge( kernel::Function, X, y, lambda::Number )

    m = size( X, 1 )
    K = Array( eltype(y), m, m )
    for i = 1:m
        for j = i:m
            K[i,j] = kernel( X[i,:], X[j,:] )
            K[j,i] = K[i,j]
        end
    end

    w,bias = ridgeKernel!( K, y, lambda )

    return w, bias, kernel, X

end

function ridge( kernel::Function, X, y, ls, gen = Kfold( size(X,1), 5 ) )

    if Base.length( ls ) == 1
        return ridge( X, y, ls[1], kernel )
    end

    m = size( X, 1 )
    K = Array( eltype(y), m, m )
    for i = 1:m
        for j = i:m
            K[i,j] = kernel( X[i,:], X[j,:] )
            K[j,i] = K[i,j]
        end
    end

    l_errs = Array( Float64, size(ls,1) )
    for i in 1:size(ls,1)
        l = ls[i]
        train_f(is) = ridgeKernel( K[is,is], y[is,:], l ), is

        function error_f(v,is)
            model, train_is = v
            yh = predictKernel( model..., K[is,train_is] )
            return mse( yh, y[is,:] )
        end

        l_errs[i] = mean( cross_validate( train_f, error_f, gen ) )
    end
    l = ls[ indmin( l_errs ) ]
    w,bias = ridgeKernel!( K, y, l )
    return (w, bias, kernel, X), l, l_errs
end

function predict( w, bias, kernel, X_model, X )
    m = size( X, 1 )
    n = size( X_model, 1 )
    K = Array( eltype(w), m, n )
    for i = 1:m
        for j = 1:n
            K[i,j] = kernel( X[i,:], X_model[j,:] )
        end
    end
    return predictKernel( w, bias, K )
end

function predict( model, X )
    predict( model..., X )
end


export getNodalMassMatrix

function getNodalMassMatrix(mesh::TetraMesh, sigma::Array{Float64,1})

	nc = mesh.nc
  nn = mesh.nn

	M = zeros(4,4,nc)

	for i=1:nc
		Ti = mesh.Tetras[i,:]
	  	Xi = mesh.Points[Ti,:]

		M[:,:,i] = getLocalNodalMassMatrix(sigma[i],Xi)

	end
	nodalInd = mesh.Tetras

	ii = zeros(Int64,16*nc)
	jj = zeros(Int64,16*nc)
	# mm = zeros(Float64,16*nc)

	cnt = 1
	for i=1:nc
		for i2=1:4
			for i1=1:4
				ii[cnt] = nodalInd[i,i1]
				jj[cnt] = nodalInd[i,i2]
				# mm[cnt] = M[i1,i2,i]
				cnt    += 1
			end
		end
	end
  mm = vec(M)

	return sparse(ii,jj,mm,nn,nn)

end

# Nodal mass matrix is independent of geometry. Store mass matrix for unit coefficient and unit volume.
const M = (1/20) * [
  2 1 1 1
  1 2 1 1
  1 1 2 1
  1 1 1 2]

function getLocalNodalMassMatrix(sigma::Float64,X::Array{Float64})
	v =  getCellVolume(X[1,:], X[2,:], X[3,:], X[4,:])
	return (v * sigma) * M
end


"""
Compute the principal eigenvector and eigenvalue of a linear operator 𝕋, where 𝕋 is a Metzler matrix (i.e. off-diagonal components are nonnegative)

Denote a = -minimum(Diagonal(V)), which implies 𝕋 + a * I has all positive entries. Applying Perron Frobenus, there a unique largest eigenvalue for aI + 𝕋, which is real, and the correspondind eigenctor is strictly positive.
Note that, in particular, it is the eigenvalue with largest real part, and so this also correspoinds to the eigenvalue with largest real part of 𝕋, which happens to be real.
Denote η(𝕋) the eigenvalue with largest real part of a matrix and ρ(𝕋) the eigenvalue with largest modulus. We have

    η(𝕋) = ρ(𝕋 + a * I) - a

Moreover, the associated eigenvector is real and strictly positive.

Note that, when 𝕋 is generator, its rows sum to zero. This means that eigenvalue with largest real part is
    η(𝕋) = 0
In other words, all eigenvalues of 𝕋 have real part <= 0. This means that 𝕋 is a singular M matrix.
(another proof is to say that, for any s, sI - 𝕋 is a non-singular M-Matrix for any s> 0, since there exists x = e such that (sI - 𝕋) * x > 0).

"""
function principal_eigenvalue(𝕋; r0 = ones(size(𝕋, 1)))
    η, r = 0.0, r0
    a = - minimum(diag(𝕋))
    try
        vals, vecs = Arpack.eigs(𝕋 + a * I; v0 = collect(r0), nev = 1, which = :LM)
        η = vals[1]
        r = vecs[:, 1]
    catch
        vals, vecs = KrylovKit.eigsolve(𝕋 + a * I, collect(r0), 1, :LM; maxiter = size(𝕋, 1))
        η = vals[1]
        r = vecs[1]
    end
    abs(imag(η)) <= eps() || @warn "Principal Eigenvalue has an imaginary part"
    maximum(abs.(imag.(r))) <= eps() || @warn "Principal Eigenvector has an imaginary part"
    real(η) - a, abs.(r)
end


function logsumexp(logx::Array{Float64,1})
    maxlog = maximum(logx)
    maxlog + log(sum(exp.(logx - maxlog)))
end

function logsumexp(logx::Array{Float64,2}, dim::Int)
    maxlog = maximum(logx, dim)
    maxlog + log.(sum(exp.(logx .- maxlog), dim))
end


function check(d, M, as)
    # as is sorted
    # check if all the cows can have a distance longer than d
    N = length(as)

    last = 1
    for i in 1:(M-1)
        crt = last + 1
        while crt ≤ N && as[crt] - as[last] < d
            crt += 1
        end

        if crt ≥ N
            return false
        else
            last = crt
        end
    end

    return true
end

function main()
    N, M = parse.(Int, split(readline()))
    as = [parse(Int, readline()) for _ in 1:N]
    sort!(as)

    d̲, d̄ = 0, maximum(as)
    while d̄ - d̲ > 1
        d = (d̄ + d̲) ÷ 2
        if check(d, M, as)
            d̲ = d
        else
            d̄ = d
        end
    end

    if check(d̄, M, as)
        println(d̄)
    else
        println(d̲)
    end
end

main()


# ============================================================
# Thermodynamics
# ============================================================

"""
$(SIGNATURES)

Calculate heat capacity ratio (monatomic gas)
"""
function heat_capacity_ratio(K, D::T) where {T<:Integer}
    γ = begin
        if D == 1
            (K + 3.0) / (K + 1.0)
        elseif D == 2
            (K + 4.0) / (K + 2.0)
        elseif D == 3
            (K + 5.0) / (K + 3.0)
        end
    end

    return γ
end

"""
$(SIGNATURES)

Calculate heat capacity ratio (diatomic gas)
"""
function heat_capacity_ratio(K, Nr, D::T) where {T<:Integer}
    γ = begin
        if D == 1
            (K + 3.0 + Nr) / (K + 1.0 + Nr)
        elseif D == 2
            (K + 4.0 + Nr) / (K + 2.0 + Nr)
        elseif D == 3
            (K + 5.0 + Nr) / (K + 3.0 + Nr)
        end
    end

    return γ
end

"""
$(SIGNATURES)

Calculate speed of sound
"""
sound_speed(λ::Real, γ::Real) = (0.5 * γ / λ)^0.5

"""
$(SIGNATURES)
"""
sound_speed(prim::AV, γ) = sound_speed(prim[end], γ)

"""
$(TYPEDSIGNATURES)

Calculate sound speed in mixture
"""
function sound_speed(prim::AM, γ)
    c = similar(prim, axes(prim, 2))
    for j in eachindex(c)
        c[j] = sound_speed(prim[end, j], γ)
    end

    return maximum(c)
end


"Brownian motion process."
immutable BrownianMotion{Y} <: AItoProcess
  y0::Y
end

BrownianMotion() = BrownianMotion(0.0)

convert(::Type{ItoProcess}, bm::BrownianMotion) =
    ItoProcess((t,dt,b,db,y)->db, bm.y0)

function distribution(bm::BrownianMotion, t)
  if t == 0
    Constant(bm.y0)
  elseif ndims(bm.y0) == 0
    Normal(bm.y0, sqrt(t))
  else
    MvNormal(bm.y0, eye(length(bm.y0)) * t)
  end
end

# BrownianMotionWithDrift

immutable BrownianMotionWithDrift{Y,S} <: AItoProcess
  mu::Y
  sigma::S
  y0::Y
end

BrownianMotionWithDrift(mu, sigma) = BrownianMotionWithDrift(mu, sigma, 0.0)

convert(::Type{ItoProcess}, bm::BrownianMotionWithDrift) =
    ItoProcess((t,dt,b,db,y)-> (bm.mu * dt) .+ (bm.sigma * db), bm.y0)

function distribution(bm::BrownianMotionWithDrift, t)
    if t == 0
      Constant(bm.y0)
    elseif ndims(bm.y0) == 0
      Normal(bm.y0 + bm.mu * t, bm.sigma * sqrt(t))
    else
      MvNormal(bm.y0 + bm.mu * t, bm.sigma *bm.sigma' * t)
    end
end

solution(p::BrownianMotionWithDrift, t, b) =
  p.y0 + t * p.mu + p.sigma * b

# Geometric Brownian Motion

type GeometricBrownianMotion <: AItoProcess
    mu::Float64
    sigma::Float64
    y0::Float64
end

convert(::Type{ItoProcess}, bm::GeometricBrownianMotion) =
    ItoProcess((t,dt,b,db,y)-> (bm.mu * dt) * y + bm.sigma * (y .* db), bm.y0)

distribution(bm::GeometricBrownianMotion, t) =
    t == 0 ? Constant(bm.y0) :
    LogNormal(log(bm.y0) + t * (bm.mu - bm.sigma^2 / 2), bm.sigma * sqrt(t))

solution(p::GeometricBrownianMotion, t, b) =
    p.y0 * exp(t * (p.mu - p.sigma^2 / 2) + p.sigma * b)

"Ito integral of f(t, B)"
immutable ItoIntegral{F} <: AItoProcess
  f::F
end

convert{F}(::Type{ItoProcess}, ii::ItoIntegral{F}) =
    ItoProcess((t,dt,b,db,y)->ii.f(t, b).*db, 0.)

"Process corresponding to solutions of stochastic differential equation
 dy = f(t, y) dt + g(t,y) db."
immutable SDE{F,G} <: AItoProcess
  f::F
  g::G
  y0::Float64
end

convert{F,G}(::Type{ItoProcess}, sde::SDE{F,G}) =
    ItoProcess((t,dt,b,db,y)->sde.f(t, y).*dt + sde.g(t, y).*db, sde.y0)


mutable struct ListNode{T<:Number}
    val :: T
    next:: Union{ListNode, Nothing}

    ListNode(data::Array{T,1}) where T<:Number = begin
        (head, rest) = Iterators.peel(data)
        if isempty(rest)
            new{T}(head::T, nothing)
        else
            new{T}(head, ListNode(collect(rest) :: Array{T,1}))
        end
    end
end

function node2list(x::Union{ListNode{T}, Nothing}) :: Array{T} where {T<:Number}
    result = []
    while x.next != nothing
        push!(result, x.val)
        x = x.next
    end
    push!(result, x.val)
    result
end

function addTwoNumbers(a::ListNode{T}, b::ListNode{T}) :: ListNode{T} where T<:Number
    this_a::Union{ListNode, Nothing}, this_b::Union{ListNode, Nothing} = a, b
    flag = false
    result::Array{T} = []
    while true
        (this_a == nothing && this_b == nothing) && begin
            flag && push!(result,1)
            return ListNode(result)
        end

        this_a == nothing && begin
            append!(result, begin
                    cache = node2list(this_b)
                    if flag cache[1] += 1 end
                    cache
                    end)
            return ListNode(result)
        end

        this_b == nothing && begin
            append!(result, begin
                    cache = node2list(this_a)
                    if flag cache[1] += 1 end
                    cache
                    end)
            return ListNode(result)
        end

        push!(result, this_a.val + this_b.val + (flag ? 1 : 0))
        if last(result) >= 10
            result[lastindex(result)] -= 10
            flag = true
        else
            flag = false
        end

        this_a, this_b = this_a.next, this_b.next
    end
    ListNode(result)
end


using LinearAlgebra
function bernsteinBasis(u, deg)
    """
    Function returns the values of  Bernstein basis of degree deg at points u
    Algorithm A1.3 in Piegl & Tiller
    xi is a 1D array
    """
    lenU = length(u)
    B = zeros(lenU, deg+1)
    B[:, 1] = ones(lenU)
    u1 = ones(lenU) - u
    u2 = ones(lenU) + u

    for j in 1:deg
        saved = zeros(lenU)
        for k in 1:j
            temp = B[:,k]
            B[:,k] = saved + u1.*temp
            saved  = u2.*temp
        end
        B[:,j+1] = saved
    end
    B = B./(2^deg)

    #calculate the 1st derivative of Bernstein polynomials
    dB = zeros(lenU, deg)
    dB[:,1] = ones(lenU)
    for j in 1:deg-1
        saved = zeros(lenU)
        for k in 0:j-1
            temp = dB[:,k+1]
            dB[:,k+1] = saved + u1.*temp
            saved = u2.*temp
        end
        dB[:,j+1] = saved
    end
    dB = dB./(2^deg)
    dB = hcat(zeros(lenU,1), dB, zeros(lenU, 1))
    dB = (dB[:,1:end-1]-dB[:,2:end])*deg

    #calculate the 2nd derivative of Bernstein polynomials
    if deg>1
        ddB = zeros(lenU, deg-1)
        ddB[:,1] = ones(lenU)
        for j=1:deg-2
            saved = zeros(lenU)
            for k in 0:j-1
                temp = ddB[:,k+1]
                ddB[:,k+1] = saved + u1.*temp
                saved = u2.*temp
            end
            ddB[:,j+1] = saved
        end
        ddB = ddB./(2^deg)
        ddB = hcat(zeros(lenU,2), ddB, zeros(lenU,2))
        ddB = (ddB[:,1:end-2]-2*ddB[:,2:end-1]+ddB[:,3:end])*deg*(deg-1)
    else
        ddB = zeros(lenU,deg+1)
    end
    return B, dB, ddB

end

function bezierExtraction(knot, deg)
    """
    Bezier extraction
    Based on Algroithm 1, from Borden - Isogeometric finite element data
    structures based on Bezier extraction
    """
    m = length(knot)-deg-1;
    a = deg + 1;
    b = a + 1;
    nb = 1;
    nb_final = length(unique(knot))-1
    C = zeros(deg+1,deg+1,nb_final)
    C[:,:,1] = Matrix{Float64}(I, deg+1, deg+1)

    while b<=m
        C[:,:,nb+1] = Matrix{Float64}(I, deg+1, deg+1)
        i = b
        while (b<=m) && (knot[b+1] == knot[b])
            b = b + 1
        end
        multiplicity = b - i + 1
        alphas = zeros(deg-multiplicity)
        if multiplicity < deg
            numerator = knot[b] - knot[a]
            for j in deg:-1:multiplicity+1
                alphas[j-multiplicity] = numerator/(knot[a+j]-knot[a])
            end
            r = deg - multiplicity
            for j in 1:r
                save = r - j + 1
                s = multiplicity + j
                for k in deg+1:-1:s+1
                    alpha = alphas[k-s]
                    C[:,k,nb] = alpha*C[:,k,nb]+(1-alpha)*C[:,k-1,nb]
                end
                if b<=m
                    C[save:save+j,save,nb+1] = C[deg-j+1:deg+1,deg+1,nb]
                end
            end
            nb = nb + 1
            if b <= m
                a = b
                b = b + 1
            end
        elseif multiplicity==deg
            if b <= m
                nb = nb + 1
                a = b
                b = b + 1
            end
        end
    end
    @assert nb_final == nb
    return C, nb
end

"""
Create the extended knot vector (Subsection 4.3.2 in Scott - Isogeometric
data structures based on the Bézier extraction of T-Splines)
"""
function formExtendedKnot(localKnot, p)
    # Repeat the first knot (if needed) so that it appears p+1 times
    firstKnot = localKnot[1]
    indexFirst = findall(localKnot.==firstKnot)
    numRep = length(indexFirst)
    numNewRepFirst = p+1-numRep

    #repeat the last knot (if needed) so that it appears p+1 times
    lastKnot = localKnot[end]
    indexLast = findall(localKnot.==lastKnot)
    numRep = length(indexLast)
    numNewRepLast = p+1-numRep

    #form the extended knot vector
    extendedKnot = vcat(firstKnot*ones(numNewRepFirst), localKnot, lastKnot*ones(numNewRepLast))
    indexFun = numNewRepFirst + 1
    return extendedKnot, indexFun
end

"""
Compute the Bézier extraction operator corresponding to the basis functions of
degree p with local knot vector localKnot
"""
function bezierExtractionLocal(localKnot, p)
    extendedKnot, indexFun = formExtendedKnot(localKnot, p)
    #perform Bézier extraction and return the basis with index numNewRepFirst+1
    C_temp, nb = bezierExtraction(extendedKnot, p)
    IEN, _ = makeIEN(extendedKnot, nb, p)
    C = zeros(p+1, nb)
    for indexSpan = 1:nb
        C_index = findfirst(IEN[indexSpan,:].==indexFun)
        C[:, indexSpan] = C_temp[C_index,:,indexSpan]
    end
    return C
end


module problem107

description = """
Minimal network
Problem 107

The following undirected network consists of seven vertices and twelve edges with a total weight of 243.

The same network can be represented by the matrix below.
    	A	B	C	D	E	F	G
A	-	16	12	21	-	-	-
B	16	-	-	17	20	-	-
C	12	-	-	28	-	31	-
D	21	17	28	-	18	19	23
E	-	20	-	18	-	-	11
F	-	-	31	19	-	-	27
G	-	-	-	23	11	27	-

However, it is possible to optimise the network by removing some edges and still ensure that all points on the network remain connected. The network which achieves the maximum saving is shown below. It has a weight of 93, representing a saving of 243 − 93 = 150 from the original network.

Using network.txt (right click and 'Save Link/Target As...'), a 6K text file containing a network with forty vertices, and given in matrix form, find the maximum saving which can be achieved by removing redundant edges whilst ensuring that the network remains connected.
"""

immutable Edge
  from :: Int
  to :: Int
  cost :: Int
end

immutable Tree
  edges :: Vector{Edge}
end

immutable Forest
  trees :: Vector{Tree}
end

# Find the index of the tree containing node in the forest, or -1 if not found
function tree_index(node :: Int, f :: Forest)
  for i = 1:length(f.trees)
    for j = 1:length(f.trees[i].edges)
      if (f.trees[i].edges[j].from == node || f.trees[i].edges[j].to == node)
        return i
      end
    end
  end
  return -1
end

# Tests if adding Edge e to Forest would connect two previously disconnected trees
function makes_new_connection(e :: Edge, f :: Forest)
  t1 = tree_index(e.from, f)
  t2 = tree_index(e.to, f)
  return t1 < 0 || t2 < 0 || t1 != t2
end

# Creates Add Edge to Forest, joining any trees that become linked
function add_edge(e :: Edge, f :: Forest)
  t1 = tree_index(e.from, f)
  t2 = tree_index(e.to, f)

  # Simple case, neither from/to existed previously, add a new tree
  if t1 < 0 && t2 < 0
    return Forest([Tree([e]); f.trees])
  end

  # One of the trees exists - make a copy and update
  newtrees = copy(f.trees)
  if t1 > 0 && t2 < 0
    newtrees[t1] = Tree([e; f.trees[t1].edges])
  elseif t1 < 0 && t2 > 0
    newtrees[t2] = Tree([e; f.trees[t2].edges])
  else
    newtrees[t1] = Tree([e; f.trees[t1].edges; f.trees[t2].edges])
    splice!(newtrees, t2, Tree[])
  end
  return Forest(newtrees)
end

# Convert from matrix representation to Edge vector
function matrix_to_edges(connections :: Matrix{Int})
  edges = Edge[]
  for i = 1:size(connections, 1)
    for j = i+1:size(connections, 2)
      if connections[i, j] > 0
        push!(edges, Edge(i, j, connections[i,j]))
      end
    end
  end
  return edges
end

# Calculate the minimum spanning tree by adding the minimum cost edge to a forest if it links up previously disconnected trees
function minimum_spanning_tree(connections :: Matrix{Int})
  edges = matrix_to_edges(connections)
  sort!(edges, by = x -> x.cost)
  f = Forest(Tree[])
  for e in edges
    if makes_new_connection(e, f)
      f = add_edge(e, f)
    end
  end
  return f.trees[1]
end

function read_network()
  m = readcsv("network.txt")
  m[m .== "-"] = -1
  connections = int(m)
  return connections
end

function solve107()
  network = read_network()
  t = minimum_spanning_tree(network)
  initial_cost = sum([e.cost for e in matrix_to_edges(network)])
  final_cost = sum([e.cost for e in t.edges])
  println("$initial_cost - $final_cost = $(initial_cost - final_cost)")
end

using Base.Test

testcase = [
-1	16	12	21	-1	-1	-1
16	-1	-1	17	20	-1	-1
12	-1	-1	28	-1	31	-1
21	17	28	-1	18	19	23
-1	20	-1	18	-1	-1	11
-1	-1	31	19	-1	-1	27
-1	-1	-1	23	11	27	-1]

@test sum([e.cost for e in minimum_spanning_tree(testcase).edges]) == 93

end


# # Kernel Ridge Regression
#
# Building on linear regression, we can fit non-linear data sets by introducing a feature space. In a higher-dimensional feature space, we can overfit the data; ridge regression introduces regularization to avoid this. In this notebook we show how we can use KernelFunctions.jl for *kernel* ridge regression.

## Loading and setup of required packages
using KernelFunctions
using LinearAlgebra
using Distributions

## Plotting
using Plots;
default(; lw=2.0, legendfontsize=11.0, ylims=(-150, 500));

using Random: seed!
seed!(42);

##
# ## Toy data
# Here we use a one-dimensional toy problem. We generate data using the fourth-order polynomial $f(x) = (x+4)(x+1)(x-1)(x-3)$:

f_truth(x) = (x + 4) * (x + 1) * (x - 1) * (x - 3)

x_train = -5:0.5:5
x_test = -7:0.1:7

noise = rand(Uniform(-20, 20), length(x_train))
y_train = f_truth.(x_train) + noise
y_test = f_truth.(x_test)

plot(x_test, y_test; label=raw"$f(x)$")
scatter!(x_train, y_train; seriescolor=1, label="observations")

##
# ## Linear regression
# For training inputs $\mathrm{X}=(\mathbf{x}_n)_{n=1}^N$ and observations $\mathbf{y}=(y_n)_{n=1}^N$, the linear regression weights $\mathbf{w}$ using the least-squares estimator are given by
# ```math
# \mathbf{w} = (\mathrm{X}^\top \mathrm{X})^{-1} \mathrm{X}^\top \mathbf{y}
# ```
# We predict at test inputs $\mathbf{x}_*$ using
# ```math
# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w}
# ```
# This is implemented by `linear_regression`:

function linear_regression(X, y, Xstar)
    weights = (X' * X) \ (X' * y)
    return Xstar * weights
end;

# A linear regression fit to the above data set:

y_pred = linear_regression(x_train, y_train, x_test)
scatter(x_train, y_train; label="observations")
plot!(x_test, y_pred; label="linear fit")

##
# ## Featurization
# We can improve the fit by including additional features, i.e. generalizing to $\tilde{\mathrm{X}} = (\phi(x_n))_{n=1}^N$, where $\phi(x)$ constructs a feature vector for each input $x$. Here we include powers of the input, $\phi(x) = (1, x, x^2, \dots, x^d)$:

function featurize_poly(x; degree=1)
    return repeat(x, 1, degree + 1) .^ (0:degree)'
end

function featurized_fit_and_plot(degree)
    X = featurize_poly(x_train; degree=degree)
    Xstar = featurize_poly(x_test; degree=degree)
    y_pred = linear_regression(X, y_train, Xstar)
    scatter(x_train, y_train; legend=false, title="fit of order $degree")
    return plot!(x_test, y_pred)
end

plot((featurized_fit_and_plot(degree) for degree in 1:4)...)

##
# Note that the fit becomes perfect when we include exactly as many orders in the features as we have in the underlying polynomial (4).
#
# However, when increasing the number of features, we can quickly overfit to noise in the data set:

featurized_fit_and_plot(20)

##
# ## Ridge regression
# To counteract this unwanted behaviour, we can introduce regularization. This leads to *ridge regression* with $L_2$ regularization of the weights ([Tikhonov regularization](https://en.wikipedia.org/wiki/Tikhonov_regularization)).
# Instead of the weights in linear regression,
# ```math
# \mathbf{w} = (\mathrm{X}^\top \mathrm{X})^{-1} \mathrm{X}^\top \mathbf{y}
# ```
# we introduce the ridge parameter $\lambda$:
# ```math
# \mathbf{w} = (\mathrm{X}^\top \mathrm{X} + \lambda \mathbb{1})^{-1} \mathrm{X}^\top \mathbf{y}
# ```
# As before, we predict at test inputs $\mathbf{x}_*$ using
# ```math
# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w}
# ```
# This is implemented by `ridge_regression`:

function ridge_regression(X, y, Xstar, lambda)
    weights = (X' * X + lambda * I) \ (X' * y)
    return Xstar * weights
end

function regularized_fit_and_plot(degree, lambda)
    X = featurize_poly(x_train; degree=degree)
    Xstar = featurize_poly(x_test; degree=degree)
    y_pred = ridge_regression(X, y_train, Xstar, lambda)
    scatter(x_train, y_train; legend=false, title="\$\\lambda=$lambda\$")
    return plot!(x_test, y_pred)
end

plot((regularized_fit_and_plot(20, lambda) for lambda in (1e-3, 1e-2, 1e-1, 1))...)

##
# ## Kernel ridge regression
# Instead of constructing the feature matrix explicitly, we can use *kernels* to replace inner products of feature vectors with a kernel evaluation: $\langle \phi(x), \phi(x') \rangle = k(x, x')$ or $\tilde{\mathrm{X}} \tilde{\mathrm{X}}^\top = \mathrm{K}$, where $\mathrm{K}_{ij} = k(x_i, x_j)$.
#
# To apply this "kernel trick" to ridge regression, we can rewrite the ridge estimate for the weights
# ```math
# \mathbf{w} = (\mathrm{X}^\top \mathrm{X} + \lambda \mathbb{1})^{-1} \mathrm{X}^\top \mathbf{y}
# ```
# using the [matrix inversion lemma](https://tlienart.github.io/pub/csml/mtheory/matinvlem.html#basic_lemmas)
# as
# ```math
# \mathbf{w} = \mathrm{X}^\top (\mathrm{X} \mathrm{X}^\top + \lambda \mathbb{1})^{-1} \mathbf{y}
# ```
# where we can now replace the inner product with the kernel matrix,
# ```math
# \mathbf{w} = \mathrm{X}^\top (\mathrm{K} + \lambda \mathbb{1})^{-1} \mathbf{y}
# ```
# And the prediction yields another inner product,
# ```math
# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w} = \langle \mathbf{x}_*, \mathbf{w} \rangle = \mathbf{k}_* (\mathrm{K} + \lambda \mathbb{1})^{-1} \mathbf{y}
# ```
# where $(\mathbf{k}_*)_n = k(x_*, x_n)$.
#
# This is implemented by `kernel_ridge_regression`:

function kernel_ridge_regression(k, X, y, Xstar, lambda)
    K = kernelmatrix(k, X)
    kstar = kernelmatrix(k, Xstar, X)
    return kstar * ((K + lambda * I) \ y)
end;

# Now, instead of explicitly constructing features, we can simply pass in a `PolynomialKernel` object:

function kernelized_fit_and_plot(kernel, lambda=1e-4)
    y_pred = kernel_ridge_regression(kernel, x_train, y_train, x_test, lambda)
    if kernel isa PolynomialKernel
        title = string("order ", kernel.degree)
    else
        title = string(nameof(typeof(kernel)))
    end
    scatter(x_train, y_train; label=nothing)
    return plot!(x_test, y_pred; label=nothing, title=title)
end

plot((kernelized_fit_and_plot(PolynomialKernel(; degree=degree, c=1)) for degree in 1:4)...)

##
# However, we can now also use kernels that would have an infinite-dimensional feature expansion, such as the squared exponential kernel:

kernelized_fit_and_plot(SqExponentialKernel())


#=
def linear(a, b, x):
    return b + a*x
=#

linear(a, b, x) = b + a * x

#=
# a linear demand function is generated for every
# pair of coefficients in vectors a_vec and b_vec
def demand_hypotheses(a_vec, b_vec):
    for a, b in itertools.product(a_vec, b_vec):
        yield {
            'd': functools.partial(linear, a, b),
            'p_opt': -b/(2*a)
        }
=#

function demand_hypothesis(f, a, b)
	f1(x) = f(a, b, x)
	return DataFrame(
		:a => a,
		:b => b,
		:d => f1,
        :d_opt => f1(-b / (2a)),
		:p_opt => -b / (2a)
	)
end

function generate_demand_hypothesis(a_range, b_range)
    h_vec = DataFrame()
    for a in a_range
    	for b in b_range
    		df1 = demand_hypothesis(linear, a, b)
    		push!(h_vec, df1[1, :])
    	end
    end
    h_vec
end


### A Pluto.jl notebook ###
# v0.14.0

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 4ea0ccfa-9622-11eb-1cf0-e9ae2f927dd2
html"""

Section 2.8

 Linear Model, Data Science, & Simulations

Lecture Video

"""

# ╔═╡ d155ea12-9628-11eb-347f-7754a33fd403
begin
    import Pkg
    Pkg.activate(mktempdir())
    Pkg.add([
        Pkg.PackageSpec(name="Plots", version="1"),
        Pkg.PackageSpec(name="PlutoUI", version="0.7"),
        Pkg.PackageSpec(name="DataFrames", version="0.22"),
        Pkg.PackageSpec(name="CSV", version="0.8"),
        Pkg.PackageSpec(name="GLM", version="1"),
        Pkg.PackageSpec(name="Distributions", version="0.24"),
    ])
    using Plots, PlutoUI, DataFrames, CSV, GLM, Statistics, LinearAlgebra, Distributions
end

# ╔═╡ 01506de2-918a-11eb-2a4d-c554a6e54631
TableOfContents(title="📚 Table of Contents", aside=true)

# ╔═╡ 877deb2c-702b-457b-a54b-f27c277928d4
md"""
# Julia concepts for data science
- Data Frames (`DataFrames.jl`)
- `CSVread`, `CSVwrite` (`CSV.jl`)
- `lm` (linear model) (`GLM.jl`)
- `@formula` (formula macro to specify variables to analyze) (`GLM.jl`)

- Underscore as digits separator (e.g. `1_000` for 1000)

- The value of fast simulations.
"""

# ╔═╡ 36ce167f-382c-4b9a-be34-83250b10c4e5
md"""
In this lecture we will simulate a real world statistical application
for the purpose of understanding what statistics is about.  It is very
	helpful in simulations to be able to run many examples fast.
"""

# ╔═╡ 83912943-a847-420a-bfdb-450027b631e8
md"""
# Fahrenheit and Celsius Data Set
"""

# ╔═╡ 280d112f-d34a-4cc4-9e3a-4ebbfcd5eb51
n = 10

# ╔═╡ b5031c96-db57-4baf-b271-6bb12e29de9b
x = sort((rand( -10:100, n)))

# ╔═╡ c2f77e8f-a8c0-4144-a8b4-b25dd98ed234
y = 5/9 .* x  .- 17.7777777 #  same as y =  5/9 .* (x .- 32)

# ╔═╡ ad161b98-f4a1-42ac-ad4f-8b71fabcfde9
begin
	plot(x,y, m=:c, mc=:red,legend=false)
	xlabel!("°F")
	annotate!(-4,16,text("°C",11))
	# plot!( x, (x.-30)./2) Dave's cool approximation
end

# ╔═╡ 8e422886-74ef-4c0f-be1e-fda238c8db44
[x y]

# ╔═╡ ca21122a-2522-482a-b7ef-bd73e96cb5a9
md"""
## Julia: Data Frames
I like to think of a Data Frame as a matrix with labels.
"""

# ╔═╡ 41e05b1e-8b5e-45e3-91bb-01355ade9f3d
md"""
### Data Frame by Columns with labels
"""

# ╔═╡ 9d2e3861-ca36-406e-952d-831ca3947e44
data = DataFrame(°F=x,°C=y) # Label = data

# ╔═╡ e73854ed-3581-41c4-ada5-e48242033759
md"""
### Data Frame with a matrix
"""

# ╔═╡ 9873d944-b611-46f9-82a7-0cf714a3078c
begin
	data2 = DataFrame([x  y]) # convert Matrix to DataFrame
    rename!(data2,["°F","°C"]) # add column labels
end

# ╔═╡ 2be44753-afee-4125-b6bc-8866d2293dc2
Matrix(data2) # Convert back to a matrix (lose label information)

# ╔═╡ 6e07e8fb-fe51-4b37-bfb2-d1466e768754
md"""
## Julia:Comment about types

Notice that [x y] converts all the data to floats, but columns of a data frame can have different types.
"""

# ╔═╡ a755e58a-b16c-4d3b-a85f-81ccf374793f
md"""
# Reading/Writing CSV (comma separated values) Files
"""

# ╔═╡ f1e10fb7-adac-4083-8977-616a505fd591
md"""
  ## Writing Data to a CSV  file
readable by spreadsheet software.
"""

# ╔═╡ 2e42986c-2de3-49e6-9c29-a7313c0b1da8
CSV.write("testCSVwrite.csv", data)

# ╔═╡ 22758dd6-9d04-4616-ba99-1430f2dedf9a
md"""
 ## Reading Data from a CSV file to a DataFrame
"""

# ╔═╡ aff6a616-6d8b-4584-a6f2-195decef7774
data_again = CSV.read("testCSVwrite.csv", DataFrame )

# ╔═╡ 338da13a-3c26-4366-a669-ac3e24f31577
data_again[:,"°F" ] #or data_again[:,1]

# ╔═╡ 5a742546-1e4d-4aee-bed1-cb10c543e439
data_again[:,1]

# ╔═╡ 6a9c8c9a-fac7-42f7-976d-3168132cae48
md"""
# Noisy Data
## Add some random noise to the celsius readings
"""

# ╔═╡ 3c038b68-8676-4877-9720-38da7c4e0e0e
begin
	noisy_data = copy(data)  # Noisy DataFrame
	noisy_data[:, "°C" ] .+= noise * randn(n)
	yy = noisy_data[:, "°C" ]
	noisy_data
end

# ╔═╡ 5a877e40-a101-4f7d-b2a1-ef4cfe5d8807
begin

	scatter(x, yy,m=:c,mc=:red, label="noisy data", ylims=(-40,40))
	for i=1 : length(data[:,2])
		plot!([x[i],x[i]], [m*x[i]+b,yy[i]], color=:gray, ls=:dash, label=false)
	end
	xlabel!("°F")
	annotate!(-15,16,text("°C",11))
	plot!(x, m.*x .+ b,  color=:blue, label="best fit line")
	plot!(x,y,alpha=.5, color=:red, label="theory") # theoretical
	plot!(legend=:top)
end

# ╔═╡ 83c28c76-2eab-49f9-9999-05df85054520
md"""
# The noise slider (so I can find it easily)
"""

# ╔═╡ ba671804-dc6d-415c-89de-9cf6294907b3
md"""
noise = $(@bind noise Slider(0:.5:1000, show_value = true ))
"""

# ╔═╡ e8683a71-5822-4491-9ccd-20e0fc3bf531
md"""
## Statistics Software Outputs Mysterious Tables
example output from the "linear model"  (`lm`) which we store in the variable `ols` for ordinary least squares (by contrast weighted least squares treats the vertical displacements with unequally with differing weights.)
"""

# ╔═╡ 0489e5d8-51ca-4955-83e1-95ea353d9cf2
ols = lm(@formula(°C ~ °F), noisy_data)

# ╔═╡ 9a65aee4-ab8e-4ab7-be6f-cc2a2e9d5127
noisy_data

# ╔═╡ c3539f42-6ca7-47fb-9707-4d11c9e76643
md"""
This lecture is about explaining the meaning and significance to every part of this table.
"""

# ╔═╡ 469d809f-424f-4595-ad43-a5b2cc055304
md"""
# Regression a few ways
"""

# ╔═╡ 6128b8fd-9b85-4896-a0bf-934a0733fafb
md"""
## The "Coef." column in the table gives the slope and intercept of the best fit line
"""

# ╔═╡ 9eb7caaa-438d-4bcb-9c54-4a0fa72c61de
b, m = [ one.(x) x]\ yy  # The mysterious linear algebra solution using "least squares"

# ╔═╡ 0e8fce45-f1c0-41d4-996a-d6093182afee
function linear_regression(x,y)   # a direct computation from the data
	n = length(x)
	x0  = x.-mean(x)
	y0 = y.-mean(y)

	mᵉ = sum( x0 .* y0 ) / sum(  x0.^2 ) # slope estimate
	bᵉ = mean(y) - mᵉ * mean(x) # intercept estimate

	s2ᵉ = sum(  (mᵉ.*x .+ bᵉ .- y).^2 ) /(n-2) # noise estimate
	bᵉ,mᵉ,s2ᵉ
end

# ╔═╡ 71590890-38b6-440e-b61b-ece6c49ac602
linear_regression(x,yy)

# ╔═╡ f7cc7146-9ee6-4d87-b024-2a91863f4b24
md"""
[So why is it called "Regression" anyway?](http://blog.minitab.com/blog/statistics-and-quality-data-analysis/so-why-is-it-called-regression-anyway) Dalton's original meaning not quite what it means today.
"""

# ╔═╡ f64815e2-44b8-4585-9269-9a62655c984c
md"""
# Demystifying the word "Model"

    Step I:  The Model is y = m*x + b + σ*randn() .
    This means that out there in the real world are b, m, and σ.  You
    don't know them.

    Step II: You do, however, have data points x and y which allow you
    to compute an bᵉ,  mᵉ, and σᵉ.  A statistician would call these estimates
    based on your data points. If you ran the experiment again, you would
    get different data points.

    The computer lets us run the experiment as many times as we want just to see what happens.

        In summary, there are three kinds of variables.  The model variables b, m, and σ which are unknown.  The predictor variable x which is considered fixed and known.  The response variable y which is considered noisy.
"""

# ╔═╡ feb3c45e-88f4-4ffc-a4a0-e89489187c8d
md"""
## Understanding the relationship `°C ~ 1 + °F`
"""

# ╔═╡ 99069dd7-e088-4626-aa29-e48d6f9a474e
ols

# ╔═╡ 051a9e38-9a84-4ead-96fa-24c86c2b9f2d
md"""
`°C ~ 1 + °F` means the celsius (y) is (Coef1)*1 + (Coef2)*(°F),

in general `y ~ 1 + x1 + x2 + x3` is shorthand for
``y = c_0 + c_1 x_1 + c_2 x_2 + c_3 x_3``, etc.
"""

# ╔═╡ 2f33ee51-0725-46c2-9f1b-a61cd68abab1
md"""
# Simulating the real world: running many noisy models
"""

# ╔═╡ e4acd97b-22f7-4812-9898-1a485887a5f2
function simulate(σ,howmany)
	[linear_regression(x,y .+ σ * randn(length(x)))   for i=1:howmany]
	#[linear_regression(x,y .+ (σ * sqrt(12)) * (-.5 .+ rand(length(x))))   for i=1:howmany]
	# [linear_regression(x,y .+ (σ ) * ( rand([-1,1],length(x))))   for i=1:howmany]

end

# ╔═╡ 4e413b40-81c4-4160-9d01-046c2d179a06
howmany = 100_000

# ╔═╡ 7b94db0d-f46b-4621-9413-1dc787ae9a39
md"""
## Julia: underscore as a digits separator
"""

# ╔═╡ 51a28b67-ad64-4cf2-a0e6-a78fb101eb15
s = simulate(σ, howmany)

# ╔═╡ d451af49-3139-4329-a885-a210b1760f74
s[1] # first simulation,  intercept, slope, estimation of noise σ

# ╔═╡ c7455f7a-9c72-42f5-8238-1799cad96f6c
md"""
## Simulated intercepts ($howmany simulations)
"""

# ╔═╡ d2971801-2cdb-4b9f-8ec8-c74cbb2a0b31
md"""
σ = $(@bind σ Slider(0:.1:3, show_value=true, default=1))
"""

# ╔═╡ e1e8c140-bc4e-400d-beb2-0986e071c3a3
begin
	histogram( first.(s) , alpha=.6, bins=100, norm=true)
	vline!([-17.777777],color=:white)
	title!("intercept")
	xlims!(-17.7777-3,-17.7777+3)
	ylims!(0,1)
	plot!(legend=false)

end

# ╔═╡ 1429be09-a31f-415f-9c3d-f32b085ef68d
md"""
Experimental mean of the intercept
"""

# ╔═╡ da321202-0dc5-44ad-aac0-f3ea0d229243
mean(first.(s)), -17.777777

# ╔═╡ 2aceb366-a067-4271-9362-c320f4735ed1
md"""
Experimental std of the intercept
"""

# ╔═╡ 58f548fd-f6d0-479d-8469-bc886783f9a7
std( first.(s))

# ╔═╡ 07be9435-bc07-4a18-aad8-3ff19f5bcce4
md"""
Statisticians know an exact formula for the theoretical std of the intercept
"""

# ╔═╡ 1a6ad08d-c3bb-47e7-bdee-156bbff3aeda
    sb = σ * norm(x)  / norm(x.-mean(x)) / sqrt(n)

# ╔═╡ c55e4894-db71-4729-a1a1-5f68b45e3bf5
md"""
## Simulated slopes ($howmany simulations)
"""

# ╔═╡ f50d66eb-0357-4017-ac9b-99e63cd52dc0
begin
	histogram( getindex.(s,2), alpha=.6, bins=100, norm=true, legend=false )
	title!("slope")
	vline!([5/9],color=:white)
	xlims!(5/9-.1, 5/9+.1)
	ylims!(0,100)
end

# ╔═╡ 5c7a7361-f0e7-473a-9e38-226828aa00ca
md"""
Sample mean of the slope
"""

# ╔═╡ acf0e90e-8f1f-451f-9f0f-70a0bcc7efca
mean(getindex.(s,2)), .555555

# ╔═╡ c9f65e15-f222-4a88-98c2-9e1d8b5ec3eb
md"""
Sample std of the slope.
"""

# ╔═╡ 2589a369-8b21-406d-906d-71b18e4c7895
std( getindex.(s,2))

# ╔═╡ ed6a0e6a-2d0c-4f77-9b08-1a5b5d56dd34
md"""
Statisticians know a formula for the theoretical std of the slope.
"""

# ╔═╡ 61d1c1f7-e070-413b-8a92-76f44d237206
 σ  / norm(x.-mean(x))

# ╔═╡ 94d80ad6-0403-4322-aa9f-647c291c19d7
md"""
## Simulated σ ($howmany simulations)
"""

# ╔═╡ ce89b805-39a2-49e6-8781-c557aa73ed27
begin
	histogram( last.(s) ./ (σ^2/(n-2)) , alpha=.6, bins=100, norm=true,legend=false)
	vline!([1],color=:white)
	title!("residual")
	vline!([n-2],color=:white, lw=4)
	#xlims!(0,20)
	#ylims!(0,.13)
	plot!( x-> pdf(Chisq(n-2),x) , lw=4 , color=:red )
	plot!()

end

# ╔═╡ 75f9b5e9-775d-4767-9da6-222f977da686
mean( last.(s)  )

# ╔═╡ 797c9f2f-0b85-4435-b1c0-edc8cf67f738
σ^2

# ╔═╡ 6e0b2452-9f8b-4730-8072-a663704893c5
std(last.(s))

# ╔═╡ bf537a3a-b7c6-4c64-8b44-85511c3d492e
 (σ^2/ sqrt((n-2)/2))

# ╔═╡ 1340818c-3391-420b-aa94-acaea8a47d7d
md"""
# The Linear Model Table
"""

# ╔═╡ 829607ff-25e0-4585-9c5c-d132ecb86cc8
ols # = lm(@formula(°C ~ °F), noisy_data)

# ╔═╡ 3fc0a4a8-6719-4920-99c7-bd576225214e
-24.3784  / 19.0397

# ╔═╡ 24a7ad28-936c-47dc-bc53-d1ddbf39d05d
0.686156 / 0.330459

# ╔═╡ 9233dc6a-7578-4d72-b0c2-c3bb110a9fbe
md"""
## The Coef column is just the regression formula for the best line
"""

# ╔═╡ 07e02bb6-380d-40dd-86ad-19d713cd1657
mᵉ, bᵉ, σ²ᵉ =  linear_regression(x, yy)

# ╔═╡ b14593ba-cb8c-4f28-8fb0-2d2df479357b
md"""
## The Std. error column
"""

# ╔═╡ ac204681-b9df-471b-a22e-9d8f68679151
md"""
Above we saw that statisticians had formulas for the exact std of the slope and intercept:

 `std(intercept) = σ * norm(x)  / norm(x.-mean(x)) / sqrt(n)`

` std(slope) =  σ  / norm(x.-mean(x))`
"""

# ╔═╡ 08f43fff-fbd8-468f-8b3b-efd1829f4fc0
md"""
Let's replace σ with our estimate √σ²ᵉ
"""

# ╔═╡ 43ec6124-c3e5-4f34-b0d9-1a0b069aa3e0
sqrt(σ²ᵉ) * norm(x) / norm(x.-mean(x)) / sqrt(n)

# ╔═╡ 3fe71215-bbf2-40e9-bcfc-0bc9b3ac94c8
sqrt(σ²ᵉ) / norm(x.-mean(x))

# ╔═╡ a2b27841-256e-4898-aeca-04c4f44138fb
md"""
See those are the numbers in the magic table above.  I always love when I can reproduce the numbers myself.  It makes me feel I understand it.
"""

# ╔═╡ 8851dca3-e1a6-46b2-9745-f175ef0b0fae
md"""
## The t column
"""

# ╔═╡ ccfcb4d9-5a88-48fb-9568-1147a74f6eec
md"""
The t column, is simply the Coeff column divided by the Std. error column which we will use in a hypothesis test in the upcoming column.
"""

# ╔═╡ 13858c0a-3e7a-4742-a821-97dd9a45109d
md"""
### The t-distribution
"""

# ╔═╡ b2c3c1e5-e569-4c6f-bad9-055a25d73dce
md"""
In a statistics class you will likely see a random variable known as a t-distribution.
(with parameter k). It is the ratio of a standard normal to a χ distribution with parameter k. Let's just use `randn` to simulate. For the data sets of most of today's experiments, the normal distribution is close enough to t, that nobody needs to even use t much anymore.  In any event, with a t or a normal we are using this distribution because we are cognizant of the fact that the true σ is unknown and is merely being estimated.
"""

# ╔═╡ 305e4dfc-af7d-4667-8da8-a7ba5fd20fa6
rand_t(k) = sqrt(k)* randn() / norm( randn(k))

# ╔═╡ a648ba4f-fec4-4fa7-b328-1b52070224eb
md"""
k = $(@bind k Slider(3:100, show_value=true))
"""

# ╔═╡ d652df7d-7364-4da4-b51e-9fc88b978cda
begin

	histogram([rand_t(k) for i=1:100000], norm=true, bins=500, label=false)
	plot!( x-> pdf(TDist(k),x) , lw=4 , color=:red, label="t dist" )
	plot!( x->pdf(Normal(),x), color=:green, lw=2, label="normal dist")
	xlims!(-3, 3)
	ylims!(0, .4)
end

# ╔═╡ 2e530106-57a8-46a9-8f99-49a871d43255
md"""
## The Pr(>|t|)  column
is the area of the curve outside of the interval [-t,t].
"""

# ╔═╡ a990b133-ce50-4edf-81e1-1e78aeff8cd6
md"""
In statistics we ask if the coefficient ought to be considered 0 (which means in this case the data has no intercept or does not depend on x) or whether the coefficients are signficant with some probability.  The Pr(>|t|) column gives us the probablity that we should accept the hypothesis that the coefficients might reasonably be just 0.

In a proper statistical test, you should decide at what level you might be willing to accept the hypothesis, example .99, .95, or .9  might be a reasonable level, and if the test gives a smaller probability, you will accept that the coefficients are signficant.  It is not proper to produce the able and then decide whether to use .99, say.
"""

# ╔═╡ 3d0ea801-d66b-4e4e-90da-3a7dce28140d
md"""
# Degrees of Freedom
"""

# ╔═╡ 6fb223bb-f193-414d-9144-df180d09bea1
md"""
It is interesting to see that the sum of squares of a demeaned Gaussian vector is the size -1.  This is the reason for the (n-1) in the sample mean for variance.
"""

# ╔═╡ fb495ba4-52e6-4e0d-bd9c-981700edfebc
md"""
How many degrees of freedom are in a "demeaned" vector of normals?
"""

# ╔═╡ cdc4b25d-d05f-40c8-9c79-265876f01523

mean([ (v = randn(17);v.-=mean(v);sum(v.^2)) for i=1:1_000_000])

# ╔═╡ 967c5e3e-ab4c-45de-953c-aff6d16229af
md"""
If you ever wondered why the sample variance always has you dividing by (n-1)
and not n, this is the crux of the reason.
"""

# ╔═╡ Cell order:
# ╟─4ea0ccfa-9622-11eb-1cf0-e9ae2f927dd2
# ╠═d155ea12-9628-11eb-347f-7754a33fd403
# ╠═01506de2-918a-11eb-2a4d-c554a6e54631
# ╟─877deb2c-702b-457b-a54b-f27c277928d4
# ╟─36ce167f-382c-4b9a-be34-83250b10c4e5
# ╟─83912943-a847-420a-bfdb-450027b631e8
# ╠═280d112f-d34a-4cc4-9e3a-4ebbfcd5eb51
# ╠═b5031c96-db57-4baf-b271-6bb12e29de9b
# ╠═c2f77e8f-a8c0-4144-a8b4-b25dd98ed234
# ╟─ad161b98-f4a1-42ac-ad4f-8b71fabcfde9
# ╠═8e422886-74ef-4c0f-be1e-fda238c8db44
# ╟─ca21122a-2522-482a-b7ef-bd73e96cb5a9
# ╟─41e05b1e-8b5e-45e3-91bb-01355ade9f3d
# ╠═9d2e3861-ca36-406e-952d-831ca3947e44
# ╟─e73854ed-3581-41c4-ada5-e48242033759
# ╠═9873d944-b611-46f9-82a7-0cf714a3078c
# ╠═2be44753-afee-4125-b6bc-8866d2293dc2
# ╟─6e07e8fb-fe51-4b37-bfb2-d1466e768754
# ╟─a755e58a-b16c-4d3b-a85f-81ccf374793f
# ╟─f1e10fb7-adac-4083-8977-616a505fd591
# ╠═2e42986c-2de3-49e6-9c29-a7313c0b1da8
# ╟─22758dd6-9d04-4616-ba99-1430f2dedf9a
# ╠═aff6a616-6d8b-4584-a6f2-195decef7774
# ╠═338da13a-3c26-4366-a669-ac3e24f31577
# ╠═5a742546-1e4d-4aee-bed1-cb10c543e439
# ╟─6a9c8c9a-fac7-42f7-976d-3168132cae48
# ╟─3c038b68-8676-4877-9720-38da7c4e0e0e
# ╟─5a877e40-a101-4f7d-b2a1-ef4cfe5d8807
# ╟─83c28c76-2eab-49f9-9999-05df85054520
# ╟─ba671804-dc6d-415c-89de-9cf6294907b3
# ╟─e8683a71-5822-4491-9ccd-20e0fc3bf531
# ╠═0489e5d8-51ca-4955-83e1-95ea353d9cf2
# ╠═9a65aee4-ab8e-4ab7-be6f-cc2a2e9d5127
# ╟─c3539f42-6ca7-47fb-9707-4d11c9e76643
# ╟─469d809f-424f-4595-ad43-a5b2cc055304
# ╟─6128b8fd-9b85-4896-a0bf-934a0733fafb
# ╠═9eb7caaa-438d-4bcb-9c54-4a0fa72c61de
# ╠═0e8fce45-f1c0-41d4-996a-d6093182afee
# ╠═71590890-38b6-440e-b61b-ece6c49ac602
# ╟─f7cc7146-9ee6-4d87-b024-2a91863f4b24
# ╟─f64815e2-44b8-4585-9269-9a62655c984c
# ╟─feb3c45e-88f4-4ffc-a4a0-e89489187c8d
# ╠═99069dd7-e088-4626-aa29-e48d6f9a474e
# ╟─051a9e38-9a84-4ead-96fa-24c86c2b9f2d
# ╟─2f33ee51-0725-46c2-9f1b-a61cd68abab1
# ╠═e4acd97b-22f7-4812-9898-1a485887a5f2
# ╠═4e413b40-81c4-4160-9d01-046c2d179a06
# ╟─7b94db0d-f46b-4621-9413-1dc787ae9a39
# ╠═51a28b67-ad64-4cf2-a0e6-a78fb101eb15
# ╠═d451af49-3139-4329-a885-a210b1760f74
# ╟─c7455f7a-9c72-42f5-8238-1799cad96f6c
# ╟─d2971801-2cdb-4b9f-8ec8-c74cbb2a0b31
# ╠═e1e8c140-bc4e-400d-beb2-0986e071c3a3
# ╟─1429be09-a31f-415f-9c3d-f32b085ef68d
# ╠═da321202-0dc5-44ad-aac0-f3ea0d229243
# ╟─2aceb366-a067-4271-9362-c320f4735ed1
# ╠═58f548fd-f6d0-479d-8469-bc886783f9a7
# ╟─07be9435-bc07-4a18-aad8-3ff19f5bcce4
# ╠═1a6ad08d-c3bb-47e7-bdee-156bbff3aeda
# ╟─c55e4894-db71-4729-a1a1-5f68b45e3bf5
# ╠═f50d66eb-0357-4017-ac9b-99e63cd52dc0
# ╟─5c7a7361-f0e7-473a-9e38-226828aa00ca
# ╠═acf0e90e-8f1f-451f-9f0f-70a0bcc7efca
# ╟─c9f65e15-f222-4a88-98c2-9e1d8b5ec3eb
# ╠═2589a369-8b21-406d-906d-71b18e4c7895
# ╟─ed6a0e6a-2d0c-4f77-9b08-1a5b5d56dd34
# ╠═61d1c1f7-e070-413b-8a92-76f44d237206
# ╟─94d80ad6-0403-4322-aa9f-647c291c19d7
# ╠═ce89b805-39a2-49e6-8781-c557aa73ed27
# ╠═75f9b5e9-775d-4767-9da6-222f977da686
# ╠═797c9f2f-0b85-4435-b1c0-edc8cf67f738
# ╠═6e0b2452-9f8b-4730-8072-a663704893c5
# ╠═bf537a3a-b7c6-4c64-8b44-85511c3d492e
# ╟─1340818c-3391-420b-aa94-acaea8a47d7d
# ╠═829607ff-25e0-4585-9c5c-d132ecb86cc8
# ╠═3fc0a4a8-6719-4920-99c7-bd576225214e
# ╠═24a7ad28-936c-47dc-bc53-d1ddbf39d05d
# ╟─9233dc6a-7578-4d72-b0c2-c3bb110a9fbe
# ╠═07e02bb6-380d-40dd-86ad-19d713cd1657
# ╟─b14593ba-cb8c-4f28-8fb0-2d2df479357b
# ╟─ac204681-b9df-471b-a22e-9d8f68679151
# ╟─08f43fff-fbd8-468f-8b3b-efd1829f4fc0
# ╠═43ec6124-c3e5-4f34-b0d9-1a0b069aa3e0
# ╠═3fe71215-bbf2-40e9-bcfc-0bc9b3ac94c8
# ╟─a2b27841-256e-4898-aeca-04c4f44138fb
# ╟─8851dca3-e1a6-46b2-9745-f175ef0b0fae
# ╟─ccfcb4d9-5a88-48fb-9568-1147a74f6eec
# ╟─13858c0a-3e7a-4742-a821-97dd9a45109d
# ╟─b2c3c1e5-e569-4c6f-bad9-055a25d73dce
# ╠═305e4dfc-af7d-4667-8da8-a7ba5fd20fa6
# ╟─a648ba4f-fec4-4fa7-b328-1b52070224eb
# ╠═d652df7d-7364-4da4-b51e-9fc88b978cda
# ╟─2e530106-57a8-46a9-8f99-49a871d43255
# ╟─a990b133-ce50-4edf-81e1-1e78aeff8cd6
# ╟─3d0ea801-d66b-4e4e-90da-3a7dce28140d
# ╟─6fb223bb-f193-414d-9144-df180d09bea1
# ╟─fb495ba4-52e6-4e0d-bd9c-981700edfebc
# ╠═cdc4b25d-d05f-40c8-9c79-265876f01523
# ╟─967c5e3e-ab4c-45de-953c-aff6d16229af


#---------------------------------------------------------
# # [LazyGrids ndgrid](@id 1-ndgrid)
#---------------------------------------------------------

# This page explains the `ndgrid` method(s) in the Julia package
# [`LazyGrids`](https://github.com/JuliaArrays/LazyGrids.jl).

# ### Setup

# Packages needed here.

using LazyGrids: ndgrid, ndgrid_array
using LazyGrids: btime, @timeo # not exported; just for timing tests here
using BenchmarkTools: @benchmark
using InteractiveUtils: versioninfo

# ### Overview

# We begin with simple illustrations.

# The Julia method `ndgrid_array` in this package
# is comparable to Matlab's `ndgrid` function.
# It is given a long name here
# to discourage its use,
# because the lazy `ndgrid` version is preferable.
# The package provides `ndgrid_array`
# mainly for testing and timing comparisons.

(xa, ya) = ndgrid_array(1.0:3.0, 1:2)

#
xa

#
ya

# This package provides a "lazy" version of `ndgrid` that appears to the user
# to be the same, but under the hood it is not storing huge arrays.

(xl, yl) = ndgrid(1.0:3.0, 1:2)

#
xl

#
yl

# The following example illustrates the memory savings
# (thanks to Julia's powerful `AbstractArray` type):

(xl, yl) = ndgrid(1:100, 1:200)
(xa, ya) = ndgrid_array(1:100, 1:200)
sizeof(xl), sizeof(xa)

# One can do everything with a lazy array that one would expect
# from a "normal" array, e.g., multiplication and summation:

sum(xl * xl'), sum(xa * xa')

# ### Using lazy `ndgrid`

# Many applications with multiple variables
# involve evaluating functions over a *grid* of values.

# As a simple example (for illustration),
# one can numerically approximate the area of the unit circle
# by sampling that circle over a grid of x,y values,
# corresponding to numerical evaluation of the double integral
# ``∫ ∫ 1_{\{x^2 + y^2 < 1\}} \, dx \, dy``.
# There are many ways to implement this approximation in Julia,
# given a vector of `x` and `y` samples.

Δ = 1/2^10
x = range(-1, stop=1, step=Δ)
y = copy(x)

@inline circle(x::Real, y::Real) = abs2(x) + abs2(y) < 1
@inline circle(xy::NTuple{2}) = circle(xy...)

# The documentation below has many timing comparisons.
# The times in the Julia comments are on a 2017 iMac with Julia 1.6.1;
# the times printed out are whatever server GitHub actions uses.
# Using a trick to [capture output](https://fredrikekre.github.io/Literate.jl/v2/generated/example/#Output-Capturing),
# let's find out:

io = IOBuffer()
versioninfo(io)
split(String(take!(io)), '\n')

# A basic double loop is the C/Fortran way.
# It uses minimal memory (only 48 bytes) but is somewhat slow.

function method0(x,y) # basic double loop
    sum = 0.0
    for x in x, y in y
        sum += circle(x,y)
    end
    return sum * Δ^2
end

area0 = method0(x,y)
t = @benchmark method0($x,$y) # 10.5 ms (3 allocations: 48 bytes)
btime(t)

# The loop version does not look much like the math.
# It often seems natural to think of a grid of x,y values
# and simply sum over that grid, accounting for the grid spacing,
# using a function like this:

area(xx,yy) = sum(circle.(xx,yy)) * Δ^2

# Users coming from Matlab who are unfamiliar with its newer broadcast
# capabilities might use an `ndgrid` of arrays, like in the following code,
# to compute the area.
# But this array approach is much slower and uses much more memory,
# so it does not scale well to higher dimensions.

function area_array(x, y)
    (xa, ya) = ndgrid_array(x, y)
    return area(xa, ya)
end
@assert area_array(x, y) ≈ area0
t = @benchmark area_array($x, $y) # 21.4 ms (11 allocations: 64.57 MiB)
btime(t)

# To be fair, one might have multiple uses of the grids `xa,ya`
# so perhaps they should be excluded from the timing.
# Separating that allocation makes the timing look faster,
# but it still uses a lot of memory,
# both for allocating the grids, and for the `circle.` broadcast
# in the `area` function above:

(xa, ya) = ndgrid_array(x, y)
@assert area(xa, ya) ≈ area0
t = @benchmark area($xa, $ya) # 5.2 ms (7 allocations: 516.92 KiB)
btime(t)

# The main point of this package is to provide
# a lazy version of `ndgrid` that uses minimal memory.

(xl, yl) = ndgrid(x, y)
@assert xl == xa
@assert yl == ya
sizeof(xa), sizeof(xl)

# Now there is essentially no memory overhead for the grids,
# but memory is still used for the `circle.` broadcast.

@assert area(xl,yl) ≈ area0
t = @benchmark area($xl,$yl) # 3.7 ms (7 allocations: 516.92 KiB)
btime(t)

# Furthermore, creating this lazy ndgrid is so efficient
# that we can include its construction time
# and still have performance comparable to the array version
# that had pre-allocated arrays.

function area_lazy(x, y)
    (xl, yl) = ndgrid(x, y)
    return area(xl, yl)
end
@assert area_lazy(x, y) ≈ area0
t = @benchmark area_lazy($x, $y) # 3.7 ms (7 allocations: 516.92 KiB)
btime(t)

# ### More details

# The comparisons below here might be more
# for the curiosity of the package developers
# than for most users...

# One can preallocate memory to store the `circle.` array,
# to avoid additional memory during the area calculation:

out = Array{Float64}(undef, length(x), length(y))
function area!(xx, yy)
    global out .= circle.(xx,yy)
    return sum(out) * Δ^2
end
@assert area!(xl,yl) ≈ area0
t = @benchmark area!(xl,yl) # 4.8 ms (4 allocations: 128 bytes)
btime(t)

# Interestingly, the lazy version is *faster* than the array version,
# presumably because of the overheard of moving data from RAM to CPU:

@assert area!(xa,ya) ≈ area0
t = @benchmark area!(xa,ya) # 6.2 ms (4 allocations: 80 bytes)
btime(t)

# One can avoid allocating the output array by using a loop
# with [CartesianIndices](https://julialang.org/blog/2016/02/iteration):

function area_ci(xx, yy)
    size(xx) == size(yy) || throw("size")
    sum = 0.0
    @inbounds for c in CartesianIndices(xx)
        sum += circle(xx[c], yy[c])
    end
    return sum * Δ^2
end

# With this approach the lazy version is a bit faster
# than the array version:

@assert area_ci(xl,yl) ≈ area0
t = @benchmark area_ci(xl,yl) # 5.2 ms (3 allocations: 48 bytes)
btime(t)

#
@assert area_ci(xa,ya) ≈ area0
t = @benchmark area_ci(xa,ya) # 5.9 ms (3 allocations: 48 bytes)
btime(t)

# Alternatively one can use a linear index for loop,
# that also avoids the extra memory of `circle.` above,
# but is slower, especially for the lazy arrays
# that are optimized for Cartesian indexing:

function area_for2(xx,yy)
    size(xx) == size(yy) || throw("size")
    sum = 0.0
    @inbounds for i in 1:length(xx)
        sum += circle(xx[i], yy[i])
    end
    return sum * Δ^2
end
@assert area_for2(xa, ya) ≈ area0
t = @benchmark area_for2($xa, $ya) # 5.9 ms (3 allocations: 48 bytes)
btime(t)

#
@assert area_for2(xl, yl) ≈ area0
t = @benchmark area_for2($xl, $yl) # 15.4 ms (3 allocations: 48 bytes)
btime(t)

# Some Julia users would
# [recommend using broadcast](https://discourse.julialang.org/t/meshgrid-function-in-julia/48679/25).
# In this case, broadcast is reasonably fast, but still uses a lot of memory
# for the `circle.` output in the simplest implementation.

areab(x,y) = sum(circle.(x,y')) * Δ^2
@assert areab(x,y) ≈ area0
t = @benchmark areab($x,$y) # 11.6 ms (7 allocations: 516.92 KiB)
btime(t)

# Using `zip` can avoid the "extra" memory beyond the grids,
# but seems to have some undesirable overhead,
# presumably because `zip` uses linear indexing:

areaz(xa, ya) = sum(circle, zip(xa,ya)) * Δ^2
@assert areaz(xa, ya) ≈ area0
t = @benchmark areaz($xa, $ya) # 3.9 ms (3 allocations: 48 bytes)
btime(t)

#
@assert areaz(xl, yl) ≈ area0
t = @benchmark areaz($xl, $yl) # 12.2 ms (3 allocations: 48 bytes)
btime(t)

# One can also ensure low memory by using a product iterator,
# but the code starts to look pretty different from the math at this point
# and it is not much faster than broadcast here.

areap(x,y) = sum(circle, Iterators.product(x, y)) * Δ^2
@assert areap(x,y) ≈ area0
t = @benchmark areap($x, $y) # 9.9 ms (3 allocations: 48 bytes)
btime(t)

# ### 3D case

# A 3D example is finding (verifying) the volume of a unit sphere.

sphere(x::Real,y::Real,z::Real) = abs2(x) + abs2(y) + abs2(z) < 1
sphere(r::NTuple) = sum(abs2, r) < 1

# Storing three 3D arrays of size 2049^3 Float64 would take 192GB,
# so already we must greatly reduce the sampling to use either
# `broadcast` or `ndgrid_array`.
# Furthermore, the `broadcast` requires annoying `reshape` steps:

Δc = 1/2^8 # coarse grid
xc = range(-1, stop=1, step=Δc)
yc = xc
zc = xc
nc = length(zc)
3 * nc^3 * 8 / 1024^3 # GB prediction

# Here is broadcast in 3D (yuch!):

vol_br(x,y,z,Δ) = sum(sphere.(
        repeat(x, 1, length(y), length(z)),
        repeat(reshape(y, (1, :, 1)), length(x), 1, length(z)),
        repeat(reshape(z, (1, 1, :)), length(x), length(y), 1),
    )) * Δ^3
vol_br([0.],[0.],[0.],Δc) # warm-up
@timeo vol0 = vol_br(xc,yc,zc,Δc) # 2.7 sec, 3.0 GiB, roughly (4/3)π

function vol_ci(xx, yy, zz, Δ)
    size(xx) == size(yy) == size(zz) || throw("size")
    sum = 0.0
    @inbounds for c in CartesianIndices(xx)
        sum += sphere(xx[c], yy[c], zz[c])
    end
    return sum * Δ^3
end

# Here is the lazy version:
(xlc, ylc, zlc) = ndgrid(xc, yc, zc) # warm-up
@timeo (xlc, ylc, zlc) = ndgrid(xc, yc, zc); # 0.000022 sec (1.8 KiB)

#
vol_ci([0.], [0.], [0.], Δc) # warm-up
@timeo vol_ci(xlc, ylc, zlc, Δc) # 0.2 sec, 1.4 MiB

# Creating the grid of arrays itself is quite slow, even for the coarse grid:
(xac, yac, zac) = ndgrid_array(xc, yc, zc) # warm-up
@timeo (xac, yac, zac) = ndgrid_array(xc, yc, zc) # 1.8 sec 3.0GiB

# Once created, the array version is no faster than the lazy version:
@timeo vol_ci(xac, yac, zac, Δc) # 0.2 seconds (1.1 MiB)

# Using `zip` is more concise (but slower):
vol_zip(xx, yy, zz, Δ) = sum(sphere, zip(xx,yy,zz)) * Δ^3

#
vol_zip([0.], [0.], [0.], Δc) # warm-up
@timeo vol_zip(xlc, ylc, zlc, Δc) # 1.0 sec, 26 MiB

# Using zip for the array version seems to have less overhead
# so that is a potential for future improvement:

@assert vol_zip(xac, yac, zac, Δc) ≈ vol0
@timeo vol_zip(xac, yac, zac, Δc) # 0.19 sec, 16 byte

# Importantly, with the lazy ndgrid now we can return to the fine scale;
# it takes a few seconds, but it is feasible because of the low memory.
z = copy(x)
@timeo (xlf, ylf, zlf) = ndgrid(x, y, z) # 0.000023 sec

#
@timeo vol_ci(xlf, ylf, zlf, Δ) # 12.7 sec, 16 bytes

# I was hoping that with a lazy grid, now we could explore
# higher-dimensional spheres.  But with the current `zip` overhead
# it was too slow, even with coarse grid.
# @timeo (π^2/2, sum(sphere, zip(ndgrid(xc,xc,xc,xc)...)) * Δc^4)

# Probably I need to learn more about stuff like `pairs(IndexCartesian(), A)`
# [e.g., this PR](https://github.com/JuliaLang/julia/pull/38150).
# Another day...


using JuMP
using Clp

CARBS = 50
PROTEIN = 40
FAT = 60

m = Model(Clp.Optimizer)

@variable(m, steak ≥ 0)
@variable(m, potatoes ≥ 0)

@objective(m, Min, 8 * steak + 4 * potatoes)

@constraint(m, 5 * steak + 15 * potatoes ≥ CARBS)
@constraint(m, 20 * steak + 5 * potatoes ≥ PROTEIN)
@constraint(m, 15 * steak + 2 * potatoes ≤ FAT)

print(m)

optimize!(m)

status = termination_status(m)

println("Solution status: ", status)

println("Objective value: ", objective_value(m))
println("potatoes = ", value(steak))
println("steak = ", value(potatoes))


export RandomPatch

"""
    RandomPatch()

A random patch. For this we first draw entries of a vector `v` independently from a
complex normal distribution (`randn(ComplexF64)`). And then normalize `v` with respect
to the 2-norm.
"""
struct RandomPatch <: AbstractAffinePatch end

struct RandomPatchState{T,N} <: AbstractAffinePatchState{N}
    v::PVector{T,N}
end

is_global_patch(::RandomPatch) = true

function state(::RandomPatch, x::PVector)
    v = similar(x, ComplexF64)
    Random.randn!(v)
    LinearAlgebra.normalize!(v)
    RandomPatchState(v)
end

onpatch!(x::AbstractVector, state::RandomPatchState) = onpatch!(x, state.v)
evaluate!(u, state::RandomPatchState, x::PVector) = evaluate_patch!(u, state.v, x)
jacobian!(U, state::RandomPatchState, x::PVector) = jacobian_patch!(U, state.v, x)