### A Pluto.jl notebook ###
# v0.12.7

using Markdown
using InteractiveUtils

# ╔═╡ 6fd616f6-1ea1-11eb-3814-8bfb4a096c49
using Pkg, DrWatson

# ╔═╡ 6ffe2628-1ea1-11eb-24ea-57f985146a72
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Distributions, Random, StatsBase, DataFrames, Plots, LaTeXStrings
	Random.seed!(1)
end;

# ╔═╡ ed174bc4-1ea0-11eb-1e2f-a32874cec549
md"## Listing 3.21"

# ╔═╡ 70179b62-1ea1-11eb-13ce-7fb4d54bad4b
begin
	cUnif = Uniform(0,2π)
	xGrid, N = 0:0.1:2π, 10^6
end

# ╔═╡ 287be85a-2165-11eb-2609-eba593feae95
begin
	stephist( rand(N)*2π, bins=xGrid,
		normed=:true, c=:blue,
		label="MC Estimate")
	plot!( xGrid, pdf.(cUnif,xGrid),
		c=:red,ylims=(0,0.2),label="PDF", ylabel="Density",xticks=([0:π/2:2π;],
		["0", L"\dfrac{\pi}{2}", L"\pi", L"\dfrac{3\pi}{2}", L"2\pi"]))
end

# ╔═╡ 70182cd8-1ea1-11eb-094d-8d0d49cf15f3
md"## End of listing 3.21"

# ╔═╡ Cell order:
# ╟─ed174bc4-1ea0-11eb-1e2f-a32874cec549
# ╠═6fd616f6-1ea1-11eb-3814-8bfb4a096c49
# ╠═6ffe2628-1ea1-11eb-24ea-57f985146a72
# ╠═70179b62-1ea1-11eb-13ce-7fb4d54bad4b
# ╠═287be85a-2165-11eb-2609-eba593feae95
# ╟─70182cd8-1ea1-11eb-094d-8d0d49cf15f3


using ModelingToolkit, DiffEqOperators, LinearAlgebra, OrdinaryDiffEq
using ModelingToolkit: operation, istree, arguments

# Define some variables
@parameters t x
@variables u(..) v(..)
Dt = Differential(t)
Dx = Differential(x)
Dxx = Differential(x)^2
eqs  = [Dt(u(t,x)) ~ Dxx(u(t,x)),
        Dt(v(t,x)) ~ Dxx(v(t,x))]
bcs = [u(0,x) ~ - x * (x-1) * sin(x),
       v(0,x) ~ - x * (x-1) * sin(x),
       u(t,0) ~ 0.0, u(t,1) ~ 0.0,
       v(t,0) ~ 0.0, v(t,1) ~ 0.0]

domains = [t ∈ IntervalDomain(0.0,1.0),
           x ∈ IntervalDomain(0.0,1.0)]

pdesys = PDESystem(eqs,bcs,domains,[t,x],[u(t,x),v(t,x)])
discretization = MOLFiniteDifference([x=>0.1],t;grid_align=edge_align)
prob = discretize(pdesys,discretization) # This gives an ODEProblem since it's time-dependent
sol = solve(prob,Tsit5())

@parameters t x y
@variables u(..)
Dxx = Differential(x)^2
Dyy = Differential(y)^2
Dt = Differential(t)
t_min= 0.
t_max = 2.0
x_min = 0.
x_max = 2.
y_min = 0.
y_max = 2.

# 3D PDE
eq  = Dt(u(t,x,y)) ~ Dxx(u(t,x,y)) + Dyy(u(t,x,y))

analytic_sol_func(t,x,y) = exp(x+y)*cos(x+y+4t)
# Initial and boundary conditions
bcs = [u(t_min,x,y) ~ analytic_sol_func(t_min,x,y),
       u(t,x_min,y) ~ analytic_sol_func(t,x_min,y),
       u(t,x_max,y) ~ analytic_sol_func(t,x_max,y),
       u(t,x,y_min) ~ analytic_sol_func(t,x,y_min),
       u(t,x,y_max) ~ analytic_sol_func(t,x,y_max)]

# Space and time domains
domains = [t ∈ IntervalDomain(t_min,t_max),
           x ∈ IntervalDomain(x_min,x_max),
           y ∈ IntervalDomain(y_min,y_max)]
pdesys = PDESystem([eq],bcs,domains,[t,x,y],[u(t,x,y)])

# Method of lines discretization
dx = 0.1; dy = 0.2
discretization = MOLFiniteDifference([x=>dx,y=>dy],t)
prob = ModelingToolkit.discretize(pdesys,discretization)
sol = solve(prob,Tsit5())

# Diffusion in a sphere
@parameters t r
@variables u(..)
Dt = Differential(t)
Dr = Differential(r)
Drr = Dr^2
eq  = Dt(u(t,r)) ~ (1/r^2 * Dr(r^2 * Dr(u(t,r))))
bcs = [u(0,r) ~ - r * (r-1) * sin(r),
       Dr(u(t,0)) ~ 0.0, u(t,1) ~ sin(1)]

domains = [t ∈ IntervalDomain(0.0,1.0),
           r ∈ IntervalDomain(0.0,1.0)]

pdesys = PDESystem(eq,bcs,domains,[t,r],[u(t,r)])
discretization = MOLFiniteDifference([r=>0.1],t)
prob = discretize(pdesys,discretization) # This gives an ODEProblem since it's time-dependent
sol = solve(prob,Tsit5())


precision(proposal, goldstandard) =
    num_common_cons(proposal, goldstandard) / num_cons(proposal)
recall(proposal, goldstandard) =
    num_common_cons(proposal, goldstandard) / num_cons(goldstandard)
f1_measure(proposal, goldstandard) =
    2 * precision(proposal, goldstandard) * recall(proposal, goldstandard) /
       (precision(proposal, goldstandard) + recall(proposal, goldstandard))

"number of constituents of tree"
function num_cons(tree::Tree)
    M = boolean_dependency_matrix(tree)
    count(x->x, M)
end

"number of common constituents"
function num_common_cons(tree1::Tree, tree2::Tree)
    A = boolean_dependency_matrix(tree1)
    B = boolean_dependency_matrix(tree2)
    sum(a && b for (a,b) in zip(A,B))
end

"number of crossed bracket pairs"
function num_crossed_brackets(tree1, tree2)
    num_cons(tree1) - num_common_cons(tree1, tree2)
end

# t1 = tree("[0maj[0maj[0maj][1maj]][0maj[7maj][0maj]]]")
# t2 = tree("[0maj[1maj[2min][1maj]][0maj[7maj][0maj]]]")
# t3 = tree("[0maj[0maj][0maj[0maj][0maj[7maj][0maj]]]]")
#
# num_crossed_brackets(t1, t3)


# p15.jl - solve eigenvalue BVP u_xx = lambda*u, u(-1)=u(1)=0

N = 36; (D,x) = cheb(N); D2 = D^2; D2 = D2[2:N,2:N];
(lam,V) = eig(D2);
ii = sortperm(-lam);          # sort eigenvalues and -vectors
lam = lam[ii]; V = V[:,ii]; clf();
for j = 5:5:30                  # plot 6 eigenvectors
    u = [0;V[:,j];0]; subplot(7,1,j/5)
    plot(x,u,".",markersize=6); grid(true);
    xx = -1:.01:1; uu = polyval(polyfit(x,u),xx);
    plot(xx,uu); axis("off");
    text(-.4,.1,"eig $j = $(lam[j]*4/pi^2) π^2/4");
    text(.7,.1,"$(signif(4*N/(pi*j),2))  ppw");
end


N=10000000
cnt=0
for i = 1:N
    x=rand()
    y=rand()
    if x^2+y^2<1.0
        cnt+=1
    end
end

@printf "Estimate of PI for %d trials is %8.5f\n" N 4.0*(cnt / N)


using Plots
using Formatting

function logistic(r,x)
    r*x*(1.0-x)
end

function logisticmap(;x0=0.2,r=4,n=10)
    results = [(x0,logistic(r,x0))];
    for i in 1:(n-1)
      xold = results[end][2]
      xnew = logistic(r,xold)
      append!(results,[(xold,xold)])
      append!(results,[(xold,xnew)])
  end
  results
end

function vary(dname;x0_vals=[],r_vals=[], nvals=[])
 if !isdir(dname)
  mkdir(dname)
 end
 fno = 1
 for x0 in x0_vals
  for r in r_vals
      for n in nvals
          fname = format(dname*"/frame{:05d}.png",fno)
          fno = fno + 1
          simres = logisticmap(x0=x0, r=r, n=n)
          xvals = [x for (x, y) in simres]
          yvals = [y for (x, y) in simres]
          plot(xvals,yvals,legend=false,xaxis=false,yaxis=false,xlim=(0,1),ylim=(0,1));
          savefig(fname)
        end
    end
end
end

# name/number each iteration then plot 1, 1+2, 1+2+3 ...

# To make frames varying r, specify x0_vals as an array with a single value
# along with whatever values of r you want to loop through
vary("frames_r";x0_vals=[0.5],r_vals=0:0.1:5, n=[100])

# To make frames varying x0, specify r_vals as an array with a single value
# along with whatever values of x0 you want to loop through
vary("frames_x0";x0_vals=0:0.05:1,r_vals=[4.0], n=[100])

vary("frames_n";x0_vals=[0.2],r_vals=[4], nvals = 1:500)

r_vals = 4.1
vary("loop3";x0_vals=[(r_vals+sqrt(r_vals-4)*sqrt(r_vals))/(2*r_vals)],r_vals=[4], nvals=1:10)

vary("loop2";x0_vals=[(r_vals-sqrt(r_vals-4)*sqrt(r_vals))/(2*r_vals)],r_vals=[4], nvals=1:10)

ffmpeg -i frame%05d.png -vf "fps=12,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 output.gif


# # Control

#-

# A simple control problem on a system usually involves a variable $x(t)$
# that denotes the state of the system over time, and a variable $u(t)$ that
# denotes the input into the system over time. Linear constraints are used to
# capture the evolution of the system over time:
#
# $$
# x(t) = Ax(t - 1) + Bu(t), \ \text{for} \ t = 1,\ldots, T,
# $$
#
# where the numerical matrices $A$ and $B$ are called the dynamics and input matrices,
# respectively.
#
# The goal of the control problem is to find a sequence of inputs
# $u(t)$ that will allow the state $x(t)$ to achieve specified values
# at certain times. For example, we can specify initial and final states of the system:
#
# $$
# \begin{aligned}
#   x(0) &= x_i \\
#   x(T) &= x_f
# \end{aligned}
# $$
#
# Additional states between the initial and final states can also be specified. These
# are known as waypoint constraints. Often, the input and state of the system will
# have physical meaning, so we often want to find a sequence inputs that also
# minimizes a least squares objective like the following:
#
# $$
#   \sum_{t = 0}^T \|Fx(t)\|^2_2 + \sum_{t = 1}^T\|Gu(t)\|^2_2,
# $$
#
# where $F$ and $G$ are numerical matrices.
#
# We'll now apply the basic format of the control problem to an example of controlling
# the motion of an object in a fluid over $T$ intervals, each of $h$ seconds.
# The state of the system at time interval $t$ will be given by the position and the velocity of the
# object, denoted $p(t)$ and $v(t)$, while the input will be forces
# applied to the object, denoted by $f(t)$.
# By the basic laws of physics, the relationship between force, velocity, and position
# must satisfy:
#
# $$
#   \begin{aligned}
#     p(t+1) &= p(t) + h v(t) \\
#     v(t+1) &= v(t) + h a(t)
#   \end{aligned}.
# $$
#
# Here, $a(t)$ denotes the acceleration at time $t$, for which we we use
# $a(t) = f(t) / m + g - d v(t)$,
# where $m$, $d$, $g$ are constants for the mass of the object, the drag
# coefficient of the fluid, and the acceleration from gravity, respectively.
#
# Additionally, we have our initial/final position/velocity conditions:
#
# $$
#   \begin{aligned}
#     p(1) &= p_i\\
#     v(1) &= v_i\\
#     p(T+1) &= p_f\\
#     v(T+1) &= 0
#   \end{aligned}
# $$
#
# One reasonable objective to minimize would be
#
# $$
#   \text{objective} = \mu \sum_{t = 1}^{T+1} (v(t))^2 + \sum_{t = 1}^T (f(t))^2
# $$
#
# We would like to keep both the forces small to perhaps save fuel, and keep
# the velocities small for safety concerns.
# Here $\mu$ serves as a parameter to control which part of the objective we
# deem more important, keeping the velocity small or keeping the force small.
#
# The following code builds and solves our control example:

using Convex, SCS, Plots

## Some constraints on our motion
## The object should start from the origin, and end at rest
initial_velocity = [-20; 100]
final_position = [100; 100]

T = 100 # The number of timesteps
h = 0.1 # The time between time intervals
mass = 1 # Mass of object
drag = 0.1 # Drag on object
g = [0, -9.8] # Gravity on object

## Declare the variables we need
position = Variable(2, T)
velocity = Variable(2, T)
force = Variable(2, T - 1)

## Create a problem instance
mu = 1

## Add constraints on our variables
constraints = Constraint[
    position[:, i+1] == position[:, i] + h * velocity[:, i] for i in 1:T-1
]

for i in 1:T-1
    acceleration = force[:, i] / mass + g - drag * velocity[:, i]
    push!(constraints, velocity[:, i+1] == velocity[:, i] + h * acceleration)
end

## Add position constraints
push!(constraints, position[:, 1] == 0)
push!(constraints, position[:, T] == final_position)

## Add velocity constraints
push!(constraints, velocity[:, 1] == initial_velocity)
push!(constraints, velocity[:, T] == 0)

## Solve the problem
problem = minimize(sumsquares(force), constraints)
solve!(problem, SCS.Optimizer; silent_solver = true)

# We can plot the trajectory taken by the object.

pos = evaluate(position)
plot([pos[1, 1]], [pos[2, 1]], st = :scatter, label = "initial point")
plot!([pos[1, T]], [pos[2, T]], st = :scatter, label = "final point")
plot!(pos[1, :], pos[2, :], label = "trajectory")

# We can also see how the magnitude of the force changes over time.

plot(vec(sum(evaluate(force) .^ 2, dims = 1)), label = "force (magnitude)")


using SummationByParts
include("nodecalc3.jl")

function nodecalc(sbp::TriSBP, isDG::Bool)
  vtx = [0.0 0; 1 0; 0 1]  # why is this 0 to 1 but the other method is -1 to 1
  r1 = vtx[1, :]
  r2 = vtx[2, :]
  r3 = vtx[3, :]

  T = zeros(2,2)
  T[:, 1] = r2 - r1
  T[:, 2] = r3 - r1

  if isDG
    coords = SummationByParts.SymCubatures.calcnodes(sbp.cub, vtx)
  else
    coords = calcnodes(sbp, vtx)
  end

  xi = zeros(coords)

  for i=1:size(coords,2)
    xi[:, i] = T\(coords[:, i] - r1)
  end

  return xi, coords
end

function nodecalc(sbp::TetSBP, isDG::Bool)

  vtx = [-1.0 -1.0 -1.0
        1.0 -1.0 -1.0
       -1.0 1.0 -1.0
       -1.0 -1.0 1.0]

  r1 = vtx[1, :]
  r2 = vtx[2, :]
  r3 = vtx[3, :]
  r4 = vtx[4, :]
  T = zeros(3,3)
  T[:, 1] = r2 - r1
  T[:, 2] = r3 - r1
  T[:, 3] = r4 - r1

  if isDG
    coords = SummationByParts.SymCubatures.calcnodes(sbp.cub, vtx)
  else
    coords = calcnodes(sbp, vtx)
  end

  xi = zeros(coords)

  for i=1:size(coords,2)
    xi[:, i] = T\(coords[:, i] - r1)
  end

  return xi, coords
end

function minNodeDist(sbp, isDG::Bool)
# get the minimum distance between nodes on a reference element of degree p

    xi, coords = nodecalc(sbp, isDG)
    min_dist = typemax(Float64)
#    println("coords = ", coords)
    for i=1:size(coords, 2)
      for j=(i+1):size(coords, 2)
	# calculate distance between node i and node j
	dist_j = norm(coords[:, i] - coords[:, j])

	if dist_j < min_dist
	  min_dist = dist_j
	end

      end  # end loop j
    end  # end loop i

#    println("for p=$p elements, min node distance = ", min_dist)

  return min_dist
end

# sbp = TriSBP{Float64}(degree=1, reorder=false, internal=false)
# sbp = getTriSBPWithDiagE(degree=4)
# xi, coords = nodecalc(sbp, true)
# printCaseStatement(xi)
# printBaryCoords(xi, coords)
# writedlm("coordsout.dat", coords, ' ')
#minNodeDist(2)


# Methods from Moukalled et al. FVM - OpenFOAM, Matlab

mutable struct Cell
    faceIndices::Vector{Int64}
    pointIndices::Vector{Int64}
end

######################### Mesh/Cell Geometry ###########################
function triangleCentroid(points)
    center = [ 0.0, 0.0, 0.0 ]
    nPts = size(points, 1)
    for pt in 1:nPts
        center = center .+ points[pt]
    end
    center /= nPts

    return center
end

#Alternate name for same function
geometricCenter = triangleCentroid

function triangleArea(points::Array{Array{Float64, 1}})
    side1 = points[2] .- points[1]
    side2 = points[3] .- points[1]
    fAVec = cross(side1, side2) ./ 2
    return fAVec
end

#=
    Calculates face area vector and centroid from the points that make up the face
    Points must be ordered sequentially

    How it works:
        1. Splits face into subtriangles
        2. Area and centroid is computed for each subtriangle
        3. Areas vectors are summed and returned
        4. The centroid returned is obtained from an area-weighted of sum of the subtriangle centroids
=#
function faceAreaCentroid(points::Array{Array{Float64, 1}})
    gC = geometricCenter(points)
    nPts = size(points, 1)

    fAVec = [ 0.0, 0.0, 0.0 ]
    centroid = [ 0.0, 0.0, 0.0 ]

    for i in 1:nPts
        if i < nPts
            subTriPts = [ gC, points[i], points[i+1] ]
        else
            subTriPts = [ gC, points[i], points[1] ]
        end

        triCentroid = triangleCentroid(subTriPts)
        subFAVec = triangleArea(subTriPts)

        fAVec += subFAVec
        centroid += triCentroid .* mag(subFAVec)
    end

    centroid /= mag(fAVec)

    return fAVec, centroid
end

#=
    Calculates cell volume (scalar) and centroid (vector) from the points and face area vectors (fAVecs) that make up the cell
        fAVecs can be computed using the faceAreaCentroids function

    How it works:
        1. Splits cell into polygonal pyramids, each incorporating a single face and the geometric center of the cell
        2. Computes volume and centroid of each sub-pyramid
        3. Resulting volume is sum of sub-pyramid volumes, centroid is the volume-weighted sum of sub-pyramid centroids
=#
function cellVolCentroid(points, fAVecs, faceCentroids)
    gC = geometricCenter(points)
    nFaces = size(fAVecs,1)

    vol = 0.0
    centroid = [ 0.0, 0.0, 0.0 ]

    for f in 1:nFaces
        cellCenterVec = faceCentroids[f] .- gC
        subPyrVol = abs(sum(fAVecs[f] .* cellCenterVec) / 3)
        subPyrCentroid = 0.75.*faceCentroids[f] .+ 0.25.*gC

        vol += subPyrVol
        centroid += subPyrCentroid .* subPyrVol
    end

    centroid /= vol

    return vol, centroid
end

# For each face, calculates the vectors from its owner and neighbour cell centers to its own center
function cellCentroidToFaceVec(faceCentroids::Array{Array{Float64, 1}}, cellCentroids::Array{Array{Float64, 1}})
    #TODO: Understand ordering to make sure we're accepting faces in the order we need to pass them back in??
    nFaces = size(faceCentroids, 1)

    cellToFaceVec = zeros(nFaces, 3)

    for f in 1:nFaces
        cellToFaceVec[f,:] = faceCentroids[f] - cellCentroids
    end

    return cellToFaceVec
end

######################### Utility Functions ###########################
# Returns basic info about the mesh: nCells, nFaces, nBoundaries, nBdryFaces
function unstructuredMeshInfo(mesh::Mesh)
    nCells = size(mesh.cells, 1)
    nFaces = size(mesh.faces, 1)
    nBoundaries = size(mesh.boundaryFaces, 1)

    # Count boundary faces
    nBdryFaces = 0
    for bdry in 1:nBoundaries
        nBdryFaces += size(mesh.boundaryFaces[bdry], 1)
    end

    return nCells, nFaces, nBoundaries, nBdryFaces
end

# Checks whether a string represents a number (using a regular expression)
function isNumber(str)
    re = r"^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)$"
    return occursin(re, str)
end

######################### Parse OpenFOAM Meshes ###########################
#=
    In the points, faces, owner, and neighbour files, the beginning of useful information in the file looks like this:

        ...
        object      faces;
        }
        // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

        5135
        (
        4(79 1516 1380 37)
        4(1381 1422 1249 39)
        ...

    The 5135 indicates how many lines of information follow it.
    This function would return 5135, and the line number of the first piece of information in the file ((line number of 5135) + 2)
=#
function OFFile_FindNItems(fileLines)
    nLines = size(fileLines, 1)
    lineCounter = 1
    itemCount = 0
    startLine = 0
    for line in fileLines
        if isNumber(strip(line))
            itemCount = parse(Int64, line)
            startLine = lineCounter+2
            break
        end
        lineCounter += 1
    end
    return startLine, itemCount
end

#=
    Takes the file path to an OpenFOAM mesh 'points' file, returns a 2D array of point coordinates:
            X   Y   Z
        P1  x1  y1  z1
        P2  x2  y2  z2
        ...
=#
function readOFPointsFile(filePath)
    f = open(filePath)
    pointsLines = readlines(f)
    startLine, pCount = OFFile_FindNItems(pointsLines)

    points = zeros(pCount, 3)
    for line in startLine:(startLine+pCount-1)
        pLine = pointsLines[line]

        bracketsRemoved = pLine[2:end-1]
        coords = split(bracketsRemoved)
        for d in 1:3
            points[line+1-startLine, d] = parse(Float64, coords[d])
        end
    end

    return points
end

#=
    Takes the file path to an OpenFOAM mesh 'faces' file, returns a 1D array of 1D arrays.
    Each entry in the upper level array represents a single face
    Each face-array contains the indices of the points that make up that face
    Note that all indices from the file are incremented by 1, to switch from OpenFOAM's 0-based indexing to Julia's 1-based indexing

    Example return value:
        [ [face1_pt1Index, face1_pt2Index, face1_pt3Index], [face2_pt1Index, face2_pt2Index, face2_pt3Index, face2_pt4Index], ... ]
=#
function readOFFacesFile(filePath)
    f = open(filePath)
    facesLines = readlines(f)
    startLine, fCount = OFFile_FindNItems(facesLines)

    faces = []
    for line in startLine:(startLine+fCount-1)
        fLine = facesLines[line]
        facePts = []

        bracketIndex = findfirst("(", fLine)[1]
        bracketsRemoved = fLine[bracketIndex+1:end-1]
        ptNumbers = split(bracketsRemoved)
        nPts = length(ptNumbers)
        for p in 1:nPts
            # Add one to convert to 1-based indexing
            push!(facePts, parse(Int64, ptNumbers[p])+1)
        end
        push!(faces, facePts)
    end

    return faces
end

#=
    Takes the file path to an OpenFOAM 'owner' file, returns a 1D array of owner cell indices
    Example return value:
        [ face1_ownerCellIndex, face2_ownerCellIndex, ...  ]
=#
function readOFOwnerFile(filePath)
    f = open(filePath)
    ownerLines = readlines(f)
    startLine, oCount = OFFile_FindNItems(ownerLines)

    faceOwnerCells = zeros(Int64, oCount)
    for line in startLine:(startLine+oCount-1)
        # Add one to convert to 1-based indexing
        faceOwnerCells[line-startLine+1] = parse(Int64, ownerLines[line])+1
    end

    return faceOwnerCells
end

#=
    Takes the file path to an OpenFOAM 'neighbour' file, returns a 1D array of neighbour cell indices
    Note that all indices from the file are incremented by 1, to switch from OpenFOAM's 0-based indexing to Julia's 1-based indexing

    Example return value:
        [ face1_neighbourCellIndex, face2_neighbourCellIndex, ...  ]
=#
function readOFNeighbourFile(filePath)
    f = open(filePath)
    neighbourLines = readlines(f)
    startLine, nCount = OFFile_FindNItems(neighbourLines)

    faceNeighbourCells = zeros(Int64, nCount)
    for line in startLine:(startLine+nCount-1)
        # Add one to convert to 1-based indexing
        faceNeighbourCells[line-startLine+1] = parse(Int64, neighbourLines[line])+1
    end

    return faceNeighbourCells
end

#=
    Returns index of first line containing str
    Otherwise returns -1
=#
function findInLines(str, lines, startLine)
    nLines = size(lines, 1)
    for i in startLine:nLines
        if occursin(str, lines[i])
            return i
        end
    end
    return -1
end

#=
    Takes the file path to an OpenFOAM 'boundary' file, returns 3 arrays:
    boundaryNames:          Array of boundary names (strings)
    boundaryNumFaces:       Array containing the number of faces in each boundary
    boundaryStartFaces:     Array containing the index of the first face in the boundary (boundaries always occupy a continuous sequence of face indices, and are usually numbered last)
=#
function readOFBoundaryFile(filePath)
    f = open(filePath)
    bLines = readlines(f)
    startLine, bCount = OFFile_FindNItems(bLines)
    nLines = size(bLines, 1)

    boundaryNames = Array{String, 1}(undef, bCount)
    boundaryStartFaces = Array{Int64, 1}(undef, bCount)
    boundaryNumFaces = Array{Int64, 1}(undef, bCount)
    for i in 1:bCount
        bNameLine = findInLines("{", bLines, startLine)-1
        boundaryNames[i] = strip(bLines[bNameLine])

        bNFacesLine = findInLines("nFaces", bLines, startLine)
        boundaryNumFaces[i] = parse(Int64, split(bLines[bNFacesLine])[2][1:end-1])

        bStartFaceLine = findInLines("startFace", bLines, startLine)
        boundaryStartFaces[i] = parse(Int64, split(bLines[bStartFaceLine])[2][1:end-1])+1

        startLine = findInLines("}", bLines, startLine)+1
    end

    return boundaryNames, boundaryNumFaces, boundaryStartFaces
end

#=
    Input: Path to an OpenFOAM mesh FOLDER.
    Output: Calls readOFPoints/Faces/Owner/NeighbourFile and returns all of their results (reads all OpenFOAM's mesh file data into arrays)
=#
function readOpenFOAMMesh(polyMeshPath)
    println("Reading mesh: $polyMeshPath")
    pointsFilePath = "$polyMeshPath/points"
    points = readOFPointsFile(pointsFilePath)
    facesFilePath = "$polyMeshPath/faces"
    faces = readOFFacesFile(facesFilePath)
    ownerFilePath = "$polyMeshPath/owner"
    owner = readOFOwnerFile(ownerFilePath)
    neighbourFilePath = "$polyMeshPath/neighbour"
    neighbour = readOFNeighbourFile(neighbourFilePath)
    boundaryFilePath = "$polyMeshPath/boundary"
    boundaryNames, boundaryNumFaces, boundaryStartFaces = readOFBoundaryFile(boundaryFilePath)

    return points, faces, owner, neighbour, boundaryNames, boundaryNumFaces, boundaryStartFaces
end

#=
    Function used to find all the points in each cell.
    Not required for CFD, which is face-based, but required for cell-based .vtk file output.

    Input: Path to an OpenFOAM mesh FOLDER.
    Returns:
        points:         Array of points obtained from readOFPoints()
        cellPtIndices:  Array of arrays, where each subarrary represents a cell, and each entry in a cell's array is the index of one of its points
=#
function OpenFOAMMesh_findCellPts(polyMeshPath)
    pointLocations, pointIndicesByFace, faceOwnerCellIndices, faceNeighborCellIndices, boundaryNames, boundaryNumFaces, boundaryStartFaces = readOpenFOAMMesh(polyMeshPath)
    nCells = maximum(faceOwnerCellIndices)
    nFaces = size(pointIndicesByFace, 1)
    nBoundaries = size(boundaryNames, 1)

    # Output
    cells = Array{Cell, 1}(undef, nCells)
    for i in eachindex(cells)
        cells[i] = Cell(Vector{Int64}(undef, 0), Vector{Int64}(undef, 0))
    end

    ### Populate the cells array ###
    function addCellFaceIndices(adjacentCellList)
        nFaces = length(adjacentCellList)
        for f in 1:nFaces
            ownerCellIndex = adjacentCellList[f]
            push!(cells[ownerCellIndex].faceIndices, f)
        end
    end

    # Start by a just adding face indices, want to order the points appropriately before adding them
    addCellFaceIndices(faceOwnerCellIndices)
    addCellFaceIndices(faceNeighborCellIndices)

    function addAllNewPoints!(cell::Cell, faceIndex)
        for pointIndex in pointIndicesByFace[faceIndex]
            if !any(x->x==pointIndex, cell.pointIndices)
                push!(cell.pointIndices, pointIndex)
            end
        end
    end

    function disjoint(f1Points, f2Points)
        allPoints = vcat(f1Points, f2Points)
        pointSet = Set(allPoints)
        return length(allPoints) == length(pointSet)
    end

    function intersection(points1, points2)
        return intersect(Set(points1), Set(points2))
    end

    function index(element, array)
        for i in eachindex(array)
            if element == array[i]
                return i
            end
        end
        return -1
    end

    function addEdges!(cellPoints, facePoints, endFacePoints, endFacePointsSet, unusedFaces, pointOffset=4)
        oppositeFaceIndex = -1
        for i in 1:length(unusedFaces)
            fi = unusedFaces[i]
            fiPoints = pointIndicesByFace[fi]
            commonPoints = intersection(fiPoints, facePoints)

            if length(commonPoints) == 2
                # These two points form an edge connecting faces one and two
                points = collect(commonPoints)

                if points[1] in endFacePointsSet
                    p1 = points[1]
                    p2 = points[2]
                else
                    p1 = points[2]
                    p2 = points[1]
                end

                position = index(p1, endFacePoints) # Find position over the point on the face one side
                if position == -1
                    throw(ErrorException("Point $p1 not found in $endFacePoints"))
                end
                cellPoints[ position + pointOffset ] = p2 # The point on the face two side goes in the matching spot
            else
                oppositeFaceIndex = i # This is the face opposite to the one containing facePoints
            end
        end
        return oppositeFaceIndex
    end

    function populatePointIndices_Tet!(cell::Cell)
        # For a tetrahedron, the order of points is unimportant
        # And all points from an arbitrary face
        addAllNewPoints!(cell, cell.faceIndices[1])

        # Choose another face, it will contain the fourth point we need to complete the tetrahedron
        addAllNewPoints!(cell, cell.faceIndices[2])
    end

    function populatePointIndices_Pyramid!(cell::Cell)
        # Pyramids need to have the points that make up their square base ordered 1-4, with the tip of the pyramid coming last
        # Find the quad face
        for faceIndex in cell.faceIndices
            if length(pointIndicesByFace[faceIndex]) == 4
                addAllNewPoints!(cell, faceIndex)

                # There is only one square face, and all other faces contain the pyramid tip
                # Find any other face and add its points to make the pyramid tip the final point
                if faceIndex != cell.faceIndices[1]
                    addAllNewPoints!(cell, cell.faceIndices[1])
                else
                    addAllNewPoints!(cell, cell.faceIndices[2])
                end

                # All done
                break
            end
        end
    end

    function populatePointIndices_Wedge!(cell::Cell)
        # Wedges need the triangular faces numbered 1-3 and 4-6 respectively, where 1 is aligned with 4, 2 with 5, and 3 with 6
        # Using similar strategy it to that used for hex cells: Identify two end faces, then use the faces that connect them to ensure their points are aligned

        # First identify the two triangular faces, keep track of one and get rid of the other
        unusedFaces = deepcopy(cell.faceIndices)
        t1 = -1 # Triangular face one, we will base the ordering off this one

        for j in 1:2
            for i in eachindex(unusedFaces)
                faceIndex = unusedFaces[i]

                if length(pointIndicesByFace[faceIndex]) == 3
                    # Found a triangular face
                    if t1 == -1
                        t1 = faceIndex
                    end
                    deleteat!(unusedFaces, i)
                    break
                end
            end
        end

        if length(unusedFaces) != 3
            throw(ErrorException("Expected three quadrilateral faces remaining after removal of two triangular end faces, got $(length(unusedFaces))"))
        end

        t1Points = pointIndicesByFace[t1]
        t1PointsSet = Set(t1Points)
        cell.pointIndices = vcat(t1Points, [0, 0, 0]) # Still have to determine proper ordering of points in the other triangular end face, 0's will be replaced below

        # Now all the remaining faces (3) will be quadrilaterals connecting the two triangular end faces
        q1 = pop!(unusedFaces) # Select one at random
        q1Points = pointIndicesByFace[q1]
        addEdges!(cell.pointIndices, q1Points, t1Points, t1PointsSet, unusedFaces, 3) # Use it to populate 2/3 of the unknown points

        q2 = pop!(unusedFaces) # Select another at random
        q2Points = pointIndicesByFace[q2]
        addEdges!(cell.pointIndices, q2Points, t1Points, t1PointsSet, unusedFaces, 3) # Use it to populate the last unknown point
    end

    function populatePointIndices_Hex!(cell::Cell)
        # Need to order point such that the points that make up a single quadrilateral face are ordered 1-4
            # The face opposite to it must have its points ordered 5-8, with point 1 connected to/aligned with point 5, 2 with 6, etc...
            # We will accomplish this by first picking a starting face at random, this face's points will be numbered 1-4
            # Then we will identify its opposite face and remove it
            # Then all the remaining faces form the connections between the first face we picked and its opposite Face
            # Pairs of these faces will contain 0 or 2 points in common
                # When zero, they are opposite each other
                # When two, they are adjacent and the two points they have in common form an edge connecting the two end faces
                    # This indicates that these two points should be aligned, and is used to properly order the remaining points

        # Start by choosing an arbitrary face
        unusedFaces = deepcopy(cell.faceIndices)
        f1 = pop!(unusedFaces)
        f1Points = pointIndicesByFace[f1]
        f1PointsSet = Set(f1Points)

        # Find the opposite face (no points in common) and remove it
        for i in 1:length(unusedFaces)
            fi = unusedFaces[i]
            fiPoints = pointIndicesByFace[fi]

            if disjoint(f1Points, fiPoints)
                deleteat!(unusedFaces, i)
                break
            end
        end

        ### Now find the correct orientation
            # Point f1_i needs to be aligned (spatially) with Point f2_i
            # We can check for this by using one of the other faces as a guide, since it forms part of the connection between f1 and f2
        cell.pointIndices = vcat(f1Points, [0, 0, 0, 0])

        # Pick an arbitrary other face
        f3 = pop!(unusedFaces)
        f3Points = pointIndicesByFace[f3]
        lastFace = addEdges!(cell.pointIndices, f3Points, f1Points, f1PointsSet, unusedFaces)

        # Add points from the edges of the face opposite f3
        lastFacePoints = pointIndicesByFace[unusedFaces[lastFace]]
        deleteat!(unusedFaces, lastFace)
        noResult = addEdges!(cell.pointIndices, lastFacePoints, f1Points, f1PointsSet, unusedFaces)

        if noResult != -1
            throw(ErrorException("Failure to appropriately order hexahedral cell points for writing to .vtk"))
        end
    end

    # Now use the face indices to gather and order each cell's points appropriately
        # Correct point ordering is important for .vtk output: (See figure 2) https://vtk.org/wp-content/uploads/2015/04/file-formats.pdf
    for cell in cells
        nFaces = length(cell.faceIndices)
        if nFaces == 4
            populatePointIndices_Tet!(cell)
        elseif nFaces == 5
            quadFaceCount = 0

            for f in cell.faceIndices
                if length(pointIndicesByFace[f]) == 4
                    quadFaceCount += 1
                end
            end

            if quadFaceCount == 1
                populatePointIndices_Pyramid!(cell)
            elseif quadFaceCount == 3
                populatePointIndices_Wedge!(cell)
            else
                throw(ErrorException("Unrecognized cell type, cell: $cell. Expecting only hex, tet, wedge, or pyramid cells for vtk output."))
            end
        elseif nFaces == 6
            populatePointIndices_Hex!(cell)
        end
    end

    return pointLocations, cells
end

#=
    Reads an OpenFOAM mesh and returns a Mesh object suitable for calculations in FvCFD.jl.

    Mesh defined in dataStructures.jl, documented in dataStructureDefintions.md
=#
function OpenFOAMMesh(polyMeshPath)
    points, OFfaces, owner, neighbour, boundaryNames, boundaryNumFaces, boundaryStartFaces = readOpenFOAMMesh(polyMeshPath)
    nCells = maximum(owner)
    nFaces = size(OFfaces, 1)
    nBoundaries = size(boundaryNames, 1)

    cells = Array{Array{Int64, 1}, 1}(undef, nCells)                # Indices of faces that make up a cell
    cVols = zeros(nCells)                                           # Cell Volumes
    cCenters = Array{Array{Float64, 1}, 1}(undef, nCells)           # Cell Centroids
    cellSizes = Matrix{Float64}(undef, nCells, 3)                   # Cell sizes (x,y,z) direction

    faces = Array{Array{Int64, 1}, 1}(undef, nFaces)                # Indices of cells adjacent to each face always owner first, then neighbour
    fAVecs = Array{Array{Float64, 1}, 1}(undef, nFaces)             # Face Area Vectors
    fCenters = Array{Array{Float64, 1}, 1}(undef, nFaces)           # Face Centroids
    boundaryFaces = Array{Array{Int64, 1}, 1}(undef, nBoundaries)   # Indices of faces that make up each boundary

    # Calculate face area vectors and face centroids from list of points in each face
    for f in 1:nFaces
        fPts = [ points[pt,:] for pt in OFfaces[f] ]
        fAVecs[f], fCenters[f] = faceAreaCentroid(fPts)
    end
    #fAVecs and fCenters now complete

    # Build 'cells' and 'faces' arrays from info from 'owner' file
    nOwners = size(owner, 1)
    for f in 1:nOwners
        ownerCell = owner[f]
        if isassigned(cells, ownerCell)
            push!(cells[ownerCell], f)
        else
            cells[ownerCell] = [f,]
        end

        faces[f] = [ownerCell, -1]
    end

    # Finish building 'cells' and 'faces' arrays, adding info from 'neighbour' file
    nNeighbours = size(neighbour, 1)
    for f in 1:nNeighbours
        neighbourCell = neighbour[f]
        if isassigned(cells, neighbourCell)
            push!(cells[neighbourCell], f)
        else
            cells[neighbourCell] = [f,]
        end

        faces[f][2] = neighbourCell
    end
    # fAVecs, fCenters, faces, cells now complete

    # For each cell, use the face area vectors and face centroids for all of its faces, to calculate it's volume and centroid location
    for c in 1:nCells
        pts = []
        for f in cells[c]
            for pt in OFfaces[f]
                if !(points[pt,:] in pts)
                    push!(pts, points[pt,:])
                end
            end
        end
        fCs = [ fCenters[f] for f in cells[c] ]
        cell_fAVecs = [ fAVecs[f] for f in cells[c] ]

        cVols[c], cCenters[c] = cellVolCentroid(pts, cell_fAVecs, fCs)
    end
    # fAVecs, fCenters, faces, cells, cVols, cCenters now complete

    # Create boundaryFaces array
    for b in 1:nBoundaries
        startF = boundaryStartFaces[b]
        endF = startF + boundaryNumFaces[b] - 1
        boundaryFaces[b] = Array(startF:endF)
    end

    # Compute cell sizes in the x, y, z directions
    # In the past, cell sizes were used for a shitty CFL calculation, may be able to get rid of them now
    maxCoords = [ -1000000.0, -1000000.0, -1000000.0 ]
    minCoords = [ 1000000.0, 1000000.0, 1000000.0 ]
    for c in 1:nCells
        fill!(maxCoords, -10000000)
        fill!(minCoords, 10000000)

        pts = Vector{Vector{Float64}}()

        # Add points
        for f in cells[c]
            for pt in OFfaces[f]
                if !any(x->x==pt, pts)
                    push!(pts, points[pt, :])
                end
            end
        end

        for pt in pts
            for d in 1:3
                maxCoords[d] = max(maxCoords[d], pt[d])
                minCoords[d] = min(minCoords[d], pt[d])
            end
        end

        for d in 1:3
            cellSizes[c,d] = maxCoords[d] - minCoords[d]
        end
    end

    return Mesh(cells, cVols, cCenters, cellSizes, faces, fAVecs, fCenters, boundaryFaces)
end


"""
    Cylinder(x, y, zmin, zmax, radius)

Create a `Cylinder` that can be used for spatial lookup.

See also `AbstractRegion`
"""
struct Cylinder{T <: Real} <: AbstractRegion{T}
    x::T
    y::T
    zmin::T
    zmax::T
    radius::T
end

"""
    volume(cylinder::Cylinder)

Returns the volume of the `cylinder`.
"""
@inline function volume(cylinder::Cylinder)
    π * cylinder.radius * cylinder.radius * (cylinder.zmax - cylinder.zmin)
end

function boundingbox(cylinder::Cylinder{T}) where T
    xmin = cylinder.x - cylinder.radius
    xmax = cylinder.x + cylinder.radius
    ymin = cylinder.y - cylinder.radius
    ymax = cylinder.y + cylinder.radius
    return BoundingBox{T}(xmin, ymin, cylinder.zmin, xmax, ymax, cylinder.zmax)
end

function in(p::StaticVector{3, <:Real}, cylinder::Cylinder)
    r² = cylinder.radius * cylinder.radius
    Δx = p[1] - cylinder.x
    Δy = p[2] - cylinder.y
    return Δx*Δx + Δy*Δy <= r² && p[3] >= cylinder.zmin && p[3] <= cylinder.zmax
end


#
# asian-option.jl
#
function run-asian(N = 100000, PutCall = 'C';)
# European Asian option.
# Uses geometric or arithmetic average.
# Euler and Milstein discretization for Black-Scholes.
# Option features.
  println("Setting option parameters");
  S0  = 100;      # Spot price
  K   = 100;      # Strike price
  r   = 0.05;     # Risk free rate
  q   = 0.0;      # Dividend yield
  v   = 0.2;      # Volatility
  tma = 0.25;     # Time to maturity

  Averaging = 'A';  # 'A'rithmetic or 'G'eometric
  OptType = (PutCall == 'C' ? "CALL" : "PUT");
  println("Option type is $OptType");
# Simulation settings.
  println("Setting simulation parameters");
  T = 100;         # Number of time steps
  dt = tma/T;      # Time increment

# Initialize the terminal stock price matrices
# for the Euler and Milstein discretization schemes.
 S = zeros(Float64,N,T);
  for n=1:N
    	S[n,1] = S0;
  end

# Simulate the stock price under the Euler and Milstein schemes.
# Take average of terminal stock price.
  println("Looping $N times.");
  A = zeros(Float64,N);
  for n=1:N
    	for t=2:T
      		dW = (randn(1)[1])*sqrt(dt);
     		 z0 = (r - q - 0.5*v*v)*S[n,t-1]*dt;
     		 z1 = v*S[n,t-1]*dW;
      		z2 = 0.5*v*v*S[n,t-1]*dW*dW;
      		S[n,t] = S[n,t-1] + z0 + z1 + z2;
   	 end
   	 if cmp(Averaging,'A') == 0
      		A[n] = mean(S[n,:]);
   	 elseif cmp(Averaging,'G') == 0
      		A[n] = exp(mean(log(S[n,:])));
   	 end
  end

# Define the payoff
  P = zeros(Float64,N);
  if cmp(PutCall,'C') == 0
    	for n = 1:N
      		P[n] = max(A[n] - K, 0);
  	  end
  elseif cmp(PutCall,'P') == 0
    	for n = 1:N
     		 P[n] = max(K - A[n], 0);
   	 end
  end
# Calculate the price of the Asian option
 AsianPrice = exp(-r*tma)*mean(P);
 @printf "Price: %10.4f\n" AsianPrice;
end

run-asian()


@doc raw"""
    ℛ(z::Number, tableau::AbstractButcherTableau) :: Number
    ℛ(z::Number, solver::AbstractRungeKuttaSolver) :: Number

returns the stability function of an [`AbstractRungeKuttaSolver`](@ref):
```math
    R(z) = \frac{\det(I - z(A - \mathbb{1}b^\intercal))}{\det(I - zA)}.
```
"""
function ℛ(z::Number, tableau::AbstractButcherTableau)
    @↓ A, b, s = tableau
    e = ones(s)
    return det(I - z * (A - e * b')) / det(I - z * A)
end
ℛ(z::Number, solver::AbstractRungeKuttaSolver) = ℛ(z, solver.tableau)

"""
    ℛ(Z::AbstractMatrix, tableau::AbstractButcherTableau) :: AbstractMatrix
    ℛ(Z::AbstractMatrix, solver::AbstractRungeKuttaSolver) :: AbstractMatrix

returns the stability function of an [`AbstractRungeKuttaSolver`](@ref).
"""
function ℛ(Z::AbstractMatrix, tableau::AbstractButcherTableau)
    @↓ A, b, s = tableau
    e = ones(s)
    tmp = kron(e, Z)
    tmp = (I - kron(A, Z)) \ tmp
    tmp = kron(b', Matrix(1.0I, size(Z)...)) * tmp
    return I + tmp
end
ℛ(Z::AbstractMatrix, solver::AbstractRungeKuttaSolver) = ℛ(Z, solver.tableau)


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, DataFrames, Plots
	Random.seed!(0)
end;

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing6.14"

# ╔═╡ 1aa41898-3afa-11eb-2f49-570853ce6435
begin
	lambda = 0.1
	dist = Exponential(1/lambda)
	actualMedian = median(dist)

	M = 10^3
	N = 10^4
	nRange = 2:2:10
	alpha = 0.05
end

# ╔═╡ fba0be84-3b2f-11eb-1e0f-5f8a1f264dba
begin
	df = DataFrame()
	for n in nRange
		coverageCount = 0
		for _ in 1:M
			sampleData = rand(dist, n)
			bootstrapSampleMeans = [median(rand(sampleData, n)) for _ in 1:N]
			L = quantile(bootstrapSampleMeans, alpha/2)
			U = quantile(bootstrapSampleMeans, 1-alpha/2)
			coverageCount += L < actualMedian && actualMedian < U
		end
		append!(df, DataFrame(:n => n, :coverage => coverageCount/M))
	end
	df
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing6.14"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═1aa41898-3afa-11eb-2f49-570853ce6435
# ╠═fba0be84-3b2f-11eb-1e0f-5f8a1f264dba
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# ╔═╡ bca1c92c-8561-11eb-0420-6f0123bacb7f
begin
	# dependencies
	using Plots
	using LightGraphs, SimpleWeightedGraphs, LinearAlgebra, Random
	import StatsBase:sample
end

# ╔═╡ 774fa124-8560-11eb-2317-53b0d106b67a
md"""
# Local Search
"""

# ╔═╡ 2b5e6876-8561-11eb-0685-dfe1cca39c45
md"""
## Simulated annealing
Simulated annealing is similar to hill climbing, but it allows downhill moves. There are several aspects to consider.
* What is considered a neighbor? And how to chose them?
* How to evaluate the energy (i.e. the fitness)?
* How to let the temperature evolve. Several options exist here:
    - exponential decrease: $T = T_0(1-\alpha)^k$, where $\alpha$ is the ‘cooling rate’ and $k$ is the iteration
    - fast decrease: $T = T_0/k$
    - boltzmann decrease: $T = T_0/log(k)$
"""

# ╔═╡ 4e190a2e-8561-11eb-3759-795e3bb4d8d2
md"""
To illustrate the principle, let's try to find the maximum of
\\[ f(x) = -\dfrac{1}{4}x^4 +  \dfrac{7}{3} x^3 - 7x^2 + 8x \\]
in the interval $[0, 5]$. You can observe that there is a global maximum in $x=4$, however, there is a local maximum in $x=1$.
"""

# ╔═╡ b49f0528-8561-11eb-36a7-fd42574edc81
begin
	f = x-> -1/4*x^4 + 7/3 *x^3 - 7*x^2 + 8*x
	x = range(0,5, length=100);
	plot(x,f.(x),label="",size=(300,200))
end

# ╔═╡ d5019d8a-8561-11eb-2a43-a9ce616532f5
begin
	"""
	Evaluate the fitness of a specific point
	"""
	function fitness(x::Float64,f::Function)
		return f(x)
	end

	"""
	Determine successors for a given location (with a fixed step size)
	You could include automatic step size modification if you want.
	"""
	function successors(x::Float64; step=0.1)
		return x .+ step*[-1; 1]
	end

	"""
	mapping of time to temperature. You might make this a wrapper function for
	different cooling schemes. Currently only has exponential cooling.
	"""
	function scheduletemp(t::Int64;T0::Float64=273.0, α::Float64=0.005)
		return T0*(1-α)^t
	end

	"""
		simulated_annealing(args; kwargs)

	Simulated annealing implementation for a 1D problem.

	Since this is stochastic (sort of a random walk), it will not always reach the optimal value, therefore, one typically runs several iterations and the best one is retained.

	The probabiliy of accepting a value that lead to a decrease in energy is given by exp(ΔE/T), which is between 0 (temp = 0) and one (ΔE for any T ≠ 0).

	In the literature you might also find  1/(1 + exp(ΔE/T)) ∈ [1/2, 1] as a treshold.

	arguments:
	- x0: starting point
	keywords:
	- tmax: maximum iterations
	- T0: initial temperature
	- α: cooling rate
	- fitness: fitness function
	- optimfun: the function we want to optimize
	"""
	function simulated_annealing(x0;
				tmax=1000, T0::Float64=273.0, α::Float64=0.005,
				fitness::Function=fitness, optimfun::Function=f)
		current = x0
		next = x0
		T = T0
		best = x0
		for t = 1:tmax
			T = scheduletemp(t,T0=T0,α=α)
			if T == 0
				return current
			else
				next = rand(successors(current))
				ΔE = fitness(next, optimfun) - fitness(current, optimfun)
				if ΔE > 0
					current = next
				else
					if rand() < exp(ΔE/T)
						current = next
					end
				end
				if fitness(current, optimfun) > fitness(best, optimfun)
					best = current
				end
			end
		end
		return best
	end

	"""
		SAsearch

	Wrapper function for multiple SA iterations. Returns the best n found value for N runs of simulated annealing.
	"""
	function SAsearch(x0, n::Int64=1, N::Int64=10;kwargs...)
		return sort!([simulated_annealing(x0; kwargs...) for _ in 1:N],
					 by=x->kwargs[:fitness](x,kwargs[:optimfun]),
					 rev=true)[1:n]
	end
end

# ╔═╡ 9c2e2032-8568-11eb-103f-fdc57c11ab1f
md"""
### Solving the TSP problem with simulated annealing
Use simulated annealing to solve the travelling saleman problem again. You will need to think in particalar about:
* an appropriate inital solution (e.g. a tour by using a nearest neigbor approach or purely random)
* generating appropriate neigbors (e.g. by flipping a subpart of a tour)
"""

# ╔═╡ c84f6af2-8568-11eb-06c4-53f7664294ac
begin
	"""
		plotgraph(g::SimpleWeightedGraph, d::Dict,draw_MST::Bool=true,wf::Real=5)

	Visualize the network with its edges and their weights

	parameters:
		- g : SimpleWeightedGraph
		- d : Dict
			dictionary that links a node to a coordinate pair
		- draw_MST : Bool
			if true, also draws the MST on the figure
		- wf : Real
			multiplication factor for the edge weigth to obtain a nicer visualisation
	"""
	function plotgraph(g::SimpleWeightedGraph, d::Dict;wf::Real=5,draw_MST::Bool=true)
		p = plot(legend=:bottomright)
		for e in edges(g)
			plot!([d[e.src][1], d[e.dst][1]],[d[e.src][2], d[e.dst][2]],
				markershape=:circle, linewidth=e.weight*wf, color=:black, markeralpha = 0.5,
				linealpha=0.2, label="", markercolor=:black, markersize=5)
		end
		plot!([],[],markershape=:circle, color=:black,
				linealpha=0.2, label="Full network", markercolor=:black, markersize=5, markeralpha = 0.5)
		if draw_MST
			MST = prim_mst(g, weights(g))
			for e in MST
				plot!([d[e.src][1], d[e.dst][1]],[d[e.src][2], d[e.dst][2]],
						markershape=:circle, linewidth=weights(g)[e.src,e.dst]*wf, color=:red,
						linealpha=0.8, label="", linestyle=:solid, markeralpha = 0)

			end
			plot!([],[],linealpha=0.8, label="MST", color=:red, linestyle=:solid)
		end
		xlims!(0,1)
		ylims!(0,1)
		plot!(legend=:best)
		return p
	end

	"""
		plotsol(fp::Array{Int64,1}, nodedict::Dict)

	Function that plots the final TSP solution
	"""
	function plotsol(fp::Array{Int64,1}, nodedict::Dict)
		p = plot()
		for i in 1:length(fp)-1
			x = [nodedict[fp[i]][1]; nodedict[fp[i+1]][1]]
			y = [nodedict[fp[i]][2]; nodedict[fp[i+1]][2]]
			plot!(x,y,color=:blue,label="", marker=:circle)
		end
		# close loop
		x = [nodedict[fp[1]][1]; nodedict[fp[end]][1]]
		y = [nodedict[fp[1]][2]; nodedict[fp[end]][2]]
		plot!(x,y,color=:red,label="", marker=:circle)
		xlims!(0,1)
		ylims!(0,1)
		return p
	end

	"plot functions"
end

# ╔═╡ 46e468e0-8569-11eb-0a30-d11e814ccf7e
begin
	"""
		problemgenerator(n::Int)

	generate n points in the unit square
	"""
	function problemgenerator(n::Int)
		return [(rand(), rand()) for _ in 1:n]
	end

	"""
		distancematrix(x::Array{Tuple{Float64,Float64},1}, p::Real=1)

	Generate distancematrix between all points.

	x is an array [n x 1] and p is the L_p-norm you want to use for the distancematrix
	"""
	function distancematrix(x::Array{Tuple{Float64,Float64},1}, p::Real=1)
		n = length(x)
		A = zeros(n,n)
		for (i,j) in collect(Base.Iterators.product(1:n, 1:n))
			A[i,j] = norm(x[i] .- x[j], p)
		end
		return A
	end
end

# ╔═╡ 7c5282fa-8569-11eb-0dcf-abc7b3b795ef
begin
	N = problemgenerator(15) # nodes
	nodedict = Dict( i => N[i] for i in 1:length(N)) # nodes dict
	A = distancematrix(N,2) # distance matrix using manhattan norm
	G = SimpleWeightedGraph(A) # graph
	p = plotgraph(G,nodedict;draw_MST=false) # layout of the current graph
end

# ╔═╡ eb27eada-8569-11eb-0739-7717087a5240
begin
	"""
		finit(A::Array{Float64,2})

	Generate an initial distribution based and a random order
	"""
	function finit(A::Array{Float64,2})
		return shuffle(1:size(A,1))
	end

	finit(A)
end

# ╔═╡ 1692c654-856a-11eb-1b02-77b0ab027c29
begin
	"""
		fitness(T::Array{Int64,1},A::Array{Float64,2})

	Given a tour, compute the tour distance based on the distance matrix
	"""
	function fitness(T::Array{Int64,1},A::Array{Float64,2})
		return -sum([A[T[i],T[i+1]] for i in 1:length(T)-1]) - A[T[1],T[end]]
	end

	fitness(finit(A),A)
end

# ╔═╡ 3c58cc1a-856a-11eb-17bd-a31c244f6229
begin
	"""
	Determine successors for a given tour (simple random permutation of partial tour)
	"""
	function successors(T::Array{Int64,1}; nsuc::Int64=5)
		res = Array{Array{Int64,1},1}()
		for _ in 1:nsuc
			i,j = sample(1:length(T),2,replace=false,ordered=true)
			cand = copy(T)
			cand[i:j] = shuffle(T[i:j])
			push!(res, cand)
		end

		return res
	end
	successors(finit(A))
end

# ╔═╡ 7ab96168-856c-11eb-166a-b7531078ac90
begin
	"""
		simulated_annealing(args; kwargs...)

	Simulated annealing implementation for TSP

	Since this is stochastic (sort of a random walk), it will not always reach the optimal value, therefore, one typically runs several iterations and the best one is retained.

	The probabiliy of accepting a value that lead to a decrease in energy is given by exp(ΔE/T). In the literature you might also find  1/(1 + exp(ΔE/T)) ∈ [1/2, 1] as a treshold.
	"""
	function simulated_annealing(A::Array{Float64,2}, init::Array{Int64,1};
								tmax=1000, T0::Float64=273.0, α::Float64=0.005,
								fitness::Function=fitness,nsuc::Int64=10, kwargs...)
		current = init
		next = init
		T = T0
		best = init
		for t = 1:tmax
			T = scheduletemp(t,T0=T0)
			if T == 0
				return current
			else
				next = rand(successors(current, nsuc=nsuc))
				ΔE = fitness(next, A) - fitness(current, A)
				if ΔE > 0
					current = next
				else
					if rand() < exp(ΔE/T)
						current = next
					end
				end
				if fitness(current, A) > fitness(best, A)
					best = current
				end
			end
		end
		return best
	end

	"""
	wrapper function for multiple SA iterations

	Returns the best n found value for N runs of simulated annealing.
	"""
	function SAsearch(A::Array{Float64,2}, n::Int64=1, N::Int64=10;kwargs...)
		sort!([simulated_annealing(A, finit(A); kwargs...) for _ in 1:N],
				by=x->kwargs[:fitness](x,A),rev=false)[1:n]
	end

	simulated_annealing(A, finit(A))
end

# ╔═╡ 934aa9e4-856e-11eb-1924-fd1df8fd3fe2
md"""
## Genetic algorithms
a genetic algorithm (GA) is inspired by the process of natural selection. In order to implement this you need the following items:
* an initial population (e.g. random)
* a fitness function that allows you to rank the population members
* a selection method (e.g. roulette based proportions, ‘elite selection’)
* a cross-over method (cutting; 50/50 per gene etc.)
* mutation rate and method
* \\(\dots\\)
"""

# ╔═╡ 8753fd9a-856f-11eb-2d91-13657e302031
md"""
### Toy example
Let's try write a message by using a genetic algorithm. E.g "172 Pol evolved!" starting from a random set of characters (a-zA-Z0-9. !)
"""

# ╔═╡ 48c05e06-8570-11eb-0367-1f3db0aa20d0
begin
	chars = vcat(65:90,97:122, 48:57,[Int(' ');Int('.'); Int('!')])
	const genes = [Char(i) for i = chars]
	const pop_size = 100
	const goal = collect("172 Pol evolved!");
end

# ╔═╡ 6faf987e-8570-11eb-1861-7de0f661bd63
begin
	function fitness(p::Array{Char,1}; goal=goal)
		sum(p.==goal)
	end

	testcase = rand(genes,length(goal))
	testcase, fitness(testcase)
end

# ╔═╡ 728b94da-8566-11eb-1b02-e94c64424845
begin
	options = Dict(	:tmax=>1000, :T0=>273.0, :α=>0.005,
               		:fitness=>fitness, :optimfun=>f)
	x0 = 0.0;
	simulated_annealing(x0; options...)
end

# ╔═╡ a72d7c3c-8569-11eb-36a3-cb68ab2622cf
let
	n = 10;
	res = SAsearch(x0,n;options...)
end

# ╔═╡ 0ce6432c-8567-11eb-1465-432d66051360
begin
	n = 10;
	res = SAsearch(x0,n;options...)
	plot(x,f.(x),label="")
	scatter!([x0],[f.(x0)],label="start",marker=(:circle, :green),size=(300,200), legend=:bottom)
	scatter!([res],[f.(res)],label="optimal values",marker=(:circle, :red),size=(300,200), legend=:bottom)
end

# ╔═╡ ef64ca86-856c-11eb-0cd0-178565e85576
let
	options = Dict(:tmax=>1000, :T0=>273.0, :α=>0.005, :fitness=>fitness)
	@show res = SAsearch(A; options...)[1];
	init = finit(A)
	println("Cost of a random trip: $(-fitness(init,A))")
	println("Cost of a optimised trip: $(-fitness(res,A))")
	plotsol(res, nodedict)
	title!("TSP solution using simulated annealing")
end

# ╔═╡ 57726a64-856d-11eb-2b37-877202c47ede
let
	# Larger example
	n = 50 # number of nodes
	N = problemgenerator(n) # nodes
	options = Dict(:tmax=>1000, :T0=>273.0, :α=>0.005, :fitness=>fitness)
	nodedict = Dict( i => N[i] for i in 1:length(N)) # nodes dict
	A = distancematrix(N,2) # distance matrix using manhattan norm
	init = finit(A)

	@time res = SAsearch(A; options...)[1]
	println("Cost of a random trip: $(-fitness(init,A))")
	println("Cost of a optimised trip: $(-fitness(res,A))")
	plotsol(res, nodedict)
end

# ╔═╡ d70478c8-856d-11eb-3f27-0b565955b6ed
let
	# trajectory cost distribution
	res = Array{Float64,1}()
	for _ in 1:50
		push!(res, -fitness(SAsearch(A;options...)[1], A))
	end
	histogram(res, normalize=:pdf, alpha=0.5,label="PDF optimised tour cost")
	scatter!([-fitness(finit(A),A)],[0],label="initial non-optimised cost")
	xlabel!("Tour cost")
	ylabel!("PDF")
	title!("Histogram for tour cost ($(size(A,1)) cities)")
end

# ╔═╡ 9cc2fa40-8570-11eb-2eba-c15433dcb90e
begin
	"""
		mate(p1,p2; pmut, genes)

	mating function to obtain children. includes both crossover and mutation.
	"""
	function mate(p1::Array{Char,1}, p2::Array{Char,1}; pmut=0.1, genes=genes)
		child = typeof(p1)() # empty child
		# walk over genome sequence
		for i in 1:length(p1)
			p = rand()
			if p < (1-pmut)/2
				# parent 1 wins
				push!(child, p1[i])
			elseif p < 1-pmut
				# parent 2 wins
				push!(child, p2[i])
			else
				# mutation
				push!(child, rand(genes))
			end
		end
		return child
	end

	# demo
	p1 = ['a';'b';'c']
	p2 = ['d';'e';'f']
	mate(p1,p2, pmut=1)
end

# ╔═╡ 47153f8e-8572-11eb-1310-c932bd478721
md"""
### Some ideas for crossover:
Below you can find some algorithms that have proven to work quite well for this type of application. Both of these are described and an example and an implementation is provided. You are also invited to come up with a method yourself if you think you have a good idea.

#### The partially mapped crossover
After choosing two random cut points on parents to build offspring, the portion between cut points, one parent’s string is mapped onto the other parent’s string and the remaining information is exchanged.

E.g. for parents $P_1$ and $P_2$ with cutting positions $c_1$ and $c_2$:
* ``P_1 = [3, 4, 8 | 2, 7, 1 | 6, 5 ]``
* ``P_` = [4, 2, 5 | 1, 6, 8 | 3, 7 ]``
We get the following mappings:
* ``m_{1 \rightarrow 2}: \{2 \rightarrow 1, 7 \rightarrow 6, 1 \rightarrow 8\}``
* ``m_{2 \rightarrow 1}: \{1 \rightarrow 2, 6 \rightarrow 7, 8 \rightarrow 1\}``
which leads to the following initial offspring:
* ``O_1 = [0, 0, 0 | 1, 6, 8 | 0, 0 ] ``
* ``O_2 = [0, 0, 0 | 2, 7, 1 | 0, 0 ] ``
we then add the non-conflicting values (i.e. the values not already present the offspring due do the crossover) from the original parents:
* ``O_1 = [3, 4, 0 | 1, 6, 8 | 0, 5 ]``
* ``O_2 = [4, 0, 5 | 2, 7, 1 | 3, 0 ] ``
For the remaining values, we (recursively) make use of the mappings using $m_{2 \rightarrow 1}$ for $P_1$ and $m_{1 \rightarrow 2}$ for $P_2$. This finally leads to the following offspring:
* ``O_1 = [3, 4, 2 | 1, 6, 8 | 7, 5 ] ``
* ``O_2 = [4, 8, 5 | 2, 7, 1 | 3, 6 ] ``

#### The cycle crossover
The Cycle Crossover operator identifies a number of so-called cycles between two parent chromosomes. To form Child 1, cycle 1 is copied from parent 1, cycle 2 from parent 2, cycle 3 again from parent 1, and so on. An example might help to better understand the principle of a cycle. E.g.
* ``P_1 = [1,2,3,4,5,6,7,8] ``
* ``P_2 = [8,5,2,1,3,6,4,7] ``
For the first cycle, we start at index 1 in $P_1$. Following the links from $P_1$ to $P_2$ we find the following mappings:
* ``1 \rightarrow 8`` with index 8
* ``8 \rightarrow 7`` with index 7
* ``7 \rightarrow 4`` with index 4
* ``4 \rightarrow 1`` with index 1
after this last mapping, we are back at the starting point, so our first cycle is complete and is $\{1,4,7,8 \}$.

The second cycle will start at the first index that is not yet present in a cycle. In this case the index is equal to 2. Following the links from $P_1$ to $P_2$ we find the following mappings
* ``2 \rightarrow 5`` with index 2
* ``5 \rightarrow 3`` with index 5
* ``3 \rightarrow 2`` with index 3

So our second cycle is equal to $\{2,3,5\}$.

The third an final cycle is equal to the only remaining part, i.e. $\{6 \}$

We have now determined all cycles. The children aren obtained as follows:
* ``O_1[1,4,7,8] =  P_1[1,4,7,8] ``
* ``O_1[2,3,5] = P_2[2,3,5] ``
* ``O_1[6]  = P_1[6] ``

This leads to $O_1 = [1,5,2,4,3,6,7,8]$. In a similar fashion, but inverting the role of $P_1$ and $P_2$ one finds $O_2 = [8,2,3,1,5,6,4,7]$.

A downside of this type of crossover is that it can sometimes lead to the offspring being identical to the parents. You can verify this for $[3,4,8,2,7,1,6,5]$ & $[4,2,5,1,6,8,3,7]$.

#### Your own method
Come up with something that seems adequate...

"""

# ╔═╡ 1f5d255c-8576-11eb-1463-89f94258168d
begin
	"""
		mate(p1,p2;method=partialcrossover, kwargs...)

	Function that handles the crossover for two different tours. The method keyword argument can be any of the functions you define as long as 1. the function exists 2. the function works in a similar way as the other ones (i.e. return two children from two parents.)

	Additional options for your own crossover implementation can be passed via the kwargs. You should document the options for each algorithm sperately.

	"""
	function mate(p1::Array{Int64,1},p2::Array{Int64,1}; method::Function=partialcrossover, kwargs...)
			return method(p1,p2; kwargs...)
	end

	"""
		partialcrossover(p1, p2; kwargs...)

	Implementation of the partial crossover function for TSP routing problems.
	"""
	function partialcrossover(p1::Array{Int64,1},p2::Array{Int64,1}; kwargs...)
		"""
			deconflicter!(O::Array{Int64,1}, mapping::Dict)

		Function that deals with the conflicts after the initial assignments.
		"""
		function deconflicter(O::Array{Int64,1},p::Array{Int64,1}, mapping::Dict)
			for i in 1:length(O)
				if O[i] == 0
					cand = p[i]
					while cand in O
						cand = mapping[cand]
					end
					O[i] = cand
				end
			end
			return O
		end

		# choose cut points
		c1, c2 = sort(sample(1:length(p1),2, replace=false))
		# initiate offspring
		O1, O2 = [zeros(Int64,length(p1)) for _ in 1:2]
		# determine mapping
		map12 = Dict(p1[i]=> p2[i] for i in c1:c2)
		map21 = Dict(value => key for (key, value) in map12)
		# set offspring initial values
		O1[c1:c2] = p2[c1:c2]
		O2[c1:c2] = p1[c1:c2]
		# fill non-conflicting positions
		for i in setdiff(1:length(p1),c1:c2)
			O1[i] = p1[i] in O1 ? 0 : p1[i]
			O2[i] = p2[i] in O2 ? 0 : p2[i]
		end
		# fill conflicting positions using the mapping
		O1, O2 = deconflicter(O1, p1, map21), deconflicter(O2, p2, map12)

		return O1, O2
	end

	"""
		cyclecrossover(p1, p2)

	Implementation of the cycle crossover function for TSP routing problems.
	"""
	function cyclecrossover(p1::Array{Int64,1},p2::Array{Int64,1})
		# initiate children
		o1=zeros(Int64,length(p1))
		o2=zeros(Int64,length(p2))

		# Pt. 1 - find all cycles

		# make hashtable: value => index
		h1 = Dict{Int64, Int64}(p1[i]=>i for i in 1:length(p1))
		h2 = Dict{Int64, Int64}(p2[i]=>i for i in 1:length(p2))
		# make hashtable to track usage in a cycle: index => bool
		u1 = Dict{Int64, Bool}(i=>false for i in 1:length(p1))
		# loop over parent creating cycles on the go
		# a cycle is stored in a Set because this is hashable. This is (much) faster than an array for ∉ check
		# the values stores in the cycles are the indices to be used
		cycles = Array{Set{Int64},1}()
		for i in 1:length(p1)
			if !u1[h1[p1[i]]] # if not used yet, start a new cycle
				cycle = Set{Int64}()
				next = h1[p1[i]] # add initial values
				while next ∉ cycle
					push!(cycle, next)  # update cycle
					u1[next] = true     # update tracking
					next = h1[p2[next]] # next location
				end
				push!(cycles, cycle)
			else
				continue
			end
		end

		# Pt. 2 - make children from cycles
		for i in 1:length(cycles)
			inds = collect(cycles[i])
			if isodd(i)
				o1[inds] = p1[inds]
				o2[inds] = p2[inds]
			else
				o1[inds] = p2[inds]
				o2[inds] = p1[inds]
			end
		end
		return o1, o2
	end

	"""
		mycrossover(p1, p2; myoption::Float64=0.5, kwargs...)

	Your own method that takes predefined options and also accepts others.
	"""
	function mycrossover(p1::Array{Int64,1},p2::Array{Int64,1}; myoption::Float64=0.5, kwargs...)
		error("Function <mycrossover> not defined yet... Get creative!")
	end
end

# ╔═╡ fd95d464-8570-11eb-1a38-879d8079d127
begin
	function textga(;goal::Array{Char,1}=goal, popsize::Int64=pop_size,
                	 ngen::Int64=500, next::Float64=0.4, pmut=0.1, kwargs...)
		# initiate population
		population = [rand(genes, length(goal)) for _ in 1:popsize]
		gen = 0
		for _ in 1:ngen
			# sort by fitness
			sort!(population; by=x->fitness(x), rev=true)
			if population[1] == goal
				break
			end
			gen += 1
			# select adequate parents
			goodparents = population[1: round(Int,popsize*next)]
			# make children
			new_generation = typeof(population)()
			for _ in 1:popsize
				push!(new_generation, mate(rand(goodparents), rand(goodparents),
											pmut=pmut ))
			end
			population = new_generation
		end
		@info "stopped after $(gen) generations"

		return (gen, prod(population[1]), fitness(population[1])/fitness(goal))
	end

	textga(ngen=500, popsize=100, pmut=0.1)
end

# ╔═╡ 7c4ef658-8576-11eb-3f6d-d3587a1af7e8
begin
	let
		# small demo matching the explanation of the algorithm above.
		# Matches the detailed example if you force c1, c2 = 4, 6 in the partialcrossover algorithm
		p1 = [3,4,8,2,7,1,6,5]
		p2 = [4,2,5,1,6,8,3,7]
		@time mate(p1,p2)
	end

	let
		# Longer route does not necessarily imply greater time for offspring generation
		# with the partial crossover algorithm (why could that be?)
		@info "100 items"
		N = 100
		p1 = collect(1:N)
		p2 = sample(p1,N,replace=false)
		@time mate(p1,p2);
	end

	let
		# demo from the method description
		p1 = [1,2,3,4,5,6,7,8]
		p2 = [8,5,2,1,3,6,4,7]
		@time res = mate(p1,p2, method=cyclecrossover)
		@assert res ==  ([1,5,2,4,3,6,7,8],  [8,2,3,1,5,6,4,7])
	end

	let
		# demo for children equal to parents
		p1 = [3,4,8,2,7,1,6,5]
		p2 = [4,2,5,1,6,8,3,7]
		@time res = mate(p1,p2, method=cyclecrossover)
		@assert res == (p1 ,p2)
	end

	let
		# Longer routes scales nicely for cyclecrossover as well (≈10x longer route and only ≈2x slower)
		N = 100
		p1 = collect(1:N)
		p2 = sample(p1,N,replace=false)
		@time res = mate(p1,p2, method=cyclecrossover);
	end
end

# ╔═╡ 8b77cf1c-8571-11eb-2ea7-531abac106cc
md"""
### Genetic algorithm - TSP
Now that you are familiar with a genetic algorithm, let's try to solve the TSP problem again, but this time, using a genetic algorithm.

We should think about:
* representation
* an initial configuration
* a fitness function
* a selection method (e.g. roulette based proportions, ‘elite selection’ etc.)
* cross-over method: (what is the difference with the text application?)
* mutation rate and method (is this required? why (not)?)
* when do you stop making new generations?
* \\( \dots \\)

Tasks:
* Come up with your own suggestions for cross-over/mutation
* Implement the genetic algorithm to solve the TSP
* Compare the performance of the tree cross-over algorithms to deal with eachother. Give that there is some randomness involved, a statistical approached might be more appropriate. Since the initial population will most likely be chosen at random, you should impose the random seed to make sure you have the same starting point for the comparisons.
* Compare the scalability with what you know from previous practical sessions (tree search etc.) What is better and what is worse?
"""

# ╔═╡ 712cb76c-8576-11eb-37c3-87beed1c3fa1
begin
	function TSP_ga(d::Array{Float64,2};
					popsize::Int64=30, ngen::Int64=500, next::Float64=0.4,
					method::Function=partialcrossover, kwargs...)
		# initiate population
		n = size(d,1)
		population = [sample(1:n,n,replace=false) for _ in 1:popsize]
		gen = 0
		# go over the generations
		for _ in 1:ngen
			# sort by fitness
			sort!(population; by=x->fitness(x,d), rev=true)
			#@show "current tour cost: $(fitness(population[1],d))"
			# include a stopcondition?
			gen += 1
			# select adequate parents
			goodparents = population[1: round(Int,popsize*next)]
			# make children
			new_generation = typeof(population)()
			for _ in 1:2:popsize
				p1,p2 = rand(goodparents,2)
				offspring = mate(p1,p2;method=method, kwargs...)
				append!(new_generation, offspring)
			end
			population = new_generation
		end

		return (gen, population[1], fitness(population[1],d))
	end
end

# ╔═╡ fa780328-8576-11eb-3a16-b7109ffbad63
let
	n = 50 # number of nodes
	N = problemgenerator(n) # nodes
	nodedict = Dict( i => N[i] for i in 1:length(N)) # nodes dict
	A = distancematrix(N,1) # distance matrix using manhattan norm
	@info fitness(finit(A), A)
	G = SimpleWeightedGraph(A) # graph
	@time begin
    	D = Dict()
    	methods = [cyclecrossover, partialcrossover]
		for method in methods
			res = []
			for _ in 1:100
				(_, tour, cost) = TSP_ga(A, ngen=20, popsize=n,method=method)
				push!(res,(tour,cost))
			end
			D[method] = [x[2] for x in res]
		end
	end

	# illustration
	histogram()
	for (key, val) in D
		histogram!(-val, label=string(key), normalize=:pdf, alpha=0.5)
	end
	scatter!([-fitness(finit(A),A)],[0],label="initial non-optimised cost")
	xlabel!("Tour cost")
	ylabel!("PDF")
	title!("Histogram for tour cost ($(size(A,1)) cities)")
end

# ╔═╡ 1a79eb5a-8577-11eb-168e-8fa9230f26fb

# ╔═╡ Cell order:
# ╟─774fa124-8560-11eb-2317-53b0d106b67a
# ╠═bca1c92c-8561-11eb-0420-6f0123bacb7f
# ╠═2b5e6876-8561-11eb-0685-dfe1cca39c45
# ╠═4e190a2e-8561-11eb-3759-795e3bb4d8d2
# ╠═b49f0528-8561-11eb-36a7-fd42574edc81
# ╠═d5019d8a-8561-11eb-2a43-a9ce616532f5
# ╠═728b94da-8566-11eb-1b02-e94c64424845
# ╠═a72d7c3c-8569-11eb-36a3-cb68ab2622cf
# ╠═0ce6432c-8567-11eb-1465-432d66051360
# ╟─9c2e2032-8568-11eb-103f-fdc57c11ab1f
# ╟─c84f6af2-8568-11eb-06c4-53f7664294ac
# ╠═46e468e0-8569-11eb-0a30-d11e814ccf7e
# ╠═7c5282fa-8569-11eb-0dcf-abc7b3b795ef
# ╠═eb27eada-8569-11eb-0739-7717087a5240
# ╠═1692c654-856a-11eb-1b02-77b0ab027c29
# ╠═3c58cc1a-856a-11eb-17bd-a31c244f6229
# ╠═7ab96168-856c-11eb-166a-b7531078ac90
# ╠═ef64ca86-856c-11eb-0cd0-178565e85576
# ╠═57726a64-856d-11eb-2b37-877202c47ede
# ╠═d70478c8-856d-11eb-3f27-0b565955b6ed
# ╟─934aa9e4-856e-11eb-1924-fd1df8fd3fe2
# ╟─8753fd9a-856f-11eb-2d91-13657e302031
# ╠═48c05e06-8570-11eb-0367-1f3db0aa20d0
# ╠═6faf987e-8570-11eb-1861-7de0f661bd63
# ╠═9cc2fa40-8570-11eb-2eba-c15433dcb90e
# ╠═fd95d464-8570-11eb-1a38-879d8079d127
# ╟─47153f8e-8572-11eb-1310-c932bd478721
# ╠═1f5d255c-8576-11eb-1463-89f94258168d
# ╠═7c4ef658-8576-11eb-3f6d-d3587a1af7e8
# ╟─8b77cf1c-8571-11eb-2ea7-531abac106cc
# ╠═712cb76c-8576-11eb-37c3-87beed1c3fa1
# ╠═fa780328-8576-11eb-3a16-b7109ffbad63
# ╠═1a79eb5a-8577-11eb-168e-8fa9230f26fb


### A Pluto.jl notebook ###
# v0.12.11

using Markdown
using InteractiveUtils

# ╔═╡ 8dbcf370-1dec-11eb-0a2d-e967ec1f2fde
md" ### Nomenclatura"

# ╔═╡ c4a8db60-1dec-11eb-0c39-73a8aac79092
md"Seja $X$ uma matriz com dimensões $n x m$ com $n$ linhas e $m$ colunas que contém nossos exemplos. $x_i^{(j)}$ corresponde ao elemento de $X$ em que $i$ é uma linha da matriz (exemplo) e $j$ uma coluna da matriz (atributo), cujo acesso na linguagem representamos por x[i,j]. Para pegarmos um vetor linha desta matriz (exemplo) representamos formalmente por $X_i$ e na linguagem por x[i,:]."

# ╔═╡ cd532e00-1dec-11eb-06d9-9b7c76e9c1ba
md"Seja $\theta$ um vetor linha com $m$ elementos tal que $m$ corresponde ao número de atributos mais um. $\theta = [\theta^{(0)},\theta^{(1)}, ...,\theta^{(m)}]$ em que cada elemento corresponde ao j-ésimo parâmetro associado ao j-ésimo atributo. Na nossa linguagem acessamos um elemento por $\theta[i]$ e o vetor por $\theta$"

# ╔═╡ dc365e10-1dec-11eb-0e4f-95624ee468ea
md"Seja $y$ nosso vetor alvo em que representamos o i-ésimo elemento por $y_i$. Na nossa linguagem representamos por $y[i]$."

# ╔═╡ 571a5ea0-1dee-11eb-0eaf-bf746a6faa86
md"### Algumas libs importantes usadas neste experimento"

# ╔═╡ 317bbe60-1ded-11eb-2faa-c3e47c0df176
begin
   import Statistics: mean,std
   import BenchmarkTools: @benchmark, @btime
   import StaticArrays: @SMatrix
   import Plots: plot,plot!,scatter,scatter!
   import StatsBase: zscore
   μ = mean
end

# ╔═╡ f9af3a70-1dec-11eb-34f0-91b7fc591b87
md" ### Hipótese e modelo linear"

# ╔═╡ 78d2c0e0-1f98-11eb-06bc-b51c3b4b9f35
md"A seguir descrevemos a hipótese que consiste da aplicação de um modelo (conjunto de parâmetros) em um exemplo. Note que a hipótese sempre se refere à aplicação de um único exemplo e não ao conjunto de dados todo. "

# ╔═╡ 2e7709e0-1ded-11eb-2abd-d770bf1afe8a
md"$H_{\theta}(X_i) = \sum_{j}^{m}{x_{i}^{(j)} \theta^{(j)}} = X_i\theta^{T}$"

# ╔═╡ b05f9930-1f97-11eb-0083-07ebd21313e0
md"Exemplificando o produto de vetores, nossa hipótese faria o seguinte cálculo para um exemplo $X_i = \begin{bmatrix} 1 & 500 \end{bmatrix}$ e um vetor de parâmetros $\theta^T = \begin{bmatrix}
    0.5 \\
    1.0
\end{bmatrix}$:"

# ╔═╡ cbf998e0-1f96-11eb-0a8b-a3bf631bfe02
md"e.g. $H_{\theta}(X_i) = \begin{bmatrix}
    1 & 500
  \end{bmatrix}\begin{bmatrix}
    0.5 \\
    1.0
\end{bmatrix} = 500.5$"

# ╔═╡ 4cc76c80-1f98-11eb-0c70-29aec0b92ac4
md"Note que o resultado é um escalar, pois estamos estimando um valor contínuo (problema de regressão)."

# ╔═╡ c0b094a0-1f98-11eb-2efb-07f680447b12
md"### Implementação a hipótese"

# ╔═╡ be0deb50-1ded-11eb-06bb-33e94217e815
H(χ,θ) = χ*Θ'

# ╔═╡ 2f5a6500-1ded-11eb-11dd-bd88bf102b98
md" ### Função de custo"

# ╔═╡ 30972ac0-1ded-11eb-1007-1d3833dfc264

md"$J(\theta)= \frac{1}{2n}\sum_{i=1}^{n}{(H_{\theta}(X_i) - y_i)²}$"

# ╔═╡ bba6d590-1e1d-11eb-2df6-67bd6c5d4777
md" A seguir estou calculando a função de custo sem loop, apenas por matrizes.Desta maneira, geramos em uma multiplicação de matrizes, um vetor de predições com a mesma dimensão da saída. Após isto, subtraímos elemento a elemento. Após isto, elevamos cada elemento ao quadrado e depois tiramos uma média. Explicado na aula anterior."

# ╔═╡ 9837a3d0-1ded-11eb-3c49-17a45387e46e
J(Θ,χ,γ) = .5μ((χ*Θ' .- γ).^2)

# ╔═╡ 2ebaad22-1dee-11eb-2fb1-f51c49ec13fb
md"### Gradiente descendente"

# ╔═╡ d1cf31b0-1e17-11eb-22d1-916c9a331700
md"A seguir, mostramos o cálculo do gradiente. Como podemos notar, o expoente 2 desce e é eliminado pela constante 2 do denominador. Adicionalemente, vemos que é uma subtração do gradiente, pois estamos minimizando a função de custo. Por isto, estamos caminhando no sentido contrário ao do gradiente. É importante também ressaltar que o viés já está embutido na matriz $X$ na forma de uma coluna com valores 1. Desta forma, o cálculo do gradiente é o mesmo para o parâmetro do viés ($θ^{(0)}$) e os demais."

# ╔═╡ cbf654c0-1dfa-11eb-0994-85a67ded79b0
md"$θ^{(j)} = θ^{(j)} - α \frac{1}{n} \sum_{i=1}^{n}{(H_θ(X_i) - y_i)x_{i}^{(j)}}$"

# ╔═╡ 46484b80-1e1d-11eb-2bdc-e50b24f6b530
md"Entretanto, podemos condensar esta fórmula usando notação de vetores e matrizes. Lembrando que $\theta$ é um vetor agora e os cálculos são feitos simultaneamente para cada eleemnto deste. Podemos observar que o índice da coluna some, pois são processadas automaticamente. A seguir, veja a fórmula:"

# ╔═╡ f1089b10-1e1d-11eb-120e-cdbd993d72dc
md"$θ = θ - α \frac{1}{n} \sum_{i=1}^{n}{(H_θ(X_i) - y_i) \circ X_{i}}$"

# ╔═╡ 79bc38d0-1e1f-11eb-066c-2929340c479e
md"Cabe observar que $\circ$ é o produto de hadamard."

# ╔═╡ f5d33090-1e1f-11eb-3f25-d10f2616803d
md"### Algoritmo "

# ╔═╡ 7a22eab0-1def-11eb-2672-495fb9cf19f8
function train(θ,χ,γ;verbose=false,α = 1e-3,ϵ=1e-5,τ=1e3)

	e₂  = J(θ,χ,γ)
	e₁  = e₂ + .1
	#Δ₁,Δ₂ = e₁ - e₂ + 2ϵ,e₁ - e₂
	💻 = [[e₂ θ]]
	i   = 1
	while e₂ < e₁ && e₂ > ϵ && i < τ
	#while Δ₁ > Δ₂  && e₂ > ϵ && i < τ

		∇     = μ((χ*θ' .- γ).* χ,dims=1)
		θ    -= α .* ∇

		e₁,e₂ = e₂,J(θ,χ,γ)
		#Δ₁,Δ₂ = Δ₂,e₁ - e₂
		verbose && if (i%10==0) append!(💻,[[e₂ θ]]) end

		i += 1
	end
	θ,vcat(💻...)
end

# ╔═╡ 1f0e51f0-1df4-11eb-0feb-b53552fa63c9
md"### Testes de unitários"

# ╔═╡ 58dd7720-1e09-11eb-1904-af5f9991ece5
md" Sempre que for desenvolver suas funções, realize uma bateria de testes para cada uma antes mesmo de executar todo o seu programa. Isso vai lhe poupar muito tempo para uma futura depuração de código. Assim, para cada função, faça pelo menos um teste. Elabore casos bons de entrada para suas funções de forma que cubra bem o espaço de possibilidades. Seus casos devem ser bem simples para que num futuro, se der erro, você consiga rapidamente resolver."

# ╔═╡ 259ac040-1e39-11eb-3670-f5c5519a1978
md"### Padronizando os Dados "

# ╔═╡ 246cc792-1e39-11eb-3c20-f1623d8ea737
md"O objetivo de padronizar os dados é acelerar a convergência do método. Aqui utilizaremos a padronização zscore. Entretanto, para amostras pequenas, a padronização não funciona muito bem."

# ╔═╡ c467ca00-1e08-11eb-3d41-75a38532652b
md"### Verificando a curva de erro"

# ╔═╡ d1a88a00-1f8f-11eb-36b7-f39dabee0657
md"### Veificando o comportamento dos parâmetros"

# ╔═╡ b405694e-1f8f-11eb-0fad-5d0c3527d206
md"### Verificando o modelo"

# ╔═╡ d497051e-1f94-11eb-2ccb-a7d3e52570d1
md"Como podemos observar, o gradiente jogou o θ₁ para zero (viés, responsável pela translação) e ajustou o θ₂ (responsável pela rotação). Cabe ressaltar que estamos trabalhando com os dados padronizados. É possível, no entanto, despadronizar e obter predições no campo dos números originais."

# ╔═╡ 37c52e40-1e48-11eb-1e13-7ba1bebfabc4
md"### Depuração "

# ╔═╡ 44997b82-1e48-11eb-1821-f7f3aecd8093
md" Costumo colocar alguma variáveis soltas para que eu veja se está tudo correto. "

# ╔═╡ a419b220-1e4b-11eb-1299-295bf39df3f4
md"### Solução direta por álgebra linear"

# ╔═╡ a39cb9f0-1e4b-11eb-296e-bf19e5438416
train(X,Y) = inv(X'X)X'Y

# ╔═╡ 628624c0-1dfa-11eb-2afb-137bf77669e1
begin
	Θ1 = [0. 4.]
	χ =  [1 50.;1 60.;1 100.; 1 200.]
	γ =  [200  ;   240  ;   400; 800]
	@assert J(Θ1,χ,γ) == 0.0
	@assert train(Θ1,χ,γ)[1] == [0. 4.] # iniciando com a solução, o θ deve continuar igual
end

# ╔═╡ 4c632fee-1e39-11eb-00c6-c3f2a0774005
begin
	χₚ = hcat(ones((4,1)), zscore(χ[:,2:end])) # padronizamos só o que não é viés
	γₚ = zscore(γ)
end

# ╔═╡ d8ab5330-1e47-11eb-06b3-af3d2ee97c7b
χₚ

# ╔═╡ 86be2920-1e48-11eb-10f1-6740a2459951
γₚ

# ╔═╡ 74604cb2-1e05-11eb-19ef-09fd04dc52e3
(θᵏ,r) = train([4 5],χₚ,γₚ,verbose=true,α = 1e-3,ϵ=1e-5,τ=1e5);

# ╔═╡ b7913600-1e0c-11eb-362e-0b4b9ef9d1aa
plot(1:length(r[:,1]),r[:,1],xlabel="iterações",ylabel="J(θ)",title="Análise de erros",label="Erro")

# ╔═╡ fa0bd460-1e23-11eb-131f-f31bec373c07
begin
	p = plot(1:length(r[:,2]),r[:,2],xlabel="iterações",ylabel="θ₁,θ₂",title="Análise de parâmetros",label="θ₁")
	plot!(p,1:length(r[:,3]),r[:,3],xlabel="iterações",label="θ₂")
end

# ╔═╡ cc73d620-1f8f-11eb-357d-b19b50c19280
begin
	f(x) = θᵏ[1] + θᵏ[2]*x # o mesmo que H(X) lá do início
	plot(χₚ[:,2],f,xlabel="m² (padronizado)",ylabel="Preço (padronizado)",title="Análise do da predição do modelo",label="θ₁ + θ₂x")
	scatter!(χₚ[:,2],γₚ,label="Dados Coletados")
end

# ╔═╡ 41505b10-1e34-11eb-06db-750eb3ab11cf
r[end,:]

# ╔═╡ 29b04780-1e44-11eb-3bcf-8b5f121d157f
J(θᵏ,χₚ,γₚ)

# ╔═╡ d0de29e0-1e36-11eb-398c-5f0b55ea281e
θᵏ

# ╔═╡ 70775a70-1e4c-11eb-3cec-11fd54e9b637
train(χₚ,γₚ)

# ╔═╡ Cell order:
# ╟─8dbcf370-1dec-11eb-0a2d-e967ec1f2fde
# ╟─c4a8db60-1dec-11eb-0c39-73a8aac79092
# ╟─cd532e00-1dec-11eb-06d9-9b7c76e9c1ba
# ╟─dc365e10-1dec-11eb-0e4f-95624ee468ea
# ╟─571a5ea0-1dee-11eb-0eaf-bf746a6faa86
# ╠═317bbe60-1ded-11eb-2faa-c3e47c0df176
# ╟─f9af3a70-1dec-11eb-34f0-91b7fc591b87
# ╟─78d2c0e0-1f98-11eb-06bc-b51c3b4b9f35
# ╟─2e7709e0-1ded-11eb-2abd-d770bf1afe8a
# ╟─b05f9930-1f97-11eb-0083-07ebd21313e0
# ╟─cbf998e0-1f96-11eb-0a8b-a3bf631bfe02
# ╟─4cc76c80-1f98-11eb-0c70-29aec0b92ac4
# ╟─c0b094a0-1f98-11eb-2efb-07f680447b12
# ╠═be0deb50-1ded-11eb-06bb-33e94217e815
# ╟─2f5a6500-1ded-11eb-11dd-bd88bf102b98
# ╟─30972ac0-1ded-11eb-1007-1d3833dfc264
# ╟─bba6d590-1e1d-11eb-2df6-67bd6c5d4777
# ╠═9837a3d0-1ded-11eb-3c49-17a45387e46e
# ╟─2ebaad22-1dee-11eb-2fb1-f51c49ec13fb
# ╟─d1cf31b0-1e17-11eb-22d1-916c9a331700
# ╟─cbf654c0-1dfa-11eb-0994-85a67ded79b0
# ╟─46484b80-1e1d-11eb-2bdc-e50b24f6b530
# ╟─f1089b10-1e1d-11eb-120e-cdbd993d72dc
# ╟─79bc38d0-1e1f-11eb-066c-2929340c479e
# ╟─f5d33090-1e1f-11eb-3f25-d10f2616803d
# ╠═7a22eab0-1def-11eb-2672-495fb9cf19f8
# ╟─1f0e51f0-1df4-11eb-0feb-b53552fa63c9
# ╟─58dd7720-1e09-11eb-1904-af5f9991ece5
# ╠═628624c0-1dfa-11eb-2afb-137bf77669e1
# ╟─259ac040-1e39-11eb-3670-f5c5519a1978
# ╟─246cc792-1e39-11eb-3c20-f1623d8ea737
# ╠═4c632fee-1e39-11eb-00c6-c3f2a0774005
# ╟─c467ca00-1e08-11eb-3d41-75a38532652b
# ╠═74604cb2-1e05-11eb-19ef-09fd04dc52e3
# ╟─b7913600-1e0c-11eb-362e-0b4b9ef9d1aa
# ╟─d1a88a00-1f8f-11eb-36b7-f39dabee0657
# ╠═fa0bd460-1e23-11eb-131f-f31bec373c07
# ╟─b405694e-1f8f-11eb-0fad-5d0c3527d206
# ╟─d497051e-1f94-11eb-2ccb-a7d3e52570d1
# ╠═cc73d620-1f8f-11eb-357d-b19b50c19280
# ╟─37c52e40-1e48-11eb-1e13-7ba1bebfabc4
# ╟─44997b82-1e48-11eb-1821-f7f3aecd8093
# ╠═41505b10-1e34-11eb-06db-750eb3ab11cf
# ╠═29b04780-1e44-11eb-3bcf-8b5f121d157f
# ╠═d0de29e0-1e36-11eb-398c-5f0b55ea281e
# ╠═d8ab5330-1e47-11eb-06b3-af3d2ee97c7b
# ╠═86be2920-1e48-11eb-10f1-6740a2459951
# ╟─a419b220-1e4b-11eb-1299-295bf39df3f4
# ╠═a39cb9f0-1e4b-11eb-296e-bf19e5438416
# ╠═70775a70-1e4c-11eb-3cec-11fd54e9b637


function arnoldi(a::AbstractArray{T,2},b::AbstractArray{T,1},m::S = length(b)) where {T <: AbstractFloat, S <: Integer} # Exact

    n = length(b)
    if m > n
        error("'m' must be no larger than the length of 'b' ")
    end

    q = zeros(n,m)
    h = zeros(m+1,m)

    β = norm(b)
    qk = copy(b)

    @views for k = 1:m
        q[:,k] .= qk/β
        qk .= a*q[:,k]
        for j = 1:k
            h[j,k] = qk'q[:,j]
            qk .-= h[j,k]*q[:,j]
        end
        β = norm(qk)
        h[k+1,k] = β
        if β < eps(T)
            return q[:,1:k], h[1:k,1:k], k
        end
    end

    return q, h[1:m,1:m], m

end

function arnoldi(f::Function,x::Array{T,1},r::Array{T,1},m::S=length(x)) where {T <: AbstractFloat, S <: Integer} #Exact

    n = length(x)
    if m > n
        error("'m' must be no larger than the length of 'b' ")
    end

    q = zeros(n,m)
    h = zeros(m+1,m)

    β = norm(r)
    qk = copy(r)

    @views for k = 1:m
        q[:,k] = qk/β
        jacvec!(qk,f,x,q[:,k])
        for j = 1:k
            h[j,k] = qk'q[:,j]
            qk -= h[j,k]*q[:,j]
        end
        β = norm(qk)
        h[k+1,k] = β
        if β < eps(T)
            return q[:,1:k], h[1:k,1:k], k
        end
    end

    return q, h[1:m,:], m

end

function arnoldi(a::AbstractArray{T,2},b::AbstractArray{T,1},tol::T,m::S=length(b)) where {T <: AbstractFloat, S <: Integer} #Inexact

    n = length(b)
    if m > n
        error("'m' must be no larger than the length of 'b' ")
    end

    q = zeros(n,m)
    h = zeros(m+1,m)

    β = norm(b)
    qk = copy(b)

    @views for k = 1:m
        q[:,k] .= qk/β
        qk .= a*q[:,k]
        for j = 1:k
            h[j,k] = qk'q[:,j]
            qk .-= h[j,k]*q[:,j]
        end
        β = norm(qk)
        h[k+1,k] = β
        if β < tol
            return q[1:n,1:k], h[1:k,1:k], k
        end
    end

    return q[1:n,1:m], h[1:m,1:m], m

end

function arnoldi(f::Function,x::Array{T,1},r::Array{T,1},tol::T,m::S=length(x)) where {T <: AbstractFloat, S <: Integer} # Inexact

    n = length(x)
    if m > n
        error("'m' must be no larger than the length of 'b' ")
    end

    q = zeros(n,m)
    h = zeros(m+1,m)

    β = norm(r)
    qk = copy(r)

    @views for k = 1:m
        q[:,k] = qk/β
        jacvec!(qk,f,x,q[:,k])
        for j = 1:k
            h[j,k] = qk'q[:,j]
            qk -= h[j,k]*q[:,j]
        end
        β = norm(qk)
        h[k+1,k] = β
        if β < tol
            return q[1:n,1:k], h[1:k,1:k], k
        end
    end

    return q[1:n,1:m], h[1:m,1:m], m

end

function arnoldi_inplace(f::Function,x::Array{T,1},r::Array{T,1},tol::T,m::S=length(x)) where {T <: AbstractFloat, S <: Integer} # Inexact

    n = length(x)
    if m > n
        error("'m' must be no larger than the length of 'b' ")
    end

    q = zeros(n,m)
    h = zeros(m+1,m)

    β = norm(r)
    qk = copy(r)

    @views for k = 1:m
        q[:,k] = qk/β
        jacvec_inplace!(qk,f,x,q[:,k])
        for j = 1:k
            h[j,k] = qk'q[:,j]
            qk -= h[j,k]*q[:,j]
        end
        β = norm(qk)
        h[k+1,k] = β
        if β < tol
            return q[1:n,1:k], h[1:k,1:k], k
        end
    end

    return q[1:n,1:m], h[1:m,1:m], m

end


using StatisticalRethinking

n_samples = 10000
a3d = ones(n_samples,1,1)
w = 6; l = 3; n = w +l
p = [0.5]
for i in 2:n_samples
  p_new = rand(Normal(p[i-1], 0.1), 1)[1]
  if  p_new < 0
    p_new = abs(p_new)
  end
  if p_new > 1
    p_new = 2 - p_new
  end
  q0 = pdf(Binomial(n, p[i-1]), w)
  q1 = pdf(Binomial(n, p_new), w)
  append!(p, [rand(Uniform(0, 1), 1)[1] < q1/q0 ? p_new : p[i-1]])
end

a3d[:, 1, 1] = p
chns = MCMCChains.Chains(a3d, ["toss"])

MCMCChains.describe(chns)

plot(chns)

w = 6; n = 9; x = 0:0.01:1
density(chns, lab="Samples")
plot!( x, pdf.(Beta( w+1 , n-w+1 ) , x ), lab="Conjugate solution")

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


# ---
# title: 888. Fair Candy Swap
# id: problem888
# author: Indigo
# date: 2021-02-01
# difficulty: Easy
# categories: Array
# link: <https://leetcode.com/problems/fair-candy-swap/description/>
# hidden: true
# ---
#
# Alice and Bob have candy bars of different sizes: `A[i]` is the size of the
# `i`-th bar of candy that Alice has, and `B[j]` is the size of the `j`-th bar
# of candy that Bob has.
#
# Since they are friends, they would like to exchange one candy bar each so that
# after the exchange, they both have the same total amount of candy.  ( _The
# total amount of candy  a person has is the sum of the sizes of candy bars they
# have._)
#
# Return an integer array `ans` where `ans[0]` is the size of the candy bar that
# Alice must exchange, and `ans[1]` is the size of the candy bar that Bob must
# exchange.
#
# If there are multiple answers, you may return any one of them.  It is
# guaranteed an answer exists.
#
#
#
# **Example 1:**
#
#
#
#     Input: A = [1,1], B = [2,2]
#     Output: [1,2]
#
#
# **Example 2:**
#
#
#
#     Input: A = [1,2], B = [2,3]
#     Output: [1,2]
#
#
# **Example 3:**
#
#
#
#     Input: A = [2], B = [1,3]
#     Output: [2,3]
#
#
# **Example 4:**
#
#
#
#     Input: A = [1,2,5], B = [2,4]
#     Output: [5,4]
#
#
#
#
# **Note:**
#
#   * `1 <= A.length <= 10000`
#   * `1 <= B.length <= 10000`
#   * `1 <= A[i] <= 100000`
#   * `1 <= B[i] <= 100000`
#   * It is guaranteed that Alice and Bob have different total amounts of candy.
#   * It is guaranteed there exists an answer.
#
#
## @lc code=start
using LeetCode

function fair_candy_swap(A::Vector{Int}, B::Vector{Int})
    set_a, set_b = Set(A), Set(B)
    dif = (sum(A) - sum(B)) ÷ 2
    for num_a in set_a
        ((num_a - dif) in set_b) && return [num_a, num_a - dif]
    end
end
## @lc code=end


"""
2-D Gradient

    gradient2(a::Array, dx::Float64=1, dy::Float=1)
    :param a: matrix
    :param dx: stepsize
    :param dy: stepsize

    :return dxdu: derivatives along x
    :return dydv: derivatives along y
"""
function gradient2(a::Array, dx::Float64=1., dy::Float64=1.)
    m,n = size(a);
    dxdu = zeros(m,n);
    dydv = zeros(m,n);

    for i=1:m
        dxdu[i,:] = gradient(vec(a[i,:]), dx)
    end

    for i=1:n
        dydv[:,i] = gradient(a[:,i], dy)
    end

    return dxdu, dydv
end

"""
Creates Rectangular Grid in 2-D space

    meshgrid(a::LinSpace,b::LinSpace)
"""
function meshgrid(a::LinSpace,b::LinSpace)
    grid_a = [i for i in a, j in b]';
    grid_b = [j for i in a, j in b]';

    return grid_a, grid_b
end

"""
Linear interpolation

    approx(xd, yd, xi)
    :param xd: x samples
    :param yd: response samples
    :param xi: new x samples
"""
function approx(xd, yd, xi)
    nd = length(xd);
    ni = length(xi);

    yi = zeros(ni);
    for i in 1:ni
        if (xi[i] <= xd[1])
            t = (xi[i]-xd[1]) / (xd[2] - xd[1]);
            yi[i] = (1.0 - t) * yd[1] + t * yd[2];
        elseif (xd[nd] <= xi[i])
            t = (xi[i] - xd[nd-1]) / (xd[nd] - xd[nd-1]);
            yi[i] = (1.0 - t) * yd[nd-1] + t * yd[nd];
        else
            for k in 2:nd
                if (xd[k-1] <= xi[i] && xi[i] <= xd[k])
                    t = (xi[i] - xd[k-1]) / (xd[k] - xd[k-1]);
                    yi[i] = (1.0 - t) * yd[k-1] + t * yd[k];
                    break
                end
            end
        end
    end

    return yi
end

"""
Trapezoidal Integration

    trapz(x, y, dim=1)
    :param x: vector of time samples
    :param y: array of response samples
    :param dim: dimension along which to integrate
"""
function trapz(x::Array{Float64, 1}, y::Array{Float64}, dim::Integer=1)
    perm = [dim:max(ndims(y),dim); 1:dim-1];
    y = permutedims(y, perm);
    if ndims(y) == 1
        m = 1;
    else
        m = size(y,1);
    end

    if m == 1
        M = length(y);
        out = sum(diff(x).*(y[1:M-1] + y[2:M])/2.);
    else
        out = diff(x).' * (y[1:m-1,:] + y[2:m,:])/2.;
        siz = size(y); siz = collect(siz); siz[1] = 1;
        out = reshape(out, tuple(siz...));
        out = ipermutedims(out, perm);
        ind = find(collect(size(out)).==1);
        out = squeeze(out,ind[1]);
        if length(out) == 1;
            out = out[1];
        end
    end

    return out
end

"""
Cumulative Trapezoidal Integration

    cumtrapz(x, y, dim=1)
    :param x: vector describing time samples
    :param y: array describing response
    :param dim: dimension to integrate over
"""
function cumtrapz(x::Array{Float64, 1}, y::Array{Float64}, dim::Integer=1)
    perm = [dim:max(length(size(y)),dim); 1:dim-1];
    y = permutedims(y, perm);
    if ndims(y) == 1
        n = 1;
        m = length(y);
    else
        m, n = size(y);
    end

    if n == 1
        dt = diff(x)/2.0;
        z = [0; cumsum(dt.*(y[1:(m-1)] + y[2:m]))];
    else
        dt = repmat(diff(x)/2.0,1,n);
        z = [zeros(1,n); cumsum(dt.*(y[1:(m-1), :] + y[2:m, :]),1)];
        z = ipermutedims(z, perm);
    end

    return z
end

"""
Cumulative Trapezoidal Integration using midpoint

    cumtrapzmid(x, y, c)
    :param x: time samples
    :param y: resposne samples
    :param c: midpoint
"""
function cumtrapzmid(x, y, c)
    a = length(x);
    mid = round(Integer, a/2);

    # case < mid
    fn = zeros(a);
    tmpx = x[(mid-1):-1:1];
    tmpy = y[(mid-1):-1:1];
    tmp = c + cumtrapz(tmpx, tmpy);
    fn[1:(mid-1)] = reverse(tmp);

    # case >= mid
    fn[mid:a] = c + cumtrapz(x[mid:a],y[mid:a]);

    return fn

end

"""
Multivariate Normal random number generation

    mvnrand(mu, C, n)
    :param mu: mean vector
    :param C: covariance matrix
    :param n: number of samples
"""
function mvnrand(mu, C, n)
    tmp = cholfact(C, :U, Val{true});
    R = tmp[:U];
    R = Array(R);
    R = R[:, tmp.piv];
    retval = randn(n, size(R,1)) * R;
    retval += transpose(repmat(mu, 1, n));
    return transpose(retval)
end

"""
Linear interpolation when response contains flat regions

    interp1_flat(x, y, xx)
    :param x: time samples
    :param y: response samples
    :param xx: new time samples
"""
function interp1_flat(x,y,xx)
    flat = find(diff(x).<=0);
    n = length(flat);

    if n==0
        tmp = interpolate((x,), y, Gridded(Linear()))
        yy = tmp[xx];
    else
        yy = zeros(size(xx));
        i1 = 1;
        if flat[1] == 1
            i2 = 1;
            j = xx.==x[i2];
            yy[j] = minimum(y[i2:i2+1]);
        else
            i2 = flat[1];
            j = (xx.>=x[i1]) & (xx.<=x[i2]);
            tmp = interpolate((x[i1:i2],), y[i1:i2], Gridded(Linear()))
            yy[j] = tmp[xx[j]];
            i1 = copy(i2);
        end
        for k = 2:n
            i2 = flat[k];
            if i2 > i1+1
                j = (xx.>=x[i1]) & (xx.<=x[i2]);
                yi = interpolate((x[i1+1:i2],), y[i1+1:i2], Gridded(Linear()))
                yy[j] = tmp[xx[j]];
            end
            j = xx.==x[i2];
            yy[j] = minimum(y[i2:i2+1]);
            i1 = copy(i2);
        end
        i2 = length(x);
        j = (xx.>=x[i1]) & (xx.<=x[i2]);
        if i1+1 == i2
            yy[j] = y[i2];
        else
            tmp = interpolate((x[i1+1:i2],), y[i1+1:i2], Gridded(Linear()))
            yy[j] = tmp[xx[j]];
        end
    end

    return yy
end


ENV["GKSwstype"] = "100" #src

# # Generalized Linear Models
#
# Generalized Linear Models (GLMs) are a family of statistical models where the prediction
# is based on a function of a linear predictor $x_^T\beta$ where $x$ is a vector of features
# and $\beta$ is the parameter vector.
#
# One of the big assumptions of GLMs is that there is a strictly increasing *link function*
# $g$ that "links" the mean of the distribution $E(Y|X) = \mu$ to the linear predictor
# $x^T\beta$.  In other words, our prediction of $y$ given $x$ is $g^{-1}(x^T\beta)$.

# The two most common GLMs are linear and logistic regression.

# The GLM package
#
#

# ## Linear Regression

# ## Logistic Regression


"""
shoot(phi,xspan,lval,lder,rval,rder,init)

Use the shooting method to solve a two-point boundary value problem. The ODE is
u'' = `phi`(x,u,u') for x in `xspan`. Specify a function value or derivative at
the left endpoint using `lval` and `lder`, respectively, and similarly for the
right endpoint  using `rval` and `rder`. (Use an empty array to denote an
unknown quantity.) The value `init` is an initial guess for whichever value is
missing at the left endpoint.

Return vectors for the nodes, the values of u, and the values of u'.
    """
function shoot(phi,xspan,lval,lder,rval,rder,init)

    # Tolerances for IVP solver and rootfinder.
    ivp_opt = 1e-6
    optim_opt = 1e-5

    # Evaluate the difference between computed and target values at x=b.
    function objective(s)
        # Combine s with the known left endpoint value.
        v_init = isempty(lder) ?  [ lval; s[1] ]  :  [ s[1]; lder ]

        # ODE posed as a first-order equation in 2 variables.
        function shootivp(v,p,x)
            [ v[2]; phi(x,v[1],v[2]) ]
        end

        IVP = ODEProblem(shootivp,v_init,xspan)
        sol = solve(IVP,abstol=ivp_opt,reltol=ivp_opt)
        x = sol.t;  v = sol;

        return isempty(rder) ? v[1,end] - rval  :  v[2,end] - rder
    end

    # Find the unknown quantity at x=a by rootfinding.
    x = [];  v = [];   # the values will be overwritten
    s = nlsolve(objective,[init],xtol=optim_opt).zero

    # Don't need to solve the IVP again. It was done within the
    # objective function already.
    u = v[1,:]            # solution
    dudx = v[2,:]         # derivative

    return x,u,dudx
end

"""
    diffmat2(n,xspan)

Compute 2nd-order-accurate differentiation matrices on `n`+1 points in the
interval `xspan`. Return a vector of nodes, and the matrices for the first
and second derivatives.
"""
function diffmat2(n,xspan)
    a,b = xspan
    h = (b-a)/n
    x = [ a + i*h for i=0:n ]   # nodes

    # Define most of Dx by its diagonals.
    dp = fill(0.5/h,n)        # superdiagonal
    dm = fill(-0.5/h,n)       # subdiagonal
    Dx = diagm(-1=>dm,1=>dp)

    # Fix first and last rows.
    Dx[1,1:3] = [-1.5,2,-0.5]/h
    Dx[n+1,n-1:n+1] = [0.5,-2,1.5]/h

    # Define most of Dxx by its diagonals.
    d0 =  fill(-2/h^2,n+1)    # main diagonal
    dp =  ones(n)/h^2         # superdiagonal and subdiagonal
    Dxx = diagm(-1=>dp,0=>d0,1=>dp)

    # Fix first and last rows.
    Dxx[1,1:4] = [2,-5,4,-1]/h^2
    Dxx[n+1,n-2:n+1] = [-1,4,-5,2]/h^2

    return x,Dx,Dxx
end

"""
    diffcheb(n,xspan)

Compute Chebyshev differentiation matrices on `n`+1 points in the
interval `xspan`. Return a vector of nodes, and the matrices for the first
and second derivatives.
"""
function diffcheb(n,xspan)
    x = [ -cos( k*pi/n ) for k=0:n ]    # nodes in [-1,1]
    Dx = zeros(n+1,n+1)
    c = [2; ones(n-1); 2];    # endpoint factors

    # Off-diagonal entries
    Dx = [ (-1)^(i+j)*c[i+1]/(c[j+1]*(x[i+1]-x[j+1])) for i=0:n, j=0:n ]

    # Diagonal entries
    Dx[isinf.(Dx)] .= 0              # fix divisions by zero on diagonal
    s = sum(Dx,dims=2)
    Dx -= diagm(0=>s[:,1])           # "negative sum trick"

    # Transplant to [a,b]
    a,b = xspan
    x = @. a + (b-a)*(x+1)/2
    Dx = 2*Dx/(b-a)

    # Second derivative
    Dxx = Dx^2

    return x,Dx,Dxx
end

"""
     bvplin(p,q,r,xspan,lval,rval,n)

Use finite differences to solve a linear bopundary value problem. The ODE is
u''+`p`(x)u'+`q`(x)u = `r`(x) on the interval `xspan`, with endpoint function
values given as `lval` and `rval`. There will be `n`+1 equally spaced nodes,
including the endpoints.

Return vectors of the nodes and the solution values.
"""
function bvplin(p,q,r,xspan,lval,rval,n)
    x,Dx,Dxx = diffmat2(n,xspan)

    P = diagm(0=>p.(x))
    Q = diagm(0=>q.(x))
    L = Dxx + P*Dx + Q     # ODE expressed at the nodes

    # Replace first and last rows using boundary conditions.
    I = Diagonal(ones(n+1))
    A = [ I[[1],:]; L[2:n,:]; I[[n+1],:] ]
    b = [ lval; r.(x[2:n]); rval ]

    # Solve the system.
    u = A\b

    return x,u
end

"""
    bvp(phi,xspan,lval,lder,rval,rder,init)

Use finite differences to solve a two-point boundary value problem. The ODE is
u'' = `phi`(x,u,u') for x in `xspan`. Specify a function value or derivative at
the left endpoint using `lval` and `lder`, respectively, and similarly for the
right endpoint  using `rval` and `rder`. (Use an empty array to denote an
unknown quantity.) The value `init` is an initial guess for whichever value is
missing at the left endpoint.

Return vectors for the nodes and the values of u.
"""
function bvp(phi,xspan,lval,lder,rval,rder,init)
    n = length(init) - 1
    x,Dx,Dxx = diffmat2(n,xspan)
    h = x[2]-x[1]

    function residual(u)
        # Compute the difference between u'' and phi(x,u,u') at the
        # interior nodes and appends the error at the boundaries.
        dudx = Dx*u                   # discrete u'
        d2udx2 = Dxx*u                # discrete u''
        f = d2udx2 - phi.(x,u,dudx)

        # Replace first and last values by boundary conditions.
        f[1] = isempty(lder) ? (u[1] - lval)/h^2 : (dudx[1] - lder)/h
        f[n+1] = isempty(rder) ? (u[n+1] - rval)/h^2 : (dudx[n+1] - rder)/h
        return f
    end

    u = levenberg(residual,init)
    return x,u[:,end]
end

"""
    fem(c,s,f,a,b,n)

Use a piecewise linear finite element method to solve a two-point boundary
value problem. The ODE is (`c`(x)u')' + `s`(x)u = `f`(x) on the interval
[`a`,`b`], and the boundary values are zero. The discretization uses `n` equal
subintervals.

Return vectors for the nodes and the values of u.
"""
function fem(c,s,f,a,b,n)
    # Define the grid.
    h = (b-a)/n
    x = @. a + h*(0:n)

    # Templates for the subinterval matrix and vector contributions.
    Ke = [1 -1; -1 1]
    Me = (1/6)*[2 1; 1 2]
    fe = (1/2)*[1; 1]

    # Evaluate coefficent functions and find average values.
    cval = c.(x);   cbar = (cval[1:n]+cval[2:n+1]) / 2;
    sval = s.(x);   sbar = (sval[1:n]+sval[2:n+1]) / 2;
    fval = f.(x);   fbar = (fval[1:n]+fval[2:n+1]) / 2;

    # Assemble global system, one interval at a time.
    K = zeros(n-1,n-1);  M = zeros(n-1,n-1);  f = zeros(n-1);
    K[1,1] = cbar[1]/h;  M[1,1] = sbar[1]*h/3;  f[1] = fbar[1]*h/2;
    K[n-1,n-1] = cbar[n]/h;  M[n-1,n-1] = sbar[n]*h/3;  f[n-1] = fbar[n]*h/2;
    for k = 2:n-1
      K[k-1:k,k-1:k] += (cbar[k]/h) * Ke
      M[k-1:k,k-1:k] += (sbar[k]*h) * Me
      f[k-1:k] += (fbar[k]*h) * fe
    end

    # Solve system for the interior values.
    u = (K+M) \ f
    u = [0; u; 0]      # put the boundary values into the result

    return x,u
end