using LinearAlgebra

"""
    angularWaveVectorComponent(n, L)

Returns the angular wave vector component for a rectangular room given the
mode index and room length along the required component cartesian direction, in
meters.
# Example
```jldoctest
julia> nx = 7
7

julia> Lx = 5
5

julia> kx = angularWaveVectorComponent(nx, Lx)
4.39822971502571
```
"""
function angularWaveVectorComponent(n::Integer, L::Real)
    return n * π / L
end

"""
    angularWaveVector(nx, ny, nz, Lx, Ly, Lz)

Returns the angular wave vector of a rectangular room mode. Mode numbers along
the cartesian directions are nx ny and nz, the room sizes along the cartesian
directions are Lx, Ly and Lz meters.
# Example
```jldoctest
julia> angularWaveVector(1, 4, 7, 5.0, 4.0, 3.0)
3-element Array{Float64,1}:
 0.6283185307179586
 3.141592653589793
 7.330382858376184
```
"""
function angularWaveVector(
    nx::Integer, ny::Integer, nz::Integer,
    Lx::Real, Ly::Real, Lz::Real
    )

    return [
        angularWaveVectorComponent(nx, Lx);
        angularWaveVectorComponent(ny, Ly);
        angularWaveVectorComponent(nz, Lz)
        ]

end

"""
    modeFrequency(nx, ny, nz, Lx, Ly, Lz, c)

Returns the modal frequency of a mode characterized by indeces nx, ny and nz
along the cartesian directions for a rectangular room of sizes Lx, Ly and Lz
meters along the cartesian directions. c is the speed of sound in air, by
default 343.0 meters per second (speed at standard conditions).
# Example
```jldoctest
julia> modeFrequency(1, 4, 7, 5.0, 4.0, 3.0)
436.7174156260672

julia> modeFrequency(1, 4, 7, 5.0, 4.0, 3.0, 340.0)
432.89772977511035
```
"""
function modeFrequency(
    nx::Integer, ny::Integer, nz::Integer,
    Lx::Real, Ly::Real, Lz::Real,
    c::Real = 343.0
    )

    return norm(angularWaveVector(nx, ny, nz, Lx, Ly, Lz)) * c / 2π

end

"""
    mode(x, y, z, nx, ny, nz, Lx, Ly, Lz)

Returns the modal shape at coordinates in x, y, z for mode characterized by
indeces nx, ny and nz along the cartesian directions for a rectangular room of
sizes Lx, Ly and Lz meters along the cartesian directions. Result returned
normalized between 1 and -1, arbitrary units.
# Example
```jldoctest
julia> mode(2.5, 2.0, 1.5, 1, 4, 7, 5.0, 4.0, 3.0)
0.0

julia> mode.([3.0; 2.5], [1.0; 2.0], [0.75; 1.5], 1, 4, 7, 5.0, 4.0, 3.0)
2-element Array{Float64,1}:
 0.2185080122244105
 0.0
```
"""
function mode(
    x::Real, y::Real, z::Real,
    nx::Integer, ny::Integer, nz::Integer,
    Lx::Real, Ly::Real, Lz::Real
    )

return cospi(nx * x / Lx) * cospi(ny * y / Ly) * cospi(nz * z / Lz)

end

"""
    meshGrid(x, y, z)

Returns the 3-D grid coordinates defined by the vectors x, y, and z.
The grid is represented by three matrices of size length(y) by length(x) by
length(z). This is similar to MATLAB's meshgrid.
# Example
```jldoctest
julia> x = [1; 2]
2-element Array{Int64,1}:
 1
 2

julia> y = [3; 4]
2-element Array{Int64,1}:
 3
 4

julia> z = [5; 6]
2-element Array{Int64,1}:
 5
 6

julia> X, Y, Z = meshGrid(x, y, z)
([1 2; 1 2]

[1 2; 1 2], [3 3; 4 4]

[3 3; 4 4], [5 5; 5 5]

[6 6; 6 6])
```
"""
function meshGrid(x::AbstractArray, y::AbstractArray, z::AbstractArray)

    X = zeros(eltype(x), length(y), length(x), length(z))
    fX(i, j, k) = X[i, j, k] = x[j]
    [fX(i, j, k) for i in 1:length(y), j in 1:length(x), k in 1:length(z)]

    Y = zeros(eltype(y), length(y), length(x), length(z))
    fY(i, j, k) = Y[i, j, k] = y[i]
    [fY(i, j, k) for i in 1:length(y), j in 1:length(x), k in 1:length(z)]

    Z = zeros(eltype(z), length(y), length(x), length(z))
    fZ(i, j, k) = Z[i, j, k] = z[k]
    [fZ(i, j, k) for i in 1:length(y), j in 1:length(x), k in 1:length(z)]

    return X, Y, Z

end

"""
    indexGrid(Nx, Ny, Nz)

Returns a 3-D grid up to indeces Nx, Ny and Nz along the 3 cartesian
directions. The grid is represented by three matrices of size Ny by Nx by Nz.
# Example
```jldoctest
julia> Gx, Gy, Gz = indexGrid(3, 4, 5)
([1 2 3; 1 2 3; 1 2 3; 1 2 3]

[1 2 3; 1 2 3; 1 2 3; 1 2 3]

[1 2 3; 1 2 3; 1 2 3; 1 2 3]

[1 2 3; 1 2 3; 1 2 3; 1 2 3]

[1 2 3; 1 2 3; 1 2 3; 1 2 3], [1 1 1; 2 2 2; 3 3 3; 4 4 4]

[1 1 1; 2 2 2; 3 3 3; 4 4 4]

[1 1 1; 2 2 2; 3 3 3; 4 4 4]

[1 1 1; 2 2 2; 3 3 3; 4 4 4]

[1 1 1; 2 2 2; 3 3 3; 4 4 4], [1 1 1; 1 1 1; 1 1 1; 1 1 1]

[2 2 2; 2 2 2; 2 2 2; 2 2 2]

[3 3 3; 3 3 3; 3 3 3; 3 3 3]

[4 4 4; 4 4 4; 4 4 4; 4 4 4]

[5 5 5; 5 5 5; 5 5 5; 5 5 5])

```
"""
function indexGrid(Nx::Integer, Ny::Integer, Nz::Integer)
    return meshGrid(0:Nx, 0:Ny, 0:Nz)
end

"""
    roomAxes(rx, ry, rz, Lx, Ly, Lz)

Returns the 1-D axes coordinates along the sides of a rectangular room with
reference to a corner. The room has sizes Lx, Ly and Lz meters along the
cartesian directions, while rx, ry and rz express the step of the axes along
each cartesian direction.
"""
function roomAxes(
    rx::Real, ry::Real, rz::Real,
    Lx::Real, Ly::Real, Lz::Real
    )

    x = range(0.0, stop = Lx, length = round(Integer, 1 + Lx / rx))
    y = range(0.0, stop = Ly, length = round(Integer, 1 + Ly / ry))
    z = range(0.0, stop = Lz, length = round(Integer, 1 + Lz / rz))

    return x, y, z

end

"""
    roomGrid(rx, ry, rz, Lx, Ly, Lz)

Returns the 3-D grid coordinates inside a rectangular room with reference to a
corner. The room has sizes Lx, Ly and Lz meters along the cartesian directions,
while rx, ry and rz express the step of the grid along each cartesian direction.
"""
function roomGrid(
    rx::Real, ry::Real, rz::Real,
    Lx::Real, Ly::Real, Lz::Real
    )

    x, y, z = roomAxes(rx, ry, rz, Lx, Ly, Lz)

    return meshGrid(x, y, z)

end


using Distributions, Random, Plots; pyplot()
Random.seed!(0)

n, N, alpha = 3, 10^7, 0.1

myT(nObs) = rand(Normal())/sqrt(rand(Chisq(nObs-1))/(nObs-1))
mcQuantile = quantile([myT(n) for _ in 1:N],alpha)
analyticQuantile = quantile(TDist(n-1),alpha)

println("Quantile from Monte Carlo: ", mcQuantile)
println("Analytic qunatile: ", analyticQuantile)

xGrid = -5:0.1:5
plot(xGrid, pdf.(Normal(), xGrid), c=:black, label="Normal Distribution")
scatter!(xGrid, pdf.(TDist(1) ,xGrid),
	c=:blue, msw=0, label="DOF = 1")
scatter!(xGrid, pdf.(TDist(3), xGrid),
	c=:red, msw=0, label="DOF = 3")
scatter!(xGrid, pdf.(TDist(100),xGrid),
	c=:green, msw=0, label="DOF = 100",
	xlims=(-4,4), ylims=(0,0.5), xlabel="X", ylabel="Density")


"""
    rkf45(f,a,b,y0,M,delta)
berisi program untuk mencari solusi persamaan
differensial `y' = f(t,y)` dengan `y(a) = y0` pada interval `[a, b]`. Program ini secara
default berisi 6 masukan, yaitu fungsi `f(t,y)`,
titik ujung interval penyelesaian `[a,b]`,
nilai awal `y0`, jumlah sub-interval `M` dan nilai toleransi error `delta`

# Example
```jl
julia> f(t,y) = 1+y^2;

julia> sol = rkf45(f,0,1.4,0,7,2e-5)
14×2 Array{Float64,2}:
 0.0     0.0
 0.2     0.20271
 0.6     0.684165
 0.8     1.02968
 ⋮
 1.35    4.45586
 1.375   5.04273
 1.3875  5.39534
 1.4     5.79895
```
return solusi masalah nilai awal `sol`.
"""
function rkf45(f,a,b,y0,M,delta)
  M = round(M)
  a2=1/4;b2=1/4;a3=3/8;b3=3/32;c3=9/32;
  a4=12/13;b4=1932/2197;c4=-7200/2197;d4=7296/2197;
  a5=1;b5=439/216;c5=-8;d5=3680/513;e5=-845/4104;
  a6=1/2;b6=-8/27;c6=2;d6=-3544/2565;e6=1859/4104;f6=-11/40;
  r1=1/360;r3=-128/4275;r4=-2197/75240;r5=1/50;r6=2/55;
  n1=25/216;n3=1408/2565;n4=2197/4104;n5=-1/5;
  big=1e15;
  h=(b-a)/M;
  hmin=h/64;
  hmax=h*64;
  maxi=200;
  j=1;
  Y = y0;
  T = a;
  br= b-0.001*abs(b);
  err=NaN
  while T[j]<b
    if (T[j]+h)>br;h=b-T[j];end
    #% Hitung koefisien
    k1=h*f(T[j],Y[j]);
    y2=Y[j]+b2*k1;
    k2=h*f(T[j]+a2*h,y2);
    y3=Y[j]+b3*k1+c3*k2;
    k3=h*f(T[j]+a3*h,y3);
    y4=Y[j]+b4*k1+c4*k2+d4*k3;
    k4=h*f(T[j]+a4*h,y4);
    y5=Y[j]+b5*k1+c5*k2+d5*k3+e5*k4;
    k5=h*f(T[j]+a5*h,y5);
    y6=Y[j]+b6*k1+c6*k2+d6*k3+e6*k4+f6*k5;
    k6=h*f(T[j]+a6*h,y6);
    err=abs(r1*k1+r3*k3+r4*k4+r5*k5+r6*k6);
    ynew=Y[j]+n1*k1+n3*k3+n4*k4+n5*k5;
    #% Perbarui ukuran langkah
    if (err<delta) || (h<2*hmin)
      Y = [Y; ynew];
      if (T[j]+h)>br
        T = [T; b];
      else
        T = [T; T[j]+h];
      end
      j=j+1;
    end
    if (err==0)
      s=0;
    else
      s=0.84*(delta*h/err)^(0.25);
    end
    if (s<0.75)&&(h>2*hmin);h = h/2;end
    if (s>1.50)&&(2*h<hmax);h = 2*h;end
    if abs(Y[j])>big || maxi==j;break;end
  end
  sol = [T Y];
  return sol
end


module GlorotUniform
    function GUMatrix(input_size::Int64, layer_size::Int64, shape::Any)
        return rand(-sqrt(6/(input_size+layer_size)):0.00001:sqrt(6/(input_size+layer_size)), shape)
    end
    export GUMatrix
end

# Source: https://keras.io/api/layers/initializers/#glorotuniform-class


const jennrich_sampson = let res_init=zeros(10), jac_init=zeros(10,2), x_init=[0.3, 0.4]

    function res(x, r)
        for i = 1:10
            ti = Float64(i)
            r[i] = 2.0e0 + 2.0e0*ti - (exp(ti*x[1]) + exp(ti*x[2]))
        end
        return r
    end

    function jac(x, j)
        for i = 1:10
            ti = Float64(i)
            j[i,1] = -ti * exp(ti*x[1])
            j[i,2] = -ti * exp(ti*x[2])
        end
        return j
    end

    f(;scale=1, verbose=false, print_steps=false) = testone("jennrich_sampson", res, jac,
                                                            res_init, jac_init, x_init;
                                                            scale=scale, verbose=verbose,
                                                            print_steps=print_steps)
end


@testset "exponentiated" begin
    rng = MersenneTwister(123456)
    x = rand(rng)*2
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)

    k = ExponentiatedKernel()
    @test kappa(k,x) ≈ exp(x)
    @test kappa(k,-x) ≈ exp(-x)
    @test k(v1,v2) ≈ exp(dot(v1,v2))
    @test metric(ExponentiatedKernel()) == KernelFunctions.DotProduct()
    @test repr(k) == "Exponentiated Kernel"
    test_ADs(ExponentiatedKernel)
end


export getEdgeLength, getFaceArea, getCellVolume

function getEdgeLength(mesh::TetraMesh)
# get edge length

if isempty(mesh.L)

  ne = mesh.ne
  L = zeros(ne)

  for i=1:ne
    i1 = mesh.edges[i,1]
    i2 = mesh.edges[i,2]

    L[i] = getEdgeLength(mesh.Points[i1,:],  mesh.Points[i2,:])
  end

  mesh.L = L

end

return mesh.L
end

function getFaceArea(mesh::TetraMesh)
# get face size

if isempty(mesh.F)
  nf = mesh.nf
  F = zeros(nf)
  for i=1:nf
    i1 = mesh.faces[i,1]
    i2 = mesh.faces[i,2]
    i3 = mesh.faces[i,3]

    F[i] = getFaceArea(mesh.Points[i1,:], mesh.Points[i2,:], mesh.Points[i3,:])
  end

mesh.F = F
end

return mesh.F
end

function getCellVolume(mesh::TetraMesh)
# get volume

if isempty(mesh.V)

  nc = mesh.nc
  T  = mesh.Tetras
  V = zeros(nc)

  for i=1:nc
    i1 = T[i,1]
    i2 = T[i,2]
    i3 = T[i,3]
    i4 = T[i,4]

    V[i] = getCellVolume(mesh.Points[i1,:], mesh.Points[i2,:], mesh.Points[i3,:], mesh.Points[i4,:])
  end

  mesh.V = V

end

return mesh.V
end

function getEdgeLength(P1::Array{Float64,1}, P2::Array{Float64,1})
	return sqrt((P1[1] - P2[1])^2 + (P1[2] - P2[2])^2 + (P1[3] - P2[3])^2)
end

function getFaceArea(P1::Array{Float64,1}, P2::Array{Float64,1}, P3::Array{Float64,1})
	return norm(cross(P3 - P1, P3 - P2))/2
end

function getCellVolume(P1::Array{Float64,1}, P2::Array{Float64,1}, P3::Array{Float64,1}, P4::Array{Float64,1})
	return 1/6.0*abs(dot(P4 - P3, cross(P4 - P1, P4 - P2)))
end


using StaticArrays: SArray, @SMatrix
using LinearAlgebra: I, norm

# This matrix-based implementation is very cool (IMO)
# and only ~25% slower than the reference implementation for addition, multiplication, subtraction
#
# We get a 10x speedup for division and abs (although, for abs in particular, base widens to guard against overflow)
struct ComplexNumber{T<:Real} <: Number
    mat::SArray{Tuple{2,2},T,2,4}
    ComplexNumber(mat) = new{eltype(mat)}(mat)
    ComplexNumber(real::T1,imag::T2) where {T1 <: Real, T2 <: Real} = new{promote_type(T1,T2)}(@SMatrix([real  -imag; imag real]))
end

Base.:*(l::ComplexNumber,r::ComplexNumber) = ComplexNumber(l.mat * r.mat)
Base.:+(l::ComplexNumber,r::ComplexNumber) = ComplexNumber(l.mat + r.mat)
Base.:-(l::ComplexNumber,r::ComplexNumber) = ComplexNumber(l.mat - r.mat)
Base.:/(l::ComplexNumber,r::ComplexNumber) = ComplexNumber(l.mat * inv(r.mat))
Base.:*(z::ComplexNumber,r::Real) = ComplexNumber(r*z.mat)
Base.:*(r::Real, z::ComplexNumber) = z*r
Base.:+(z::ComplexNumber,r::Real) = ComplexNumber(r*I + z.mat)
Base.:+(r::Real, z::ComplexNumber) = z + r

Base.show(io::IO, z::ComplexNumber) = print(io, "ComplexNumber(", Re(z), ", ", Im(z), ")")
Base.:≈(l::ComplexNumber,r::ComplexNumber) = l.mat ≈ r.mat # I had to make one of the division tests ≈ to pass it
Base.:(==)(l::ComplexNumber,r::ComplexNumber) = l.mat == r.mat

Base.abs(z::ComplexNumber) = norm(z.mat[:,1])

# Sorry about the capital letter. That's what maths does.
@inline Re(z::ComplexNumber) = @inbounds z.mat[1,1]
@inline Im(z::ComplexNumber) = @inbounds z.mat[2,1]

Base.real(z::ComplexNumber) = Re(z)
Base.imag(z::ComplexNumber) = Im(z)

Base.conj(z::ComplexNumber) = ComplexNumber(transpose(z.mat))

Base.exp(z::ComplexNumber) = ComplexNumber(exp(z.mat))

# There's not an iota of a chance that ι will take off as the sensible symbol to use for the imaginary unit
const ι = jm = ComplexNumber(0,1)


using LinAlgBigFloat.VectorSpaces
using Random

Random.seed!(1)

l, n, m = 2, 6, 4
A = randn(n, m)
B = randn(l, n)

vz = ZeroSpace(n)
@test rank(vz) == 0 && dim(vz) == n

vf = VectorSpace(n)
@test rank(vf) == n && dim(vf) == n

va = VectorSpace(A)
@test rank(va) == min(m, n) && dim(va) == n
vb = VectorSpace(B)
@test rank(vb) == min(l, n) && dim(vb) == l

@test va == image(A)
@test vb == image(B)

@test vz ⊆ va ⊆ vf

@test kernel(A) == ZeroSpace(m)
@test image(B) == VectorSpace(l)

@test kernel(A) == preimage(A)


# This file includes the MonteCarlo simulation of ASK modulation scheme and compares
# the numerical results with the theoretical results.

using DigitalCommunications
using Plots

# Simulation parameters
k = 3
M = 2^k
nsymbols = Int(1e6)
nbits = k * nsymbols
Ep = 100.                       # Modulating pulse energy.
ebno = collect(0 : 10)          # Snr per bit
esno = ebno .+ 10 * log10(k)    # Snr ber symbol

# Communcation system components
gen = BitGenerator(nbits)
coding = GrayCoding(M)
modulator = Modulator(PAM(M))
channel = AWGNChannel()
detector = Detector(alphabet(modulator))

# Monte Carlo simulation
message = coding(gen.bits)  # Message signal
symerr = zeros(length(esno))
for i in 1 : length(symerr)
    channel.esno = esno[i]  # Update channel snr
    mbar = gen.bits |> coding |> modulator |> channel |> detector  # Extracted message signal
    symerr[i] = sum(mbar .!= message) / length(message)  # Symbol error rate
end

# Plots
plt = plot(title="$M-PAM", xlabel="ebno [dB]", ylabel="Pe")
plot!(ebno, berpam.(ebno, M), marker=:circle, yscale=:log10, label="theoretical")
plot!(ebno, symerr, marker=:circle, yscale=:log10, label="montecarlo")


using SymPy
using Base.Test
using Compat

## Symbol creation
x = Sym("x")
#x = sym"x" # deprecated
x = Sym(:x)
x,y = Sym(:x, :y)
x,y = symbols("x,y")

@syms u1 u2 u3
@syms u positive=true
@test length(solve(u+1)) == 0
# make sure @syms defines in a local scope
let
    @syms w
end
@test_throws UndefVarError isdefined(w)

## extract symbols
ex = x*y
@test isa(free_symbols(ex), Vector{Sym})

## number conversions
Sym(2)
Sym(2.0)
Sym(2//1)
Sym(im)
Sym(2im)
Sym(1 + 2im)
Sym(pi)
Sym(e)
Sym(catalan)

## function conversion
f1 = convert(Function, x^2)
@test f1(2) == Sym(4)

### Subs
## subs, |> (x == number)
f(x) = x^2 - 2
y = f(x)
@assert float(subs(y, x, 1)) == f(1)
@assert float( y |> subs(x,1) ) == f(1)

## interfaces
ex = (x-1)*(y-2)
@assert subs(ex, x, 1) == 0
@assert subs(ex, (x,1)) == 0
@assert subs(ex, (x,2),(y,2)) == 0

if VERSION >= v"0.4.0"
    @assert subs(ex, x=>1) == 0
    @assert subs(ex, x=>2, y=>2) == 0
    @assert subs(ex, Dict(x=>1)) == 0
    @assert ex(x=>1) == 0
end

## match, replace, xreplace, rewrite
x,y,z = symbols("x, y, z")
a,b,c = map(Wild, (:a,:b,:c))
## match: we have pattern, expression to follow `match`
d = match(a^a, (x+y)^(x+y))
@test d[a] == x+y
d = match(a^b, (x+y)^(x+y))
@test d[b] == x + y
ex = (2x)^2
pat = a*b^c
d = match(pat, ex)
@test d[a] == 4 && d[b] == x && d[c] == 2
@test xreplace(pat, d) == 4x^2

## replace
if VERSION >= v"0.4.0"
ex = log(sin(x)) + tan(sin(x^2))
@test replace(ex, func(sin(x)), func(cos(x))) == log(cos(x)) + tan(cos(x^2))
@test replace(ex, func(sin(x)), u ->  sin(2u)) == log(sin(2x)) + tan(sin(2*x^2))
@test replace(ex, sin(a), tan(a)) ==  log(tan(x)) + tan(tan(x^2))
@test replace(ex, sin(a), a) == log(x) + tan(x^2)
@test replace(x*y, a*x, a) == y
end
## xreplace
if VERSION >= v"0.4.0"
@test xreplace(1 + x*y, x => PI) == 1 + PI*y
@test xreplace(x*y + z, x*y => PI) == z + PI
@test xreplace(x*y * z, x*y => PI) == x* y * z
@test xreplace(x +2 + exp(x + 2), x+2=>y) == x + exp(y) + 2
end

#Test subs for pars and dicts
ex = 1
dict1 = Dict{Compat.String,Any}()
dict2 = Dict{Any,Any}()
#test subs
for i=1:4
    x = Sym("x$i")
    ex=ex*x
    dict2[x] = i
    dict1[string(x)] = i
end
for d in [dict1, dict2]
    @test ex |> subs(d) == factorial(4)
    @test subs(ex, d) == factorial(4)
    @test subs(ex, d...) == factorial(4)
    if VERSION >= v"0.4.0"
        @test ex |> subs(d...) == factorial(4)
        @test ex(d) == factorial(4)
        @test ex(d...) == factorial(4)
    end
end

a = Sym("a")
b = Sym("b")
line = x -> a + b * x
sol = solve([line(0)-1, line(1)-2],[a,b])
ex = line(10)
@test ex |> subs(sol) == 11
if VERSION >= v"0.4.0"
    @test ex(sol) == ex(sol...) == 11
end

## Conversion
x = Sym("x")
p = subs(x,x,pi)
q = subs(x,x,1//2)
r = subs(x,x,1.2)
z = subs(x,x,1)
@assert isa(N(p), Float64)
@assert isa(N(p, 60), BigFloat)
@assert isa(evalf(p), Sym)
@assert isa(N(q), Rational)
@assert isa(N(r), Float64)
@assert isa(N(z), Int)

## method calls via getindex
p = (x-1)*(x-2)
p[:roots]()  # sympy.roots
p = Poly(p, x)
p[:coeffs]() # p.coeffs

## algebra
expand((x + 1)*(x+2))
x1 = (x + 1)*(x+2)
expand(x1)
expand_trig(sin(2x))

## solve
x,y,a = symbols("x,y,a", real=true)
solve(x^2 - 2x)
solve(x^2 - 2a, x)
solve(x^2 - 2a, a)
solve(Lt(x-2, 0))
solve( x-2 ≪ 0)
exs = [x-y-1, x+y-2]
d = solve(exs)
@assert d[x] == 3//2
@assert map(ex -> subs(ex, d), exs) == [0,0]
solve([x-y-a, x+y], [x,y])

## limits
@assert limit(sin(x)/x, x, 0) |> float == 1
(x, h) = @syms x h
out = limit((sin(x+h) - sin(x))/h, h, 0)
@assert (out |> replace(x, pi) |> float) == -1.0

## diff
diff(sin(x), x)
out = diff(sin(x), x, 2)
@assert abs((out |> replace(x, pi/4) |> float) - - sin(pi/4)) < sqrt(eps())

@syms x y
diff(x^2 + x*y^2, x, 1)         # partial derivatives

t = symbols("t", real=true)     # vector-valued functions
r1(t) = [sin(t), cos(t), t]
u = r1(t)
#kappa = norm(diff(u) × diff(u,t,2)) / norm(diff(u))^3 |> simplify
#@assert convert(Rational,kappa) == 1//2

u = SymFunction("u")
eqn = Eq(x^2 + u(x)^2, x^3 - u(x))
diff(eqn, x)

## integrate
integrate(sin(x))
integrate(sin(x), (x, 0, pi))
@vars a b t
integrate(sin(x), (x, a, b))
integrate(sin(x), (x, a, b)) |> replace(a, 0) |> replace(b, pi)
integrate(sin(x) * DiracDelta(Sym(0))) # sin(0)
integrate(Heaviside(x), (x, -1, 1))
C = Curve([exp(t)-1, exp(t)+1], (t, 0, log(Sym(2))))
line_integrate(x + y, C, [x,y])

## summation
summation(1/x^2, (x, 1, 10))
out = summation(1/x^2, (x, 1, 10))
out1 = sum([1//x^2 for  x in 1:10])
@assert @compat round(Integer, out.x[:p]) == out1.num
@assert @compat round(Integer, out.x[:q]) == out1.den

## matrices
## these can also be tested by matrix-tests.jl
@syms x
A = [x 1; 1 x]
b = [x, 2]

## Ops
s = 3
x = Sym("x")
v = [x, 1]
rv = [x 1]
a = [x 1; 1 x]
b = [x 1 2; 1 2 x]

const DIMERROR = VERSION < v"0.4.0-dev" ? ErrorException : DimensionMismatch

## scalar, [vector, matrix]
s + v
v + s
s + rv
rv + s
s + a
a + s

s - v
v - s
s - rv
rv - s
s - a
a - s

2v
2rv
2a
s .* v
v .* s
s .* rv
rv .* s
s .* a
a .* s

## s / v  ## broadcasts s Depreacated
s ./ v
v / s
v .\ s
s \ v
## s / rv ## broadcasts s Deprecated
s ./ rv
rv / s
rv .\ s
s \ rv
## s / a ## broadcasts s Deprecated
s ./ a
a / s
a .\ s
s \ a

@test_throws MethodError  s ^ v ## error
s .^ v
@test_throws MethodError  v ^ s ## error
v .^ s
@test_throws MethodError  s ^ rv ## error
s .^ rv
@test_throws DimensionMismatch  rv ^ s ## error
rv .^ s
@test_throws MethodError  s ^ a ## error
s .^ a
a ^ s
a .^ s

## vector vector
v .+ v
##@test_throws MethodError  v .+ rv ##  no longer an error, broadcase
v .- v
@test_throws DIMERROR  v - rv
@test_throws MethodError  v * v ## error
v .* v
dot(v, v)
v * rv ## 2x1 1x2 == 2x2
rv * v ## 1x2 2 x 1 == 1x1
v .* rv ## XXX ?? should be what? -- not 2 x 2
rv .* v ## XXX ditto
## @test_throws MethodError  v / v ## error
v ./ v ## ones()
v .\ v
## @test_throws MethodError  v / rv ## error
v ./ rv  ## ??
rv .\ v
@test_throws MethodError  v ^ v ## error
v .^ v
@test_throws MethodError  v ^ rv ## error
v .^ rv ## ??

## vector matrix
@test_throws DIMERROR v + a ## error (Broadcast?)
@test_throws DIMERROR a + v ## error
v .+ a ## broadcasts
a .+ v
@test_throws DIMERROR  v - a ## error
v .- a
@test_throws DimensionMismatch  v * a ## error
v .* a
#@test_throws MethodError  v / a ## error
v ./ a
a .\ v
@test_throws MethodError  v ^ a ## error
v .^ a
v
## matrix matrix
a + a
@test_throws DIMERROR  a + b ## error
a + 2a
a - a
@test_throws DIMERROR  a - b ## error
a * a
a .* a
a * b ## 2x2 * 2*3 -- 2x3
@test_throws DIMERROR  a .* b ## error -- wrong size
#@test_throws MethodError  a / a
a ./ a ## ones
a .\ a
##@test_throws MethodError  a / b ## error
@test_throws DIMERROR  a ./ b ## error
@test_throws DIMERROR  b .\ a ## error
@test_throws MethodError  a ^ a ## error
a .^ a
@test_throws MethodError  a ^ b ## error
@test_throws DIMERROR  a .^ b ## error

## Number theory
#@test isprime(100) == isprime(Sym(100))
#@test factorint(Sym(100)) == factor(100)
@test prime(Sym(100)) == 541
@test multiplicity(Sym(10), 100) == 2

## polynomials
@syms x y
f1 = 5x^2  + 10x + 3
g1 = 2x + 2
q,r = polydiv(f1,g1, domain="QQ") # not div, as can't disambiguate div(Sym(7), 5)) to do integer division
@test r == Sym(-2)
@test simplify(q*g1 + r - f1) == Sym(0)

## piecewise
x = Sym("x")
p = piecewise((x, Ge(x,0)), (0, Lt(x,0)), (1, Eq(x,0)))
## using infix \ll<tab>, \gt<tab>, \Equal<tab>
p = piecewise((x, x ≥ 0), (0, x ≪ 0), (1, x ⩵ 0))
@assert @compat Int(subs(p,x,2)) == 2
@assert @compat Int(subs(p,x,-1)) == 0
@assert @compat Int(subs(p,x,0)) == 0

u = ifelse(Lt(x, 0), "neg", ifelse(Gt(x, 0), "pos", "zero"))
@assert subs(u,x,-1) == Sym("neg")
@assert subs(u,x, 0) == Sym("zero")
@assert subs(u,x, 1) == Sym("pos")

## relations
x,y=symbols("x, y")
ex = Eq(x^2, x)
@assert lhs(ex) == x^2
@assert rhs(ex) == x
@assert args(ex) == (x^2, x)

## mpmath functions
if isdefined(:mpmath)
    x = Sym("x")
    Sym(big(2))
    Sym(big(2.0))                   # may need mpmath (e.g., conda install mpmath)

    @assert limit(besselj(1,1/x), x, 0) == Sym(0)
    complex(hankel2(2, pi))
    bei(2, 3.5)
    bei(1+im, 2+3im)
end

## Assumptions
@assert ask(Q.even(Sym(2))) == true
@assert ask(Q.even(Sym(3))) == false
@assert ask(Q.nonzero(Sym(3))) == true

## sets
s = FiniteSet("H","T")
s1 = powerset(s)
VERSION >= v"0.4.0" && @assert length(collect(convert(Set, s1))) == length(collect(s1.x))
a, b = Interval(0,1), Interval(2,3)
@assert is_disjoint(a, b) == true
@assert measure(union(a, b)) == 2

## Issue # 56
@assert Sym(1+2im) == 1+2IM

## Issue #59
cse(sin(x)+sin(x)*cos(x))
cse([sin(x), sin(x)*cos(x)])
cse([sin(x) sin(x)*cos(x); cos(x) sin(x)*cos(x)])

## Issue #60, lambidfy
if VERSION >= v"0.4.0"
    @vars x,y
    lambdify(sin(x)*cos(2x) * exp(x^2/2))
    lambdify(sin(x)*asin(x)*sinh(x))(0.25)
    lambdify(real(x)*imag(x))
#    @assert lambdify(Min(x,y))(3,2) == 2

    ex = 2*x^2/(3-x)*exp(x)*sin(x)*sind(x)
    map(lambdify(ex), rand(10))
    ex = x - y
    @assert lambdify(ex)(3,2) == 1

    i = Indicator(x, 0, 1)
    u = lambdify(i)
    @assert u(.5) == 1
    @assert u(1.5) == 0
end

## issue #67
@assert N(Sym(4//3)) == 4//3

## issue #71
@test log(Sym(3), Sym(4)) == log(Sym(4)) / log(Sym(3))


module TestHess
    using Calculus2
    using Base.Test

    # multivariate test function
    function f(x::Vector)
        return sin(x[1]) + cos(x[2])
    end
    x = [0.0, 0.0]
    n = length(x)
    H = Array(eltype(x), n, n)

    # Test in-place multivariate grad!
    hess!(H, f, x)
    @test Calculus2.maxdiff(
        H,
        [-sin(0.0) 0.0; 0.0 -cos(0.0)]
    ) < 10e-8

    # Test pure multivariate grad
    @test Calculus2.maxdiff(
        hess(f, x),
        [-sin(0.0) 0.0; 0.0 -cos(0.0)]
    ) < 10e-8

    # Test in-place multivariate gradfunc!
    fill!(H, 0.0)
    h! = hessfunc!(f, x)
    h!(H, x)
    @test Calculus2.maxdiff(
        H,
        [-sin(0.0) 0.0; 0.0 -cos(0.0)]
    ) < 10e-8

    # Test pure multivariate gradfunc
    h = hessfunc(f, x)
    @test Calculus2.maxdiff(
        h(x),
        [-sin(0.0) 0.0; 0.0 -cos(0.0)]
    ) < 10e-8

    # Test pure univariate grad
    @test isapprox(hess(sin, 1.0), -sin(1.0))

    # Test pure univariate gradfunc
    @test isapprox(hessfunc(sin, 0.0)(1.0), -sin(1.0))

    # TODO: Test automatic differentation
    # TODO: Test symbolic differentiation
    # TODO: Test direction options
end


#==============================================================================
    Code for solving the Hamiltonian Jacboi Bellman for
	   an basic model: a neoclassical growth model: ρV(k) = max_{c} U(c) + V'(k)[F(k)-δk - c]
	   Where s(k) = F(k) - δk - c(k) and c(k) = U'^{-1}(V'(k))

	Translated Julia code from Matlab code by Ben Moll:
        http://www.princeton.edu/~moll/HACTproject.htm

		Updated to julia 1.0.0
==============================================================================#

using Distributions, Plots

σ= 2.0 #
ρ = 0.05 #the discount rate
δ = 0.05 # the depreciation rate
A = 1.0
α= 0.3

k_ss = (α*A/(ρ+δ))^(1/(1-α))

H= 150
k_min = 0.001*k_ss
k_max = 2.0*k_ss

k = LinRange(k_min, k_max, H)
k = convert(Array, k) # create grid for a values
dk = (k_max-k_min)/(H-1)

maxit = 1000
ε = 10e-6

dVf, dVb = [zeros(H,1) for i=1:2]

#initial guess for V
v0 = (A.*k.^α).^(1-σ)/(1-σ)/ρ
v= v0

dist=[]

for n=1:maxit
	V=v

    # forward difference
	dVf[1:H-1] = (V[2:H]-V[1:H-1])/dk
	dVf[H] = 0

	# backward difference
	dVb[2:H] = (V[2:H]-V[1:H-1])/dk
	dVb[1] = 0 # the boundary condition

	I_concave = dVb .> dVf

    # consumption and savings with forward difference
    cf = dVf.^(-1/σ)
    μ_f = A.*k.^α - δ.*k -cf

    # consumption and savings with backward difference
    cb = dVb.^(-1/σ)
    μ_b = A.*k.^α - δ.*k -cb

	c0 = A.*k.^α - δ.*k
    dV0 = c0.^(-σ)

    # Now to make a choice between forward and backward difference
    If = μ_f .> 0
    Ib = μ_b .< 0
    I0 = 1.0.-If-Ib
    Ib[1] = false
    If[1] = true
    Ib[H] = true
    If[H] = false

    global dV_Upwind= dVf.*If + dVb.*Ib + dV0.*I0

    global c = dV_Upwind.^(-1/σ)
    V_change = c.^(1-σ)/(1-σ) + dV_Upwind.*(A.*k.^α - δ.*k-c) -ρ.*V

	# update
	Δ = .9*dk/(findmax(A.*k.^α- δ.*k-c)[1])
	global v = v + Δ*V_change
	push!(dist,findmax(abs.(V_change))[1])
	if dist[n] .< ε
		println("Value Function Converged Iteration=")
		println(n)
		break
	end

end

plot(dist, grid=false,
		xlabel="Iteration", ylabel="||V^{n+1} - V^n||",
		ylims=(-0.001,0.030),
		legend=false, title="")
png("Convergence")

v_err = c.^(1-σ)/(1-σ) + dV_Upwind.*(A.*k.^α - δ.*k -c) - ρ.*v

plot(k, v_err, grid=false,
		xlabel="k", ylabel="Error in the HJB equation",
		xlims=(k_min,k_max),
		legend=false, title="")
png("HJB_error")

plot(k, v, grid=false,
		xlabel="k", ylabel="V(k)",
		xlims=(k_min,k_max),
		legend=false, title="")
png("Value_function_vs_k")

plot(k, c, grid=false,
		xlabel="k", ylabel="c(k)",
		xlims=(k_min,k_max),
		legend=false, title="")
png("c(k)_vs_k")

# approximation at the borrowing constraint
k_dot = (A.*k.^α - δ.*k -c)

plot(k, k_dot, grid=false,
		xlabel="k", ylabel="s(k)",
		xlims=(k_min,k_max), title="", label="s(k)", legend=:topright)
plot!(k, zeros(H,1), label="", line=:dash)
png("stateconstraint")


using Distributions
using StatsBase

function Binom(k, N, p)
    return pdf(Binomial(N, p), k)
end

function Binom_sum(N, p)
    if iseven(N)
        N -= 1
    end
    return sum([Binom(k, N, p) for k = N:-1:N/2])
end

function compute_analytical_accuracy(; N, p_GC_sig, p_GC_bkg, weight = [1, 1])
    p_α_sig = 1 - p_GC_sig
    p_α_bkg = 1 - p_GC_bkg
    return mean([Binom_sum(N, p_α_sig), Binom_sum(N, p_α_bkg)], weights(weight))
end

function compute_analytical_accuracies(;
    N_half,
    p_GC_sig = 0.4,
    p_GC_bkg = 0.5,
    weight = [1, 1],
)
    bases_included = 1:N_half
    analytical_accuracies = [
        compute_analytical_accuracy(
            N = i * 2,
            p_GC_sig = p_GC_sig,
            p_GC_bkg = p_GC_bkg,
            weight = weight,
        ) for i in bases_included
    ]
    return analytical_accuracies
end

#%%

using CairoMakie
using ColorSchemes

N_half = 38
x = 1:N_half

analytical_accuracies = compute_analytical_accuracies(;N_half=N_half)

colormap = [x for x in ColorSchemes.Set1_9.colors]

#%%

f = Figure()
ax = Axis(
    f[1, 1],
    title = "Analytical accuracy",
    xlabel = "# bases included (centered)",
    ylabel = "Accuracy",
    # limits = (0.5, half_seq_length + 0.5, 0.634, 0.701),
    # limits = (0.5, half_seq_length + 0.5, ylimits...),
    xticks = 1:2:N_half,
)

scatterlines!(
    ax,
    x,
    analytical_accuracies,
    color = colormap[3],
    markercolor = colormap[3],
    label = "Analytical",
)

axislegend(position = :rb)
f

#%%

compute_analytical_accuracy(N=1, p_GC_sig = 0.4, p_GC_bkg = 0.5)
compute_analytical_accuracy(N=2, p_GC_sig = 0.4, p_GC_bkg = 0.5)
compute_analytical_accuracy(N=3, p_GC_sig = 0.4, p_GC_bkg = 0.5)
compute_analytical_accuracy(N=4, p_GC_sig = 0.4, p_GC_bkg = 0.5)
compute_analytical_accuracy(N=5, p_GC_sig = 0.4, p_GC_bkg = 0.5)
compute_analytical_accuracy(N=6, p_GC_sig = 0.4, p_GC_bkg = 0.5)


"""
    dualgraph(dist::Matrix{Float64}; method::Symbol = :inverse, σ::Float64 = 1.0)

build the dual graph's weight matrix based on the given non-trivial eigenvector
metric.

# Input Arguments
- `dist::Matrix{Float64}`: eigenvector distance matrix
- `method::Symbol`: default is by taking inverse of the distance between
    eigenvectors. Ways to build the dual graph edge weights. Option: `:inverse`,
    `:gaussian`.
- `σ::Float64`: default is `1.0`. Gaussian variance parameter.

# Output Argument
- `G_star::GraphSig`: A `GraphSig` object containing the weight matrix of the
    dual graph.

"""
function dualgraph(dist::Matrix{Float64}; method::Symbol = :inverse, σ::Float64 = 1.0)
    N = Base.size(dist, 1)
    W_star = zeros(N, N)
    if method == :inverse
        for i = 1:(N - 1), j = (i + 1):N
            W_star[i, j] = 1 / dist[i, j]
        end
    elseif method == :gaussian
        for i = 1:(N - 1), j = (i + 1):N
            W_star[i, j] = exp(-dist[i, j] / σ^2)
        end
    else
        error("method must be :inverse or :gaussian.")
    end
    W_star = W_star + W_star'
    G_star = GraphSig(sparse(W_star); name = "dual graph")
    return G_star
end


using Random, Statistics
Random.seed!(1)
allData = [rand(24), rand(15), rand(73)]

xBarArray = mean.(allData)
nArray = length.(allData)
xBarTotal = mean(vcat(allData...))
L = length(nArray)

ssBetween=sum([nArray[i]*(xBarArray[i] - xBarTotal)^2 for i in 1:L])
ssWithin=sum([sum([(ob - xBarArray[i])^2 for ob in allData[i]]) for i in 1:L])
ssTotal=sum([sum([(ob - xBarTotal)^2 for ob in allData[i]]) for i in 1:L])

println("Sum of squares between groups: ", ssBetween)
println("Sum of squares within groups: ", ssWithin)
println("Sum of squares total: ", ssTotal)


using MadMax
using Test

# BisectionEval
# RegulaFalsiEval
# NewtonEval
# GaussJacobiEval
# GaussSeidelEval
# SOREval
# SOREval2
# EulerODE
# ModifiedEulerODE
# RK2ODE
# RK4ODE

X = [5 5 15 -40
      5 15 35 -104
      15 35 99 -290]

X2 = [3  2  0 -5
      2  3 -1 -4
      0 -1  2 -1]

x, a = GaussSeidelEval(X2, [0.9;-3.1;0.9], N=3)
x, a = GaussJacobiEval(X, [0;0;0], N=7)
x, a = SOREval(X2, [0;0;0], 1.25, N=3)
x, a = SOREval2(X2, [0;0;0], 0.8, N=7)

f(x) = x^24-2.6x+1.6
RegulaFalsiEval(f, 0, 1, tol=0.5*10^-3)
NewtonEval(f, 0.05, N=4, tol=0.5*10^-4)

f(t, x) = -x*sin(t) + sin(t)
ModifiedEulerODE(f, 0, 2+ℯ, 2π, 2π/10)
RK2ODE(f, 0, 2+ℯ, 2π, 2π/10)


import Random
using Plots

println("Pool Testing Optimization\nMonte Carlo Simulation")

min_total = 1
max_total = 100
min_infected = 0
iterations = 100

function run(total, infected, n_groups)
	# Trivial cases
	# No one is infected or testing everybody
	if infected==0 || n_groups==total
		return n_groups
	# Only 1 group (there is only 1 grouping step)
	elseif n_groups==1
		return total+1
	end

	people = zeros(Bool, total)
	people[1:infected] .= true
	Random.shuffle!(people)

	n_per_group = total ÷ n_groups
	n_plus_groups = total % n_groups

	groups = zeros(UInt, n_groups)

	for i in 1:infected
		j = 1+(Random.rand(UInt) % n_groups)
		groups[j] += 1
	end

	infected_groups = count(g -> g>0, groups)

	tests = n_groups

	for (index, value) in enumerate(groups)
		if value>0
			if index<=n_plus_groups
				tests += n_per_group+1
			else
				tests += n_per_group
			end
		end
	end

	#=
	println("Total   \t", total)
	println("Infected\t", infected)
	println("People\t", people)
	println("People per group\t", n_per_group)
	println("# groups with 1 more\t", n_plus_groups)
	println("Infected per group\t", groups)
	println("Number of infected groups\t", infected_groups)
	println("Number of tests\t", tests)
	=#

	return tests
end

n = binomial(max_total+2-min_infected, 2) - binomial(min_total+1-min_infected, 2)
results = Array{UInt, 2}(undef, n, 3)

i = 1
for total in min_total:max_total
	for infected in min_infected:total
		println("total ", total, "  infected ", infected)

		optimum_n_groups = total
		previous_sum_tests = iterations*total

		for n_groups in 1:total
			sum_tests = 0

			for iteration in 1:iterations
				tests = run(total, infected, n_groups)
				sum_tests += tests
				#println("total ", total, "  infected ", infected, "  n_groups ", n_groups, "  iteration ", iteration, "  tests ", tests)
			end

			if sum_tests<previous_sum_tests
				optimum_n_groups = n_groups
			end
		end

		global i
		results[i, :] = [total, infected, optimum_n_groups]
		i += 1
	end
end

println("Calculated")

matrix = zeros(Float32, (max_total+1-min_total, max_total+1-min_infected))
for k = 1:size(results, 1)
	local i = results[k, 1] + 1 - min_total
	local j = results[k, 2] + 1 - min_infected
	matrix[i, j] = results[k, 3]/results[k, 1]
end

#println("results ", results)
#println("matrix ", matrix)

heatmap(min_total:max_total, min_infected:max_total, matrix',
		xlabel="Total", ylabel="Infected",
		title="Optimal number of groups / Total",
		aspect_ratio=:equal,
		show=true)
println("Plotted")
readline()

println("End")


### A Pluto.jl notebook ###
# v0.14.2

using Markdown
using InteractiveUtils

# ╔═╡ 0a5335e0-ab9e-11eb-0a85-139ff5cf0b1d
md"# Chapter 02 from NNFS book"

# ╔═╡ 0c5d667f-0ea5-4d21-80ad-58d144470a73
md"### A Single Neuron (4 inputs, 1 output)"

# ╔═╡ d85b097d-c7c7-439f-80ec-a1b75c696cb1
md"""
for **each input** we need **one weigth** and for **each output** we need **one bias**.
So, for this simple example with 1 neuron with 4 inputs we are gonna have:

- **3** inputs
- **3** weights
- **1** bias.
"""

# ╔═╡ ef746509-f29b-42ae-b9ec-a2e18ad4b327
begin
	inputs = [1; 2; 3; 2.5]
	weights = [0.2; 0.8; -0.5; 1.0]
	bias = 2.0

	inputs'*weights + bias # or dot(inputs, weights) + bias
end

# ╔═╡ a3acafe8-6a4b-4438-84d9-16a00cb21997
md"### A Layer of 03 Neurons (4 inputs, 3 outputs)"

# ╔═╡ 9fbc1e4c-52c0-4b99-92dc-4969b55f645d
begin
	#inputs = [1, 2, 3, 2.5]
	weights1 = [0.2, 0.8, -0.5, 1.0]
	weights2 = [0.5, -0.91, 0.26, -0.5]
	weights3 = [-0.26, -0.27, 0.17, 0.87]
	bias1 = 2.0
	bias2 = 3.0
	bias3 = 0.5

	[inputs'*weights1 + bias1,
	inputs'*weights2 + bias2,
	inputs'*weights3 + bias3]
end

# ╔═╡ 3613059e-501d-4cb0-bd73-8e42e6395e16
md" with only one unique matrix of weights we use matrix multiplication from linear algebra"

# ╔═╡ 0e309e2a-59e8-4fce-819d-afeec8c21ca8
begin
	weights_array = [
		0.2    0.8  -0.5   1.0;
		0.5   -0.91  0.26 -0.5;
		-0.26 -0.27  0.17  0.87;
	]
	bias_vec = [2.0 3.0 0.5]

	(weights_array * inputs) .+ bias_vec
end

# ╔═╡ d27568ac-934c-4408-880d-4bd34ccead72
md"### A Layer of Neurons with multiple batch of Data"

# ╔═╡ 91d6b3e6-5124-4963-b6c2-f2c65b67b045
md"Now we are gonna need use transpose operation to match the right vectors in matrix multiplication"

# ╔═╡ 7ae39d13-e448-4c91-81be-d44a15f1a414
begin
	inputs_batch = [
		1.0 2.0 3.0 2.5
		2.0 5.0 -1.0 2.0
		-1.5 2.7 3.3 -0.8
	]

	(inputs_batch * weights_array') .+ bias_vec
end

# ╔═╡ Cell order:
# ╟─0a5335e0-ab9e-11eb-0a85-139ff5cf0b1d
# ╟─0c5d667f-0ea5-4d21-80ad-58d144470a73
# ╟─d85b097d-c7c7-439f-80ec-a1b75c696cb1
# ╠═ef746509-f29b-42ae-b9ec-a2e18ad4b327
# ╟─a3acafe8-6a4b-4438-84d9-16a00cb21997
# ╠═9fbc1e4c-52c0-4b99-92dc-4969b55f645d
# ╟─3613059e-501d-4cb0-bd73-8e42e6395e16
# ╠═0e309e2a-59e8-4fce-819d-afeec8c21ca8
# ╟─d27568ac-934c-4408-880d-4bd34ccead72
# ╟─91d6b3e6-5124-4963-b6c2-f2c65b67b045
# ╠═7ae39d13-e448-4c91-81be-d44a15f1a414


using TuringModels, StatsFuns

Turing.setadbackend(:reverse_diff)

# outcome and predictor almost perfectly associated

x = repeat([-1], 9); append!(x, repeat([1],11))
y = repeat([0], 10); append!(y, repeat([1],10))

@model m_good_stan(x,y) = begin
    α ~ Normal(0,10)
    β ~ Normal(0,10)

    for i ∈ 1:length(y)
        p = logistic(α + β * x[i])
        y[i] ~ Binomial(1, p)
    end
end

chns = sample(m_good_stan(x,y), Turing.NUTS(0.95), 1000)

# Stan results

m_10_x,_results = "
    mean   sd   5.5% 94.5% n_eff Rhat
 a -5.09 4.08 -12.62 -0.25   100 1.00
 b  7.86 4.09   2.96 15.75   104 1.01
";

# Look at the proper draws (in corrected chn2)

describe(chns)

# End of 10/m10.xxt.jl


# function nint(x::Float64)
#     return round(Int, x)
# end

# function geo_coordinate(x, y)
#     PI = 3.141592

#     deg = nint(x)
#     m = x - deg
#     latitude = PI * (deg + 5.0 * m / 3.0) / 180.0

#     deg = nint(y)
#     m = y - deg
#     longitude = PI * (deg + 5.0 * m / 3.0) / 180.0

#     return latitude, longitude
# end

# function distance2D(xi, yi, xj, yj; dist="EUC_2D")
#     if dist == "EUC_2D"
#         xd = xi - xj
#         yd = yi - yj
#         return nint(sqrt(xd*xd + yd*yd))
#     elseif dist == "MAN_2D"
#         xd = abs(xi - xj)
#         yd = abs(yi - yj)
#         return nint(xd + yd)
#     elseif dist == "MAX_2D"
#         xd = abs(xi - xj)
#         yd = abs(yi - yj)
#         return max(nint(xd), nint(yd))
#     elseif dist == "GEO"
#         lat_i, long_i = geo_coordinate(xi, yi)
#         lat_j, long_j = geo_coordinate(xj, yj)
#         RRR = 6378.388
#         q1 = cos(long_i - long_j)
#         q2 = cos(lat_i - lat_j)
#         q3 = cos(lat_i + lat_j)
#         dij =  RRR * acos( 0.5*((1.0+q1)*q2 - (1.0-q1)*q3) ) + 1.0
#         return floor(Int, dij)
#     else
#         error("Distance function $dist is not supported.")
#     end
# end

# function dist_matrix(x::Vector{Float64}, y::Vector{Float64}; dist="EUC_2D")
#     n_nodes = length(x)
#     @assert length(x) == length(y)

#     M = Matrix{Int}(undef, n_nodes, n_nodes)

#     for i in 1:n_nodes
#         for j in i:n_nodes
#             if i == j
#                 M[i, j] = 0
#             else
#                 M[i, j] = distance2D(x[i], y[i], x[j], y[j]; dist=dist)
#                 M[j, i] = M[i, j]
#             end
#         end
#     end
#     return M
# end


#' ---
#' title: Geographical Clustering With Additional Constraint
#' ---

#' **Originally Contributed by**: Matthew Helm ([with help from Mathieu Tanneau on Julia Discourse](https://discourse.julialang.org/t/which-jump-jl-solver-for-this-problem/43350/17?u=mthelm85))

#' The goal of this exercise is to cluster $n$ cities into $k$ groups, minimizing the total pairwise distance between cities
#' *and* ensuring that the variance in the total populations of each group is relatively small.

#' For this example, we'll use the 20 most populous cities in the United States.

using Cbc
using DataFrames
using Distances
using JuMP
using LinearAlgebra

cities = DataFrame(
    city=[ "New York, NY", "Los Angeles, CA", "Chicago, IL", "Houston, TX", "Philadelphia, PA", "Phoenix, AZ", "San Antonio, TX", "San Diego, CA", "Dallas, TX", "San Jose, CA", "Austin, TX", "Indianapolis, IN", "Jacksonville, FL", "San Francisco, CA", "Columbus, OH", "Charlotte, NC", "Fort Worth, TX", "Detroit, MI", "El Paso, TX", "Memphis, TN"],
    population=[8405837,3884307,2718782,2195914,1553165,1513367,1409019,1355896,1257676,998537,885400,843393,842583,837442,822553,792862,792727,688701,674433,653450],
    lat=[40.7127,34.0522,41.8781,29.7604,39.9525,33.4483,29.4241,32.7157,32.7766,37.3382,30.2671,39.7684,30.3321,37.7749,39.9611,35.2270,32.7554,42.3314,31.7775,35.1495],
    lon=[-74.0059,-118.2436,-87.6297,-95.3698,-75.1652,-112.0740,-98.4936,-117.1610,-96.7969,-121.8863,-97.7430,-86.1580,-81.6556,-122.4194,-82.9987,-80.8431,-97.3307,-83.0457,-106.4424,-90.0489])

#' ### Model Specifics

#' We will cluster these 20 cities into 3 different groups and we will assume that the ideal or target population $P$ for a
#' group is simply the total population of the 20 cities divided by 3:

n = size(cities,1)
k = 3
P = sum(cities.population) / k

#' ### Obtaining the distances between each city

#' Let's leverage the *Distances.jl* package to compute the pairwise Haversine distance between each of the cities in our data
#' set and store the result in a variable we'll call `dm`:

dm = Distances.pairwise(Haversine(6372.8), Matrix(cities[:, [3,4]])', dims=2)

#' Our distance matrix is symmetric so we'll convert it to a `LowerTriangular` matrix so that we can better interpret the
#' objective value of our model (if we don't do this the total distance will be doubled):

dm = LowerTriangular(dm)

#' ### Build the model
#' Now that we have the basics taken  care of, we can set up our model, create decision variables, add constraints, and then
#' solve.

#' First, we'll set up a model that leverages the [Cbc](https://github.com/coin-or/Cbc) solver. Next, we'll set up a binary
#' variable $x_{i,k}$ that takes the value $1$ if city $i$ is in group $k$ and $0$ otherwise. Each city must be in a group, so
#' we'll add the constraint $\sum_kx_{i,k} = 1$ for every $i$.

model = Model(Cbc.Optimizer)

@variable(model, x[1:n, 1:k], Bin)

for i in 1:n
    @constraint(model, sum(x[i,:]) == 1)
end

#'The total population of a group $k$ is $Q_k = \sum_ix_{i,k}q_i$ where $q_i$ is simply the $i$th value from the `population`
#' column in our `cities` DataFrame. Let's add constraints so that $\alpha \leq (Q_k - P) \leq \beta$. We'll set $\alpha$
#' equal to -2,500,000 and $\beta$ equal to 2,500,000. By adjusting these thresholds you'll find that there is a tradeoff
#' between having relatively even populations between groups and having geographically close cities within each group. In
#' other words, the larger the absolute values of $\alpha$ and $\beta$, the closer together the cities in a group will be but
#' the variance between the group populations will be higher.

α = -2_500_000
β = 2_500_000

for i in 1:k
    @constraint(model, (x' * cities.population)[i] - P <= β)
    @constraint(model, (x' * cities.population)[i] - P >= α)
end

#' Now we need to add one last binary variable $z_{i,j}$ to our model that we'll use to compute the total distance between the
#' cities in our groups, defined as  $\sum_{i,j}d_{i,j}z_{i,j}$. Variable $z_{i,j}$ will equal $1$ if cities $i$ and $j$ are
#' in the same group, and $0$ if they are not in the same group.

#' To ensure that $z_{i,j} = 1$ if and only if cities $i$ and $j$ are in the same group, we add the constraints $z_{i,j} \geq
#' x_{i,k} + x_{j,k} - 1$ for every pair $i,j$ and every $k$:

@variable(model, z[1:n,1:n], Bin)

for k in 1:k, i in 1:n, j in 1:n
    @constraint(model, z[i,j] >= x[i,k] + x[j,k] - 1)
end

#' We can now add an objective to our model which will simply be to minimize the dot product of $z$ and our distance matrix,
#' `dm`. We can then call `optimize!` and review the results.

@objective(model, Min, dot(z,dm));

optimize!(model)

#' ### Reviewing the Results

#' Now that we have results, we can add a column to our `cities` DataFrame for the group and then loop through our $x$
#' variable to assign each city to its group. Once we have that, we can look at the total population for each group and also
#' plot the cities and their groups to verify visually that they are grouped by geographic proximity.

cities.group = zeros(n)

for i in 1:n, j in 1:k
    if round(value.(x)[i,j]) == 1.0
        cities.group[i] = j
    end
end

for group in groupby(cities, :group)
    @show group
    println("")
    @show sum(group.population)
    println("")
end

#' The populations of each group are fairly even and we can see from the plot below that the groupings look good in terms of
#' geographic proximity:

#' <img src="img/geo_clusters.png" style="width: auto; height: auto" alt="Geographic Clusters">


"""
    id(k::Int64) -> SparseMatrixCSC{ℤ₂, Int64}

Generate and return a `k × k` identity-matrix.
"""
function id(k::Int64)::SparseMatrixCSC{ℤ₂, Int64}
    I = sparse(zeros(ℤ₂, k, k))
    for i = 1:k
        I[i,i] = one(ℤ₂)
    end
    return I
end

"""
    anti_id(k::Int64) -> SparseMatrixCSC{ℤ₂, Int64}

Generate and return a `k × k` anti-identity-matrix.
"""
function anti_id(k::Int64)::SparseMatrixCSC{ℤ₂, Int64}
    antiid = ones(ℤ₂, k, k)
    for i = 1:k
        antiid[i,i] = zero(ℤ₂)
    end
    return sparse(antiid)
end


using Images, Colors

function filter_sobel(image::AbstractMatrix, direction::String="-")

  tipo = typeof(image)

  try
    image = convert(Array{Float32}, image)
  catch er
    error(er,"\nProblem converting the image into an array.\n")
  end

  if direction=="-"
    mask_x = [-1 0 1; -2 0 2; -1 0 1];
    mask_y = [1 2 1; 0 0 0; -1 -2 -1];

    Gx = imfilter(image,centered(mask_x));
    Gy = imfilter(image,centered(mask_y));

    d1 = size(Gx);
    d2 = size(Gy);

    if d1!=d2
      error("Problem during the function's execution.")
    end

    img_final = zeros(d1[1],d1[2])

    for i = 1:d1[1]
      for j = 1:d1[2]

        img_final[i,j] = sqrt((Gx[i,j]^2)+(Gy[i,j]^2))

        if img_final[i,j]>1.0
          img_final[i,j]=1.0;
        elseif img_final[i,j]<0.0
          img_final[i,j]=0.0;
        end

      end
    end
elseif direction=="W->E"
    mask = [1 2 1; 0 0 0; -1 -2 -1];
    img_final = imfilter(image,centered(mask));
    d1 = size(img_final);
    for i = 1:d1[1]
      for j = 1:d1[2]
        if img_final[i,j]>1.0
          img_final[i,j]=1.0;
        elseif img_final[i,j]<0.0
          img_final[i,j]=0.0;
        end
      end
    end
elseif direction=="N->S"
    mask = [-1 0 1; -2 0 2; -1 0 1];
    img_final = imfilter(image,centered(mask));
    d1 = size(img_final);
    for i = 1:d1[1]
      for j = 1:d1[2]
        if img_final[i,j]>1.0
          img_final[i,j]=1.0;
        elseif img_final[i,j]<0.0
          img_final[i,j]=0.0;
        end
      end
    end
  elseif direction=="NW->SE"
    mask = [0 1 2; -1 0 1; -2 -1 0];
    img_final = imfilter(image,centered(mask));
    d1 = size(img_final);
    for i = 1:d1[1]
      for j = 1:d1[2]
        if img_final[i,j]>1.0
          img_final[i,j]=1.0;
        elseif img_final[i,j]<0.0
          img_final[i,j]=0.0;
        end
      end
    end
  elseif direction=="SW->NE"
    mask = [-2 -1 0; -1 0 1; 0 1 2];
    img_final = imfilter(image,centered(mask));
    d1 = size(img_final);
    for i = 1:d1[1]
      for j = 1:d1[2]
        if img_final[i,j]>1.0
          img_final[i,j]=1.0;
        elseif img_final[i,j]<0.0
          img_final[i,j]=0.0;
        end
      end
    end
  else
    error("The position value must be an Text variable. Available values are: W->E, N->S, NW->SE, and SW->NE. This is not a mandatory field.\n");
  end

  img_final = convert(tipo, img_final)
  return(img_final)
end


module Quadratic

using STMO: Tracker, notrack, PathTrack, trace

export fquad, plot_quadratic, solve_quadratic, quadratic_ls, gradient_descent

using LinearAlgebra, Plots

"""Compute a quadratic function."""
fquad(x::Vector, P::Matrix, q::Vector, r::Real=0.0) = 0.5x' * P * x + q' * x + r

"""Plot a 1-D quadratic function."""
function plot_quadratic(p::Real, q::Real, r::Real, (xl, xu), stepsize=0.1; kwargs...)
    return plot(x -> p*x^2 + q*x+r, xl:stepsize:xu, xlabel="\$x\$"; kwargs...)
end

"""Plot a 2-D quadratic function."""
function plot_quadratic(P::AbstractMatrix, q::AbstractVector, r::Real,
                            (x1l, x1u), (x2l, x2u), stepsize=0.1; kwargs...)
    fun = (x1, x2) -> [x1,x2] |> x -> 0.5x' * P * x + q' * x + r
    return contour(x1l:stepsize:x1u, x2l:stepsize:x2u, fun, xlabel="\$x_1\$",
                                            ylabel="\$x_2\$"; kwargs...)
end

"""
    solve_quadratic(p::Real, q::Real[, r::Real=0.0])

Finds the minimizer of an 1-D quadratic system. If the quadratic term `p` is
negative, the function raises an error.

Inputs:
    - `p`, `q`, `r`: the terms of the 1-D quadratic system

Output:
    - xstar: the minimizer, a number
"""
function solve_quadratic(p::Real, q::Real, r::Real=0.0)
    @assert p > 0.0
    return - q / p
end

"""
    solve_quadratic(P::AbstractMatrix, q::AbstractVector, r::Real=0)

Finds the minimizer of an N-D quadratic system.
P is assumed to be a symmetric positive-definite matrix.

Inputs:
    - P, q, r: the terms of the nD quadratic system

Output:
    - xstar: the minimizer, an (n x 1) vector
"""
function solve_quadratic(P::AbstractMatrix, q::AbstractVector, r::Real=0.0)
    return - P \ q
end

"""
    quadratic_ls(P::AbstractMatrix, q::AbstractVector, Δx::AbstractVector,
                                    x::AbstractVector)

Find the exact step size that minimized a quadratic system in
a given point x for a given search direction Dx

Inputs:
    - P, q: the terms of the nD quadratic system
    - x: starting point
    - Δx: search direction

Output:
    - t: optimal step size
"""
function quadratic_ls(P::AbstractMatrix, q::AbstractVector, Δx::AbstractVector,
                                    x::AbstractVector)
    t = solve_quadratic(Δx' * P * Δx, Δx' * P * x + Δx ⋅ q)
    return t
end

"""
    gradient_descent(P::AbstractArray, q::AbstractVector,
            x₀::AbstractVector; β::Real=0.0, ϵ::Real=1e-6,
            tracker::Tracker=notrack)

Computes the minimizes of a quadratic system using gradient descent. Optionally
provide momentum.

Inputs:
    - P, q: the terms of the nD quadratic system
    - x₀: starting point
    - ϵ: convergence parameter
    - β: momentum parameter
    - tracker: object of the type `Tracker` to save the steps

Outputs:
    - xstar: the found minimum
"""
function gradient_descent(P::AbstractArray, q::AbstractVector,
            x₀::AbstractVector; β::Real=0.0, ϵ::Real=1e-6,
            tracker::Tracker=notrack)
    @assert 0 ≤ β < 1
    x = x₀  # initial value
    Δx = zero(x)  # pre-allocate a vector for the gradient
    while true
        Δx .= (1.0 - β) * (- P * x .- q) .+ β *  Δx
        if norm(Δx) < ϵ
            break
        end
        # determine stepsize using exact line search
        t = quadratic_ls(P, q, Δx, x)
        # perform step
        x .+= t * Δx
        trace(tracker, x)  # saves the steps
    end
    return x
end

# SIGNAL RECOVERY
# ---------------

module SignalRecovery
using LinearAlgebra: I

signalfun(x, n) = 3sin(x * 2 * pi / n) +
            2cos(x * 4 * pi / n) +
            sin(x * 4 * pi / n) + 0.8 * cos(x * 12 * pi / n)

"""
    generate_noisy_measurements(m, n; σ=1.0)

Generate noisy measurements according to some function f

Inputs:
    - m : number of observations
    - n : dim of x
    - σ : normally distributed noise (default = 1)

Output:
    - y : vector of noisy measurements
    - ind : vector of indices
"""
function generate_noisy_measurements(m, n; σ=1.0)
    ind = rand(1:n, m)
    y = signalfun.(ind, n) .+ randn(m) * σ
    return y, ind
end

"""
    make_connection_matrix(n, γ=100)

Generates the kernel matrix and the inverse kernel matrix

Uses γ as a characteristic length scale of the radial basis kernel,
assumes periodic boundaries.

Inputs:
    - n : number of points
    - γ : length scale

Output:
    - K
    - Kinv
"""
function make_connection_matrix(n, γ=100)
    # cyclic distance
    d(i, j) = min((i-j)^2, ((i-n)-j)^2, ((j-n)-i)^2)
    # distance matrix
    D = [d(i, j) for i in 1:n, j in 1:n]
    # kernel matrix
    K = exp.(-D / γ^2) + 1e-2I
    return K, inv(K)
end

"""
    make_bookkeeping(I, n)

Constructs the bookkeeping matrix R

Inputs:
    - ind : the indices of the measurements
    - n : dimensionality of signal vector

Output:
    - R : m x n bookkeeping matrix
"""
function make_bookkeeping(ind, n)
    m = length(ind)
    R = zeros(Int, m, n)
    for (i, j) in enumerate(ind)
        R[i,j] = 1
    end
    return R
end

export generate_noisy_measurements, make_connection_matrix, make_bookkeeping, signalfun

end # module SignalRecovery

export SignalRecovery

end  # module Quadratic


import Random
import Test

Test.@testset "examples" begin
    Test.@testset "linear regression" begin
        Random.seed!(123)
        include(joinpath(examples_directory, "linear_regression.jl"))
        include(joinpath(test_directory, "examples", "linear_regression.jl"))

    end

    Test.@testset "logistic regression" begin
        Random.seed!(123)
        include(joinpath(examples_directory, "logistic_regression.jl"))
        include(joinpath(test_directory, "examples", "logistic_regression.jl"))
    end

    Test.@testset "multinomial logistic regression" begin
        Random.seed!(123)
        @info("This next test may go five to fifteen minutes without producing any output")
        include(joinpath(examples_directory, "multinomial_logistic_regression.jl"))
        include(joinpath(test_directory, "examples", "multinomial_logistic_regression.jl"))
    end
end


using Distributions
using Optim
using HTTP
using GLM
using LinearAlgebra
using Random
using Statistics
using DataFrames
using CSV
using FreqTables

 #:::::::::::::::::::::::::::::::::::::::::::::::::::
 # question 1
 #:::::::::::::::::::::::::::::::::::::::::::::::::::

url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2021/master/ProblemSets/PS4-mixture/nlsw88t.csv"
df = CSV.read(HTTP.get(url).body,DataFrame)
X = [df.age df.white df.collgrad]
Z = hcat(df.elnwage1, df.elnwage2, df.elnwage3, df.elnwage4,
df.elnwage5, df.elnwage6, df.elnwage7, df.elnwage8)
y = df.occ_code

function mlogit_with_Z(theta, X, Z, y)

    alpha = theta[1:end-1]
    gamma = theta[end]
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    T = promote_type(eltype(X),eltype(theta))
    num   = zeros(T,N,J)
    dem   = zeros(T,N)
    for j=1:J
        num[:,j] = exp.(X*bigAlpha[:,j] .+ (Z[:,j] .- Z[:,J])*gamma)
        dem .+= num[:,j]
    end

    P = num./repeat(dem,1,J)

    loglike = -sum( bigY.*log.(P) )

    return loglike
end
startvals = [2*rand(7*size(X,2)).-1; .1]
td = TwiceDifferentiable(theta -> mlogit_with_Z(theta, X, Z, y), startvals; autodiff = :forward)
# run the optimizer
theta_hat_optim_ad = optimize(td, startvals, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
theta_hat_mle_ad = theta_hat_optim_ad.minimizer
# evaluate the Hessian at the estimates
H  = Optim.hessian!(td, theta_hat_mle_ad)
theta_hat_mle_ad_se = sqrt.(diag(inv(H)))
println([theta_hat_mle_ad theta_hat_mle_ad_se]) # these standard errors match Stata

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 2
#:::::::::::::::::::::::::::::::::::::::::::::::::::

#Yes. Estimated gamma in PS3 is -0.09419383447879262. Yet gamma/100 is the change in utility with a 1% increase in expected wage.
#It should not be negative. Here I got a postitive number(1.3074804109794527) so it is more reasonable.

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 3
#:::::::::::::::::::::::::::::::::::::::::::::::::::

#(a)
include("lgwt.jl") # make sure the function gets read in
# define distribution
d = Normal(0,1) # mean=0, standard deviation=1
# get quadrature nodes and weights for 7 grid points
nodes, weights = lgwt(7,-4,4)
# now compute the integral over the density and verify it
println(sum(weights.*pdf.(d,nodes)))
# now compute the expectation and verify it
println(sum(weights.*nodes.*pdf.(d,nodes)))

#(b)
#(1)
d = Normal(0,2)
nodes, weights = lgwt(7,-5,5)
println(sum(weights.*(nodes.^2).*pdf.(d,nodes)))
#(2)
d = Normal(0,2)
nodes, weights = lgwt(10,-5,5)
println(sum(weights.*(nodes.^2).*pdf.(d,nodes)))
#(3)
#As the number of quadrature points increase, the quadrature approximates closer to the true value.

#(c)
#(1)
d = Normal(0,2)
a = -5*2
b = 5*2
z = Uniform(a,b)
x = rand(z,1000000)
println((b-a)*mean((x.^2).*pdf.(d,x)))
#(2)
println((b-a)*mean(x.*pdf.(d,x)))
#(3)
println((b-a)*mean(pdf.(d,x)))
#(4)
x = rand(z,1000)
println((b-a)*mean((x.^2).*pdf.(d,x)))
println((b-a)*mean(x.*pdf.(d,x)))
println((b-a)*mean(pdf.(d,x)))
#when D = 1000000, the simulated integral approximates the true value better than that when D = 1000

function mixed_logit_normal(theta, X, Z, y)

    alpha = theta[1:end-2]
    mu_gamma = theta[end-1]
    sigma_gamma = theta[end]
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    T = promote_type(eltype(X),eltype(theta))
    num   = zeros(T,N,J)
    dem   = zeros(T,N)
    d = Normal(mu_gamma,sigma_gamma)
    gammas, weights = lgwt(10,-4,4)
    M = size(gammas,1)
    F = ones(N,M)
    for i =1:M
        for j=1:J
            num[:,j] = exp.(X*bigAlpha[:,j] .+ (Z[:,j] .- Z[:,J]).*gammas[i])
            dem .+= num[:,j]
        end
        P = (num./repeat(dem,1,J))

        for j=1:J
            P[:,j]=P[:,j].^bigY[:,j]
            F[:,i].*=P[:,j]
        end
    end

    loglike = -sum([log(sum(weights.*F[i,:].*pdf.(d,gammas))) for i = 1:N])

    return loglike
end

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 4
#:::::::::::::::::::::::::::::::::::::::::::::::::::

startvals = [2*rand(7*size(X,2)).-1; 0; 1]
obj = optimize(alpha->mixed_logit_normal(alpha, X, Z, y), startvals, LBFGS(), Optim.Options(g_tol=1e-5, iterations=100_000, show_trace=true, show_every=50))
beta = obj.minimizer[1:end-2]
mu_gamma = obj.minimizer[end-1]
sigma_gamma = obj.minimizer[end]
println('beta is ',beta)
println('mu_gamma is ',mu_gamma)
println('sigma_gamma is ', sigma_gamma)

#I also tried this code yet MethodError: no method matching Float64(::ForwardDiff.Dual{ForwardDiff.Tag{var"#7#8", Float64}, Float64, 12})
# startvals = [2*rand(7*size(X,2)).-1; 0; 1]
# td = TwiceDifferentiable(theta -> mixed_logit_normal(theta, X, Z, y), startvals; autodiff = :forward)
# theta_hat_optim_ad = optimize(td, startvals, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100000, show_trace=true, show_every=50))
# theta_hat_mle_ad = theta_hat_optim_ad.minimizer
# H  = Optim.hessian!(td, theta_hat_mle_ad)
# theta_hat_mle_ad_se = sqrt.(diag(inv(H)))
# println([theta_hat_mle_ad theta_hat_mle_ad_se]) # these standard errors match Stata

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 5
#:::::::::::::::::::::::::::::::::::::::::::::::::::

function mixed_logit_normal2(theta, X, Z, y)

    alpha = theta[1:end-2]
    mu_gamma = theta[end-1]
    sigma_gamma = theta[end]
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    T = promote_type(eltype(X),eltype(theta))
    num   = zeros(T,N,J)
    dem   = zeros(T,N)
    d = Normal(mu_gamma,sigma_gamma)
    gammas, weights = lgwt(10,-4,4)
    M = size(gammas,1)
    F = ones(N,M)
    for i =1:M
        for j=1:J
            num[:,j] = exp.(X*bigAlpha[:,j] .+ (Z[:,j] .- Z[:,J]).*gammas[i])
            dem .+= num[:,j]
        end
        P = (num./repeat(dem,1,J))

        for j=1:J
            P[:,j]=P[:,j].^bigY[:,j]
            F[:,i].*=P[:,j]
        end
    end

    a = -5*sigma_gamma
    b = 5*sigma_gamma
    z = Uniform(a,b)
    x = rand(z,1000000)
    loglike = -sum([log((b-a)*mean(F[i,:].*pdf.(d,gammas))) for i = 1:N])

    return loglike
end

startvals = [2*rand(7*size(X,2)).-1; 0; 1]
obj = optimize(alpha->mixed_logit_normal2(alpha, X, Z, y), startvals, LBFGS(), Optim.Options(g_tol=1e-5, iterations=100_000, show_trace=true, show_every=50))
beta = obj.minimizer[1:end-2]
mu_gamma = obj.minimizer[end-1]
sigma_gamma = obj.minimizer[end]
println('beta is ',beta)
println('mu_gamma is ',mu_gamma)
println('sigma_gamma is ', sigma_gamma)


# utilitaries
using LinearAlgebra

include("quadrature.jl")
# include("connectique.jl")

@inline function mod_wrap(x, a)
    #= return the index x modulo a =#
    return mod(x-1, a) + 1
end

function polygonArea(vtx)
    area = 0.
    N = size(vtx, 2)
    for i=1:N
        j = mod_wrap(i+1, N)
        area += vtx[1, i]*vtx[2, j] - vtx[2, i]*vtx[1, j]
    end
    return 0.5*abs(area)
end

function polygonDiameter(vtx)
    diameter = 0.
    N = size(vtx, 2)
    for i=1:(N-1)
        for j=(i+1):N
            diameter = max(diameter, norm(vtx[:, i] - vtx[:, j]))
        end
    end
    return diameter
end

function polygonCentroid(vtx)
    N        = size(vtx, 2)
    centroid = zeros(Float64, 2)
    #
    den = 0.
    for i=1:N
        j = mod_wrap(i+1, N)
        w = (vtx[1, i]*vtx[2, j] - vtx[1, j]*vtx[2, i])
        centroid[1] += (vtx[1, i] + vtx[1, j])*w
        centroid[2] += (vtx[2, i] + vtx[2, j])*w
        den += w
    end
    #
    return centroid/(3 * den)
end

# INTEGRATION OF MULTIPLE FUNCTIONS OVER A POLYGON
function integratePolygon(pverts::Array{Float64,2}, xE::Array{Float64,1},
    ref::Quadrature2d, fs::Array{Function, 1})
    #=
    Integration of all functions in fs over a polygon by subdividing it
    into triangles
    =#
    n   = size(pverts, 2)
    nf  = length(fs)
    res = zeros(Float64, nf)
    for i=1:n
        ip = mod_wrap(i+1, n)
        @inbounds quadT = quadGL2d(ref, [pverts[1,i] pverts[1,ip] xE[1];
        pverts[2,i] pverts[2,ip] xE[2]])
        for f = 1:nf
            @inbounds res[f] += integrate(quadT, fs[f])
        end
    end
    return  res
end

function integratePolygon(pverts::Array{Float64,2}, xE::Array{Float64,1},
    ref::Quadrature2d, fs::Function)
    #= Same as before, but for a single function fs =#
    n   = size(pverts, 2)
    res = 0.
    for i=1:n
        ip = mod_wrap(i+1, n)
        @inbounds quadT = quadGL2d(ref, [pverts[1,i] pverts[1,ip] xE[1];
        pverts[2,i] pverts[2,ip] xE[2]])
        res += integrate(quadT, fs)
    end
    return  res
end


using Statistics
using BenchmarkTools
using LinearAlgebra

# Looking into pointers from http://fa.bianp.net/blog/2019/evaluate_logistic/
#

σ(x) = 1 / (1 + exp(-x))

## Order 0
function f_naive(x, A, b)
    z = A*x
    tmp = σ.(z)
    @. tmp = - b * log(tmp) - (1 - b) * log(1 - tmp)
    return sum(tmp)/length(tmp)
end

## blog logistic
function logsig_naive(t)
    return log(1 / (1 + exp(-t)))
end

function logsig_log1pexp(t)
    if t < -33.3
        return t
    elseif t <= -18
        return t - exp(t)
    elseif t <= 37
        return -log1p(exp(-t))
    else
        return -exp(-t)
    end
end

function f_logsumexp(x, A, b, logsig)
    m = size(A, 1)

    Ax = A * x
    fval = 0.0
    @inbounds @simd for i in 1:m
        fval += -logsig(b[i] * Ax[i])
    end
    fval /= m

    return fval
end

### Order 1
function ∇f_current(x, A, b)
    m = size(A, 1)

    σbAx = σ.(-b .* (A * x))
    res = transpose(A) * (σbAx .* -b)

    res /= m
    return res
end

function expit_b(x, b)
    out = zeros(size(x))

    @inbounds for (i, xi) in enumerate(x)
        if xi < 0
            out[i] = ((1 - b[i]) * exp(xi) - b[i]) / (1 + exp(xi))
        else
            out[i] = ((1 - b[i]) - b[i] * exp(-xi)) / (1 + exp(-xi))
        end
    end
    return out
end

function ∇f_prop(x, A, b)
    """Computes the gradient of the logistic loss.

    Parameters
    ----------
    x: array-like, shape (n_features,)
        Coefficients

    A: array-like, shape (n_samples, n_features)
        Data matrix

    b: array-like, shape (n_samples,)
        Labels

    Returns
    -------
    grad: array-like, shape (n_features,)
    """
    z = A*x
    s = expit_b(z, b)
    return A'*s / size(A, 1)
end

function main()
    A = [1 1;]
    b = [1]
    x = [20, 20]

    # mode = [:fvals]
    mode = [:∇f]

    if :fvals in mode
        @show f_naive(x, A, b)
        @show f_logsumexp(x, A, b, logsig_naive)
        @show f_logsumexp(x, A, b, logsig_log1pexp)
    elseif :∇f in mode
        @show ∇f_current(x, A, b)
        @show ∇f_prop(x, A, b)
    end

    println("")
    println("")

    println("Larger test")
    n, m = 500, 400
    A = rand(m, n)
    x = rand(n)
    b = rand(m)

    if :fvals in mode
        @show f_naive(x, A, b)
        @show f_logsumexp(x, A, b, logsig_naive)
        @show f_logsumexp(x, A, b, logsig_log1pexp)

        @btime f_naive($x, $A, $b)
        @btime f_logsumexp($x, $A, $b, $logsig_naive)
        @btime f_logsumexp($x, $A, $b, $logsig_log1pexp)
    elseif :∇f in mode
        ∇f_cu = ∇f_current(x, A, b)
        ∇f_pr = ∇f_prop(x, A, b)

        @show norm(∇f_cu - ∇f_pr)

        @btime ∇f_current($x, $A, $b)
        @btime ∇f_prop($x, $A, $b)
    end

    return
end

main()


@testset "basic interest rates" begin

    ## vector interest rate
    @testset "vector interest rate" begin
        i = InterestRate([0.05, 0.05, 0.05])

        @test disc(i,0) == 1.0
        @test disc(i,1) == 1 / 1.05
        @test disc(i,2) == 1 / 1.05 ^ 2
        @test disc(i,3) == 1 / 1.05 ^ 3
        @test disc(i,1,2) == 1 / 1.05
        @test rate(i,1) == 0.05
        @test rate(i,2) == 0.05

        @test disc.(i,1:3) == [1 / 1.05 ^ t for t in 1:3]
    end

    ## real interest rate
    @testset "constant interest rate" begin
        i = InterestRate(0.05)

        @test disc(i,0) == 1.0
        @test disc(i,1) == 1 / 1.05
        @test disc(i,2) == 1 / 1.05 ^ 2
        @test disc(i,3) == 1 / 1.05 ^ 3
        @test disc(i,1,2) == 1 / 1.05
        @test rate(i,1) == 0.05
        @test rate(i,2) == 0.05

        @test disc.(i,1:3) == [1 / 1.05 ^ t for t in 1:3]
    end
end

@testset "DiscountFactor Iterator" begin

    target = [1 / 1.05 ^ t for t in 0:4 ]

    @testset "contsant" begin
        df = DiscountFactor(InterestRate(0.05),1)

        ds = Iterators.take(df,5) |> collect

        @test all(ds .≈ target)

        @test length(Iterators.take(df,100)) == 100

        # can't collect an infinite series
        @test_throws MethodError collect(df)

        @test df == InterestRate(0.05)(1)
    end

    @testset "vector" begin
    df = DiscountFactor(InterestRate(repeat([0.05],4)),1)

        ds = Iterators.take(df,5) |> collect
        @test all(ds .≈ target)
        @test all(collect(df) .≈ target)
        @test length(df) == 5

        @test_broken InterestRate([0.05,0.05,0.05,0.05])(1) == df #https://stackoverflow.com/questions/62336686/struct-equality-with-arrays
    end

end

@testset "passthrough life contingency" begin
    ins = LifeContingency(
        SingleLife(
            mort = UltimateMortality([0.5]),
            issue_age = 0
        ),
        InterestRate(0.05)
    )

    @test disc(ins,1) ≈ 1/1.05
    @test disc(ins,1,2) ≈ 1/1.05
end


using FastGaussQuadrature

get_nodes(::Val{:gauss_legendre}, n, T=Float64) = convert(Array{T}, gausslegendre(n)[1])
get_nodes(::Val{:lobatto_legendre}, n, T=Float64) = convert(Array{T}, gausslobatto(n)[1])

function get_nodes(::Val{:gauss_chebyshev}, n, T=Float64; k=1)
    x = zeros(T, n)
    if k == 1
        for i in eachindex(x)
            x[i] = sin( π*(n-2i+1) / (2n) )
        end
        return x[end:-1:1]
    elseif k == 2
        for i in eachindex(x)
            x[i] = cos( π*i / (n-1) )
        end
        return x
    end
end

function get_nodes(::Val{:lobatto_chebyshev}, n, T=Float64)
    x = zeros(T, n)
    for i in eachindex(x)
        x[i] = cos( (i-1)*π / (n-1) )
    end
    return x[end:-1:1]
end


#=
Accurate simple zeros of polynomials in floating point arithmetic
Stef Graillat
Computers and Mathematics with Applications 56 (2008) 1114–1120

   poly has N+1 coefficients a_0,a_1,..,a_N for x^0,x^1,..,x^N

   poly = polynomial(coeffs[a_0,a_1,..,a_N])
   poly(x) = sum[i=0:1:N]( a_i * x^i )          this the the classical Horner scheme

   function classicalHorner{R<:Real}(p::Poly,x::R)
      n = length(p.a)
      s = zeros(R, n)       # one-based indexing
      s[:] = p.a[:]
      for i in (n-1):(-1):1
          s[i] += s[i+1]*x
      end
      s[1]
    end

Algorithms for Accurate, Validated and Fast Polynomial Evaluation
Stef Graillat, Philippe Langlois and Nicolas Louvet
Japan J. Indust. Appl. Math., 26 (2009), 191–214

eftHorner(p_n,x) computes
(i)  the floating point evaluation Horner(p_n,x)
(ii) two polynomials pa, pb, of degree n-1
such that
  Horner(p_n,x), pa, pb = eftHorner(p_n,x)
  p_n(x) == Horner(p_n,x) + (pa + pb)(x)
so
  eftHorner is an errorfree transformation for polynomial evaluation with the Horner algorithm
     (unless there is underflow)
=#

function eftHorner{T<:AbstractFloat}(p::Poly{T},x::T)
   deg = degree(p); n=deg+1
   cofs = coeffs(p)
   pa = zeros(T,n)
   pb = zeros(T,n)
   r = cofs[n]
   for i in deg:(-1):1
      t, pa[i] = eftMul(r, x)
      r, pb[i] = eftAdd(t, cofs[i])
    end
    r, pa, pb
end

# expects length(a) == length(b), length(a)>0
function sumHorner{T<:AbstractFloat}(a::Vector{T}, b::Vector{T}, x::T)
   n = length(a)
   if ((n == 0)|(n != length(b)))
      throw(ErrorException("vectors must be of the same nonzero length"))
   end
   r = a[n]+b[n]
   for i in (n-1): -1: 1
     r = r * x + (a[i]+b[i])
   end
   r
end

function csdHornerA{T<:AbstractFloat}(p::Poly{T}, x::T)
    r, pa, pb = eftHorner(p,x)
    c = sumHorner(pa,pb,x)
    r + c
end

# as accurate as if computed at twice the working precision and then rounded to working precision
function csdHorner{T<:AbstractFloat}(p::Poly{T}, x::T)
    r, pa, pb = eftHorner(p,x)
    pab = Poly(pa + pb)
    c = polyval(pab, x)
    r + c
end


# A palindromic number reads the same both ways.
# The largest palindrome made from the product
# of two 2-digit numbers is 9009 = 91 × 99.
# Find the largest palindrome made from the product of two 3-digit numbers.

function is_palindrome(n)
    s = string(n)
    return s == reverse(s)
end

@assert is_palindrome(2)
@assert is_palindrome(11)
@assert is_palindrome(9009)
@assert is_palindrome(90109)
@assert is_palindrome(90209)

function find_next_palindrome_decreasing(n)
    while !is_palindrome(n)
        n -= 1
    end
    return n
end

@assert find_next_palindrome_decreasing(10000) == 9999
@assert find_next_palindrome_decreasing(9998) == 9889

function is_divisible_by_digits(n, digits)
    # Checks if a number is divisible by two x digits number
    max = (10 ^ digits) - 1
    min = minimum([max, trunc(Int, n / (10 ^ digits) + 1)])

    for i in range(max, -1, max-min)
        if n % i == 0 && (n / i) <= max
            return true
        end
    end
    return false
end

@assert is_divisible_by_digits(9999, 2) == false
@assert is_divisible_by_digits(9779, 2) == false
@assert is_divisible_by_digits(9009, 2) == true

function compute_palindrome_by_factors(n, digit)
    while !is_palindrome(n) || !is_divisible_by_digits(n, digit)
        n = find_next_palindrome_decreasing(n-1)
    end
    return n
end
@assert compute_palindrome_by_factors(99*99,2) == 9009
println(compute_palindrome_by_factors(999*999, 3))


# Make the Convex.jl module available
using Convex, SCS, ECOS
using LinearAlgebra

# Generate random problem data
m = 4;  n = 5
A = randn(m, n); b = randn(m, 1)

# Create a (column vector) variable of size n x 1.
x = Convex.Variable(n)

# The problem is to minimize ||Ax - b||^2 subject to x >= 0
# This can be done by: minimize(objective, constraints)
problem = minimize(sumsquares(A * x - b), [x >= 0])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval

# minimum distance spheres
pa = Variable(3)
pb = Variable(3)
xa = [0.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ra = 0.1
rb = 0.1

problem = minimize(norm(pa - pb), [norm(pa - xa) <= ra, norm(pb - xb) <= rb])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

# minimum distance ellipsoids
pa = Variable(3)
pb = Variable(3)
xa = [0.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ax = 0.3
ay = 0.1
az = 0.1
bx = 0.1
by = 0.2
bz = 0.1

problem = minimize(norm(pa - pb),
    [
        quadform(pa - xa, Array(Diagonal([1.0 / ax^2, 1.0 / ay^2, 1.0 / az^2]))) <= 1.0,
        quadform(pb - xb, Array(Diagonal([1.0 / bx^2, 1.0 / by^2, 1.0 / bz^2]))) <= 1.0,
    ])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

### minimum distance cylinders
pa = Variable(3)
pb = Variable(3)
sa = Variable(3)
sb = Variable(3)
xa = [-1.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ra = 0.1
ha = 1.0
rb = 0.1
hb = 1.0

problem = minimize(norm(pa - pb),
    [
        sa == pa - xa,
        sb == pb - xb,
        norm(sa[1:2]) <= ra,
        sa[3] <= 0.5 * ha + sqrt(ra * ra - dot(sa[1:2], sa[1:2])),
        -0.5 * ha - sqrt(ra * ra - dot(sa[1:2], sa[1:2])) <= sa[3],
        norm(sb[1:2]) <= rb,
        sb[3] <= 0.5 * hb + sqrt(rb * rb - dot(sb[1:2], sb[1:2])),
        -0.5 * hb - sqrt(rb * rb - dot(sb[1:2], sb[1:2])) <= sb[3],
    ])

pa.value = [-1.0; 0.0; 0.0]
pb.value = [1.0; 0.0; 0.0]
# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = false)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

## line segment
za = Convex.Variable(3)
zb = Convex.Variable(3)
ta = Convex.Variable(1)
tb = Convex.Variable(1)

xa = [-1.0; 0.0; 0.0]
qa = Quaternion(RotY(-0.25 * π) * RotX(0.0 * π))
qa = [qa.w; qa.x; qa.y; qa.z]

xb = [1.0; 0.0; 0.0]
qb = Quaternion(RotZ(0.0 * π) * RotY(0.0 * π) * RotX(0.0))
qb = [qb.w; qb.x; qb.y; qb.z]
ra = 0.1
ha = 1.0
rb = 0.1
hb = 1.0

p1a = xa + rotation_matrix(qa) * [0.0; 0.0; 0.5 * ha]
p2a = xa + rotation_matrix(qa) * [0.0; 0.0; -0.5 * ha]

p1b = xb + rotation_matrix(qb) * [0.0; 0.0;  0.5 * hb]
p2b = xb + rotation_matrix(qb) * [0.0; 0.0; -0.5 * hb]

problem = minimize(norm(za - zb),
    [
        za == ta * p1a + (1.0 - ta) * p2a,
        zb == tb * p1b + (1.0 - tb) * p2b,
        ta >= 0.0,
        ta <= 1.0,
        tb >= 0.0,
        tb <= 1.0,
    ])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = false)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
za
zb


using NumericalIntegration
using LinearAlgebra
using FFTW

#=
This function computes the Fourier modes either integrating "by hand" or using FFTW. We split the comlex modes, associated to the complex exponentials into a and b, associated to the cos and sin functions by a(kx,ky) = 2*Re(f(kx,ky)) and b(kx,ky) = -2*Im(f(kx,ky)).
=#

function Fourier_modes_2D(e :: Array{T,2}, x :: Array{T,1}, y :: Array{T,1}, order_x :: Integer, order_y :: Integer, integration_method :: String) where {T<:Real}
    Lx, Ly = x[end]-x[1], y[end]-y[1]
    xmid, ymid = (x[end]+x[1])/2, (y[end]+y[1])/2
    nx, ny = length(x),length(y)

    if order_x > Integer(floor(nx)) || order_y > Integer(floor(ny))
        print("\n Error, requested order is higher than the number of grid points\n")
        nothing
    end

    if integration_method == "trapezoidal"

        a, b = zeros(order_x+1,order_y+1), zeros(order_x+1,order_y+1)
        xx, yy = zeros(nx,ny), zeros(nx,ny)
        #We create the 2D grid
        for i in 1:ny
            xx[:,i] = x
        end
        for i in 1:nx
            yy[i,:] = y
        end

        for my in 0:order_y
            for mx in 0:order_x
                integral_cos, integral_sin = zeros(nx), zeros(nx)
                integrand_cos = 2/(Lx*Ly).*e.*cos.(2*π*(mx/Lx.*(xx.-xmid).+my/Ly.*(yy.-ymid)))
                integrand_sin = 2/(Lx*Ly).*e.*sin.(2*π*(mx/Lx.*(xx.-xmid).+my/Ly.*(yy.-ymid)))
                for i in 1:nx
                    integral_cos[i] = integrate(y,integrand_cos[i,:], Trapezoidal())
                    integral_sin[i] = integrate(y,integrand_sin[i,:], Trapezoidal())
                end
                a[mx+1,my+1] = integrate(x,integral_cos, Trapezoidal())
                b[mx+1,my+1] = integrate(x,integral_sin, Trapezoidal())
            end
        end

    elseif integration_method == "FFTW"

        plan = plan_rfft(e)
        fft_z = (1/(nx*ny).*(plan*e)[1:order_x+1,1:order_y+1])
        a = 2 .*real(fft_z)
        b = -2 .*imag(fft_z)

    else
        print("\n No such integration method\n")
        nothing
    end
    return a, b
end


export gaussian, xavier, xavier_uniform, xavier_normal, bilinear
using Random: randn, rand

"""

    gaussian(a...; mean=0.0, std=0.01)

Return a Gaussian array with a given mean and standard deviation.  The
`a` arguments are passed to `randn`.

"""
function gaussian(a...; mean=0.0, std=0.01)
    r = randn(a...)
    T = eltype(r)
    r .* T(std) .+ T(mean)
end

"""

    xavier_uniform(a...; gain=1)
    xavier(a...; gain=1)

Return uniform random weights in the range `± gain * sqrt(6 / (fanin + fanout))`.  The `a`
arguments are passed to `rand` to specify type and dimensions.  See ([Glorot and Bengio
2010](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) or the [PyTorch
docs](https://pytorch.org/docs/stable/nn.init.html#torch.nn.init.xavier_uniform_) for a
description.  The function implements equation (16) of the referenced paper. Also known as
Glorot initialization. The function `xavier` is an alias for `xavier_uniform`. See also
`xavier_normal`.

"""
xavier, xavier_uniform

function xavier_uniform(a...; gain=1)
    w = rand(a...)
    if ndims(w) == 1
        fanout = 1
        fanin = length(w)
    elseif ndims(w) == 2
        fanout = size(w,1)
        fanin = size(w,2)
    else
        # if a is (3,3,16,8), then there are 16 input channels and 8 output channels
        # fanin = 3*3*16 = (3*3*16*8) ÷ 8
        # fanout = 3*3*8 = (3*3*16*8) ÷ 16
        fanin = div(length(w),  a[end])
        fanout = div(length(w), a[end-1])
    end
    s = convert(eltype(w), gain*sqrt(6 / (fanin + fanout)))
    return 2s .* w .- s
end

const xavier = xavier_uniform

"""

    xavier_normal(a...; gain=1)

Return normal distributed random weights with mean 0 and std `gain * sqrt(2 / (fanin +
fanout))`.  The `a` arguments are passed to `rand`.  See ([Glorot and Bengio
2010](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) and [PyTorch
docs](https://pytorch.org/docs/stable/nn.init.html#torch.nn.init.xavier_normal_) for a
description. Also known as Glorot initialization. See also `xavier_uniform`.

"""
function xavier_normal(a...; gain=1)
    w = randn(a...)
    if ndims(w) == 1
        fanout = 1
        fanin = length(w)
    elseif ndims(w) == 2
        fanout = size(w,1)
        fanin = size(w,2)
    else
        # if a is (3,3,16,8), then there are 16 input channels and 8 output channels
        # fanin = 3*3*16 = (3*3*16*8) ÷ 8
        # fanout = 3*3*8 = (3*3*16*8) ÷ 16
        fanin = div(length(w),  a[end])
        fanout = div(length(w), a[end-1])
    end
    s = convert(eltype(w), gain*sqrt(2 / (fanin + fanout)))
    return s .* w
end

"""

Bilinear interpolation filter weights; used for initializing deconvolution layers.

Adapted from https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/surgery.py#L33

Arguments:

`T` : Data Type

`fw`: Width upscale factor

`fh`: Height upscale factor

`IN`: Number of input filters

`ON`: Number of output filters

Example usage:

w = bilinear(Float32,2,2,128,128)

"""
function bilinear(T,fw,fh,IN,ON)
    @assert fw == fh "Filter must be square"
    @assert IN == ON "Number of input and output filters must be equal"
    f=fw; N=IN;

    sz = 2*f-f%2
    center = f-0.5
    if sz%2 == 1
        center = f-1
    end
    r = range(0,stop=sz-1,length=sz)
    c = range(0,stop=sz-1,length=sz)'

    kernel = (1 .- abs.(r .- center) ./ f) .* (1 .- abs.(c .- center) ./ f)
    w = zeros(T,sz,sz,N,N);
    for i=1:N
        w[:,:,i,i] = kernel
    end
    return w
end


using NumericFuns
using Base.Test

@test sqr(2) === 4
@test sqr(2.0) === 4.0

@test rcp(2) === 0.5
@test rsqrt(4) === 0.5
@test rcbrt(8) === 0.5

@test xlogx(0) === 0.0
@test_approx_eq xlogx(2) 2.0 * log(2.0)

@test xlogy(0, 1) === 0.0
@test_approx_eq xlogy(2, 3) 2.0 * log(3.0)

@test_approx_eq logistic(2) 1.0 / (1.0 + exp(-2.0))
@test_approx_eq logit(0.5) 0.0
@test_approx_eq logit(sigmoid(2)) 2.0

@test_approx_eq softplus(2.0) log(1.0 + exp(2.0))
@test_approx_eq softplus(-2.0) log(1.0 + exp(-2.0))
@test_approx_eq softplus(10000) 10000.0
@test_approx_eq softplus(-10000) 0.0

@test_approx_eq invsoftplus(softplus(2.0)) 2.0
@test_approx_eq invsoftplus(softplus(-2.0)) -2.0

@test_approx_eq logsumexp(2.0, 3.0) log(exp(2.0) + exp(3.0))
@test_approx_eq logsumexp(10002, 10003) 10000 + logsumexp(2.0, 3.0)


using SimpleGraphs, SimpleGraphAlgorithms

const DEFAULT_FILE_NAME = "tree_codes.jl"

# initialize the table of unlabeled trees

"""
    init_trees_table()
Create a new table of distinct trees on 1 and 2 vertices.
"""
function init_trees_table()
    TT = Dict{Int,Vector{SimpleGraph{Int}}}()

    TT[1] = [IntGraph(1)]
    T = IntGraph(2)
    add!(T, 1, 2)
    TT[2] = [T]

    return TT
end

"""
    check_in(G,S)
See if the set `S` contains a graph isomorphic to `G`. Return `true` if so.
"""
function check_in(G::SimpleGraph{Int}, S::Set{SimpleGraph{Int}})
    if isempty(S)
        return false
    end
    for H ∈ S
        if uhash(G) == uhash(H) #is_iso(G, H)
            return true
        end
    end
    return false
end

"""
    extend_trees_table!(TT)
Given a table of distinct trees up to size `n`, extend that table to include
all distinct trees of size `n+1`.
"""
function extend_trees_table!(TT::Dict{Int,Vector{SimpleGraph{Int}}})::Nothing
    n = maximum(keys(TT))
    outset = Set{SimpleGraph{Int}}()  # set of trees with n+1 vertices
    for T ∈ TT[n]
        for w = 1:n
            X = deepcopy(T)
            add!(X, w, n + 1)
            if check_in(X, outset)
                continue
            end
            push!(outset, X)
        end
    end

    TT[n+1] = collect(outset)

    @info "Added $(length(TT[n+1])) new trees with $(n+1) vertices"
    nothing
end

function build_trees_table(nmax::Int)
    TT = init_trees_table()
    while maximum(keys(TT)) < nmax
        @info "Adding trees of size $(maximum(keys(TT))+1)"
        extend_trees_table!(TT)
    end
    return TT
end

"""
    create_codes_table(TT)
Given a table of distinct trees, convert that into a table of Prufer codes.
This is used by `save_trees_table` and not useful to be called directly.
"""
function create_codes_table(TT::Dict{Int64,Vector{SimpleGraph{Int}}})
    codes = Dict{Int64,Vector{Vector{Int}}}()
    codes[2] = [Int[]]
    for n = 3:maximum(keys(TT))
        codes[n] = prufer_code.(TT[n])
    end
    return codes
end

"""
    save_tree_table(TT, filename)
Save a trees table into a file specified by `filename`.
If the file name is omitted, use `codes.jl`.
"""
function save_trees_table(
    TT::Dict{Int64,Vector{SimpleGraph{Int}}},
    filename::String = DEFAULT_FILE_NAME,
)
    outfile = open(filename, "w")
    codes = create_codes_table(TT)
    println(outfile, "codes = Dict{Int,Vector{Vector{Int}}}()")
    for n = 2:maximum(keys(TT))
        print(outfile, "codes[$n] = ")
        println(outfile, codes[n])
    end
    close(outfile)
    nothing
end

"""
    load_trees_table(filename)
Create a table of distinct trees by reading in a file that has been
precomputed (and presumably saved using `save_tree_table`). If `filename`
is omitted, use `codes.jl`.
"""
function load_trees_table(filename::String = DEFAULT_FILE_NAME)
    include(filename)
    TT = init_trees_table()
    nmax = maximum(keys(codes))

    for n = 3:nmax
        TT[n] = prufer_restore.(codes[n])
    end
    @info "Read in table of distinct trees up to $nmax vertices"
    return TT

end


## PCA vs LDA
## Height Weight data
# Grab data from https://www.kaggle.com/mustafaali96/weight-height
# https://opendata.stackexchange.com/questions/7793/age-weight-and-height-dataset
# https://www.reddit.com/r/datasets/comments/1dii3t/data_sets_for_heights_and_weights/

# https://vincentarelbundock.github.io/Rdatasets/datasets.html

# a related tutorial
# http://goelhardik.github.io/2016/10/04/fishers-lda/

##
using Plots
##
# https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv
using CSV
df = CSV.read(download("https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv"))
X = Float64.([df[:weight] df[:height]])

##
g = df[:sex]

##
scatter(X[:,1],X[:,2])
xlabel!("Weight")
ylabel!("Height")

## Filter the data
goodpts = X[:,1] .<= 115
Xf = X[goodpts,:]
gf = g[goodpts]

##
theme(:dark)
scatter(Xf[:,1],Xf[:,2],color=2)
xlabel!("Weight")
ylabel!("Height")

## Save a picture
pyplot()
scatter(Xf[:,1],Xf[:,2],color=2,size=(300,300), legend=false)
xlabel!("Weight")
ylabel!("Height")
savefig("weight-height-1.pdf")

## Partition by classes
s1 = gf .== "F"
s2 = gf .== "M"
X1 = Xf[s1,:]
X2 = Xf[s2,:]
gr()
scatter(X1[:,1],X1[:,2])
scatter!(X2[:,1],X2[:,2])

## Save a picture
pyplot()
scatter(X1[:,1],X1[:,2],size=(300,300), legend=false)
scatter!(X2[:,1],X2[:,2])
savefig("weight-height-2.pdf")
##
using Statistics
m = mean(Xf,dims=1)
m1 = mean(X1,dims=1)
m2 = mean(X2,dims=1)
n1 = size(X1,1)
n2 = size(X2,1)

B = (m1-m)'*n1*(m1-m) + (m2-m)'*n2*(m2-m)
W = (X1.-m1)'*(X1.-m1) + (X2.-m2)'*(X2.-m2)

## The matrices B and W add up to the total covariance
using LinearAlgebra
C = (Xf.-m)'*(Xf.-m)
norm(B+W - C)

##
lams,V = eigen(B,W)

##
v = V[:,2]
p1 = histogram((X1)*v,nbins=10)
histogram!((X2)*v,nbins=10)
## Save
plot!(size=(300,300),legend=false)
savefig("weight-height-lda.pdf")
##

## Compare to PCA
lams,V = eigen(C)
v2 = V[:,2]
p2 = histogram(X1*v2,nbins=10)
histogram!(X2*v2,nbins=10)
plot(p1,p2)
##
## Save
plot(p2)
plot!(size=(300,300),legend=false)
savefig("weight-height-pca.pdf")
##
X1c = X1.-m
X2c = X2.-m
scatter(X1c[:,1],X1c[:,2], label="")
scatter!(X2c[:,1],X2c[:,2], label="")
Plots.abline!(v[2]/v[1],0,label="LDA")
Plots.abline!(v2[2]/v2[1],0,label="PCA")
##
plot!(size=(300,300),legend=false)
savefig("weight-height-lda-pca.pdf")


"""
    _setdiff(x::Interval{T}, y::Interval{T})

Computes the set difference x\\y and always returns a tuple of two intervals.
If the set difference is only one interval or is empty, then the returned tuple contains 1
or 2 empty intervals.
"""
function _setdiff(x::Interval{T}, y::Interval{T}) where T
    intersection = x ∩ y

    isempty(intersection) && return (x, emptyinterval(T))
    intersection == x && return (emptyinterval(T), emptyinterval(T))  # x is subset of y; setdiff is empty

    x.lo == intersection.lo && return (Interval(intersection.hi, x.hi), emptyinterval(T))
    x.hi == intersection.hi && return (Interval(x.lo, intersection.lo), emptyinterval(T))

    return (Interval(x.lo, y.lo), Interval(y.hi, x.hi))

end

"""
    setdiff(A::IntervalBox{N,T}, B::IntervalBox{N,T})

Returns a vector of `IntervalBox`es that are in the set difference `A ∖ B`,
i.e. the set of `x` that are in `A` but not in `B`.

Algorithm: Start from the total overlap (in all directions);
expand each direction in turn.
"""
function setdiff(A::IntervalBox{N,T}, B::IntervalBox{N,T}) where {N,T}

    intersection = A ∩ B
    isempty(intersection) && return [A]

    result_list = fill(IntervalBox(emptyinterval(T), N), 2*N)
    offset = 0
    x = A.v
    @inbounds for i = 1:N
        tmp = _setdiff(A[i], B[i])
        @inbounds for j = 1:2
            x = setindex(x, tmp[j], i)
            result_list[offset+j] = IntervalBox{N, T}(x)
        end
        offset += 2
        x = setindex(x, intersection[i], i)
    end
    filter!(!isempty, result_list)
end


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ c96551a0-881e-11eb-0d92-61edbb6c86e6
begin
	using Pkg; Pkg.activate("MLJ_env", shared=true)
  using PlutoUI
	using Test

end

# ╔═╡ accc6f84-881d-11eb-3b49-e9cd25a3b6fd
md"""
## Linear Algebra

ref. from book **"Data Science from Scratch"**, Chap 4

$(html"
© Pascal, March 2021
")
"""

# ╔═╡ 136cdd48-8835-11eb-0482-737cf7bd5a3e
PlutoUI.TableOfContents(indent=true, depth=4, aside=true)

# ╔═╡ b22cf860-8834-11eb-02de-d3ad803d1db6
html"""

"""

# ╔═╡ b1e26cfa-8834-11eb-16f5-85efb64c6a4c
md"""
### Vectors
Points in some finite dimensional space. A vector has a direction and a magnitude.$(html"
")
Vectors are useful to represent data (which lie in some multi-finite-dimensional space).
We will want to perform operations on those vectors, so let defines and implements those operations. $(html"
")
We will be using the `Vector` type from `Julia`.
"""

# ╔═╡ 9b2a7694-881e-11eb-325c-f3edd9178641
begin
	import Base: +, -

	function +(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
		"""
		Vector Additon is component-wise
		"""
		@assert length(v₁) == length(v₂)
		v₁ .+ v₂
	end

	function +(vc::Vector{Vector{T}})::Vector{T} where T <: Number
		"""
		Sum all corresponding components of a collection of vector to return
		one vector
		"""
		@assert length(vc) > 0 "expect the collection to have at leat one vector element"
		n = length(vc[1])
		@assert all(v -> length(v) == n, vc) "Expect all vectors to be of he same length"
		v = zeros(T, n)
		for v₀ ∈ vc
			v .+= v₀
		end
		v
	end

	function -(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
		"""
		Vector Additon is component-wise
		"""
		@assert length(v₁) == length(v₂)
		v₁ .- v₂
	end
end

# ╔═╡ 21a74512-881f-11eb-31e6-974bcaf59141
begin
	@test +([1, 2, 3], [4, 5, 6]) == [5, 7, 9]
	@test [1, 2, 3] + [4, 5, 6] == [5, 7, 9]

	vᵣ = [1.0, -1.0, π, ℯ]
	@test vᵣ + [-1.0, 1.0, -π, -ℯ] == zeros(eltype(vᵣ), length(vᵣ))
end

# ╔═╡ 11ff42a0-8828-11eb-3ba5-dbd12b5ffdda
md"""
Sometimes we want to sum a collection of vector component wise. that is create a new vector whose first component is the sum of evry first comp[onent of thew collection and  so for for the other components.

Here the collection is itsef a vector in `Julia`'s term.
cf. celle above for implementation
"""

# ╔═╡ 7dc3365c-8828-11eb-3c57-69dae41619cd
begin
	@test +([[1, 2], [3, 4], [5, 6], [7, 8]]) == [16, 20]
	@test +([[1, 3, 2] for _ ∈ 1:3]) == [3, 9, 6]
end

# ╔═╡ e9b7d526-8829-11eb-34d1-e317fdc7a1d0
md"""
`Multiplication` by a scalar, `Mean` and `dot` product.
"""

# ╔═╡ 3a7741c2-882a-11eb-2ffd-65b97e75a0b2
begin
	import Base.*

	*(x::T, v::Vector{T}) where T <: Number = v .* x
	*(v::Vector{T}, x::T) where T <: Number = v .* x

	function μ(vc::Vector{Vector{T}})::Vector{AbstractFloat} where T <: Number
		"""Element-wise mean"""
		@assert length(vc) > 0
		n = length(vc[1])
		@assert all(v -> length(v) == n, vc)
		reduce(+, vc) / n
 	end

	function dot(v₁::Vector{T}, v₂::Vector{T})::T where T <: Number
		@assert length(v₁) == length(v₂)
		sum(v₁ .* v₂)
	end
end

# ╔═╡ b3f61d4e-881f-11eb-2dcb-4dd0fd7fba06
begin
	@test -([1, 2, 3], [4, 5, 6]) == [-3, -3, -3]
	@test [1, 2, 3] - [4, 5, 6] == [-3, -3, -3]

	# vᵣ = [1.0, -1.0, π, ℯ]
	@test vᵣ - [-1.0, 1.0, -π, -ℯ] == [2.0, -2.0, 2π, 2ℯ]
end

# ╔═╡ 19f5b176-882b-11eb-3713-73c85f599be1
begin
	@test *(2, [1, 2, 3]) == [2, 4, 6]
	@test *(2, [1, 2, 3]) == *([1, 2, 3], 2)
	@test 2 * [1, 2, 3] == [1, 2, 3] * 2  ==  [2, 4, 6]
end

# ╔═╡ 50444c92-882b-11eb-2893-bf66b63ee595
begin
	μ([[1, 2, 1, 2], [3, 4, 3, 4], [5, 6, 5, 6]]) == Float64[3., 4., 3., 4.]
end

# ╔═╡ d358476c-882d-11eb-26bd-c167cabd7b28
begin
	@test dot([1, 2, 3], [4, 5, 6]) == 32
end

# ╔═╡ d33d417e-882d-11eb-0aae-a1d10512920f
function sum_of_square(v::Vector{T})::T where T <: Number
	"""
	return vᵢ * vᵢ ∀ i ∈ 1:length(v)
	"""
	dot(v, v)
end

# ╔═╡ d3202a62-882d-11eb-2e24-6d874dce773d
begin
	@test sum_of_square([1, 2, 3]) == 14
	@test sum_of_square([4, 5, 6]) == 77
end

# ╔═╡ 9db8e630-882e-11eb-07e5-0d5036f7e8e2
md"""
Let's now implement L₂ norm and euclidean distance.
"""

# ╔═╡ ba1d63da-882e-11eb-1736-37a246585172
function norm(v::Vector{T})::AbstractFloat where T <: Number
	√ sum(v .* v)
end

# ╔═╡ 9da1fcfc-882e-11eb-021f-1f89149817a9
begin
	@test norm([3, 4]) == 5
	@test norm([1, 0, 2]) ≈ √5
end

# ╔═╡ 9d8634ae-882e-11eb-15ad-75b06c847379
function distance(v₁::Vector{T}, v₂::Vector{T})::AbstractFloat where T <: Number
	"""
	distance(v₁, v₂) ≡ √(Σ (v₁ᵢ - v₂ᵢ)²)
	"""
	@assert length(v₁) == length(v₂)
	√ sum((v₁ .- v₂).^2)  # == norm(v₁ .- v₂)
end

# ╔═╡ 9d69ec22-882e-11eb-0637-d7e29f7b9f37
begin
	@test distance([2, 6, 7, 7, 5, 13, 14, 17, 11, 8],
		[3, 5, 5, 3, 7, 12, 13, 19, 22, 7]) ≈ 12.409673645990857
end

# ╔═╡ 0539916a-8835-11eb-3407-41f17efc2ee4
html"""

"""

# ╔═╡ 73e7de7a-8830-11eb-05ef-b93f6c774587
md"""
### Matrices

A two-dimensional collection of numbers.
"""

# ╔═╡ 7f051430-8830-11eb-1e43-797df6b195ed
function shape(m::Matrix{T})::Tuple{Integer, Integer} where T <: Number
	size(m)
end

# ╔═╡ 7ee4e642-8830-11eb-3d15-813c4dbb6420
@test shape([1 2 3; 4 5 6]) == (2, 3) # 2 rows, 3 columns

# ╔═╡ 7ec91688-8830-11eb-18a0-d5a67e267f9b
function get_row(m::Matrix{T}, r::Integer)::Vector{T}  where T <: Number
	@assert 1 ≤ r ≤ size(m)[1]
	view(m, r, :)  ## no copy!
end

# ╔═╡ 7e9afe92-8830-11eb-3772-150255e27923
begin
	@test get_row([1 2 3; 4 5 6], 1) == [1, 2, 3]
	@test get_row([1 2 3; 4 5 6], 2) == [4, 5, 6]
	@test_throws AssertionError get_row([1 2 3; 4 5 6], 0)
end

# ╔═╡ 648af952-8831-11eb-340b-5bc7ef7aaa73
function get_column(m::Matrix{T}, c::Integer)::Vector{T}  where T <: Number
	@assert 1 ≤ c ≤ size(m)[2]
	view(m, :, c)  ## no copy!
end

# ╔═╡ 6470ad90-8831-11eb-0413-5d016bf66536
begin
	@test get_column([1 2 3; 4 5 6], 1) == [1, 4]
	@test get_column([1 2 3; 4 5 6], 2) == [2, 5]
	@test get_column([1 2 3; 4 5 6], 3) == [3, 6]
	@test_throws AssertionError get_column([1 2 3; 4 5 6], 0)
end

# ╔═╡ 6458987c-8831-11eb-2b89-efac8fabf803
md"""
We will want to be able to create a new matrix given its shape and a function generator.
"""

# ╔═╡ c832331c-8831-11eb-0b7c-9f978c82f890
function make_matrix(nrows::Integer, ncols::Integer, fn::Function;
		DT::DataType=Float64)::Matrix
	@assert 1 ≤ nrows && 1 ≤ ncols

	m = zeros(DT, (nrows, ncols))
	for c ∈ 1:ncols, r ∈ 1:nrows
		m[r, c] = fn(r, c)
	end
	m
end

# ╔═╡ be10ce56-8832-11eb-0742-d90fa7f6f669
function identity_matrix(n::Integer; DT::DataType=Float64)::Matrix
	"""Returns the n × n identity matrix"""
	make_matrix(n, n, (i, j) -> i == j ? one(DT) : zero(DT); DT)
end

# ╔═╡ c81a12dc-8831-11eb-27e9-07132c6310a1
@test identity_matrix(5) == [1. 0. 0. 0. 0.; 0. 1. 0. 0. 0.; 0. 0. 1. 0. 0.; 0. 0. 0. 1. 0.; 0. 0. 0. 0. 1.]

# ╔═╡ 3c69e83a-8835-11eb-218e-0bb108ad7b9c
html"""

Mar 2021, Corto Inc
"""

# ╔═╡ Cell order:
# ╟─accc6f84-881d-11eb-3b49-e9cd25a3b6fd
# ╟─136cdd48-8835-11eb-0482-737cf7bd5a3e
# ╟─b22cf860-8834-11eb-02de-d3ad803d1db6
# ╟─b1e26cfa-8834-11eb-16f5-85efb64c6a4c
# ╠═c96551a0-881e-11eb-0d92-61edbb6c86e6
# ╠═9b2a7694-881e-11eb-325c-f3edd9178641
# ╠═21a74512-881f-11eb-31e6-974bcaf59141
# ╠═b3f61d4e-881f-11eb-2dcb-4dd0fd7fba06
# ╟─11ff42a0-8828-11eb-3ba5-dbd12b5ffdda
# ╠═7dc3365c-8828-11eb-3c57-69dae41619cd
# ╟─e9b7d526-8829-11eb-34d1-e317fdc7a1d0
# ╠═3a7741c2-882a-11eb-2ffd-65b97e75a0b2
# ╠═19f5b176-882b-11eb-3713-73c85f599be1
# ╠═50444c92-882b-11eb-2893-bf66b63ee595
# ╠═d358476c-882d-11eb-26bd-c167cabd7b28
# ╠═d33d417e-882d-11eb-0aae-a1d10512920f
# ╠═d3202a62-882d-11eb-2e24-6d874dce773d
# ╟─9db8e630-882e-11eb-07e5-0d5036f7e8e2
# ╠═ba1d63da-882e-11eb-1736-37a246585172
# ╠═9da1fcfc-882e-11eb-021f-1f89149817a9
# ╠═9d8634ae-882e-11eb-15ad-75b06c847379
# ╠═9d69ec22-882e-11eb-0637-d7e29f7b9f37
# ╟─0539916a-8835-11eb-3407-41f17efc2ee4
# ╟─73e7de7a-8830-11eb-05ef-b93f6c774587
# ╠═7f051430-8830-11eb-1e43-797df6b195ed
# ╠═7ee4e642-8830-11eb-3d15-813c4dbb6420
# ╠═7ec91688-8830-11eb-18a0-d5a67e267f9b
# ╠═7e9afe92-8830-11eb-3772-150255e27923
# ╠═648af952-8831-11eb-340b-5bc7ef7aaa73
# ╠═6470ad90-8831-11eb-0413-5d016bf66536
# ╟─6458987c-8831-11eb-2b89-efac8fabf803
# ╠═c832331c-8831-11eb-0b7c-9f978c82f890
# ╠═be10ce56-8832-11eb-0742-d90fa7f6f669
# ╠═c81a12dc-8831-11eb-27e9-07132c6310a1
# ╟─3c69e83a-8835-11eb-218e-0bb108ad7b9c

Data Science From Scratch/04_Linear_Algebra.jl