using Statistics
using Plots

include("ga.jl")

using .GA

str_toint(s) = parse(Int, join(Int.(s)), base=2)

fitness(x::Integer) = 20 * (cos(x/3 + 1)/(x/3 + 1))
fitness(x::Vector{Bool}) = fitness(str_toint(x))

x = 0:63
y = fitness.(x)

@show x[argmin(y)]
@show y[argmin(y)]

ff_plt = plot(
    x,
    y,
    label=nothing,
    title="Fitness Function",
    size=(1024, 512),
)
scatter!(x, y, label=nothing)
savefig(ff_plt, "figs/f_f_fig.png")

let quz="uniform-1", n_pop=8, n_genes=ceil(Integer, log2(63)), uniform_p=1.0, n_repeat=100, stop_bybest=true
    ga01_his = SimpleGA[]
    ga05_his = SimpleGA[]
    ga10_his = SimpleGA[]
    ga50_his = SimpleGA[]

    for i in 1:n_repeat
        ga01 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.01,
            uniform_p=uniform_p,
        )
        optimize!(ga01, stop_bybest=stop_bybest)
        push!(ga01_his, ga01)

        ga05 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.05,
        )
        optimize!(ga05, stop_bybest=stop_bybest)
        push!(ga05_his, ga05)

        ga10 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.1,
        )
        optimize!(ga10, stop_bybest=stop_bybest)
        push!(ga10_his, ga10)

        ga50 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.5,
        )
        optimize!(ga50, stop_bybest=stop_bybest)
        push!(ga50_his, ga50)
    end

    mean_f(his) = mean(fitness.(broadcast(x -> x.best_chro, his)))
    var_f(his) = var(fitness.(broadcast(x -> x.best_chro, his)))

    mean_i(his) = mean(broadcast(x -> x.iter_n, his))
    var_i(his) = var(broadcast(x -> x.iter_n, his))

    fgm_c(his) = sum(broadcast(x -> str_toint(x.best_chro) == 5, his))

    mut_plt_gm = plot(
        [1, 5, 10, 50],
        fgm_c.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Global Minimum Count",
        size=(1024, 512),
    )
    savefig(mut_plt_gm, "figs/$(quz)_mut_gm_fig.png")

    mut_plt_f_m = plot(
        [1, 5, 10, 50],
        mean_f.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Fitness Mean",
        size=(1024, 512),
    )
    savefig(mut_plt_f_m, "figs/$(quz)_mut_f_m_fig.png")
    mut_plt_f_v = plot(
        [1, 5, 10, 50],
        var_f.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Fitness Variance",
        size=(1024, 512),
    )
    savefig(mut_plt_f_v, "figs/$(quz)_mut_f_v_fig.png")

    mut_plt_i_m = plot(
        [1, 5, 10, 50],
        mean_i.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - N.Iterations Mean",
        size=(1024, 512),
    )
    savefig(mut_plt_i_m, "figs/$(quz)_mut_i_m_fig.png")
    mut_plt_i_v = plot(
        [1, 5, 10, 50],
        var_i.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - N.Iterations Variance",
        size=(1024, 512),
    )
    savefig(mut_plt_i_v, "figs/$(quz)_mut_i_v_fig.png")
end

let quz="uniform-0", n_pop=8, n_genes=ceil(Integer, log2(63)), uniform_p=0.0, n_repeat=100, stop_bybest=true
    ga01_his = SimpleGA[]
    ga05_his = SimpleGA[]
    ga10_his = SimpleGA[]
    ga50_his = SimpleGA[]

    for i in 1:n_repeat
        ga01 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.01,
            uniform_p=uniform_p,
        )
        optimize!(ga01, stop_bybest=stop_bybest)
        push!(ga01_his, ga01)

        ga05 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.05,
        )
        optimize!(ga05, stop_bybest=stop_bybest)
        push!(ga05_his, ga05)

        ga10 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.1,
        )
        optimize!(ga10, stop_bybest=stop_bybest)
        push!(ga10_his, ga10)

        ga50 = SimpleGA(
            n_pop=n_pop,
            n_genes=n_genes,
            fitness=fitness,
            mutation_p=0.5,
        )
        optimize!(ga50, stop_bybest=stop_bybest)
        push!(ga50_his, ga50)
    end

    mean_f(his) = mean(fitness.(broadcast(x -> x.best_chro, his)))
    var_f(his) = var(fitness.(broadcast(x -> x.best_chro, his)))

    mean_i(his) = mean(broadcast(x -> x.iter_n, his))
    var_i(his) = var(broadcast(x -> x.iter_n, his))

    fgm_c(his) = sum(broadcast(x -> str_toint(x.best_chro) == 5, his))

    mut_plt_gm = plot(
        [1, 5, 10, 50],
        fgm_c.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Global Minimum Count",
        size=(1024, 512),
    )
    savefig(mut_plt_gm, "figs/$(quz)_mut_gm_fig.png")

    mut_plt_f_m = plot(
        [1, 5, 10, 50],
        mean_f.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Fitness Mean",
        size=(1024, 512),
    )
    savefig(mut_plt_f_m, "figs/$(quz)_mut_f_m_fig.png")
    mut_plt_f_v = plot(
        [1, 5, 10, 50],
        var_f.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - Fitness Variance",
        size=(1024, 512),
    )
    savefig(mut_plt_f_v, "figs/$(quz)_mut_f_v_fig.png")

    mut_plt_i_m = plot(
        [1, 5, 10, 50],
        mean_i.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - N.Iterations Mean",
        size=(1024, 512),
    )
    savefig(mut_plt_i_m, "figs/$(quz)_mut_i_m_fig.png")
    mut_plt_i_v = plot(
        [1, 5, 10, 50],
        var_i.([ga01_his, ga05_his, ga10_his, ga50_his]),
        label=nothing,
        title="Mutation Change - N.Iterations Variance",
        size=(1024, 512),
    )
    savefig(mut_plt_i_v, "figs/$(quz)_mut_i_v_fig.png")
end


using deSolveDiffEq
using Test

function lorenz(u,p,t)
 du1 = 10.0(u[2]-u[1])
 du2 = u[1]*(28.0-u[3]) - u[2]
 du3 = u[1]*u[2] - (8/3)*u[3]
 [du1, du2, du3]
end
u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob,deSolveDiffEq.lsoda())
sol = solve(prob,deSolveDiffEq.lsode())
sol = solve(prob,deSolveDiffEq.lsodes())
sol = solve(prob,deSolveDiffEq.lsodar())
sol = solve(prob,deSolveDiffEq.vode())
sol = solve(prob,deSolveDiffEq.daspk())
sol = solve(prob,deSolveDiffEq.euler())
sol = solve(prob,deSolveDiffEq.rk4())
sol = solve(prob,deSolveDiffEq.ode23())
sol = solve(prob,deSolveDiffEq.ode45())
sol = solve(prob,deSolveDiffEq.radau())
sol = solve(prob,deSolveDiffEq.bdf())
sol = solve(prob,deSolveDiffEq.bdf_d())
sol = solve(prob,deSolveDiffEq.adams())
sol = solve(prob,deSolveDiffEq.impAdams())
sol = solve(prob,deSolveDiffEq.impAdams_d())
#sol = solve(prob,deSolveDiffEq.iteration())

sol = solve(prob,deSolveDiffEq.lsoda(),saveat=0.1)
#using Plots; plot(sol,vars=(1,2,3))


@testset "moments" begin
    ind = rand([0,1], 10)
    @test sum(normalize_indicator(ind)) ≈ 1.

    Ind = diagm(ind)
    @test sum(normalize_indicator(Ind)) ≈ 1.

    @test to_indicator_matrix(ind) == Ind

    Ind = rand([0,1], 10, 10)
    @test sum(normalize_indicator(Ind, 1), dims=1) ≈ ones(Int64, 1, 10)
    @test sum(normalize_indicator(Ind, 2), dims=2) ≈ ones(Int64, 10)

    @test diag(union_diagonal!(Ind)) == ones(Int64, 10)

    ind = rand([0,1], 10)

    G = graph_filter(ind, 1)
    @test G isa SparseVector
    @test sum(G .!= 0) == sum(ind .!= 0)
    @test sum(G) ≈ 1.

    G = graph_filter(ind, 2)
    @test G isa SparseMatrixCSC
    @test sum(G .!= 0) == sum(ind .!= 0)
    @test sum(G) ≈ 1.

    adj = rand([0,1], 10, 10)

    G = graph_filter(adj, 1, dims=1)
    @test G isa SparseMatrixCSC
    @test sum(G .!= 0, dims=1) == sum(adj .!= 0, dims=1)
    @test sum(G, dims=1) ≈ ones(Int64, 1, 10)

    G = graph_filter(adj, 1, dims=2)
    @test G isa SparseMatrixCSC
    @test sum(G .!= 0, dims=2) == sum(adj .!= 0, dims=2)
    @test sum(G, dims=2) ≈ ones(Int64, 10)

    adj = [1 0 0; 1 0 1; 1 1 1]
    blocks = [diagm([1/3, 1/3, 1/3]), diagm([0, 0, 1]), diagm([0, 1/2, 1/2])]
    @test graph_filter(adj, 2, dims=1) == cat(blocks...,dims=(1,2))

    blocks = [diagm([1, 0, 0]), diagm([1/2, 0, 1/2]), diagm([1/3, 1/3, 1/3])]
    @test graph_filter(adj, 2, dims=2) == cat(blocks...,dims=(1,2))

    A = rand([0,1], 5, 5)
    X = rand(5, 10)
    @test size(moment(A, X, 1, dims=1)) == size(X)
    @test size(moment(A, Matrix(X'), 1, dims=2)) == size(X')
    # @test size(moment(A, X, 2, dims=1)) == (size(X, 1), 5*10)
    # @test size(moment(A, Matrix(X'), 2, dims=2)) == (5*10, size(X, 1))
end


@testset "set operations" begin

    Nsamps = 10^6

    a = interval(0,1) ∪ interval(2,3)

    b = interval(-1, 1) ∪ interval(2,5)

    aSamps = zeros(Nsamps); bSamps = zeros(Nsamps)

    for i = 1: Nsamps
        if rand() < 0.5
            aSamps[i] = rand()
        else
            aSamps[i] = rand() + 2
        end

        if rand() < 0.5
            bSamps[i] = (rand() * 2) -1
        else
            bSamps[i] = rand() * 3 + 2
        end
    end

    @test all(aSamps .∈ a)
    @test all(bSamps .∈ b)

    ac = complement(a)

    @test all(.~(aSamps .∈ ac))     # Test all samples are not in complement

    intersec = a ∩ b                 # Intersect

    aSampsB = aSamps[ aSamps .∈ b]
    bSampsA = bSamps[ bSamps .∈ a]

    @test all( aSampsB .∈ intersec)
    @test all( bSampsA .∈ intersec)

    diff = a \ b                 # Set difference

    @test diff[1] == interval(1)
    @test diff[2] == interval(2)

    bi = bisect(a, 0.5)         # Cut at a = 0.5

    @test bi[1] == interval(0, 0.5)
    @test bi[2] == interval(0.5, 1)
    @test bi[3] == interval(2, 3)

    @test a ⊂ interval(0, 3)     # Subset
    @test !(b ⊂ interval(0, 3))

end


### A Pluto.jl notebook ###
# v0.12.4

using Markdown
using InteractiveUtils

# ╔═╡ 3e849bdf-f13a-4ea0-aa54-b6ae89894565
md"""
# Álgebra lineal básica en Julia
Autor: Andreas Noack Jensen (MIT) (http://www.econ.ku.dk/phdstudent/noack/)
(con edición de Jane Herriman)
"""

# ╔═╡ a880f5b4-1f0f-4569-a59c-0e87d21cd203
md"""
Julia's syntax is very similar to other languages but there are some important differences. Define a matrix of random normal variates
"""

# ╔═╡ 05eff53f-64c5-4833-9eff-11c7d09b9684
A = rand(1:4,3,3)

# ╔═╡ 16ca2d04-2f46-4de7-9481-daebf5112c2f
md"""
Definir un vector de unos
"""

# ╔═╡ 018e028c-43ff-4dbe-bc83-e5f0982d6bb6
x = fill(1.0, (3))

# ╔═╡ 5cc332db-a229-4182-b860-4d276e3042a9
md"""
Notamos que $A$ tiene el tipo Array{Int64,2} pero $x$ tiene tipo Array{Int64,1}. Julia define los alias a Vector{Type}=Array{Type,1} y Matrix{Type}=Array{Type,2}.

Muchas de las operaciones básicas son idénticas a otros lenguajes

#### Multiplicación
"""

# ╔═╡ 383658c8-9d47-4a1e-ba63-66fa6e7f3311
b = A*x

# ╔═╡ e2cc7ff7-781c-4102-892c-da24291f0388
md"""
#### Traspuestas
Como en otros lenguajes, `A'` es la transpuesta conjugada mientras que `A.'` es sólo la traspuesta
"""

# ╔═╡ 5c3d3b48-6b7c-4a3b-a9b6-d83f7bc03168
Asym = A + A'

# ╔═╡ 88f5c70e-b5b1-4cf0-8f50-d1241f0a769b
md"""
#### Multiplicación traspuesta
Julia nos permite escribir esto sin *
"""

# ╔═╡ 0691212e-d372-43be-b66e-61f842df09eb
Apd = A'A

# ╔═╡ 1445f674-cfa2-44ab-920f-fe4d85d00c2e
md"""
#### Resolviendo sistemas lineales
El problema $Ax=b$ para $A$ cuadrada se resulve con la función \.
"""

# ╔═╡ c42fc4ae-bf1d-4eb7-9863-a498fb9116b8
A\b

# ╔═╡ 462e5a23-f323-4158-bf27-cf36bbc5cec5
md"""
#### Sistemas sobredeterminados
Cuando nuestra matriz es alta (número de renglones mayores al número de columnas), tenemos un sistema lineal sobredeterminado.

En este caso \ calcula la de mínimos cuadrados
"""

# ╔═╡ 8c87b3e1-b737-4179-9955-defcf0905b1b
Atall = rand(3, 2)
display(Atall)
Atall\b

# ╔═╡ 7dea4ca8-27e0-429e-bbf3-dccf2ca67df3
md"""
La función \ también sirve ocn problemas deficientes de rango de mínimos cuadrados. En este caso, la solución no es única y Julia regresa el valor con la menor norma.

Para crear un problema de rango deficiente de mínimos cuadrados, vamos a crear una matriz deficiente en rango con columnas linealmente dependientes
"""

# ╔═╡ 962df670-f9d1-441d-bf86-f4619f29ab4f
v = randn(3)

# ╔═╡ 306db6d0-193e-11eb-3bee-cbb7d8c4c5c6
rankdef = [v v]

# ╔═╡ 92b4cb6a-47c9-459b-8694-aee9158439dd
rankdef\b

# ╔═╡ 4c836857-08fc-411a-8b65-6340c6e94c3b
md"""
#### Sistemas indeterminados
cuando A es corta (número de columnas mayor al número de renglones), tenemos un sistema indeterminado

En este caso \ regresa la solución con la norma mínima
"""

# ╔═╡ 0784963e-8842-43f6-af98-781aa7e214b6
Ashort = rand(2, 3)

# ╔═╡ 3346487c-193e-11eb-15dd-4dd8241ac876
display(Ashort)

# ╔═╡ 3346b974-193e-11eb-2c77-9133e1f9b17b
Ashort\b[1:2]

# ╔═╡ 86dc3af8-2f80-48a5-b0d3-caeddc5ae8a8
md"""
### Ejercicios

```
A = [
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 1  2  3  4  5  6  7  8  9  10
 ]
```

Quieres obtener

```
A = [
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 7  8  9  10  1  2  3  4  5  6
 ]
```
"""

# ╔═╡ f86a1d32-1e1d-461e-ad40-f5b906b04cf9

# ╔═╡ 22e5c662-6bf6-48d4-b7f9-93d8ac33ffdd
md"""
10.2 Toma el producto de un vector `v` con sí mismo.
"""

# ╔═╡ 403752d3-4a6b-41bf-b4ed-fd6fe7b9ee95

# ╔═╡ 6b98b27f-f671-4c5c-af5e-6d290f0d158b
md"""
10.3 Toma el producto de un vector `v` con sí mismo.
"""

# ╔═╡ fd670b40-f44e-440e-90b0-b27522c7adbd

# ╔═╡ Cell order:
# ╟─3e849bdf-f13a-4ea0-aa54-b6ae89894565
# ╟─a880f5b4-1f0f-4569-a59c-0e87d21cd203
# ╠═05eff53f-64c5-4833-9eff-11c7d09b9684
# ╟─16ca2d04-2f46-4de7-9481-daebf5112c2f
# ╠═018e028c-43ff-4dbe-bc83-e5f0982d6bb6
# ╟─5cc332db-a229-4182-b860-4d276e3042a9
# ╠═383658c8-9d47-4a1e-ba63-66fa6e7f3311
# ╟─e2cc7ff7-781c-4102-892c-da24291f0388
# ╠═5c3d3b48-6b7c-4a3b-a9b6-d83f7bc03168
# ╟─88f5c70e-b5b1-4cf0-8f50-d1241f0a769b
# ╠═0691212e-d372-43be-b66e-61f842df09eb
# ╟─1445f674-cfa2-44ab-920f-fe4d85d00c2e
# ╠═c42fc4ae-bf1d-4eb7-9863-a498fb9116b8
# ╟─462e5a23-f323-4158-bf27-cf36bbc5cec5
# ╠═8c87b3e1-b737-4179-9955-defcf0905b1b
# ╟─7dea4ca8-27e0-429e-bbf3-dccf2ca67df3
# ╠═962df670-f9d1-441d-bf86-f4619f29ab4f
# ╠═306db6d0-193e-11eb-3bee-cbb7d8c4c5c6
# ╠═92b4cb6a-47c9-459b-8694-aee9158439dd
# ╟─4c836857-08fc-411a-8b65-6340c6e94c3b
# ╠═0784963e-8842-43f6-af98-781aa7e214b6
# ╠═3346487c-193e-11eb-15dd-4dd8241ac876
# ╠═3346b974-193e-11eb-2c77-9133e1f9b17b
# ╟─86dc3af8-2f80-48a5-b0d3-caeddc5ae8a8
# ╠═f86a1d32-1e1d-461e-ad40-f5b906b04cf9
# ╟─22e5c662-6bf6-48d4-b7f9-93d8ac33ffdd
# ╠═403752d3-4a6b-41bf-b4ed-fd6fe7b9ee95
# ╟─6b98b27f-f671-4c5c-af5e-6d290f0d158b
# ╠═fd670b40-f44e-440e-90b0-b27522c7adbd


# One step of the metropolis algorithm on a Ising's spin graph
function stepmetropolis!(grid::Array{Int, 2}; # Spin grid
                         h::Float64    = 0.0, # External field
                         temp::Float64 = 1.0) # Temperature

    # Randomly pick a position within the grid
    x   = rand(1:size(grid, 1))
    y   = rand(1:size(grid, 2))

    # Calculate the ΔE for switching the spin
    m      = nspins(grid, x, y) |> sum
    eplus  = -m - h
    ΔE     = -2eplus * grid[x,y]

    # Change spin accordingly
    if ΔE <= 0 || rand() < exp(-ΔE/temp)
        grid[x,y] *= -1
    end
end

# Several steps of the heat bath algorithm on a Ising's spin graph
function metropolis!(grid::Array{Int, 2};  # Spin grid
                     h::Float64=0.0,       # External field
                     temp::Float64=1.0,    # Temperature
                     iters::Integer=50000, # Number of iterations
                     plot::Bool=true,      # Plot flag
                     verbose::Bool=true)   # Verbose flag

    m = Float64[]
    for i in 1:iters
        stepmetropolis!(grid, h=h, temp=temp)
        push!(m, magnetization(grid))
        # Must find a better way to decide convergence
    end

    if verbose println("(T=$temp) Metropolis ended with magnetization $(m[end]) after $(length(m)) iterations") end
    if plot
        PyPlot.plot(1:length(m), m, "o", color="blue")
        PyPlot.plot(1:length(m), m, "-", color="blue")
        PyPlot.title("Metropolis for T=$temp for size $size ($(length(m)) iterations and H=$h)")
        PyPlot.xlabel("Number of Iterations")
        PyPlot.ylabel("Magnetization")
        PyPlot.ylim(0,1.1)
        PyPlot.savefig("metropolis_T=$temp.png")
        PyPlot.close()
    end

    return m
end


using CSV, DataFrames, StatsPlots, StanSample
using Distributions, Statistics, Random

ProjDir = @__DIR__

# Simulate the data

#Random.seed!(123)

countries = ["Guatamala", "Panama", "Kenya", "Ethiopia", "Papua New Guinea"]
brands = [4, 3, 4, 5, 1]
intercepts = [500, 300, 600, 700, 900] # Quantity at cheapest price_point
slopes = [-6, -8, -10, -15, -20] # Assuming constant price elasticity
price_points = [12.5, 12.75, 13.02, 14.20, 15.15] # Selected price points
costs = [4.0, 4.2, 4.6, 4.7, 5.2]
channels = ["retail", "wholesale", "shop"]

df = DataFrame()
for n in 1:1500
	yr = 2019
	wk = rand(1:52)
	c_ind = rand(1:length(countries))
	p_ind = rand(1:length(price_points))
	c = countries[c_ind]							# country
	b = rand(1:brands[c_ind])						# brand
	ch = rand(channels)								# channel
	cst = rand(Normal(costs[c_ind], 0.1))			# cost
	p = price_points[p_ind]							# ppu
	if rand(Bernoulli(0.2))
		p = rand(Normal(p, 0.07))
	end
	p = round(p, digits=2)
	q = intercepts[c_ind] + slopes[c_ind] * p_ind	# quantity sold
	if rand(Bernoulli(0.2))
		q = round(rand(Normal(q, 30)))
	end
	append!(df, DataFrame(
		year=yr, week=wk,
		country=c, brand=b, channel=ch,
		quantity=q, ppu=p)
	)
end

df[!, :ppu_l] = log.(df[:, :ppu])
df[!, :quantity_l] = log.(df[:, :quantity])

# Clean up

df = df[df.ppu_l .> 0.04, :]
df = df[df.quantity_l .> 0.0, :]

CSV.write(joinpath(ProjDir, "data", "data.csv"), df)

bglm = "
data {
  int<lower=0> N;
  vector[N] q; // Outcome
  vector[N] p; // Predictor
}
parameters {
  real<lower=0> a;
  real b;
  real<lower=0> sigma;
}
model {
  vector[N] mu;            		// mu is a vector
  a ~ normal(2.0, 10.0);		// intercept positive
  b ~ normal(-2.0, 10.0);		// demand drops as price increases
  sigma ~ uniform(0, 15);
  mu = a + b * p;
  q ~ normal(mu, sigma);
}
// generated quantities {
//    vector[N] mu_pred;
//    mu_pred = exp(p);
//  }
";

mean_q_l = mean(df[:, :quantity_l])
mean_p_l = mean(df[:, :ppu_l])
data = Dict(
  :N => size(df, 1),
  :q => df[:, :quantity_l] .- mean_q_l,
  :p => df[:, :ppu_l] .- mean_p_l
)

p1 = scatter(df[:, :ppu], df[:, :quantity],
	xlabel="ppu", ylabel="quantity sold", leg=false)
p2 = scatter(data[:p], data[:q],
	xlabel="log(ppu)", ylabel="log(quantity sold)", leg=false)
plot(p1, p2, layout=(2, 1))
savefig(joinpath(ProjDir, "plots", "data.png"))

sm = SampleModel("bglm", bglm)

rc = stan_sample(sm, data=data)

if success(rc)
	chns = read_samples(sm)
	plot(chns)
	savefig(joinpath(ProjDir, "plots", "chains.png"))

	dfs = DataFrame(chns)
	show(chns)

	p = -0.1:0.01:0.15
	p1 = scatter(data[:p], data[:q],
		xlims=(-0.1, 0.15), ylims=(-1, 1),
		xlabel="ppu_l", ylabel="quantity_l sold", leg=false)
	a_mean = mean(dfs[:, :a])
	b_mean = mean(dfs[:, :b])
	mu_q = a_mean .+ b_mean .* p
	plot!(p1, p, mu_q)
	savefig(joinpath(ProjDir, "plots", "regression.png"))

	price_points_l = log.(price_points) .- mean(log.(price_points))
	scatter(price_points_l, a_mean .+ b_mean .* price_points_l,
		xlims = (-0.1, 0.15), ylims= (-0.1, 0.1),
		leg=false, xlabel="log(ppu)", ylabel="log(quantity sold)")
	selected_rows = rand(1:size(dfs, 1), 75)
	for row in eachrow(dfs[selected_rows,:])
		plot!(p, row[:a] .+ row[:b] .* p, color=:gray)
	end
	scatter!(price_points_l, a_mean .+ b_mean .* price_points_l)
	savefig(joinpath(ProjDir, "plots", "sampling.png"))
end


@testset "Mincut" begin

    g = lg.complete_digraph(5)
    cap1 = [
       0.0 2.0 2.0 0.0 0.0
       0.0 0.0 0.0 0.0 3.0
       0.0 1.0 0.0 3.0 0.0
       0.0 0.0 0.0 0.0 1.0
       0.0 0.0 0.0 0.0 0.0
    ];
    (part1, part2, value) = LightGraphsFlows.mincut(g,1,5,cap1,LightGraphsFlows.PushRelabelAlgorithm())
    @test value ≈ 4.0
    @test part1 == [1]
    @test sort(part2) == collect(2:5)
    cap2 = [
       0.0 3.0 2.0 0.0 0.0
       0.0 0.0 0.0 0.0 3.0
       0.0 1.0 0.0 3.0 0.0
       0.0 0.0 0.0 0.0 1.5
       0.0 0.0 0.0 0.0 0.0
    ];
    (part1, part2, value) = LightGraphsFlows.mincut(g,1,5,cap2,LightGraphsFlows.PushRelabelAlgorithm())
    @test value ≈ 4.5
    @test sort(part1) == collect(1:4)
    @test part2 == [5]

    g2 = lg.DiGraph(5)
    lg.add_edge!(g2,1,2)
    lg.add_edge!(g2,1,3)
    lg.add_edge!(g2,3,4)
    lg.add_edge!(g2,3,2)
    lg.add_edge!(g2,2,5)

    (part1, part2, value) = LightGraphsFlows.mincut(g2,1,5,cap1,LightGraphsFlows.PushRelabelAlgorithm())
    @test value ≈ 3.0
    @test sort(part1) == [1,3,4]
    @test sort(part2) == [2,5]

    #non regression test
    flow_graph = lg.DiGraph(7)
    capacity_matrix = zeros(7,7)
    flow_edges = [
    (1,2,2),(1,3,2),(2,4,4),(2,5,4),
    (3,5,4),(3,6,4),(4,7,1),(5,7,1),(6,7,1)
    ]
    for e in flow_edges
       u, v, f = e
    lg.add_edge!(flow_graph, u, v)
    capacity_matrix[u,v] = f
    end
    (part1, part2, value) = LightGraphsFlows.mincut(flow_graph, 1, 7, capacity_matrix, EdmondsKarpAlgorithm())
    @test value ≈ 3.0
    @test sort(part1) == [1,2,3,4,5,6]
    @test sort(part2) == [7]

end


immutable Gamma <: ContinuousUnivariateDistribution
    shape::Float64
    scale::Float64
    function Gamma(sh::Real, sc::Real)
        sh > zero(sh) && sc > zero(sc) ||
            error("Both shape and scale must be positive")
        new(float64(sh), float64(sc))
    end
end

Gamma(sh::Real) = Gamma(sh, 1.0)
Gamma() = Gamma(1.0, 1.0) # Standard exponential distribution

scale(d::Gamma) = d.scale
rate(d::Gamma) = 1.0 / d.scale

@_jl_dist_2p Gamma gamma

function entropy(d::Gamma)
    x = (1.0 - d.shape) * digamma(d.shape)
    x + lgamma(d.shape) + log(d.scale) + d.shape
end

insupport(::Gamma, x::Real) = zero(x) <= x < Inf
insupport(::Type{Gamma}, x::Real) = zero(x) <= x < Inf

kurtosis(d::Gamma) = 6.0 / d.shape

mean(d::Gamma) = d.shape * d.scale

median(d::Gamma) = quantile(d, 0.5)

mgf(d::Gamma, t::Real) = (1.0 - t * d.scale)^(-d.shape)

cf(d::Gamma, t::Real) = (1.0 - im * t * d.scale)^(-d.shape)

function mode(d::Gamma)
    d.shape >= 1.0 ? d.scale * (d.shape - 1.0) : error("Gamma has no mode when shape < 1.0")
end

modes(d::Gamma) = [mode(d)]

# rand()
#
#  A simple method for generating gamma variables - Marsaglia and Tsang (2000)
#  http://www.cparity.com/projects/AcmClassification/samples/358414.pdf
#  Page 369
#  basic simulation loop for pre-computed d and c
function randg2(d::Float64, c::Float64)
    while true
        x = v = 0.0
        while v <= 0.0
            x = randn()
            v = 1.0 + c * x
        end
        v = v^3
        U = rand()
        x2 = x^2
        if U < 1.0 - 0.331 * x2^2 ||
           log(U) < 0.5 * x2 + d * (1.0 - v + log(v))
            return d * v
        end
    end
end

# sampling from Gamma(α, 1)
function randg(α::Float64)
    dpar = (α <= 1.0 ? α + 1.0 : α) - 1.0 / 3.0
    cpar = 1.0 / sqrt(9.0 * dpar)
    randg2(dpar, cpar) * (α > 1.0 ? 1.0 : rand()^(1.0 / α))
end

rand(d::Gamma) = d.scale * randg(d.shape)

function rand!(d::Gamma, A::Array{Float64})
    α = d.shape
    dpar = (α <= 1.0 ? α + 1.0 : α) - 1.0 / 3.0
    cpar = 1.0 / sqrt(9.0 * dpar)
    n = length(A)
    for i in 1:n
        A[i] = randg2(dpar, cpar)
    end
    if α <= 1.0
        ainv = 1.0 / α
        for i in 1:n
            A[i] *= rand()^ainv
        end
    end
    multiply!(A, d.scale)
end

skewness(d::Gamma) = 2.0 / sqrt(d.shape)

var(d::Gamma) = d.shape * d.scale * d.scale

## Fit model

immutable GammaStats
    sx::Float64      # (weighted) sum of x
    slogx::Float64   # (weighted) sum of log(x)
    tw::Float64      # total sample weight

    GammaStats(sx::Real, slogx::Real, tw::Real) = new(float64(sx), float64(slogx), float64(tw))
end

function suffstats(::Type{Gamma}, x::Array)
    sx = 0.
    slogx = 0.
    for xi = x
        sx += xi
        slogx += log(xi)
    end
    GammaStats(sx, slogx, length(x))
end

function suffstats(::Type{Gamma}, x::Array, w::Array{Float64})
    n = length(x)
    if length(w) != n
        throw(ArgumentError("Inconsistent argument dimensions."))
    end

    sx = 0.
    slogx = 0.
    tw = 0.
    for i = 1:n
        xi = x[i]
        wi = w[i]
        sx += wi * xi
        slogx += wi * log(xi)
        tw += wi
    end
    GammaStats(sx, slogx, tw)
end

function gamma_mle_update(logmx::Float64, mlogx::Float64, a::Float64)
    ia = 1.0 / a
    z = ia + (mlogx - logmx + log(a) - digamma(a)) / (abs2(a) * (ia - trigamma(a)))
    1.0 / z
end

function fit_mle(::Type{Gamma}, ss::GammaStats;
    alpha0::Float64=NaN, maxiter::Int=1000, tol::Float64=1.0e-16)

    mx = ss.sx / ss.tw
    logmx = log(mx)
    mlogx = ss.slogx / ss.tw

    a::Float64 = isnan(alpha0) ? 0.5 / (logmx - mlogx) : alpha0
    converged = false

    t = 0
    while !converged && t < maxiter
        t += 1
        a_old = a
        a = gamma_mle_update(logmx, mlogx, a)
        converged = abs(a - a_old) <= tol
    end

    Gamma(a, mx / a)
end


using MultivariateStats
using LinearAlgebra
using Test
using StatsBase
import Statistics: mean, cov
import Random

@testset "Multi-class LDA" begin

    #Test equivalence of eigenvectors/singular vectors taking into account possible
    #phase (sign) differences
    #This may end up in Base.Test; see JuliaLang/julia#10651
    function test_approx_eq_vecs(a::StridedVecOrMat{S},
        b::StridedVecOrMat{T}, error=nothing) where {S<:Real, T<:Real}
        n = size(a, 2)
        @test n==size(b, 2) && size(a, 1)==size(b, 1)
        error==nothing && (error=n^3*(eps(S)+eps(T)))
        for i=1:n
            ev1, ev2 = a[:, i], b[:, i]
            deviation = min(abs(norm(ev1-ev2)), abs(norm(ev1+ev2)))
            if !isnan(deviation)
                @test isapprox.(deviation, 0.0, atol=error)
            end
        end
    end

    Random.seed!(34568)

    ## prepare data
    d = 5
    ns = [10, 15, 20]
    nc = length(ns)
    n = sum(ns)
    Xs = Matrix{Float64}[]
    ys = Vector{Int}[]
    Ss = Matrix{Float64}[]
    cmeans = zeros(d, nc)

    for k = 1:nc
        R = qr(randn(d, d)).Q
        nk = ns[k]

        Xk = R * Diagonal(2 * rand(d) .+ 0.5) * randn(d, nk) .+ randn(d)
        yk = fill(k, nk)
        uk = vec(mean(Xk, dims=2))
        Zk = Xk .- uk
        Sk = Zk * Zk'

        push!(Xs, Xk)
        push!(ys, yk)
        push!(Ss, Sk)
        cmeans[:,k] .= uk
    end

    X = hcat(Xs...)
    y = vcat(ys...)
    mv = vec(mean(X, dims=2))
    Sw = zeros(d, d)
    for k = 1:nc
        Sw += Ss[k]
    end

    Sb = zeros(d, d)
    for k = 1:nc
        dv = cmeans[:,k] - mv
        Sb += ns[k] * (dv * dv')
    end

    Z = X .- mv
    Sa = Z * Z'
    @test Sw + Sb ≈ Sa

    @assert size(X) == (d, n)
    @assert size(y) == (n,)

    ## Stats

    S = multiclass_lda_stats(nc, X, y)

    @test S.dim == d
    @test S.nclasses == nc

    @test classweights(S) == ns
    @test classmeans(S) ≈ cmeans
    @test mean(S) ≈ mv

    @test withclass_scatter(S) ≈ Sw
    @test betweenclass_scatter(S) ≈ Sb

    covestimator = SimpleCovariance(corrected=true)
    Sce = multiclass_lda_stats(nc, X, y; covestimator_between=covestimator, covestimator_within=covestimator)

    Swcorr = Sw * sum(ns)/(sum(ns)-1)
    Sbcorr = Sb * nc/(nc-1)

    @test withclass_scatter(Sce) ≈ Swcorr
    @test betweenclass_scatter(Sce) ≈ Sbcorr

    ## Solve

    emax = maximum(eigvals(Sw))
    Sw_c = copy(Sw)
    Sb_c = copy(Sb)

    lambda = 1.0e-3
    Sw_r = Sw + (lambda * emax) * Matrix(I, d, d)

    P1 = mclda_solve(Sb, Sw, :gevd, nc-1, lambda)
    @test P1' * Sw_r * P1 ≈ Matrix(I, nc-1, nc-1)
    U = Sb * P1
    V = Sw_r * P1
    # test whether U is proportional to V,
    # which indicates P is the generalized eigenvectors
    @test U ≈ V*Diagonal(vec(mean(U./V, dims=1)))

    P2 = mclda_solve(Sb, Sw, :whiten, nc-1, lambda)
    @test P2' * Sw_r * P2 ≈ Matrix(I, nc-1, nc-1)

    test_approx_eq_vecs(P1, P2)

    ## MC-LDA

    for T in (Float32, Float64)
        M = fit(MulticlassLDA, nc, convert(Matrix{T}, X), y; method=:gevd, regcoef=convert(T, lambda))
        @test size(M) == (d, nc - 1)
        @test projection(M) ≈ P1
        @test M.pmeans ≈ M.proj'cmeans
        @test predict(M, X) ≈ M.proj'X

        M = fit(MulticlassLDA, nc, convert(Matrix{T}, X), y; method=:whiten, regcoef=convert(T, lambda))
        @test size(M) == (d, nc - 1)
        # @test projection(M) P2  # signs may change
        @test M.pmeans ≈ M.proj'cmeans
        @test predict(M, X) ≈ M.proj'X
    end

    ## High-dimensional LDA (subspace LDA)

    # Low-dimensional case (no singularities)

    centers = [zeros(5) [10.0;zeros(4)] [0.0;10.0;zeros(3)]]

    # Case 1: 3 groups of 500
    dX = randn(5,1500);
    for i = 0:500:1000
        dX[:,(1:500).+i] .= dX[:,(1:500).+i] .- mean(dX[:,(1:500).+i], dims=2)  # make the mean of each 0
    end
    dX1 = dX
    X1 = [dX[:,1:500].+centers[:,1] dX[:,501:1000].+centers[:,2] dX[:,1001:1500].+centers[:,3]]
    label1 = [fill(1,500); fill(2,500); fill(3,500)]
    # Case 2: 3 groups, one with 1000, one with 100, and one with 10
    dX = randn(5,1110);
    dX[:,   1:1000] .= dX[:,   1:1000] .- mean(dX[:,   1:1000], dims=2)
    dX[:,1001:1100] .= dX[:,1001:1100] .- mean(dX[:,1001:1100], dims=2)
    dX[:,1101:1110] .= dX[:,1101:1110] .- mean(dX[:,1101:1110], dims=2)
    dX2 = dX
    X2 = [dX[:,1:1000].+centers[:,1] dX[:,1001:1100].+centers[:,2] dX[:,1101:1110].+centers[:,3]]
    label2 = [fill(1,1000); fill(2,100); fill(3,10)]

    for (X, dX, label) in ((X1, dX1, label1), (X2, dX2, label2))
        w = [length(findall(label.==1)), length(findall(label.==2)), length(findall(label.==3))]
        M = fit(SubspaceLDA, X, label)
        @test indim(M) == 5
        @test outdim(M) == 2
        totcenter = vec(sum(centers.*w', dims=2)./sum(w))
        @test mean(M) ≈ totcenter
        @test classmeans(M) ≈ centers
        @test classweights(M) == w
        x = rand(5)
        @test predict(M, x) ≈ projection(M)'*x
        dcenters = centers .- totcenter
        Hb = dcenters.*sqrt.(w)'
        Sb = Hb*Hb'
        Sw = dX*dX'
        # When there are no singularities, we need to solve the "regular" LDA eigenvalue equation
        proj = projection(M)
        @test Sb*proj ≈ Sw*proj*Diagonal(M.λ)
        @test proj'*Sw*proj ≈ Matrix(I,2,2)

        # also check that this is consistent with the conventional algorithm
        Mld = fit(MulticlassLDA, 3, X, label)
        for i = 1:2
            @test abs(dot(normalize(proj[:,i]), normalize(Mld.proj[:,i]))) ≈ 1
        end
    end

    # High-dimensional case (undersampled => singularities)
    X = randn(10^6, 9)
    label = rand(1:3, 9); label[1:3] = 1:3
    M = fit(SubspaceLDA, X, label)
    centers = M.cmeans
    for i = 1:3
        flag = label.==i
        @test centers[:,i] ≈ mean(X[:,flag], dims=2)
        @test M.cweights[i] == sum(flag)
    end
    dcenters = centers .- mean(X, dims=2)
    Hb = dcenters*Diagonal(sqrt.(M.cweights))
    Hw = X - centers[:,label]
    @test (M.projw'*Hb)*(Hb'*M.projw)*M.projLDA ≈ (M.projw'*Hw)*(Hw'*M.projw)*M.projLDA*Diagonal(M.λ)
    Gw = projection(M)'*Hw
    @test Gw*Gw' ≈ Matrix(I,2,2)

    # Test that nothing breaks if one class has no members
    label[1:4] .= 1
    label[5:9] .= 3  # no data points have label == 2
    M = fit(SubspaceLDA, X, label)
    centers = M.cmeans
    dcenters = centers .- mean(X, dims=2)
    Hb = dcenters*Diagonal(sqrt.(M.cweights))
    Hw = X - centers[:,label]
    @test (M.projw'*Hb)*(Hb'*M.projw)*M.projLDA ≈ (M.projw'*Hw)*(Hw'*M.projw)*M.projLDA*Diagonal(M.λ)

    # Test normalized LDA
    function gen_ldadata_2(centers, n1, n2)
        d = size(centers, 1)
        X = randn(d, n1+n2)
        X[:,1:n1]       .-= vec(mean(X[:,1:n1], dims=2))
        X[:,n1+1:n1+n2] .-= vec(mean(X[:,n1+1:n1+n2], dims=2))
        dX = copy(X)
        X[:,1:n1]       .+= centers[:,1]
        X[:,n1+1:n1+n2] .+= centers[:,2]
        label = [fill(1,n1); fill(2,n2)]
        X, dX, label
    end
    centers = zeros(2,2); centers[1,2] = 10
    n1 = 100
    for n2 in (100, 1000, 10000)
        X, dX, label = gen_ldadata_2(centers, n1, n2)
        M = fit(SubspaceLDA, X, label; normalize=true)
        proj = projection(M)
        Hb = centers .- mean(centers, dims=2)   # not weighted by number in each cluster
        Sb = Hb*Hb'
        Sw = dX*dX'/(n1+n2)
        @test Sb*proj ≈ Sw*proj*Diagonal(M.λ)
    end

    # Test various input data types
    for (T, nrm) in Iterators.product((Float64, Float32), (false, true))
        n2 = 100
        X, dX, label = gen_ldadata_2(centers, n1, n2)
        M = fit(SubspaceLDA, convert(Matrix{T}, X), label; normalize=nrm)
        proj = projection(M)
        @test eltype(proj) === T
    end

end


export eikonal_const_vel

"""
`eikonal_const_vel(src, z, x, [y, ] velocity)`

Compute traveltime tables between `src` locations and subsurface model locations given by `z`, `x`, and optionally `y`. It assumes `velocity` is constant.

Parameters
----------
* `src` : `(ns, n)`, `AbstractMatrix{<:Real}`

The `(z_s, x_s [, x_y])` coordinates for every source. In 2D, `n = 2` and in 3D `n = 3`.

* `z` : `(nz,)`, `AbstractVector{<:Real}`

Model depth `z` coordinates.

* `x` : `(nx,)`, `AbstractVector{<:Real}`

Model horizontal `x` coordinates.

* `y` : `(ny,)`, `AbstractVector{<:Real}`, optional

Model horizontal `y` coordinates.

* `velocity` : `Real`

Wave speed

Returns
-------

* `trav` : `(nz, nx, [ny, ] ns)`, `AbstractArray{<:Real, M}`

Traveltime between each src and locations given by `z`, `x` and optionally `y`:

```math
t(z, x, y, s) = \\frac{\\sqrt{(z-s_z)^2 + (x-s_x)^2 + (y-s_y)^2}}{v}
```
"""
function eikonal_const_vel(src::AbstractMatrix{<:Real},
                           z::AbstractVector{<:Real},
                           x::AbstractVector{<:Real},
                           velocity::Real)
    if velocity < 0
        @warn("Velocity cannot not be negative")
        throw(DomainError())
    end
    size(src, 2) == 2 || throw(DimensionMismatch("Second dimension of `src` must match number of model arguments"))

    ns = size(src, 1)
    trav = zeros(length(z), length(x), ns)
    Z = [i for i in z, j in x]
    X = [j for i in z, j in x]
    for is in 1:ns
        src_z, src_x = src[is, :]
        trav[:,:,is] = @. sqrt.((Z - src_z)^2 + (X - src_x)^2)/velocity
    end
    return trav
end
function eikonal_const_vel(src::AbstractMatrix{<:Real},
                           z::AbstractVector{<:Real},
                           x::AbstractVector{<:Real},
                           y::AbstractVector{<:Real},
                           velocity::Real)
    if velocity < 0
        @warn("Velocity cannot not be negative")
        throw(DomainError())
    end
    size(src, 2) == 3 || throw(DimensionMismatch("Second dimension of `src` must match number of model arguments"))

    ns = size(src, 1)
    trav = zeros(length(z), length(x), length(y), ns)
    Z = [i for i in z, j in x, k in y]
    X = [j for i in z, j in x, k in y]
    Y = [k for i in z, j in x, k in y]
    for is in 1:ns
        src_z, src_x, src_y = src[is, :]
        trav[:,:,:,is] = @. sqrt((Z - src_z)^2 + (X - src_x)^2 + (Y - src_y)^2)/velocity
    end
    return trav
end


# Scikit-Learn:
#  - Python: https://scikit-learn.org/stable/
#  - Julia port/wrapper: https://github.com/cstjean/ScikitLearn.jl

# Import libraries
using Statistics, StatsPlots, ScikitLearn
using ScikitLearn.CrossValidation: cross_val_score
@sk_import datasets: load_breast_cancer
@sk_import preprocessing : scale
@sk_import linear_model: SGDClassifier

# Load and prepare data
cancer_data = load_breast_cancer()
y = cancer_data["target"] # Training labels ('malignant = 0', 'benign = 1')
X = cancer_data["data"] # 30 attributes; https://scikit-learn.org/stable/datasets/index.html#breast-cancer-dataset
X = scale(X) # scale each data attribute to zero-mean and unit variance

# Plot the first 2 attributes of training points
colors = [y == 1 ? "blue" : "red" for y in y]
labels = [y == 1 ? "benign" : "malign" for y in y]
scatter(X[:,1],X[:,2], colour=colors, title="Classified tumors",xlabel="Tumor Radius", ylabel="Tumor Texture", group=labels)

# Compute cross-validation scores
alpha = eps():0.005:1 # Range of hyperparameter values 1E-15 to 1 by 0.005
val_scores = zeros(length(alpha)) # Initialize validation score for each alpha value
for (i,a) in enumerate(alpha) # for each alpha value
    # Set up SVM with hinge loss and l2 norm regularization
    model = SGDClassifier(loss="hinge", penalty="l2", alpha=a)
    # Calculate cross validation scores for 5-fold cross-validation
    score = cross_val_score(model, X, y; cv=5)
    val_scores[i] = mean(score) # Calculate mean of the 5 scores
end

# Plot how cross-validation score changes with alpha
plot(alpha,val_scores,xlims=(0,1), xlabel="alpha",ylabel="Mean Cross-Validation Accuracy",label="")

# Determine the alpha that maximizes the cross-validation score
ind = argmax(val_scores)
alpha_star = alpha[ind]
println("alpha_star = $alpha_star")
vline!([alpha_star], label="alpha*")

# Train model with alpha_star
model_star = SGDClassifier(loss="hinge", penalty="l2", alpha=alpha_star)
model_trained = model_star.fit(X,y)
print("Training Accuracy = $(model_trained.score(X,y))")

# Plot decision boundary of trained model
slope = model_trained.coef_[1,2]/-model_trained.coef_[1,1]
x1 = -10:0.5:10
y1 = slope*x1
scatter(X[:,1],X[:,2], colour=colors, title="Classified tumors",xlims=(-4,4),ylims=(-6,6), xlabel="Tumor Radius", ylabel="Tumor Texture", group=labels)
plot!(x1,y1, linestyle = :dot, label="classifier")

Unit 01 - Linear Classifiers and Generalizations/recitation01_code_tumorDiagnosisDemo.jl


# This file is part of IntegerSequences.
# Copyright Peter Luschny. License is MIT.

(@__DIR__) ∉ LOAD_PATH && push!(LOAD_PATH, (@__DIR__))

module GeneralizedBinomial
using Nemo, PrimesIterator, NumberTheory, Products, Triangles

export ModuleGeneralizedBinomial
export Binomial, Pascal, T007318, Multinomial

"""

 P. Luschny, [Generalized Binomial](http://oeis.org/wiki/User:Peter_Luschny/ExtensionsOfTheBinomial), OEIS Wiki.

* Binomial, Pascal, T007318
"""
const ModuleGeneralizedBinomial = ""

"""

The classical binomial coefficients defined for ``n≥0`` and ``0≤k≤n`` (a.k.a. Pascal's triangle).
"""
function Pascal(n::Int, k::Int)

    (k == 0 || k == n) && return 1
    if k > div(n, 2)
        k = n - k
    end

    nk = n - k
    factors = fmpz[]
    rootN = isqrt(n)

    # Make use of Kummer's theorem.
    for prime ∈ Primes(2, n)

        if prime > nk
            push!(factors, prime)
            continue
        end

        prime > div(n, 2) && continue

        if prime > rootN
            (n % prime < k % prime) && push!(factors, prime)
            continue
        end

        r, N, K, p = 0, n, k, 1

        while N > 0
            r = N % prime < (K % prime + r) ? 1 : 0
            if r == 1
                p *= prime
            end
            N = div(N, prime)
            K = div(K, prime)
        end

        p > 1 && push!(factors, p)
    end

    ∏(factors)
end

"""

Pascal's triangle.
"""
function T007318(n::Int)
    T = zeros(QQ, div(n * (n + 1), 2))
    j = 1
    for m ∈ 0:n-1, k ∈ 0:m
        T[j] = binomial(ZZ(m), ZZ(k))
        j += 1
    end
    T
end

# See the discussion on
# [Extensions of the Binomial](http://oeis.org/wiki/User:Peter_Luschny/ExtensionsOfTheBinomial).
"""

Return the extended binomial coefficients defined for all ``n ∈ Z`` and ``k ∈ Z``. Behaves like the binomial function in Maple and Mathematica.

``\\binom{n}{k} = \\lim \\limits_{x \\rightarrow 1}(Γ(n + x) / (Γ(k + x) Γ(n - k + x))).``

"""
function Binomial(n::Int, k::Int)
    0 ≤ k ≤ n && return binomial(ZZ(n), ZZ(k))
    k ≤ n < 0 && return binomial(ZZ(-k - 1), ZZ(n - k)) * (-1)^(n - k)
    n < 0 ≤ k && return binomial(ZZ(-n + k - 1), ZZ(k)) * (-1)^k
    ZZ(0)
end

"""

Return the multinomial coefficient of a list.
"""
function Multinomial(lst::Array{Int})
    s = 0
    result = fmpz(1)
    for c ∈ lst
        s += c
        result *= binomial(ZZ(s), ZZ(c))
    end
    result
end

#START-TEST-########################################################

using Test
function test()

    @testset "Binomial" begin
        for n ∈ 0:10, k ∈ 0:n
            @test Binomial(n, k) == div(factorial(n), (factorial(n - k) * factorial(k)))
            @test Binomial(n, k) == Pascal(n, k)
        end
    end
end

function demo()
    ShowAsΔ(T007318(8))
    println()

    for n ∈ -10:10
        println([Binomial(n, k) for k ∈ -10:10])
    end
end

"""

for n ∈ 0:10000 Binomial(2*n,n) end
    0.504729 seconds (10.00 k allocations: 156.266 KiB)
for n ∈ -100:100, k ∈ -100:100 Binomial(n,k) end
    0.008669 seconds (55.55 k allocations: 867.984 KiB)
for k ∈ -10000:10000 Binomial(-5,k) end
    0.005378 seconds (40.00 k allocations: 624.969 KiB)
"""
function perf()
    @time (for n ∈ 0:10000
        Binomial(2n, n)
    end)
    @time (for n ∈ -100:100, k ∈ -100:100
        Binomial(n, k)
    end)
    @time (for k ∈ -10000:10000
        Binomial(-5, k)
    end)
end

function main()
    test()
    demo()
    perf()
end

main()

end # module


#!/usr/bin/julia

# Trizen
# 17 January 2017
# https://github.com/trizen

# Generates a Mandelbrot-like set.

# See also:
#   https://en.wikipedia.org/wiki/Mandelbrot_set
#   https://trizenx.blogspot.ro/2017/01/mandelbrot-set.html

using Images

@inline function hsv2rgb(h, s, v)
    c = v * s
    x = c * (1 - abs(((h/60) % 2) - 1))
    m = v - c

    if h < 60
        r,g,b = (c, x, 0)
    elseif h < 120
        r,g,b = (x, c, 0)
    elseif h < 180
        r,g,b = (0, c, x)
    elseif h < 240
        r,g,b = (0, x, c)
    elseif h < 300
        r,g,b = (x, 0, c)
    else
        r,g,b = (c, 0, x)
    end

    (r + m), (b + m), (g + m)
end

#
## A specific function which maps the value of `c` to some other value.
#
@inline function f(c)
    1/c
end

function mandelbrot()

    w, h = 1000, 1000

    zoom  = 1         # the zoom factor
    moveX = 0         # the amount of shift on the x axis
    moveY = 0         # the amount of shift on the y axis

    L = 100           # the maximum value of |z|
    I = 30            # the maximum number of iterations

    img = zeros(RGB{Float64}, h, w)

    for x in 1:w, y in 1:h
        c = Complex(
            (2*x - w) / (w * zoom) + moveX,
            (2*y - h) / (h * zoom) + moveY
        )
        z = c
        n = 0
        q = f(c)
        while (abs(z) < L && (n += 1) < I)
            z = z^q
        end
        v = (I - n) / I
        r,g,b = hsv2rgb(v*360, 1, v)
        img[y,x] = RGB{Float64}(r, g, b)
    end

    println("Generating image...")
    save("mandelbrot_like_set.png", img)
end

mandelbrot()


"""
    vandermonde(elem::AbstractElemShape, N, rst...)

Computes the generalized Vandermonde matrix V of degree N at points (r,s,t).
"""
vandermonde(elem::AbstractElemShape, N, rst...) = first(basis(elem,N,rst...))
vandermonde(elem::Line, N, r) = first(basis(elem,N,r)) # specialize for 1D

"""
    grad_vandermonde(elem::AbstractElemShape, N, rst...)

Computes the generalized Vandermonde derivative matrix V of degree N at points (r,s,t).
"""
grad_vandermonde(elem::AbstractElemShape, N, rst...) = basis(elem, N, rst...)[2:end]
grad_vandermonde(elem::Line, N, r) = last(basis(elem,N,r)) # specialize for 1D

"""
    nodes(elem::AbstractElemShape,N)

Computes interpolation nodes of degree N. Edge nodes coincide with (N+1)-point Lobatto points.
Default routine for elem = Tet(), Pyr(), Tri().

For Quad(), Hex(), Wedge() elements, nodes(...) returns interpolation points constructed
using a tensor product of lower-dimensional nodes.
"""
nodes(elem::AbstractElemShape,N) = build_warped_nodes(elem,N,nodes(Line(),N))

"""
    basis(elem::AbstractElemShape, N, rst...)

Computes orthonormal basis of degree N at tuple of coordinate arrays (r,s,t).
"""
basis(elem::AbstractElemShape,N,rst...)

"""
    equi_nodes(elem::AbstractElemShape, N)

Compute equispaced nodes of degree N.
"""
equi_nodes(elem::AbstractElemShape, N)

"""
    quad_nodes(elem::AbstractElemShape, N)

Compute quadrature nodes and weights exact for (at least) degree 2N polynomials.
"""
quad_nodes(elem::AbstractElemShape, N)

"""
    stroud_quad_nodes(elem::AbstractElemShape,N)

Returns Stroud-type quadrature nodes and weights constructed from the tensor product
of (N+1)-point Gauss-Jacobi rules. Exact for degree 2N polynomials
"""
stroud_quad_nodes(elem::AbstractElemShape,N)


# # dempe.mod	QQR2-MN-4-3
# # Original AMPL coding by Sven Leyffer
#
# # An MPEC from S. Dempe, "A necessary and sufficient optimality
# # condition for bilevel programming problems", Optimization 25,
# # pp. 341-354, 1992. From book Math. Progr. with Equil. Constr,
# # by Luo, Pang & Ralph, CUP, 1997, p. 354.
#
# # Number of variables:   2 + 1 multipliers
# # Number of constraints: 2
# # Nonlinear complementarity constraints
#
# var x;
# var z;
# var w >= 0;
#
# minimize f: (x - 3.5)^2 + (z + 4)^2;
#
# subject to
#     con1:  z - 3 + 2*z*w = 0;
#     con2:  0 >= z^2 - x     complements  w >= 0;
#
# data;
#
# # starting point
# let x := 1;
# let z := 1;
# let w := 1;
#
#
# let x :=   0.183193 ;
# let z := 0.428106;
# let w := 3.00379;

using JuMP, Complementarity
using Ipopt
using Test

@testset "dempe.jl" begin

    dempe = Model(with_optimizer(Ipopt.Optimizer))

    @variable(dempe, x)
    @variable(dempe, z)
    @variable(dempe, w>=0)

    @NLobjective(dempe, Min, (x - 3.5)^2 + (z + 4)^2)

    @NLconstraint(dempe, z - 3 + 2*z*w == 0)

    @complements(dempe, 0 >= z^2 - x,  w >= 0, simple)

    # Initial solutions to help reaching the optimality
    set_start_value(x, 0)
    set_start_value(z, 0)
    set_start_value(w, 1e7)

    # (xx, zz, ww) = (-3.999779029254958e-10, 9.852984581857451e-8, 1.5223813031201378e7)

    JuMP.optimize!(dempe)

    @show JuMP.objective_value(dempe)
    @test isapprox(JuMP.objective_value(dempe), 28.25, atol=1e-4)

    xx = JuMP.value(x)
    zz = JuMP.value(z)
    ww = JuMP.value(w)
    @test isapprox(zz - 3 + 2*zz*ww, 0, atol=1e-4)
end


#=
This file contains the definition of the generalized dimension and related concepts
and also uses linear_regions.jl
=#
export generalized_dim

"""
    generalized_dim(dataset [, sizes]; q = 1, base = MathConstants.e) -> Δ_q
Return the `q` order generalized dimension of the `dataset`, by calculating
the [`genentropy`](@ref) for each `ε ∈ sizes`.

The case of `q = 0` is often called "capacity" or "box-counting" dimension, while
`q = 1` is the "information" dimension.

## Description
The returned dimension is approximated by the
(inverse) power law exponent of the scaling of the [`genentropy`](@ref) ``H_q``
versus the box size `ε`, where `ε ∈ sizes`:

```math
H_q \\sim -\\Delta_q\\log(\\varepsilon)
```

Calling this function performs a lot of automated steps:

  1. A vector of box sizes is decided by calling `sizes = estimate_boxsizes(dataset)`,
     if `sizes` is not given.
  2. For each element of `sizes` the appropriate entropy is
     calculated, through `H = genentropy.(Ref(dataset), sizes; q, base)`.
     Let `x = -log.(sizes)`.
  3. The curve `H(x)` is decomposed into linear regions,
     using [`linear_regions`](@ref)`(x, h)`.
  4. The biggest linear region is chosen, and a fit for the slope of that
     region is performed using the function [`linear_region`](@ref),
     which does a simple linear regression fit using [`linreg`](@ref).
     This slope is the return value of `generalized_dim`.

By doing these steps one by one yourself, you can adjust the keyword arguments
given to each of these function calls, refining the accuracy of the result.
"""
function generalized_dim(data::AbstractDataset, sizes = estimate_boxsizes(data);
        base = Base.MathConstants.e, q = 1.0
    )
    dd = [genentropy(data, ε; q, base) for ε ∈ sizes]
    return linear_region(-log.(base, sizes), dd)[2]
end


@doc raw"""
    LorentzMetric <: Metric

Abstract type for Lorentz metrics, which have a single time dimension. These
metrics assume the spacelike convention with the time dimension being last,
giving the signature $(++...+-)$.
"""
abstract type LorentzMetric <: Metric end

@doc raw"""
    MinkowskiMetric <: LorentzMetric

As a special metric of signature  $(++...+-)$, i.e. a [`LorentzMetric`](@ref),
see [`minkowski_metric`](@ref) for the formula.
"""
struct MinkowskiMetric <: LorentzMetric end

@doc raw"""
    Lorentz{N} = MetricManifold{Euclidean{N,ℝ},LorentzMetric}

The Lorentz manifold (or Lorentzian) is a pseudo-Riemannian manifold.

# Constructor

    Lorentz(n[, metric=MinkowskiMetric()])

Generate the Lorentz manifold of dimension `n` with the [`LorentzMetric`](@ref) `m`,
which is by default set to the [`MinkowskiMetric`](@ref).
"""
const Lorentz = MetricManifold{ℝ,Euclidean{Tuple{N},ℝ},<:LorentzMetric} where {N}

function Lorentz(n, m::MT = MinkowskiMetric()) where {MT<:LorentzMetric}
    return Lorentz{n,typeof(m)}(Euclidean(n), m)
end

function local_metric(
    ::MetricManifold{ℝ,Euclidean{Tuple{N},ℝ},MinkowskiMetric},
    p,
) where {N}
    return Diagonal([ones(N - 1)..., -1])
end

function inner(::MetricManifold{ℝ,Euclidean{Tuple{N},ℝ},MinkowskiMetric}, p, X, Y) where {N}
    return minkowski_metric(X, Y)
end
@doc raw"""
    minkowski_metric(a,b)

Compute the minkowski metric on $\mathbb R^n$ is given by
````math
⟨a,b⟩_{\mathrm{M}} = -a_{n}b_{n} +
\displaystyle\sum_{k=1}^{n-1} a_kb_k.
````
"""
minkowski_metric(a, b) = -a[end] * b[end] + sum(a[1:(end - 1)] .* b[1:(end - 1)])


#Função de substituição reversa, onde retorna um x que é o o que queremos encontrar de Ax=b

function sr00(U,b);

	(m,n) = size(U);
	x = zeros(m);

	x[m] = b[m]/U[m,m];

	for i = m-1:-1:1
		soma = 0;

		for j = i+1:n

			soma = soma + U[i,j]*x[j];
		end
		x[i] = (b[i] - soma)/U[i,i];
	end

	return x;

end


export FFTOp, \

mutable struct FFTOp{T,F1<:FuncOrNothing,F2<:FuncOrNothing,F3<:FuncOrNothing} <: AbstractLinearOperator{T,F1,F2,F3}
  nrow   :: Int
  ncol   :: Int
  symmetric :: Bool
  hermitian :: Bool
  prod   :: F1 # apply the operator to a vector
  tprod  :: F2 # apply the transpose operator to a vector
  ctprod :: F3 # apply the transpose conjugate operator to a vector
end

"""
  FFTOp(T::Type, shape::Tuple, shift=true)

returns an operator which performs an FFT on Arrays of type T

# Arguments:
* `T::Type`       - type of the array to transform
* `shape::Tuple`  - size of the array to transform
* (`shift=true`)  - if true, fftshifts are performed
"""
function FFTOp(T::Type, shape::Tuple, shift=true)
  plan = plan_fft(zeros(T, shape);flags=FFTW.MEASURE)
  iplan = plan_ifft(zeros(T, shape);flags=FFTW.MEASURE)

  if shift
    return FFTOp{T,Function, Nothing,Function}(prod(shape), prod(shape), false, false
              , x->vec(fftshift(plan*fftshift(reshape(x,shape))))/sqrt(prod(shape))
              , nothing
              , y->vec(ifftshift(iplan*ifftshift(reshape(y,shape)))) * sqrt(prod(shape)) )
  else
    return FFTOp{T,Function, Nothing,Function}(prod(shape), prod(shape), false, false
            , x->vec(plan*(reshape(x,shape)))/sqrt(prod(shape))
            , nothing
            , y->vec(iplan*(reshape(y,shape))) * sqrt(prod(shape)) )
  end
end

\(A::FFTOp, x::Vector) = adjoint(A) * x


"""
Based on:
https://medium.com/vooban-ai/hyperopt-tutorial-for-optimizing-neural-networks-hyperparameters-e3102814b919
"""
module TestVoobanBasic

using Test
using TreeParzen

objective(params) = params[:x]^2 - params[:x] + 1
space = Dict(:x => HP.Uniform(:x, -5.0, 5.0))
best = fmin(objective, space, 100)
@test isapprox(best[:x], 0.5366692707330428, rtol=1e1)

end
true


@testset "Computation of c constants" begin
    # Testing right boundary
    # zero order right boundary
    @test Network_ADER.compute_c(1,0,0,0) == 1
    # 3 rd order right boundary
    @test Network_ADER.compute_c(3,1,1,2) == 1
    @test Network_ADER.compute_c(3,2,0,2) == 11/6

    # Testing left boundary
    # left boundary seond order
    @test Network_ADER.compute_c(2,0,0,1,true) == -1/2
    # left boundary 4th order
    @test Network_ADER.compute_c(4,1,1,3,true) == -1/12
    @test Network_ADER.compute_c(4,2,2,1,true) == -1/2
end

@testset "Computation of C matrices" begin
    # Testing right boundary
    # Order 3, r=0 test
    @test isapprox(Network_ADER.compute_C(3,0) , [1/3 5/6 -1/6;-1 1 0;1 -2 1])

    # Testing left boundary
    # Order 3, r=2 test
    @test isapprox(Network_ADER.compute_C(3,2,true) , [-1/6 5/6 1/3;0 -1 1;1 -2 1])
end

@testset "Computation of B matrices" begin
    # Testing right boundary
    # Order 2, r=0 test
    @test isapprox(Network_ADER.compute_B([3/2 -1/2;-1 1]) , [1 -1; -1 1])
    # Order 3, r=2 test
    @test isapprox(Network_ADER.compute_B([1/3 -7/6 11/6;1 -3 2;1 -2 1]) , [4/3 -19/6 11/6; -19/6 25/3 -31/6;11/6 -31/6 10/3])
end

@testset "Computation of protoype D coeffiencents" begin
    C= Network_ADER.compute_C_constants(5)
    @test isapprox(Network_ADER.compute_D(2,C) , [2/3; 1/3])
    @test isapprox(Network_ADER.compute_D(3,C) , [3/10; 3/5 ; 1/10])
end

@testset "Computation of a complete protoype D coeffiencent set" begin
    C= Network_ADER.compute_C_constants(5)
    D_expected = Array{Array{Float64,2},1}(undef,2);
    D_expected[1]=zeros(1,1)
    D_expected[1][1]=1
    D_expected[2]=zeros(2,1)
    D_expected[2][:]=[2/3;1/3]
    @test isapprox(Network_ADER.compute_D_constants(2,C) , D_expected)
end


module Issue743

using Gridap
using LinearAlgebra
using Gridap.FESpaces
using Test

model = CartesianDiscreteModel((0,2π,0,2π),(10,10);isperiodic=(true,true))
order = 2
reffe = ReferenceFE(lagrangian,Float64,order)
V = TestFESpace(model,reffe;constraint=:zeromean)
U = TrialFESpace(V)
degree = 2*order
Ω = Triangulation(model)
dΩ = Measure(Ω,degree)
f(x) = cos(x[1])+sin(x[2])
a(u,v) = ∫( ∇(v)⋅∇(u) )*dΩ
b(v) = ∫( v*f )*dΩ
op = AffineFEOperator(a,b,U,V)
uh = solve(op)
A = get_matrix(op)
@test cond(Matrix(A)) < 1.e5
@test abs(sum(∫(uh)dΩ)) < 1.e-10

#writevtk(Ω,"results",nsubcells=10,cellfields=["uh"=>uh])

end # module


@info "Análise Estrutural"
# Matriz de rigidez dos elementos
k1 = [300  -300
    -300   300]

k2 = [
    600  -600
    -600   600
]

k3 = [
    800  -800
    -800   800
]

k4 = [
    500  -500
    -500   500
]

k5 = [
    600  -600
    -600   600
]

k6 = [
    800  -800
    -800   800
]

# Vetor de forças aplicadas nos Nós
f =[0, 500, 500, 600, 1000, 0]
# Vetor de deslocamentos nodais
u = [0, 0, 0, 0, 0, 2]

# Nós
A = 1
B = 2
C = 3
D = 4
E = 5
F = 6

# Relacionamento entre elementos e nós
ak1 = [[A,A], [A,B], [B,A], [B,B]]
ak2 = [[B,B], [B,C], [C,B], [C,C]]
ak3 = [[B,B], [B,D], [D,B], [D,D]]
ak4 = [[B,B], [B,E], [E,B], [E,E]]
ak5 = [[D,D], [F,D], [D,F], [F,F]]
ak6 = [[E,E], [E,F], [F,E], [F,F]]

# Matriz de rigidez global
k = zeros(6, 6)

# Mapeia a matriz de rigidez local com a matriz de rigidez global
function mapear(a, b)
    l = 1
    for i in 1:2
        for j in 1:2
            k[a[l][1,1],a[l][2,1]] += b[i, j]
            l+=1
        end
    end
end

# Monta matriz de rigidez global
mapear(ak1, k1)
mapear(ak2, k2)
mapear(ak3, k3)
mapear(ak4, k4)
mapear(ak5, k5)
mapear(ak6, k6)

# Resolve sistema de equações para achar os deslocamentos nodais
# é retirada da matriz a linha e coluna correspondente ao nó que não tem deslocamento
@show _k = (k[1:end .!=A, 1:end .!=A])[1:end .!=E, 1:end .!=E] \ (f[1:end .!=A])[1:end .!=E]


makepositive(x) = exp(x)

invmakepositive(y) = log(y)

sigmoid(x) = 1.0 / (exp(-x) + 1.0)

invsigmoid(y) = - log(1/y -1.0)

transformbetween(x, lower, upper) = sigmoid(x) * (upper - lower) + lower

invtransformbetween(y, lower, upper) = invsigmoid((y - lower)/(upper-lower))


# # Section Allocation
# Suppose you have $n$ students in a class who need to be assigned to $m$
# discussion sections. Each student needs to be assigned to exactly one section.
# Each discussion section should have between 6 and 10 students. Suppose an
# $n \times m$ preference matrix $P$ is given, where $P_{ij}$ gives student
# $i$'s ranking for section $j$ (1 would mean it is the student's top choice,
# 10,000 or a large number would mean the student can not attend that section).
#
# The goal will be to get an allocation matrix $X$, where $X_{ij} = 1$ if
# student $i$ is assigned to section $j$ and $0$ otherwise.

using Convex, GLPK
aux(str) = joinpath(@__DIR__, "aux_files", str) # path to auxiliary files

# Load our preference matrix, `P`
include(aux("data.jl"))

X = Variable(size(P), :Bin)

# We want every student to be assigned to exactly one section. So, every row
# must have exactly one non-zero entry. In other words, the sum of all the
# columns for every row is 1. We also want each section to have between 6 and 10
# students, so the sum of all the rows for every column should be between these.
constraints = [sum(X, dims=2) == 1, sum(X, dims=1) <= 10, sum(X, dims=1) >= 6]

# Our objective is simple `sum(X .* P)`, which can be more efficiently
# represented as `vec(X)' * vec(P)`. Since each entry of `X` is either 0 or 1,
# this is basically summing up the rankings of students that were assigned to them.
# If all students got their first choice, this value will be the number of
# students since the ranking of the first choice is 1.
p = minimize(vec(X)' * vec(P), constraints)

solve!(p, GLPK.Optimizer)
p.optval


# Learn Polynomials

using Logging
logger = SimpleLogger(stdout, Logging.Info)
global_logger(logger)

import Polynomials
Poly = Polynomials

poly = Poly.Polynomial([1, 0, 3, 4])
using Plots
plotly()
xs = range(0, 10, length=10)
ys = @. exp(-xs)
f = Poly.fit(xs,ys) # degree = length(xs) - 1
f2 = Poly.fit(xs,ys, 2) # 2
f3 = Poly.fit(xs,ys, 1)

#pt = scatter(xs, ys, markerstrokewidth=0, label="Data")
pt = plot(xs, ys, seriestype=:scatter, markerstrokewidth=0)

plot!(pt, f, extrema(xs)..., label="Fit")
plot!(pt, f2, extrema(xs)..., label="Quadratic Fit")
plot!(pt, f3, extrema(xs)..., label="Linear Fit")

gui(pt)
nothing


# Author: Gabrio Rizzuti, rizzuti.gabrio@gatech.edu
# Date: August, 2020
# Copyright: Georgia Institute of Technology, 2020
#
# Run MALA to sample from Gaussian mixture in 2D

## Module loading

using DrWatson
@quickactivate "UncertaintyQuantificationAVP"
using UncertaintyQuantificationAVP
using LinearAlgebra, PyPlot, Statistics
using Random; Random.seed!(1)

## Setting negative log density for Gaussian mixture

# Gaussian modes parameters
nmodes = 7
muComplex = Complex{Float32}.(exp.(1im*2f0*pi*collect(range(0f0, 1f0; length=nmodes+1))))
mu = [real.(muComplex[1:end-1]) imag.(muComplex[1:end-1])]
sigma2 = zeros(Float32, nmodes); sigma2 .= Float32(0.025)

# Sampling function
sample = function sample_gaussmixture(nsamples::Int64) # Direct sampling
    x = Array{Float32, 2}(undef, (nsamples, 2))
    for i = 1:nsamples
        imode = rand(1:nmodes)
        x[i, :] = mu[imode, :]+sqrt(sigma2[imode])*randn(Float32, 2)
    end
    return x
end
x = sample(2^10)
figure()
plot(x[:, 1], x[:, 2], "*")
title("Sample from Gaussian mixture distribution")

# Negative log density
function negLogDensity_gaussmixture(x::Array{Float32, 2})
    p = zeros(Float32, size(x, 1))
    g = zeros(Float32, size(x))
    for i = 1:nmodes
        p_i = exp.(-((x[:, 1].-mu[i, 1]).^2+(x[:, 2].-mu[i, 2]).^2)/(2*sigma2[i]))/(nmodes*2f0*pi*sigma2[i])
        p += p_i
        g += -p_i.*(x.-mu[i:i, :])/sigma2[i]
    end
    return -log.(p), -g./p
end
function neglogp(x::Array{Float32, 1})
    f, g = negLogDensity_gaussmixture(reshape(x, 1, 2))
    return f[1], reshape(g, size(x))
end
function meshgrid(x, y)
    X = [i for i in x, j in 1:length(y)]
    Y = [j for i in 1:length(x), j in y]
    return X, Y
end

## MALA sampling

# Starting input
x0 = randn(Float32, 2)

# Parameters
nsamples = 2^10
ϵ = 0.02f0
flag_MALA = true

# Run MALA
time = @elapsed fval, xLD = langevinSampler(neglogp, nsamples, x0, ϵ; flag_MALA = flag_MALA)
figure()
plot(x[:, 1], x[:, 2], "r*")
plot(xLD[1,:], xLD[2,:], ".")
title("MALA samples")


import Base: rand, randn
function rand(::Type{SimpleQuaternion{T}}) where T <: Real
    args = (rand(T) for j=1:4)
    return SimpleQuaternion(args...)
end

"""
`rand(SimpleQuaternion)` returns a quaternion whose four entries
are uniform [0,1] random values.
"""
rand(::Type{SimpleQuaternion}) = rand(SimpleQuaternion{Float64})

"""
`randn(SimpleQuaternion)` returns a quaternion whose four entries are
standard normal random values.
"""
function randn(::Type{SimpleQuaternion{T}}) where T
    args = (randn(T) for j=1:4)
    return SimpleQuaternion(args...)
end

randn(::Type{SimpleQuaternion}) = randn(SimpleQuaternion{Float64})


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ 1bcf25d8-71c2-11eb-05ca-6fe7c4e1b24f
begin
	using Pkg
	Pkg.activate("MLJ_env", shared=true)
end

# ╔═╡ 1f7ea604-71c2-11eb-028a-a166f6f4dc5d
begin
	using MLJ
	using CSV
	using DataFrames
	using PlutoUI
	using Random
	using Test
	using Printf
end

# ╔═╡ f377f6d4-71c9-11eb-0249-f3d63e87062b
begin
	using LinearAlgebra

	function feature_derivative(errors::Vector{T},
			feature::Vector{T}) where {T <: Real}
    	return 2 * dot(feature, errors) # ≡ 2 * feature' * errors
	end
end

# ╔═╡ 9dcffb1c-71c1-11eb-2402-39eaa1f1b5e1
md"""
## Week 2: Multiple Regression (Gradient Descent)
"""

# ╔═╡ c39f9348-71c1-11eb-1691-1d28c6b1d916
md"""

In this notebook we will cover estimating multiple regression weights via gradient descent, performing the following:

 - Add a constant column of 1's to a DataFrame to account for the intercept
 - Write a `predict_output()` function
 - Write a function to compute the derivative of the regression weights with respect to a single feature
 - Write gradient descent function to compute the regression weights given an initial weight vector, step size and tolerance.
 - Use the gradient descent function to estimate regression weights for multiple features
"""

# ╔═╡ 3e5d907e-71c2-11eb-2eb3-4375fb21b884
md"""
### Load in house sales data
"""

# ╔═╡ 4c736078-71c2-11eb-225b-816745199225
sales = CSV.File("../../ML_UW_Spec/C02/data/kc_house_test_data.csv";
	header=true) |> DataFrame;

# ╔═╡ 58e6255a-71c2-11eb-2094-9dbe7fb64151
first(sales, 3)

# ╔═╡ 64c97662-71c2-11eb-2894-2d544642b136
typeof(sales.bedrooms)

# ╔═╡ 80fbf0f8-71c2-11eb-265a-fb539d764180
typeof(sales.bathrooms)

# ╔═╡ 35b5d05a-71cc-11eb-15a9-852c5a534fab
md"""
### Convert to Julia Matrix/Vector
"""

# ╔═╡ a2928dc6-71c2-11eb-3724-29dc1448a0f3
md"""
Now we will write a function that will accept a DataFrame, a list of feature names (e.g. [:sqft_living, :bedrooms') and a target feature e.g. (:price) and will return two things:

 - A matrix whose columns are the desired features plus a constant column (this is how we create an 'intercept')
 - An array containing the values of the output

With this in mind, let's write the `get_data` function:

"""

# ╔═╡ e4da992e-71c2-11eb-36b8-93dd614051c1
function get_data(df, features, output)
	df[:, :constant] .= 1.0 # df.constant = fill(1.0, size(df, 1))
	features = [:constant, features...]
	X_matrix = convert(Matrix, select(df, features)) # to get a matrix
	y = df[!, output]                                # => to get a vector
	return (X_matrix, y)
end

# ╔═╡ d62e0368-71c2-11eb-16f7-5fda56951d21
begin
	(ex_features, ex_output) = get_data(sales, [:sqft_living], :price)
	with_terminal() do
		println(ex_features[1:5, :], " / ", typeof(ex_features))
		println(ex_output[1], " / ", typeof(ex_output))
	end
end

# ╔═╡ 498b3458-71cc-11eb-3b55-8381d58d04bb
md"""
### Predicting output given regression weights
"""

# ╔═╡ daf8aefc-71c7-11eb-04fa-e71d7b2cde21
md"""
The predictions from all the observations are just the dot product between the features matrix (on the left) and the weights vector (on the right).

With this in mind write the following `predict_output` function to compute the predictions given the feature matrix (X) and the weights:
"""

# ╔═╡ 02f0c99e-71c5-11eb-1fe1-b3f820998d54
function predict_output(X::Matrix{T}, weights::Vector{T}) where {T <: Real}
    # assume feature_matrix is a matrix containing the features as columns
	# and weights is a corresponding array
    X * weights
end

# ╔═╡ 9cd636f2-71c5-11eb-2957-afbaf437aaa5
begin
	my_weights = Float64[1., 1.]
	test_predictions = predict_output(ex_features, my_weights)

	@test test_predictions[1] == 1431.0  # should be 1431.0
	@test test_predictions[2] == 2951.0  # should be 2951.0
end

# ╔═╡ 21635b6e-71c8-11eb-2272-edec98c58339
md"""
### Computing the derivative

We are now going to compute the derivative of the regression cost function. Recall that the cost function is the sum over the data points of the squared difference between an observed output and a predicted output.

Since the derivative of a sum is the sum of the derivatives we can compute the derivative for a single data point and then sum over data points. We can write the squared difference between the observed output and predicted output for a single point as follows:

$$(w[0]\times[CONST] + w[1]\times[feature_1] + ... + w[i]\times[feature_i] + ... +  w[k]\times[feature_k] - output)^2$$

Where we have $k$ features and a constant. So the derivative with respect to weight $w[i]$ by the chain rule is:

$$2\times(w[0]\times[CONST] + w[1]\times[feature_1] + ... + w[i]\times[feature_i] + ... +  w[k]\times[feature_k] - output)\times[feature_i]$$

The term inside the parenthesis is just the error (difference between prediction and output). So we can re-write this as:

$$2\times error\times[feature_i]$$

That is, the derivative for the weight for feature $i$ is:
  - the sum (over data points) of 2 × the product of the error and the feature itself.
    In the case of the constant then this is just twice the sum of the errors!

Recall that twice the sum of the product of two vectors is just:
  - twice the dot product of the two vectors.
    Therefore the derivative for the weight for $feature_i$ is just two times the dot product between the values of $feature_i$ and the current errors.

With this in mind let's write the following derivative function which computes the derivative of the weight given the value of the feature (over all data points) and the errors (over all data points).

`doc ref. https://docs.julialang.org/en/v1/stdlib/LinearAlgebra/`
"""

# ╔═╡ 1301a4d2-71ca-11eb-339c-d1f747a3ffe7
begin
	(nex_features, nex_output) = get_data(sales, [:sqft_living], :price)
	ya_weights = Float64[0., 0.] # this makes all the predictions 0
	test_preds = predict_output(nex_features, ya_weights)

	# just like SFrames 2 numpy arrays can be elementwise subtracted with '-':
	errors = test_preds .- nex_output # prediction errors in this case is just the -example_output
	ya_feature = nex_features[:, 1]   # let's compute the derivative with respect to 'constant', the ":" indicates "all rows"
	der = feature_derivative(errors, ya_feature)
	alt_der = -sum(nex_output) * 2

	@test der == alt_der  # should be the same as derivative
end

# ╔═╡ 49ce0c84-71cb-11eb-3947-6141a9498ffb
with_terminal() do
	@printf("derivative: %2.5e / alt_derivative: %2.5e\n", der, alt_der)
end

# ╔═╡ 239ff6ca-71cc-11eb-341a-df1453d60a15
md"""
### Gradient Descent [GD]

Now we will write a function that performs a gradient descent. The basic premise is simple. Given a starting point we update the current weights by moving in the negative gradient direction. Recall that the gradient is the direction of increase and therefore the negative gradient is the direction of decrease and we're trying to minimize a cost function.

The amount by which we move in the negative gradient direction is called the *step size* denoted by η. We stop when we are 'sufficiently close' to the optimum. We define this by requiring that the magnitude (length) of the gradient vector to be smaller than a fixed *tolerance* denoted by ϵ.

With this in mind, wriet the following gradient descent function below using the derivative function above. For each step in the gradient descent we update the weight for each feature befofe computing our stopping criteria
"""

# ╔═╡ 958f81a6-71cc-11eb-13e8-61dddeb014a7
function regression_gradient_descent(f_matrix::Matrix{T},
		output::Vector{T}, init_weights::Vector{T}, η::T, ϵ::T) where {T <: Real}
    weights = init_weights # make sure it's a vector

    while true
		preds = predict_output(f_matrix, weights) # compute the predictions
        errors = preds .- output                  # compute the errors
        ∇_sum_squares = 0.0                       # init the gradient sum of squares

		## Update the weights
        for ix ∈ 1:length(weights) # loop over each weight
            ## Recall that feature_matrix[:, ix] is the feature column
		    ## associated with weights[i]
            deriv_ix = feature_derivative(errors, f_matrix[:, ix])
			## add squared value of derivative to ∇_sum_squares (convergence)
            ∇_sum_squares += deriv_ix * deriv_ix

			## subtract the step size times the derivative from the CURRENT weight
            weights[ix] -= η * deriv_ix
		end
        ##  compute the square-root of the ∇_sum_squares to get the ∇ magnitude:
        ∇_magnitude = √(∇_sum_squares)
        ∇_magnitude < ϵ && break
	end
    weights
end

# ╔═╡ 4454239e-71ce-11eb-295e-1755bc0549e6
md"""
### Running the Gradient Descent as Simple Regression
"""

# ╔═╡ 0623d920-71ce-11eb-0936-870685f2e788
begin
function train_test_split(df; split=0.8, seed=42, shuffled=true)
	Random.seed!(seed)
	(nr, nc) = size(df)
	nrp = round(Int, nr * split)
	row_ixes = shuffled ? shuffle(1:nr) : collect(1:nr)
	df_train = view(df[row_ixes, :], 1:nrp, 1:nc)
	df_test = view(df[row_ixes, :], nrp+1:nr, 1:nc)
	(df_train, df_test)
end

sales_train, sales_test = train_test_split(sales);
end

# ╔═╡ 6a1ee460-71ce-11eb-1b49-4de942aefe9d
begin
	# let's test out the gradient descent
	s_features = [:sqft_living]
	s_out = :price
	(simple_f_matrix, out) = get_data(sales_train, s_features, s_out)

	init_weights = [-47000., 1.]
	η = 7e-12
	ϵ = 2.5e7
end

# ╔═╡ 9883c1ee-71cf-11eb-3b48-a14df232588f
gd_weights = regression_gradient_descent(simple_f_matrix, Vector{Float64}(out), init_weights, η, ϵ)

# ╔═╡ 06cf79e0-71d0-11eb-3eab-ad040213a64b
md"""
Use your newly estimated weights and your `predict_output()` function to compute the predictions on all the test data.

you will need to create a julia array of the test feature_matrix and test output first, as follows:
"""

# ╔═╡ 26199ccc-71d0-11eb-0b4e-5f2f8ebc7b76
(test_s_feature_matrix, test_out) = get_data(sales_test, s_features, s_out);

# ╔═╡ 4be11b60-71d0-11eb-1550-6d2854691b05
md"""
Now compute your predictions using `test_s_feature_matrix` and the `gd_weights` from above.
"""

# ╔═╡ 65c26930-71d0-11eb-248a-3751d927b2f6
ya_preds = predict_output(test_s_feature_matrix, gd_weights)

# ╔═╡ 749979f6-71d0-11eb-0b3a-459257d95ca9
md"""
**Quiz Question: What is the predicted price for the 1st house in the TEST data set for model 1 (round to nearest dollar)?**

"""

# ╔═╡ 898fa5b2-71d0-11eb-18c3-ad6d0dbf9bf8
begin
	pred_price_first_house = ya_preds[1]

	with_terminal() do
		@printf("price: %6.2f / price rounded to nearest dollar: %6.0f\n", pred_price_first_house, round(pred_price_first_house))
	end
end

# ╔═╡ 5f0bdea2-71d3-11eb-150c-8722eee3f23a
md"""
Now that you have the predictions on test data, compute the RSS on the test data set. Save this value for comparison later. Recall that RSS is the sum of the squared errors (difference between prediction and output).
"""

# ╔═╡ 6d6eb4b0-71d3-11eb-24f0-f9f796040473
begin
	rss_test = sum((ya_preds - test_out).^2)

	with_terminal() do
		@printf("rss on test set: %15.2f / in scientific notation: %2.4e\n", rss_test, rss_test)
	end
end

# ╔═╡ e13d4ca6-71d0-11eb-1ad7-119b4bf3c830
md"""
### Running a multiple regression

Now we will use more than one actual feature. Use the following code to produce the weights for a second model with the following parameters:

"""

# ╔═╡ ec505642-71d0-11eb-2619-0f410fee1f94
begin
	mr_model_features = [:sqft_living, :sqft_living15]
	mr_target = :price
	(mr_feature_matrix, mr_output) = get_data(sales_train, mr_model_features,
			mr_target)
	mr_init_weights = [-100000., 1., 1.]
	ηᵣ = 4e-12
	ϵᵣ = 1e9
end

# ╔═╡ 59a6c7bc-71d1-11eb-2836-71ec957bd538
md"""
Use the above parameters to estimate the model weights. Record these values for your quiz.
"""

# ╔═╡ 6a907c30-71d1-11eb-3adb-1355e084f741
weights_mr = regression_gradient_descent(mr_feature_matrix,
	Vector{Float64}(mr_output), mr_init_weights, ηᵣ, ϵᵣ)

# ╔═╡ c47ea9ec-71d1-11eb-0491-9ba85adf1e0a
md"""
Use your newly estimated weights and the predict_output function to compute the predictions on the test data.

*Don't forget to create a Julia array for these features from the test set first!*
"""

# ╔═╡ da6f7c6a-71d1-11eb-381f-5f75054e14e7
begin
	(test_mr_features_matrix, test_mr_output) = get_data(sales_test,
		mr_model_features, mr_target)

	preds_mr = predict_output(test_mr_features_matrix, weights_mr)
end

# ╔═╡ 05450824-71d2-11eb-0a05-2b895313421f
md"""
**Quiz Question: What is the predicted price for the 1st house in the TEST data set for model 2 (round to nearest dollar)?**

"""

# ╔═╡ 1706d680-71d2-11eb-3cd6-4d156d964fd7
begin
	pred_mr_price_first_house = preds_mr[1]

	with_terminal() do
		@printf("price: %6.2f / price rounded to nearest dollar: %6.0f\n", pred_mr_price_first_house, round(pred_mr_price_first_house))
	end
end

# ╔═╡ 44c15f32-71d2-11eb-2232-5fd5a07b2bfc
md"""
What is the actual price for the 1st house in the test data set?
"""

# ╔═╡ 4eeb7218-71d2-11eb-1fa7-21fec4e8c846
sales_test[!, :price][1]

# ╔═╡ 84462372-71d2-11eb-1627-1fc0ccb0d650
md"""
**Quiz Question: Which estimate was closer to the true price for the 1st house on the *test* data set, model 1 or model 2?**

  - [x] Model 1
  - [ ] Model 2

"""

# ╔═╡ 8ef08466-71d2-11eb-0cbd-17d088b840e0
md"""
Now use your predictions and the output to compute the RSS for model 2 on *test* data.
"""

# ╔═╡ 9db31f5e-71d2-11eb-1527-29c0a844ce47
begin
	rss_test_mr = sum((preds_mr - test_mr_output).^2)

	with_terminal() do
		@printf("rss on test set: %15.2f / in scientific notation: %2.4e\n", rss_test_mr, rss_test_mr)
	end
end

# ╔═╡ 0d254b32-71d3-11eb-2c92-b1a2d1125c5b
md"""
**Quiz Question: Which model (1 or 2) has lowest RSS on all of the test data?**

  - [ ] model 1
  - [x] model 2
"""

# ╔═╡ Cell order:
# ╟─9dcffb1c-71c1-11eb-2402-39eaa1f1b5e1
# ╟─c39f9348-71c1-11eb-1691-1d28c6b1d916
# ╠═1bcf25d8-71c2-11eb-05ca-6fe7c4e1b24f
# ╠═1f7ea604-71c2-11eb-028a-a166f6f4dc5d
# ╟─3e5d907e-71c2-11eb-2eb3-4375fb21b884
# ╠═4c736078-71c2-11eb-225b-816745199225
# ╠═58e6255a-71c2-11eb-2094-9dbe7fb64151
# ╠═64c97662-71c2-11eb-2894-2d544642b136
# ╠═80fbf0f8-71c2-11eb-265a-fb539d764180
# ╟─35b5d05a-71cc-11eb-15a9-852c5a534fab
# ╟─a2928dc6-71c2-11eb-3724-29dc1448a0f3
# ╠═e4da992e-71c2-11eb-36b8-93dd614051c1
# ╠═d62e0368-71c2-11eb-16f7-5fda56951d21
# ╟─498b3458-71cc-11eb-3b55-8381d58d04bb
# ╟─daf8aefc-71c7-11eb-04fa-e71d7b2cde21
# ╠═02f0c99e-71c5-11eb-1fe1-b3f820998d54
# ╠═9cd636f2-71c5-11eb-2957-afbaf437aaa5
# ╟─21635b6e-71c8-11eb-2272-edec98c58339
# ╠═f377f6d4-71c9-11eb-0249-f3d63e87062b
# ╠═1301a4d2-71ca-11eb-339c-d1f747a3ffe7
# ╠═49ce0c84-71cb-11eb-3947-6141a9498ffb
# ╟─239ff6ca-71cc-11eb-341a-df1453d60a15
# ╠═958f81a6-71cc-11eb-13e8-61dddeb014a7
# ╟─4454239e-71ce-11eb-295e-1755bc0549e6
# ╠═0623d920-71ce-11eb-0936-870685f2e788
# ╠═6a1ee460-71ce-11eb-1b49-4de942aefe9d
# ╠═9883c1ee-71cf-11eb-3b48-a14df232588f
# ╟─06cf79e0-71d0-11eb-3eab-ad040213a64b
# ╠═26199ccc-71d0-11eb-0b4e-5f2f8ebc7b76
# ╟─4be11b60-71d0-11eb-1550-6d2854691b05
# ╠═65c26930-71d0-11eb-248a-3751d927b2f6
# ╟─749979f6-71d0-11eb-0b3a-459257d95ca9
# ╠═898fa5b2-71d0-11eb-18c3-ad6d0dbf9bf8
# ╠═5f0bdea2-71d3-11eb-150c-8722eee3f23a
# ╠═6d6eb4b0-71d3-11eb-24f0-f9f796040473
# ╟─e13d4ca6-71d0-11eb-1ad7-119b4bf3c830
# ╠═ec505642-71d0-11eb-2619-0f410fee1f94
# ╟─59a6c7bc-71d1-11eb-2836-71ec957bd538
# ╠═6a907c30-71d1-11eb-3adb-1355e084f741
# ╟─c47ea9ec-71d1-11eb-0491-9ba85adf1e0a
# ╠═da6f7c6a-71d1-11eb-381f-5f75054e14e7
# ╟─05450824-71d2-11eb-0a05-2b895313421f
# ╠═1706d680-71d2-11eb-3cd6-4d156d964fd7
# ╟─44c15f32-71d2-11eb-2232-5fd5a07b2bfc
# ╠═4eeb7218-71d2-11eb-1fa7-21fec4e8c846
# ╟─84462372-71d2-11eb-1627-1fc0ccb0d650
# ╟─8ef08466-71d2-11eb-0cbd-17d088b840e0
# ╠═9db31f5e-71d2-11eb-1527-29c0a844ce47
# ╟─0d254b32-71d3-11eb-2c92-b1a2d1125c5b


# Example from Improved relaxations for the parametric solutionsof ODEs using differential inequalities
id = :ScottA2013b

pL = [1.0; 1.0; 0.001]
pU = [1200.0; 1200.0; 100.0]
ScottA2013b_x0(p::Vector{T}) where T = [1.53*10^(-4)*one(T); 0.4*one(T); zero(T);
                                        zero(T); 0.0019*one(T); zero(T); zero(T);
                                        zero(T); zero(T); zero(T)]
function ScottA2013b_f!(du::Vector{T}, u::Vector{T}, p::Vector{T}, t) where T

    k = T[53.0*one(T); p[1]; p[2]; p[3]; 1200.0*one(T)]
    K2 = 2081.0
    K3 = 4162.0

    S = zeros(10,5)
    S[1,1] = -1.0; S[2,1] = -1.0; S[3,1] = 1.0;  S[3,2:3] = -1.0; S[3,5] = -2.0;
    S[4,1] = 1.0; S[5,2] = -1.0;  S[5,3] = -1.0; S[6,2] = 1.0;
    S[7,3] = 1.0; S[7,4] = -1.0;  S[8,4] = 1.0;  S[9,4] = 1.0;  S[10,5] = 1.0

    r = zeros(T, 5)
    r[1] = k[1]*u[1]*u[2]
    r[2] = k[2]*(u[3]*u[5] - (1.0/K2)*u[6])
    r[3] = k[3]*(u[3]*u[5] - (1.0/K3)*u[7])
    r[4] = k[4]*u[7]
    r[5] = k[5]*u[3]^2

    du[:] = S*r
    return
end
tspan = (0.0, 2.0)

prob = ODERelaxProb(ScottA2013b_f!, tspan, ScottA2013b_x0, pL, pU)
url = "https://link.springer.com/article/10.1007%2Fs10898-012-9909-0"
source = "ScottA2013b"
desc = "Cyclohexadienyl Radical Oxidation"


# Load Julia packages (libraries) needed.

using StatisticalRethinking, CmdStan
#gr(size=(600,600));

ProjDir = rel_path("..", "scripts", "05")
cd(ProjDir)

# ### snippet 5.1

wd = CSV.read(rel_path("..", "data", "WaffleDivorce.csv"), delim=';');
df = convert(DataFrame, wd);
df[!, :A] = scale(df[!, :MedianAgeMarriage]);
df[!, :D] = scale(df[!, :Divorce]);
first(df, 5)

# ### snippet 5.1

std(df[!, :MedianAgeMarriage])

ad = "
data {
 int < lower = 1 > N; // Sample size
 vector[N] D; // Outcome
 vector[N] A; // Predictor
}

parameters {
 real a; // Intercept
 real bA; // Slope (regression coefficients)
 real < lower = 0 > sigma; // Error SD
}

model {
  vector[N] mu;
  a ~ normal(0, 0.2);
  bA ~ normal(0, 0.5);
  sigma ~ exponential(1);
  mu = a + bA * A;
  D ~ normal(mu , sigma);
}
";

# Define the Stanmodel and set the output format to :mcmcchains.

m5_1s = Stanmodel(name="MedianAgeDivorce", model=ad);

# Input data for cmdstan

data = Dict("N" => length(df[!, :D]), "D" => df[!, :Divorce],
    "A" => df[!, :A]);

# Sample using cmdstan

rc, chn, cnames = stan(m5_1s, data, ProjDir, diagnostics=false,
  summary=true, CmdStanDir=CMDSTAN_HOME);

# Describe the draws

MCMCChains.describe(chn)

# Plot the density of posterior draws

plot(chn)

# Result rethinking

rethinking = "
       mean   sd  5.5% 94.5% n_eff Rhat
a      9.69 0.22  9.34 10.03  2023    1
bA    -1.04 0.21 -1.37 -0.71  1882    1
sigma  1.51 0.16  1.29  1.79  1695    1
"

# Plot regression line using means and observations

xi = -3.0:0.01:3.0
rws, vars, chns = size(chn)
alpha_vals = convert(Vector{Float64}, reshape(chn.value[:, 1, :], (rws*chns)))
beta_vals = convert(Vector{Float64}, reshape(chn.value[:, 2, :], (rws*chns)))
yi = mean(alpha_vals) .+ mean(beta_vals)*xi

scatter(df[!, :A], df[!, :D], color=:darkblue,
  xlab="Standardized median age of marriage",
  ylab="Standardize divorce rate")
plot!(xi, yi, lab="Regression line")

# shade(), abline() and link()

mu = link(xi, chn, [1, 2], mean(xi));
yl = [minimum(mu[i]) for i in 1:length(xi)];
yh =  [maximum(mu[i]) for i in 1:length(xi)];
ym =  [mean(mu[i]) for i in 1:length(xi)];
pi = hcat(xi, yl, ym, yh);
pi[1:5,:]

plot!((xi, yl), color=:lightgrey, leg=false)
plot!((xi, yh), color=:lightgrey, leg=false)
for i in 1:length(xi)
  plot!([xi[i], xi[i]], [yl[i], yh[i]], color=:lightgrey, leg=false)
end
scatter!(df[!, :A], df[!, :D], color=:darkblue)
plot!(xi, yi, lab="Regression line")

# End of `05/m5.1s.jl`


"""
Abstract decomposition of N-mode tensor.
"""
abstract type TensorDecomposition{T<:Number, N} end

"""
Returns relative error between the re-composed and the original tensors.
"""
rel_residue{T,N}(recomposed::StridedArray{T, N}, tensor::StridedArray{T, N}) = vecnorm(recomposed - tensor) / vecnorm(tensor)

"""
Returns relative error between re-composed and the original tensor.
"""
rel_residue{T,N}(decomp::TensorDecomposition{T, N}, tensor::StridedArray{T, N}) = rel_residue(compose(decomp), tensor)

"""
Returns relative tensor decomposition error, NaN if not available.
"""
rel_residue(decomp::TensorDecomposition) = get(decomp.props, :rel_residue, NaN)

_set_rel_residue(decomp::TensorDecomposition, error::Float64) = setindex!(decomp.props, error, :rel_residue)

_set_rel_residue{T,N}(decomp::TensorDecomposition{T,N}, tensor::StridedArray{T,N}) =
    _set_rel_residue(decomp, rel_residue(decomp, tensor))


#
#  Denoising a cyclic (S1-valued) Signal with first and second order TV.
#  using the Cyclic Proximal Point Algorithm
#
# This example was in a similar form part of the paper
#
# > Bergmann, R., Laus, F., Steidl, G., Weinmann, A.:
# > _Second Order Differences of Cyclic Data and Applications in Variational Denoising_
# > SIAM J. Imaging Sci., 7(4), 2916–2953, 2014.
# > doi: [10.1137/140969993](https://dx.doi.org/10.1137/140969993)
# > arXiv: [1405.5349](https://arxiv.org/abs/1405.5349)
#
using Manopt, Manifolds, Plots
#
# Settings
results_folder = joinpath(@__DIR__, "Signal_TV")
experiment_name = "S1_Signal_TV12_CPPA"
export_orig = true
export_result = true
!isdir(results_folder) && mkdir(results_folder)
#
# Parameters
n = 500
σ = 0.2
α = 0.5
β = 1.0
#
# Coloes
data_color = RGBA{Float64}(colorant"#BBBBBB")
result_color = RGBA{Float64}(colorant"#EE7733") # data Color: Tol Vibrant Orange
orig_color = RGBA{Float64}(colorant"#33BBEE") # tangent vector: Tol Vibrant Teal
#
# Manifolds and Data
M = Circle()
N = PowerManifold(M, n)
f = artificial_S1_signal(n)
xCompare = f
fn = exp.(Ref(M), f, random_tangent.(Ref(M), f, Val(:Gaussian), σ))
data = fn
t = range(0.0, 1.0; length=n)

if export_orig
    scene = scatter(
        t,
        f;
        markersize=2,
        markercolor=data_color,
        markerstrokecolor=data_color,
        lab="original",
    )
    scatter!(
        scene,
        t,
        fn;
        markersize=2,
        markercolor=orig_color,
        markerstrokecolor=orig_color,
        lab="noisy",
    )
    yticks!(
        [-π, -π / 2, 0, π / 2, π],
        [raw"$-\pi$", raw"$-\frac{\pi}{2}$", raw"$0$", raw"$\frac{\pi}{2}$", raw"$\pi$"],
    )
    png(scene, joinpath(results_folder, experiment_name * "-original.png"))
end
#
# Setup and Optimize
F = (N, x) -> costL2TVTV2(N, data, α, β, x)
proxes = (
    (N, λ, x) -> prox_distance(N, λ, data, x, 2),
    (N, λ, x) -> prox_TV(N, α * λ, x),
    (N, λ, x) -> prox_TV2(N, β * λ, x),
)

o = cyclic_proximal_point(
    N,
    F,
    proxes,
    data;
    λ=i -> π / (2 * i),
    debug=[
        :Iteration,
        " | ",
        DebugProximalParameter(),
        " | ",
        :Cost,
        " | ",
        :Change,
        "\n",
        1000,
        :Stop,
    ],
    record=[:Iteration, :Cost, :Change, :Iterate],
    return_options=true,
)
fR = get_solver_result(o)
r = get_record(o)
#
# Result
if export_result
    scene = scatter(
        t,
        f;
        markersize=2,
        markercolor=data_color,
        markerstrokecolor=data_color,
        lab="original",
    )
    scatter!(
        scene,
        t,
        fR;
        markersize=2,
        markercolor=result_color,
        markerstrokecolor=result_color,
        lab="reconstruction",
    )
    yticks!(
        [-π, -π / 2, 0, π / 2, π],
        [raw"$-\pi$", raw"$-\frac{\pi}{2}$", raw"$0$", raw"$\frac{\pi}{2}$", raw"$\pi$"],
    )
    png(scene, joinpath(results_folder, experiment_name * "-result.png"))
end

print("MSE (input):  ", 1 / n * distance(N, xCompare, data)^2, "\n")
print("MSE (result): ", 1 / n * distance(N, xCompare, fR)^2, "\n")


function mutual_information_contingency(probs::AbstractMatrix{Float64}; normalize::Bool=false)
    ee = entropy(ProbabilityWeights(probs[:]))
    ex = entropy(ProbabilityWeights(sum(probs, dims=1)[:]))
    ey = entropy(ProbabilityWeights(sum(probs, dims=2)[:]))

    mi = ex + ey - ee
    return normalize ? min(mi / min(ex, ey), 1.) : mi
end

function mutual_information(
    x::AbstractVector, y::AbstractVector, ex::Float64, ey::Float64; method=:Naive, adjusted=false, normalize=false)

    # Compute the joint entropy without adjustment nor normalization
    ee = estimate_joint_entropy(x, y; method=method)

    if adjusted
        ee_shuffle = estimate_joint_entropy(x, shuffle(y); method=method)
        mi_shuffle = ex + ey - ee_shuffle
        mi = ee_shuffle - ee
        normalize ? mi / (min(ex, ey) - mi_shuffle) : mi
    else
        mi = ex + ey - ee
        normalize ? min(mi / min(ex, ey), 1.) : mi
    end
end

function mutual_information(
    x::AbstractVector, y::AbstractVector;
    method=:Naive, adjusted=false, normalize=false)

    ex = estimate_entropy(x)
    ey = estimate_entropy(y)
    mutual_information(
        x, y, ex, ey; method=method, adjusted=adjusted, normalize=normalize)
end

function mutual_information(data::AbstractMatrix; method::Symbol=:Naive, adjusted::Bool=false, normalize::Bool=false)
    M = size(data, 2)
    mi_sym = Array{Float64, 2}(undef, M, M)

    for i =1:M
        mi_sym[i,i] = estimate_entropy(@view data[:,i]; method=method)
    end

    for i = 1:M, j=i+1:M
        ex = mi_sym[i,i]
        ey = mi_sym[j,j]

        x = @view data[:,i]
        y = @view data[:,j]

        mi_sym[i, j] = mutual_information(
            x, y, ex, ey;
            method=method, adjusted=adjusted, normalize=normalize)

        # Drop null or negative values
        if mi_sym[i, j] < Base.eps()
            mi_sym[i, j] = 0.
        end

        mi_sym[j,i] = mi_sym[i,j]  # tbr
    end
    mi_sym
end


using Plots
pyplot() # Use PyPlot as a backend (may already be the default)

#x = collect(1:7)
#y = []

#for item in x
    #push!(y, 2 - 2*item + item^2/4)
#end

#plot(x,y)
#plot!(x, y, marker = :diamond, linewidth=2)
#plot!(title = "Sample plot", leg=false)

x = [1 2 3 4 5 6]'
#y = (x-3).^2/4
y = []

for item in x
    push!(y, (item-3).^2/4)
end

plot(x,y, marker = :hex, leg=false, linewidth = 2, linecolor=:black)
plot!(title="Plot for graded quiz")
gui()    # To show plot on atom


"""
    RiemannMatrix

Type containing information about the Riemann Matrix, for use in computing the Riemann theta function.
"""
struct RiemannMatrix
    τ::Array{<:Number} # matrix in Siegel upper-half space
    g::Integer # genus
    X::Array{<:Real} # real part of τ
    Y::Array{<:Real} # imaginary part of τ
    T::Array{<:Real} # upper triangular matrix in Cholesky factorization of Y
    ellipsoid::Array{} # points to sum over for computing theta functions

    """
        RiemannMatrix(τ, siegel=false, ϵ=1.0e-12, nderivs=4)

    Construct a RiemannMatrix type using a matrix τ in the Siegel upper-half space, for use in computing the Riemann theta function.

    # Arguments
    - `τ::Array{<:Number}`: 2-dimensional array of complex numbers
    - `siegel::Bool=false`: do a siegel transformation on τ if true, and use the input matrix otherwise
    - `ϵ::Real=1.0e-12`: absolute error in value of theta function or its derivatives
    - `nderivs::Integer=4`: highest order of the derivatives of the theta function

    # Examples
    ```julia
    julia> RiemannMatrix([1+im -1; -1 1+im], siegel=true, nderivs=2)
    ```
    """
    function RiemannMatrix(τ::Array{<:Number}; siegel::Bool=false, ϵ::Real=1.0e-12, nderivs::Integer=4)
        τ = (siegel ? siegel_transform(τ)[2] : τ);
        X = real(τ);
        Y = imag(τ);
        g = size(Y)[1];
        T = (g > 1 ? convert(Array{Float64}, cholesky(Symmetric(Y)).U) : sqrt.(Y));
        ρ = svp(T)[2]*sqrt(π); # length of shortest vector in lattice generated by √π*T
        radius = [radius_ellipsoid(ϵ, T, Y, ρ, i) for i=0:nderivs]; # radius of ellipsoids for computing theta function, and up to nderivs derivatives.
        ellipsoid = [ellipsoid_uniform(T, r) for r in radius];
        new(τ, g, X, Y, T, ellipsoid)
    end
end

"""
    random_siegel(g)

Sample a random matrix in the Siegel upper half space with genus g.

# Arguments
- `g::Integer`: genus.

# Examples
```julia
julia> random_siegel(5)
```
"""
function random_siegel(g::Integer)
    Mx = 2*rand(g, g) - ones(g,g); # entries are random between [-1,1)
    My = 2*rand(g, g) - ones(g,g); # entries are random between [-1,1)
    τ = Symmetric((1/2)*(transpose(Mx) + Mx)) + Symmetric(transpose(My)*My)*im; # symmetric real part and psd imaginary part
    return convert(Array, τ);
end


export rrefx

"""
`rref(A)`  returns the row-reduced echelon form of the exact
matrix `A`.
"""
function rrefx(A::AbstractMatrix{T}) where {T<:IntegerX}
    AA = big.(A // 1)
    return rrefx(AA)
end

function rrefx(A::AbstractMatrix{T}) where {T}
    AA = copy(A)
    rrefx!(AA)
    return AA
end

function _pivot!(A::AbstractMatrix{T}, i::Int, j::Int) where {T}
    r, c = size(A)
    s = A[i, j]
    row_scale!(A, i, invx(s))

    for k = 1:r
        if k != i
            s = -A[k, j]
            row_add_mult!(A, i, s, k)
        end
    end
end

function rrefx!(A::AbstractMatrix{T}) where {T}
    rn, nc = size(A)

    r = 1
    for j = 1:nc
        # see if column j's first one is at level r or below
        col = A[r:end, j]
        if all(col .== 0)
            continue
        end
        k = findfirst(col .!= 0)
        row_swap!(A, r, k + r - 1)
        _pivot!(A, r, j)
        r += 1
    end

end


using JLD2
@load "2/data.jld2" c d A B b D f G g

using JuMP
using Clp
using LinearAlgebra

p1 = 25
p2 = 100
p3 = 200
n = 50
m = 100

# Primal

primal = Model(Clp.Optimizer)

@variables(primal, begin
        x[1:n]
        y[1:m]
    end
)

@objective(primal, Min, c'x + d'y)

@constraints(
	primal,
	begin
        A * x + B * y .== b
		D * x .<= f
        G * y .<= g
    end
)

optimize!(primal)

x̂ = value.(x)
ŷ = value.(y)

# Dual

dual = Model(Clp.Optimizer)

@variables(dual, begin
        p[1:p1]
        q[1:p2] <= 0
        r[1:p3] <= 0
    end
)

@objective(dual, Max, p'b + q'f + r'g)

@constraints(
	dual,
	begin
        p'A + q'D .== c'
        p'B + r'G .== d'
    end
)

optimize!(dual)

println("primal: ", objective_value(primal))
println("dual: ", objective_value(dual))
@assert objective_value(primal) ≈ objective_value(dual)


using LinearAlgebra

include("../quadrature/quadgrid.jl")

function param_grid_comp(n_param, param_priorInfo)
    # extract param info
    thetamin = param_priorInfo.theta.min
    thetamax = param_priorInfo.theta.max
    lambdamin = param_priorInfo.lambda.min
    lambdamax = param_priorInfo.lambda.max
    n_theta = n_param.theta
    n_lambda = n_param.lambda

   # form a 2d theta grid
   theta1_grid, wt1_theta = quadgrid(thetamin[1], thetamax[1], n_theta[1])
   theta2_grid, wt2_theta = quadgrid(thetamin[2], thetamax[2], n_theta[2])
   n_theta_totl = n_theta[1] * n_theta[2]
   theta_grid = zeros(n_theta_totl, 2)
   wt_theta = zeros(n_theta_totl)
   for i in 1:n_theta[1], j in 1:n_theta[2]
       theta_grid[j + (i-1) * n_theta[2], :] = [theta1_grid[i] theta2_grid[j]]
       wt_theta[j + (i-1) * n_theta[2]] = wt1_theta[i] * wt2_theta[j]
   end

   # form a 1d lambda grid
   lambda_grid, wt_lambda = quadgrid(lambdamin, lambdamax, n_lambda)

   thetaInfo = (grid = theta_grid, wt = wt_theta)
   lambdaInfo = (grid = lambda_grid, wt = wt_lambda)
   param_gridInfo = (theta = thetaInfo, lambda = lambdaInfo)

   return param_gridInfo
end


@testset "distance" begin
    p1 = Position([-180, -90])
    p2 = Position([180, -90])
    @test distance(p1, p1) == 0

    p3 = Position([-75.343, 39.984])
    p4 = Position([-75.534, 39.123])

    @test distance(p3, p4, "radians") ≈ 0.015 atol=0.001
    @test distance(p3, p4, "degrees") ≈ 0.87 atol=0.01
    @test distance(p3, p4, "miles") ≈ 60.35 atol=0.01

    @test rhumb_distance(p3, p4, "miles") ≈ 60.35 atol=0.01
    @test rhumb_distance(p3, p4, "degrees") ≈ 0.87 atol=0.01
    @test rhumb_distance(p3, p4, "nauticalmiles") ≈ 52.44 atol=0.01

    @test rhumb_distance(Point([-75.343, 39.984]), Point([-75.534, 39.123]), "miles") == rhumb_distance(p3, p4, "miles")

    a =  Point([-75.833, 39.284])
    b =  Point([-75.6, 39.984])
    c =  Point([-75.221, 39.125])
    d =  Point([-75.9221, 39.27])
    e =  Point([-75.534, 39.123])
    f = Point([-75.33, 39.44])
    target = Point([-75.4, 39.4])

    @test nearestpoint(target, [a, b, c, d, e, f]).coordinates == [-75.33, 39.44]

    @test pnorm_distance(Point([2, 0]), Point([0, 0]), 2) == 2
    @test pnorm_distance(Point([1, 1]), Point([0, 0]), 1) == 2

    mid = midpoint(Point([0, 10]), Point([0, 0]))

    @test distance(Point([0, 10]), mid) ≈ distance(Point([0, 0]), mid)

    point = Point([0, 0])
    line = LineString([[1, 1],[-1, 1]])

    @test point_to_line_distance(point, line, "miles") ≈ 69.11854715938406 atol=0.05

end


"""
    BlackScholes()

[Black-Scholes model](https://en.wikipedia.org/wiki/Black%E2%80%93Scholes_model).
"""
struct BlackScholes <: Model end

"""
    evaluate(O, BlackScholes())

Evaluate option `O` using `BlackScholes` model.

# Arguments
- `O::Option`: Option
"""
function evaluate(O::EuropeanOption, m::BlackScholes)
    d1 = (log(O.s / O.k) + (O.r + O.σ * O.σ / 2) * O.t) / (O.σ * √O.t)
    d2 = d1 - O.σ * √O.t

    if O.call == -1
        return cdf(Normal(), -d2) * O.k * exp(-O.r * O.t) - cdf(Normal(), -d1) * O.s
    elseif O.call == 1
        return O.s * cdf(Normal(), d1) - exp(-O.r * O.t) * O.k * cdf(Normal(), d2)
    end
end

"""
    evaluate(o)

Evaluate option `o` using Back-Scholes model as default valuation model.
"""
evaluate(O::EuropeanOption) = evaluate(O::EuropeanOption, m::BlackScholes)