# ==============================================================================
# Five-dimensional system
#
# Example from Althoff's thesis, Section 3.2.3
#
#
# system type: continuous LTI system
# state dimension: 5
# input dimension: 1
#
# This five-dimensional system is taken from Example 4.1 in [1].
#
# [1] Colas Le Guernic: Reachability analysis of hybrid systems with linear
#     continuous dynamics. PhD thesis, 2009.
# ==============================================================================

function linear5D_homog()
    A = Float64[-1 -4   0   0   0;
                 4 -1   0   0   0;
                 0  0  -3   1   0;
                 0  0  -1  -3   0;
                 0  0   0   0  -2]
    X0 = Hyperrectangle(low=fill(0.9, 5), high=fill(1.1, 5))
    prob = @ivp(x' = Ax, x(0) ∈ X0)
    tspan = (0.0, 5.0)
    return prob, tspan
end

function linear5D()
    # system matrix
    D = [-1.0 -4.0  0.0  0.0  0.0;
          4.0 -1.0  0.0  0.0  0.0;
          0.0  0.0 -3.0  1.0  0.0;
          0.0  0.0 -1.0 -3.0  0.0;
          0.0  0.0  0.0  0.0 -2.0]
    P = [ 0.6 -0.1  0.1  0.7 -0.2;
         -0.5  0.7 -0.1 -0.8  0.0;
          0.9 -0.5  0.3 -0.6  0.1;
          0.5 -0.7  0.5  0.6  0.3;
          0.8  0.7  0.6 -0.3  0.2]
    A = P * D * inv(P)
    X = Universe(5) # state domain
    U = Ball2(zeros(5), 0.01) # input domain

    X0 = BallInf([1.0, 0.0, 0.0, 0.0, 0.0], 0.1)
    prob = @ivp(x' = Ax + u, x ∈ X, u ∈ U, x(0) ∈ X0) # continuous LTI system
    tspan = (0.0, 5.0)
    return prob, tspan
end


using Test, JLD2, LinearAlgebra
#
# gradient computation by parial automatic differentiation and adjoint merthods
# of
#           E(v) = ∫ |U(v)|² dx
# where
# A(v) U(v) = F(v) the discrete version of - Δ u = f
# and v are the vertices of the voronoi cells
#

# load polygonal mesh
filename, nc = "squarepolmesh_coarse", 100
mesh_filename = "$(@__DIR__)/data/$(filename)_$(nc).jld2"
println("read file $(mesh_filename)")
JLD2.@load(mesh_filename, pv, cellsb, cellsbt, t, pb, tb)
 # boundary points / rhs
 rhs, boundary_condition = if occursin("square", mesh_filename)
    PolygonalFem.rhs_sqr, PolygonalFem.boundary_condition_sqr
else
    PolygonalFem.rhs_L, PolygonalFem.boundary_condition_L
end

np, ε = size(pv, 1), 1e-4
x, dp = pv, 2 * rand(size(pv)...) .- 1
Ib = unique(PolygonalFem.btri(t)[:])
xp, xm = x .+ ε * dp, x .- ε * dp
# finite differences
f = x -> PolygonalFem.costnormU(x, cellsb, t, rhs, boundary_condition, Ib)
fp, fm = f(xp), f(xm)
@time g = PolygonalFem.∇costnormU(x, cellsb, t, rhs, boundary_condition, Ib)
@time g = PolygonalFem.∇costnormU(x, cellsb, t, rhs, boundary_condition, Ib)
println("")
println(fp)
println(fm)
println(" ")
println((fp - fm) / (2 * ε))
println(dot(g, dp))
println("")
@testset "test eval gradient dot function for polygonal costnormU" begin
    @test abs(dot(g, dp) - (fp - fm) / (2 * ε)) < 1e-4
end


#some random matrices for testing
module Randm
export randsym, randposdef, randstable, randnormal, randorth, randunitary

#%  .. currentmodule:: Randm
#%

#%  Module Randm
#%  ------------
#%
#%
#%  Introduction
#%  ~~~~~~~~~~~~
#%

#%  Random matrices for testing purposes. I did not figure out the actual distributions
#%  the matrices are drawn from.
#%

#%  Reference
#%  ~~~~~~~~~
#%

##%  .. function:: randsym(d)
##%
##%  	Random symmetric matrix. Note: randsym
##%

#function randsym(d)
#	x = 0.5*randn(d, d)
#	x' + x
#end

#%  .. function:: randposdef(d)
#%
#%  	Random positive definite matrix of dimension ``d``.
#%

function randposdef(d)
	x = randn(d, d)
	x'* x
end

#%  .. function:: randstable(d)
#%
#%  	Random stable matrix (matrix with eigenvalues with negative real part) with
#%  	dimension ``d``.

function randstable(d)
	# positive definite matrix
	x = randn(d, d)
	a = x'*x

	# anti symmetric matrix
	above_diag =	[ i<j ? 1 : 0 for i in 1:d, j in 1:d] # nonzero pattern for above diagonal
	y = randn(d, d)
	b = above_diag.*y - above_diag'.*y'

	# return stable matrix
	b - a
end

#%  .. function:: randunitary(d)
#%
#%  	Random unitary matrix of dimension ``d``.
#%

randunitary(d) = expm(im* randposdef(d))

#%  .. function:: randorth(d)
#%
#%  	Orthogonal matrix drawn according to the Haar measure on the group of orthogonal matrices.
#%

randorth(d) = qr(randn(d,d))[1]

#%  .. function:: randnormal(d)
#%
#%  	Random normal matrix.
#%

function tridiag(di, u, l)
	assert(length(di) -1 == length(u) == length(l))
	M = diagm(di)
	for i in 1:length(u)
	  M[i,i+1] = l[i]
  	  M[i+1,i] = u[i]
	end
	M

end
tridiag(alpha, beta) = tridiag(alpha, beta, -beta)

function randnormal(d)
	Q = randorth(d)
	B = schur(randn(d,d))[1] #same eigenvalues as a randn(d,d)
	m = d
	alpha = zeros(d)
	beta = zeros(d-1)
	while m > 1
	   s = abs(B[m-1,m-1]) + abs(B[m,m])
	   if s + abs(B[m,m-1]) > s # if significant offdiagonal value: evaluate submatrix m-1:m;
		spur = B[m,m] + B[m-1,m-1]
		dis2 = abs2(B[m,m]- B[m-1,m-1]) + 4.0*B[m,m-1]*B[m-1,m] # = spur^2 - 4det
		alpha[m] = alpha[m-1] = 0.5*spur

		beta[m-1] = 0.5*sqrt(-dis2)
		m -= 2
	   else
	        alpha[m] = B[m,m]
	        beta[m-1] = 0
	        m -= 1
	   end

	end
	if (m == 1)
		alpha[1] = B[1,1]
	end

	Q' * tridiag(alpha, beta) * Q
	#tridiag(alpha, beta)

end

end #module


module Functions

#=
two-variable function
x = 5.90133
y = 0.5
fmin = -43.3159
-30 <= x, y <= 30
=#
function chichinadze(pars)
    @assert(length(pars) == 2)
    x = pars[1]
    y = pars[2]
    return x^2 - 12.0x + 11.0 + 10.0 * cos(pi * x / 2.0) + 8.0 * sin(5 * pi * x) -
           (1 / sqrt(5)) * exp(-0.5 * (y - 0.5)^2)
end

#=
n-variable function
x = [1, 1, ..., 1]
fmin = 0
-10 <= x_i <= 10
=#
function levy(pars)
    d = length(pars)

    w = zeros(Float64, d)

    for ii = 1:d
        w[ii] = 1 + (pars[ii] - 1.0) / 4.0
    end

    term1 = (sin(pi * w[1]))^2
    term3 = (w[d] - 1)^2.0 * (1 + (sin(2.0 * pi * w[d]))^2.0)

    sum = 0
    for ii = 1:(d-1)
        wi = w[ii]
        new = (wi - 1)^2 * (1 + 10 * (sin(pi * wi + 1))^2)
        sum = sum + new
    end

    return term1 + sum + term3
end

#=
n-variable function
x = [0, 0, ..., 0]
fmin = 0
-32.768 <= x_i <= 32.768
=#
function ackley(xx)
    d = length(xx)
    a = 20
    b = 0.2
    c = 2 * pi

    sum1 = 0
    for i in 1:d
        sum1 = sum1 + xx[i]^2
    end

    sum2 = 0
    for i in 1:d
        sum2 = sum2 + cos(c * xx[i])
    end

    term1 = -a * exp(-b*sqrt(sum1/d))
    term2 = -exp(sum2/d)

    y = term1 + term2 + a + exp(1)

    return(y)
end

end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    ∠(A, B, C)

Angle ∠ABC between rays BA and BC.
See https://en.wikipedia.org/wiki/Angle.

Uses the two-argument form of `atan` returning value in range [-π, π].
See https://en.wikipedia.org/wiki/Atan2.

## Examples

```julia
∠(Point(1,0), Point(0,0), Point(0,1)) == π/2
```
"""
∠(A::P, B::P, C::P) where {P<:Point{2}} = ∠(A-B, C-B)
∠(A::P, B::P, C::P) where {P<:Point{3}} = ∠(A-B, C-B)

"""
    ∠(u, v)

Angle between vectors u and v.
See https://en.wikipedia.org/wiki/Angle.

Uses the two-argument form of `atan` returning value in range [-π, π].
See https://en.wikipedia.org/wiki/Atan2.

## Examples

```julia
∠(Vec(1,0), Vec(0,1)) == π/2
```
"""
∠(u::V, v::V) where {V<:Vec{2}} = atan(u × v, u ⋅ v)  # preserve sign
∠(u::V, v::V) where {V<:Vec{3}} = atan(norm(u × v), u ⋅ v)  # discard sign


# Uncomment below if the package is not already installed
#using Pkg; Pkg.add("NBInclude")

# Import code from previous section
using NBInclude
@nbinclude("Graph_Code.ipynb");

function dfs(g::Graph, start)
    visited = falses(length(g.vertices))
    function visit(ivertex)
        visited[ivertex] = true
        println("Visiting vertex #$ivertex")
        for nb in g.vertices[ivertex].neighbors
            if !visited[nb]
                visit(nb)
            end
        end
    end
    visit(start)
    return nothing
end

plot_graph(g)

dfs(g,1)

function find_path_dfs(g::Graph, start, finish)
    visited = falses(length(g.vertices))
    path = Int64[]
    function visit(ivertex)
        visited[ivertex] = true
        if ivertex == finish
            pushfirst!(path, ivertex)
            return true
        end
        for nb in g.vertices[ivertex].neighbors
            if !visited[nb]
                if visit(nb)
                    pushfirst!(path, ivertex)
                    return true
                end
            end
        end
        return false
    end
    visit(start)
    return path
end

function plot_path(g, path)
    xypath = [v.coordinates for v in g.vertices][path]
    plot(first.(xypath), last.(xypath), color="red", linewidth=2)
end

plot_graph(g)
path = find_path_dfs(g, 10, 5)
plot_path(g, path)
println("Path from 10 to 5: $path")

function bfs(g::Graph, start)
    visited = falses(length(g.vertices))
    S = [start]
    visited[start] = true
    while !isempty(S)
        ivertex = popfirst!(S)
        println("Visiting vertex #$ivertex")
        for nb in g.vertices[ivertex].neighbors
            if !visited[nb]
                visited[nb] = true
                push!(S, nb)
            end
        end
    end
end

bfs(g,1)

function shortest_path_bfs(g::Graph, start, finish)
    parent = zeros(Int64, length(g.vertices))
    S = [start]
    parent[start] = start
    while !isempty(S)
        ivertex = popfirst!(S)
        if ivertex == finish
            break
        end
        for nb in g.vertices[ivertex].neighbors
            if parent[nb] == 0 # Not visited yet
                parent[nb] = ivertex
                push!(S, nb)
            end
        end
    end
    # Build path
    path = Int64[]
    iv = finish
    while true
        pushfirst!(path, iv)
        if iv == start
            break
        end
        iv = parent[iv]
    end
    return path
end

path = shortest_path_bfs(g, 10, 5)
plot_graph(g)
plot_path(g, path)
println("Shortest path from 10 to 5: $path")

field = """
....O....
O.O.O.O..
..O...O.O
.OOOOOO.O
....OO..O
.O...OOOO
.O.......
.OOOOOO..
......O..
......O..
"""
n = 10

F = Int.(reverse(reshape(collect(field), n, n) .== 'O', dims=2))

g1 = Graph([])
nbr = 0
for j = 1:n
    for i = 1:n
        nbr += 1
        nb = Int64[]
        for dj = -1:1
            for di = -1:1
                if 1 ≤ i+di ≤ n && 1 ≤ j+dj ≤ n
                    if F[i,j] == 0 && F[i+di,j+dj] == 0
                        push!(nb, nbr + di + n*dj)
                    end
                end
            end
        end
        v = Vertex(nb, coordinates=[i,j])
        push!(g1.vertices, v)
    end
end

path = shortest_path_bfs(g1, 6, 57)
xypath = [v.coordinates for v in g1.vertices][path]
plot_graph(g1, scale=0.5)
plot(first.(xypath), last.(xypath), color="red", linewidth=2)


### A Pluto.jl notebook ###
# v0.11.14

using Markdown
using InteractiveUtils

# ╔═╡ 57f367d6-025a-11eb-182a-773094da4307
using Yao, YaoPlots #calling the Yao and YaoPlots package

# ╔═╡ 3c9cf124-0257-11eb-16ce-0d857dce310f
md"# Using Yao - The basics of quantum computing in Julia using Yao.jl"

# ╔═╡ 5d5644b2-0257-11eb-36e9-7f4803cb3558
md"At the current moment, we don't have have quantum computers. How do we make quantum circuits then? Well, two things we can do right now are, simulate a few qubits or use the qubits created by corporates like IBM and D'Wave. Using Yao, we can simulate the qubits, without having a quantum computer\(based on the known mathematical and physics rules\), although the support to run your circuits on Yao using real qubits is coming to Yao soon."

# ╔═╡ 0f7d5bdc-0258-11eb-05bb-074d1b38c551
md"We can make a cicuit in Yao using *chain* function. For parameters we define the number of qubits and the operations we've to perform on them. Lets say we want to pass two qubits through two X gates. We do this by chain(number of qubits, operations). To use the X gate, we use the put() parameter. Run the cell below to see what happens."

# ╔═╡ ee15c64e-025a-11eb-2158-bd9cbea5f262
md" **\#\#** or single **#** sign mean comments in julia. It means that anything written after # or ## won't be read as a part of the program, in the line you used them."

# ╔═╡ dcf63604-0258-11eb-1466-fb21170c2e1a
let
	circuit = chain(2, put(1=>X), put(2=>X)); #define a variable "circuit" and "put" an X gate on the first qubit, and then put an X gate on the second qubit
	plot(circuit) #plot function, which takes a circuit for a parameter and prints the circuit diagram.
end

# ╔═╡ 841aa488-0259-11eb-351a-c1f1770b3a51
md"Assume we have 5 qubits and we have to pass each through an X gate. We can use the *repeat\(\)* parameter to pass the given number of qubits through the same gate."

# ╔═╡ af1aa2b4-0259-11eb-29fe-19ba9462deb9
plot(chain(5, repeat(X,1:5))) #plot function takes a circuit, which repeats the X gate on the qubits 1:5 or from 1st qubit to 5th qubit

# ╔═╡ 1efc3566-025a-11eb-2aae-fd953f22edfb
md"What about the Y, Z and H gate? Figure that out yourself. The next cell is left for you to play around."

# ╔═╡ 443afbfa-025a-11eb-03c6-e74bb5344e36
let
	##circuit =
	##plot(circuit)
end

# ╔═╡ 686ec0a0-025b-11eb-339e-1158d9b25529
md"What about multiqubit gates? We can use the control gate in Yao using the control\( \) parameter."

# ╔═╡ 22f24af0-025c-11eb-23d3-b9945bc05a36
plot(chain(2, control(1, 2=>X))) #Which translates to if the state of the 1st qubit is |1>, perform X gate to the 2nd qubit or "put" the 2nd qubit through the X gate.

# ╔═╡ Cell order:
# ╟─3c9cf124-0257-11eb-16ce-0d857dce310f
# ╟─5d5644b2-0257-11eb-36e9-7f4803cb3558
# ╟─0f7d5bdc-0258-11eb-05bb-074d1b38c551
# ╟─ee15c64e-025a-11eb-2158-bd9cbea5f262
# ╠═57f367d6-025a-11eb-182a-773094da4307
# ╠═dcf63604-0258-11eb-1466-fb21170c2e1a
# ╟─841aa488-0259-11eb-351a-c1f1770b3a51
# ╠═af1aa2b4-0259-11eb-29fe-19ba9462deb9
# ╟─1efc3566-025a-11eb-2aae-fd953f22edfb
# ╠═443afbfa-025a-11eb-03c6-e74bb5344e36
# ╟─686ec0a0-025b-11eb-339e-1158d9b25529
# ╠═22f24af0-025c-11eb-23d3-b9945bc05a36


immutable Uniform <: ContinuousUnivariateDistribution
    a::Float64
    b::Float64
    function Uniform(a::Real, b::Real)
	a < b || error("a < b required for range [a, b]")
	new(float64(a), float64(b))
    end
    Uniform() = new(0.0, 1.0)
end

## Support
@distr_support Uniform d.a d.b

## Properties
mean(d::Uniform) = (d.a + d.b) / 2.0

median(d::Uniform) = (d.a + d.b) / 2.0

mode(d::Uniform) = mean(d)
modes(d::Uniform) = Float64[]

function var(d::Uniform)
    w = d.b - d.a
    w * w / 12.0
end

skewness(d::Uniform) = 0.0
kurtosis(d::Uniform) = -1.2

entropy(d::Uniform) = log(d.b - d.a)

## Functions
pdf(d::Uniform, x::Float64) = insupport(d,x) ? 1/(d.b-d.a) : 0.0
logpdf(d::Uniform, x::Float64) = insupport(d,x) ? -log(d.b-d.a) : -Inf

function cdf(d::Uniform, q::Float64)
    if isnan(q)
        return NaN
    elseif q <= d.a
        return 0.0
    elseif q >= d.b
        return 1.0
    end
    (q-d.a)/(d.b-d.a)
end
function ccdf(d::Uniform, q::Float64)
    if isnan(q)
        return NaN
    elseif q <= d.a
        return 1.0
    elseif q >= d.b
        return 0.0
    end
    (d.b-q)/(d.b-d.a)
end

quantile(d::Uniform, p::Float64) = @checkquantile p d.a+p*(d.b-d.a)
cquantile(d::Uniform, p::Float64) = @checkquantile p d.b+p*(d.a-d.b)

function mgf(d::Uniform, t::Real)
    u = 0.5*(d.b-d.a)*t
    u == zero(u) && return one(u)
    v = 0.5*(d.a+d.b)*t
    exp(v)*(sinh(u)/u)
end
function cf(d::Uniform, t::Real)
    u = 0.5*(d.b-d.a)*t
    u == zero(u) && return complex(one(u))
    v = 0.5*(d.a+d.b)*t
    exp(im*v)*(sin(u)/u)
end

## Sampling
rand(d::Uniform) = d.a + (d.b - d.a) * rand()

## Fitting
function fit_mle{T <: Real}(::Type{Uniform}, x::Vector{T})
    if isempty(x)
        throw(ArgumentError("x cannot be empty."))
    end

    xmin = xmax = x[1]
    for i = 2:length(x)
        xi = x[i]
        if xi < xmin
            xmin = xi
        elseif xi > xmax
            xmax = xi
        end
    end

    Uniform(xmin, xmax)
end


using Distributions, StatsBase, Plots, LaTeXStrings; pyplot()

mu0, mu1, sd, tau  = 15, 18, 2, 17.5
dist0, dist1 = Normal(mu0,sd), Normal(mu1,sd)
grid = 5:0.1:25
h0grid, h1grid = tau:0.1:25, 5:0.1:tau

println("Probability of Type I error: ", ccdf(dist0,tau))
println("Probability of Type II error: ", cdf(dist1,tau))

plot(grid, pdf.(dist0,grid),
	c=:blue, label="Bolt type 15g")
plot!(h0grid, pdf.(dist0, h0grid),
	c=:blue, fa=0.2, fillrange=[0 1], label="")
plot!(grid, pdf.(dist1,grid),
	c=:green, label="Bolt type 18g")
plot!(h1grid, pdf.(dist1, h1grid),
	c=:green, fa=0.2, fillrange=[0 1], label="")
plot!([tau, 25],[0,0],
	c=:red, lw=3, label="Rejection region",
	xlims=(5, 25), ylims=(0,0.25) , legend=:topleft,
    xlabel="x", ylabel="Density")
annotate!([(16, 0.02, text(L"\beta")),(18.5, 0.02, text(L"\alpha")),
            (15, 0.21, text(L"H_0")),(18, 0.21, text(L"H_1"))])


include("TypeDefine.jl")

#see (4.46) from Toro
function p_guess(left::HydroStatus, right::HydroStatus, ::TwoRarefaction)
		@assert left.gamma == right.gamma
		gamma = left.gamma
		gamma_power = (gamma-1) / 2. / gamma
		delta_u = right.u - left.u
		return ( (left.c + right.c - 0.5 * (gamma - 1) * delta_u) /
			   (left.c/left.p^gamma_power + right.c/right.p^gamma_power) ) ^
		       (1. / gamma_power)
end

#see (4.47) from Toro
function p_guess(left::HydroStatus, right::HydroStatus, ::PrimitiveValue)
	delta_u = right.u - left.u
	mean_pressure = 0.5 * (left.p + right.p)
	mean_density = 0.5 * (left.rho + right.rho)
	mean_speed_of_sound = 0.5 * (left.c + right.c)
	return mean_pressure - delta_u * mean_density * mean_speed_of_sound
end

#see (4.48) from Toro
g(p, status::HydroStatus) = sqrt(status.A / (p + status.B))
function p_guess(left::HydroStatus, right::HydroStatus, ::TwoShock)
	p_hat = p_guess(left, right, PrimitiveValue)
	g_left = g(p_hat, left)
	g_right = g(p_hat, right)
	delta_u = right.u - left.u
	return (g_right * left.p + g_left * right.p - delta_u)/(g_right + g_left)
end

#see (4.49) from Toro
p_guess(left::HydroStatus, right::HydroStatus, ::MeanPressure) = 0.5*(left.p + right.p)


"""
    node_clustering(s,v)

Return the clustering coefficient of the given node v in stream s:
```math
cc \\left(v \\right) = \\frac{ \\sum_{uw \\in V \\otimes V} \\left| T_{vu} \\cap T_{vw} \\cap T_{uw} \\right| }{\\sum_{uw \\in V \\otimes V} \\left| T_{vu} \\cap T_{vw} \\right|}
```
Note: If node v never has two neighbors present at the same
time in the stream, this function returns 0.

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function node_clustering(s::AbstractUndirectedStream, v::AbstractString)
    nomin=0
    denom=0
    N=Set(AbstractString[])
    for n in keys(neighborhood(s,v))
        push!(N,n)
    end
    for (u,w) in N ⊗ N
        nomin+=length((times(s,v,u) ∩ times(s,v,w)) ∩ times(s,u,w))
        denom+=length(times(s,v,u) ∩ times(s,v,w))
    end
    denom != 0 ? nomin/denom : 0.0
end

"""
    node_clustering(s,v,t)

Return the instantaneous clustering coefficient of the given node v
at the given time t in the given stream:
```math
cc_t \\left(v \\right)= \\frac{\\sum_{uw} vu_t vw_t uw_t}{\\sum_{uw} vu_t vw_t}
```
Note: If node v does not have two neighbors at the given
time t, this function returns 0.

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function node_clustering(s::AbstractUndirectedStream,v::AbstractString,t::Float64)
    Nt=Set(AbstractString[])
    for n in neighborhood(s,v,t)
        push!(Nt,n)
    end
    nom=sum([1.0 for (u,w) in Nt ⊗ Nt if ((t ∈ times(s,v,u)) & (t ∈ times(s,v,w)) & (t ∈ times(s,u,w)))])
    denom=sum([1.0 for (u,w) in Nt ⊗ Nt if ((t ∈ times(s,v,u)) & (t ∈ times(s,v,w)))])
    denom != 0 ? nom/denom : 0.0
end

"""
    node_clustering(s)

Return the node clustering coefficient of the given stream:
```math
cc \\left( V \\right)= \\frac{1}{n} \\sum_{v \\in V} n_v cc \\left( v \\right) = \\sum_{v \\in V} \\frac{\\left| T_v \\right|}{\\left|W \\right|} cc \\left(v \\right)
```
Where n is the number of nodes in the stream.
Note: If there is no node in the stream, this function returns 0.

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function node_clustering(s::AbstractUndirectedStream)
    if length(s.V)>0
        1.0/length(s.V)*sum([contribution(s,v)*node_clustering(s,v) for v in s.V])
    else
        0.0
    end
end

"""
    time_clustering(s,t)

Return time clustering coefficient of a given time instant in the stream:
```math
cc \\left( t \\right)= \\frac{\\sum_{v}cc_t\\left(v \\right) \\sum_{uw} vu_t vw_t}{\\sum_{v} \\sum_{uw} vu_t vw_t}
```

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function time_clustering(s::AbstractUndirectedStream,t::Float64)
    nom::Float64=0.0
    denom::Float64=0.0
    for v in s.V
        acc::Float64=0.0
        for (u,w) in s.V ⊗ s.V
            if ((t ∈ times(s,v,u)) & (t ∈ times(s,v,w)))
                acc+=1.0
            end
        end
        nom+=node_clustering(s,v,t) * acc
        denom+=acc
    end
    denom != 0 ? nom/denom : 0.0
end

"""
    time_clustering(s)

Return the time clustering coefficient of the given stream:
```math
cc \\left( T \\right) = \\int_{t} \\frac{\\left| V_t \\right|}{\\left| W \\right|} cc \\left( t \\right) dt
```
Note: If nodes are never present in the stream, this function returns 0.

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function time_clustering(s::AbstractUndirectedStream)
    τ = times(s)
    dW = duration(s.W)
    dW != 0 ? 1.0/dW*sum([time_clustering(s,0.5*(t[2]+t[1])) * length(nodes(s,0.5*(t[2]+t[1]))) for t in zip(τ[1:end-1],τ[2:end])]) : 0.0
end

"""
    time_clustering(s)

Return the time clustering coefficient of the given stream using a TimeCursor.
"""
function time_clustering(s::AbstractUndirectedStream,tc::TimeCursor)
    throw("Not Implemented")
end

"""
    clustering(s)

Return the clustering coefficient of the given stream:
```math
cc \\left( S \\right) = \\int_t \\frac{1}{\\left|T \\right|} \\sum_{v} \\frac{cc_t\\left(v \\right)}{\\left|V \\right|} dt
```

### Reference
- Matthieu Latapy, Tiphaine Viard and Clémence Magnien Social
  Networks Analysis and Mining, 8: 61, 2018. "Stream Graphs and
  Link Streams for the Modeling of Interactions over Time".
  [(arXiv)](https://arxiv.org/pdf/1710.04073.pdf)
"""
function clustering(s::AbstractUndirectedStream)
    card_TV = duration(s)*length(s.V)
    if card_TV==0
        return 0.0
    end
    τ = times(s)
    acc::Float64 = 0.0
    for v in s.V
        for t in zip(τ[1:end-1],τ[2:end])
            acc+=node_clustering(s,v,0.5*(t[1]+t[2]))*(t[2]-t[1])
        end
    end
    1.0/card_TV*acc
end


## fit logistic regression

using DataFrames, GLM, Plots
temp = [53, 57, 58, 63, 66, 67, 67, 67, 68, 69, 70, 70, 70, 70, 72, 73, 75, 75, 76, 76, 78, 79, 81]
failure = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0]

data = DataFrame(temp = temp, failure = failure)

logit_fit = glm(@formula(failure ~ temp), data, Binomial(), LogitLink())
#plot(temp, predict(logit_fit), legend = false, xlabel = "Temperature", ylab = "Probability")
#scatter!(temp, predict(logit_fit))

## metropolis-hastings
using Distributions
γ = 0.57721
function ll(α::Float64, β::Float64)
    a = exp.(α .+ β*temp)
    return prod( (a ./ (1 .+ a) ).^failure .* (1 ./ (1 .+ a)).^(1 .- failure) )
end
function mh_logit(T::Int, α_hat::Float64, β_hat::Float64, σ_hat::Float64)
    φ = Normal(β_hat, σ_hat)
    π = Exponential(exp(α_hat+γ))
    Α = ones(T)
    Β = ones(T)
    for t = 1:T-1
        α = log(rand(π))
        β = rand(φ)
        r = ( ll(α, β) / ll(Α[t], Β[t]) ) * ( pdf(φ, Β[t]) / pdf(φ, β) )
        if rand() < r
            Α[t+1] = α
            Β[t+1] = β
        else
            Α[t+1] = Α[t]
            Β[t+1] = Β[t]
        end
    end
    return Α, Β
end

# trace plot

Α, Β = mh_logit(10000, 15.04, -0.233, 0.108)

p1 = plot(Α, legend = false, xlab = "Intercept")
hline!([15.04])

p2 = plot(Β, legend = false, xlab = "Slope")
hline!([-0.233])

plot(p1, p2, layout = (1,2))

# mean trace plot

Αmean = cumsum(Α) ./ collect(1:length(Α))
Βmean = cumsum(Β) ./ collect(1:length(Β))

p1 = plot(Αmean, legend = false, xlab = "Intercept")
hline!([15.04])

p2 = plot(Βmean, legend = false, xlab = "Slope")
hline!([-0.233])

plot(p1, p2, layout = (1,2))


# Function for FAVAR by Bernanke et al. (2015)
import Pkg;
using LinearAlgebra, Distributions, Statistics, MultivariateStats;
using ProgressMeter;
using Gadfly, Colors;

function func_std(mat)
    # Function to standardize data
    T, k = size(mat);
    μ = mean(mat; dims = 1);
    σ = std(mat; dims = 1);
    return (mat .- μ) ./ σ;
end

function func_favar(yₘ, yₚ, p, ζ)
    # yₘ: data - main block
    # yₚ: data - factor block - to be
    # p: lag term for VAR
    # ζ: Number of factors to be extracted

    T, kₘ = size(yₘ);

    # For principal component analysis (PCA), always use standardized data.
    res_pca = fit(PCA, func_std(yₚ); maxoutdim = ζ);
    𝔽ₒ = projection(res_pca);

    Q = yₚ';
    Z = [yₘ 𝔽ₒ]';
    ℾₒ = (Q * Z')/(Z * Z');
    ℾₒ𝑦 = ℾₒ[:, (1:kₘ)];
    x̃ = yₚ - (ℾₒ𝑦 * yₘ')';

    ssrₒ = sum((Q - ℾₒ * Z).^2, dims = 2)

    # Start the iteraction process
    𝔽₁ = 𝔽ₒ;
    ℾ₁ = ℾₒ; ℾ₁𝑦 = ℾₒ𝑦;
    ssr₁ = ssrₒ

    n_ϝ = 0; # Count iterations;
    while sum(abs(ssr₁ .- ssrₒ) > 10^(-6))
        # If no convergence, repeat the process
        # Update the criteria
        𝔽ₒ = 𝔽₁;
        ssrₒ = ssr₁;

        # Redraw component
        res_pca = fit(PCA, func_std(x̃); maxoutdim = ζ);
        𝔽₁ = projection(res_pca);

        Q = yₚ';
        Z = [yₘ 𝔽₁]';
        ℾ₁ = (Q * Z')/(Z * Z');
        ℾ₁𝑦 = ℾ₁[:, (1:kₘ)];
        x̃ = yₚ - (ℾ₁𝑦 * yₘ')';

        ssr₁ = sum((Q - ℾ₁ * Z).^2, dims = 2);

        n_ϝ = n_ϝ + 1;
    end

    # conditional check for factor loading (optional)
    #=
    #Check on elements of factor loading that belongs to first series in the
    # information set matrix - yₚ
    # If an element is negative, multiply whole factor loading with -1 and
    # then respective factor

    for iₚ = (kₘ + 1):size(Gamma_final, 2)
        if ℾ₁[1, iₚ] < 0
            ℾ₁[:, iₚ] = -1 .* ℾ₁[:, iₚ];
            𝔽₁[:, (iₚ - kₘ)] = -1 .* 𝔽₁[:, (iₚ - kₘ)];
        end
    end
    =#

    Q = yₚ';
    Z = [yₘ 𝔽₁]';
    ℾ₁ = (Q * Z')/(Z * Z');
    ℾ₁𝑦 = ℾ₁[:, (1:kₘ)];
    x̃ = yₚ - (ℾ₁𝑦 * yₘ')';

    # compute VAR(p)
    𝔇 = [yₘ 𝔽₁];

    𝚩, 𝞄, 𝝨 = func_VAR(𝔇, p);

    return 𝚩, 𝞄, 𝝨, 𝔽₁, ℾ₁;
end


using ChooseOptimizer, JuMP, Multisets

"""
`matrix_match(A,B)` returns permutation matrices `P` and `Q`
so that `P*A==B*Q` or throws an error if not possible.
"""
function matrix_match(A::AbstractMatrix, B::AbstractMatrix)
    err_msg = "Matrices do not match"

    # Check that A and B have same dimensions and same elements
    r, c = size(A)
    if (r, c) != size(B) || Multiset(collect(A)) != Multiset(collect(B))
        error(err_msg)
    end

    m = Model(get_solver())

    # Set up P and Q are permutation matrices

    @variable(m, P[1:r, 1:r], Bin)
    @variable(m, Q[1:c, 1:c], Bin)

    for i = 1:r
        @constraint(m, sum(P[i, j] for j = 1:r) == 1)
        @constraint(m, sum(P[j, i] for j = 1:r) == 1)
    end

    for i = 1:c
        @constraint(m, sum(Q[i, j] for j = 1:c) == 1)
        @constraint(m, sum(Q[j, i] for j = 1:c) == 1)
    end

    # require P*A == B*Q

    for i = 1:r
        for j = 1:c
            @constraint(
                m,
                sum(P[i, k] * A[k, j] for k = 1:r) == sum(B[i, k] * Q[k, j] for k = 1:c)
            )
        end
    end

    optimize!(m)
    status = Int(termination_status(m))
    if status != 1
        error(err_msg)
    end

    PP = Int.(value.(P))
    QQ = Int.(value.(Q))
    return PP, QQ
end


using DataInterpolations, Test
using FiniteDifferences
using DataInterpolations: derivative

function test_derivatives(func, tspan, name::String)
  trange = range(minimum(tspan), maximum(tspan), length=32)[2:end-1]
  @testset "$name" begin
    for t in trange
      # Linearly spaced points might lead to evaluations outside
      # trange
      cdiff = central_fdm(5, 1; geom=true)(_t -> func(_t), t)
      adiff = derivative(func, t)
      @test isapprox(cdiff, adiff, atol=1e-8)
    end
  end
end

# Linear Interpolation
u = 2.0collect(1:10)
t = 1.0collect(1:10)
A = LinearInterpolation(u,t)

test_derivatives(A, t, "Linear Interpolation (Vector)")

u = vcat(2.0collect(1:10)', 3.0collect(1:10)')
A = LinearInterpolation(u,t)

test_derivatives(A, t, "Linear Interpolation (Matrix)")

# Quadratic Interpolation
u = [1.0, 4.0, 9.0, 16.0]
t = [1.0, 2.0, 3.0, 4.0]
A = QuadraticInterpolation(u,t)

test_derivatives(A, t, "Quadratic Interpolation (Vector)")

u = [1.0 4.0 9.0 16.0; 1.0 4.0 9.0 16.0]
A = QuadraticInterpolation(u,t)

test_derivatives(A, t, "Quadratic Interpolation (Matrix)")

# Lagrange Interpolation
u = [1.0, 4.0, 9.0]
t = [1.0, 2.0, 3.0]
A = LagrangeInterpolation(u,t)

test_derivatives(A, t, "Lagrange Interpolation (Vector)")

# Lagrange Interpolation
u = [1.0 4.0 9.0; 1.0 2.0 3.0]
t = [1.0, 2.0, 3.0]
A = LagrangeInterpolation(u,t)

test_derivatives(A, t, "Lagrange Interpolation (Matrix)")

# Lagrange Interpolation
u = [[1.0, 4.0, 9.0], [3.0, 7.0, 4.0], [5.0, 4.0, 1.0]]
t = [1.0, 2.0, 3.0]
A = LagrangeInterpolation(u,t)

test_derivatives(A, t, "Lagrange Interpolation (Vector of Vectors)")

# Lagrange Interpolation
u = [[3.0 1.0 4.0; 1.0 5.0 9.0], [2.0 6.0 5.0; 3.0 5.0 8.0], [9.0 7.0 9.0; 3.0 2.0 3.0]]
t = [1.0, 2.0, 3.0]
A = LagrangeInterpolation(u,t)

test_derivatives(A, t, "Lagrange Interpolation (Vector of Matrices)")

# Akima Interpolation
u = [0.0, 2.0, 1.0, 3.0, 2.0, 6.0, 5.5, 5.5, 2.7, 5.1, 3.0]
t = collect(0.0:10.0)
A = AkimaInterpolation(u, t)

test_derivatives(A, t, "Akima Interpolation")

# QuadraticSpline Interpolation
u = [0.0, 1.0, 3.0]
t = [-1.0, 0.0, 1.0]

A = QuadraticSpline(u,t)

test_derivatives(A, t, "Quadratic Interpolation (Vector)")

# QuadraticSpline Interpolation
u = [[1.0, 2.0, 9.0], [3.0, 7.0, 5.0], [5.0, 4.0, 1.0]]
t = [-1.0, 0.0, 1.0]

A = QuadraticSpline(u,t)

test_derivatives(A, t, "Quadratic Interpolation (Vector of Vectors)")

# QuadraticSpline Interpolation
u = [[1.0 4.0 9.0; 5.0 9.0 2.0], [3.0 7.0 4.0; 6.0 5.0 3.0], [5.0 4.0 1.0; 2.0 3.0 8.0]]
t = [-1.0, 0.0, 1.0]

A = QuadraticSpline(u,t)

test_derivatives(A, t, "Quadratic Interpolation (Vector of Matrices)")

# CubicSpline Interpolation
u = [0.0, 1.0, 3.0]
t = [-1.0, 0.0, 1.0]

A = CubicSpline(u,t)

test_derivatives(A, t, "Cubic Spline Interpolation (Vector)")

# CubicSpline Interpolation
u = [[1.0, 2.0, 9.0], [3.0, 7.0, 5.0], [5.0, 4.0, 1.0]]
t = [-1.0, 0.0, 1.0]

A = CubicSpline(u,t)

test_derivatives(A, t, "Cubic Spline Interpolation (Vector of Vectors)")

# CubicSpline Interpolation
u = [[1.0 4.0 9.0; 5.0 9.0 2.0], [3.0 7.0 4.0; 6.0 5.0 3.0], [5.0 4.0 1.0; 2.0 3.0 8.0]]
t = [-1.0, 0.0, 1.0]

A = CubicSpline(u,t)

test_derivatives(A, t, "Cubic Spline Interpolation (Vector of Matrices)")

# BSpline Interpolation and Approximation
t = [0,62.25,109.66,162.66,205.8,252.3]
u = [14.7,11.51,10.41,14.95,12.24,11.22]

A = BSplineInterpolation(u,t,2,:Uniform,:Uniform)

test_derivatives(A, t, "BSpline Interpolation (Uniform, Uniform)")

A = BSplineInterpolation(u,t,2,:ArcLen,:Average)

test_derivatives(A, t, "BSpline Interpolation (Arclen, Average)")

A = BSplineApprox(u,t,3,4,:Uniform,:Uniform)

test_derivatives(A, t, "BSpline Approx (Uniform, Uniform)")


#Prezentacja wyników do sprawozdania: Zadanie P1.2
#Kacper Kulczak 279079

#Pomocnicza funkcja pomgająca. Potegowanie
function pow(n,w)
    result = one(n)
    for i = one(w):w
        result *= n
    end
    return result
end

# Funkcja obliczajaca stala e za pomoca przyblizenia granicy (1+1/n)^n
function definition_e(n)
    result = (one(n) + one(n) / n)
    for i = one(n):log2(n)
        result *= result
    end
    return result
end
# Funkcja obliczajaca stala e za pomoca szeregu taylora sum(1/ n!)
function taylor_series(n)
    result = zero(n)
    for i = zero(n):n
        result += one(n) / factorial(n-i)
    end
    return result
end
# Funkcja obliczajaca stala e za pomoca ulamka lancuchowego
function continued_fraction(n)
    i = n + n
    result = i
    result = one(n) + one(n) / result
    result = one(n) + one(n) / result
    i -= one(n) + one(n)
    while i > one(n) + one(n)
        result = i + one(n) / result
        result = one(n) + one(n) / result
        result = one(n) + one(n) / result
        i -= one(n) + one(n)
    end
    result = one(n) + one(n) + one(n) / result
    result = one(n) + one(n) / result
    result = one(n) + one(n) + one(n) / result
    return result

end
#Drugi szereg SUM (3k+1)/(3k)!
function second_series(n)
    result = zero(n)
    for i = zero(n):n
        result += (one(n) + (convert(typeof(n),3) * (n-i) ) * ( convert(typeof(n),3) * (n-i) ) )/ factorial(convert(typeof(n),3) *(n-i))
    end
    return result
end

# Trzeci szreg SUM (4k + 3) / 2^(2k+1) * (2k+1)!
function third_series(n)
    result = zero(n)
    for i = zero(n):n
        temp = convert(typeof(n),4) * (n-i) + one(n) + one(n) + one(n)
        temp /= factorial(convert(typeof(n),2) *(n-i) + one(n)) * pow( convert(typeof(n),2), convert(typeof(n),2)*(n-i) + one(n))
        result += temp
    end
    return result * result
end
#Test zapisujace wyniki na zadanym zasiegu 1->range
function test(fun,range)
    container= [[],[],[],[]]
    for i = one(range) : range
        temp = fun(i)
        err = convert(typeof(range),eu)-temp
        push!(container[1], i)
        push!(container[2], temp)
        push!(container[3],err)
        push!(container[4], -log10(abs( err )) )
    end
    return container
end

#Test dla definicji, przedstawiajacy tylko niektore iteracje
function test_def(fun,range)
    container= [[],[],[],[]]
    x = one(range)
    for i = one(range) : range
        x *= convert(typeof(range),8.)
        temp = fun(x)
        err = convert(typeof(range),eu)-temp
        push!(container[1], x)
        push!(container[2], temp)
        push!(container[3],err)
        push!(container[4], -log10(abs( err )) )
    end
    return container
end

#Funkcja formatujaca wyjscie
function myshow(list)
    for i = 1:length(list[1])
        @printf "n=%d val=%.f error=%.4e #cyfr=%.0f\n" list[1][i] list[2][i] list[3][i] list[4][i]
    end
end

setprecision(256)
@show precision(BigFloat)
range = BigFloat(60.)
data1 = test_def(definition_e, BigFloat(30.) )
data2 = test(taylor_series, range)
data3 = test(continued_fraction, range )
data4 = test(second_series, range)
data5 = test(third_series, range)
@printf "\n\n\n"
println("Definition of e")
myshow(data1)
@printf "\n\n\n"
println("Taylor series")
myshow(data2)
@printf "\n\n\n"
println("Contiuned fraction")
myshow(data3)
@printf "\n\n\n"
println("Second Series")
myshow(data4)
@printf "\n\n\n"
println("Third Series")
myshow(data5)

#Funckje do obliczenia ilości mnożeń i dzieleń w zależnosci od iteracji w wykorzytanych
# przeze mnie metodach
function Tdef(n)
    return one(n)+ one(n)+log2(n)
end
function TCF(n)
    return n * convert(typeof(n),3)
end
function Tser1(n)
    return n * convert(typeof(n),2)
end
function Tser2(n)
    return n * convert(typeof(n),4)
end
function Tser3(n)
    return n * convert(typeof(n),5) + one(n)
end


using MultivariateStats
using Base.Test

srand(34568)

## data

d = 3
n = 5

X = rand(d, n)
mv = vec(mean(X, 2))
C = cov(X, 2)
C0 = copy(C)

emax = maximum(eigvals(C))
rc = 0.01
Cr = C + (emax * rc) * eye(d)

## cov_whitening

cf = cholfact(C, :U)
W = cov_whitening(cf)
@test istriu(W)
@test W'C * W ≈ eye(d)

cf = cholfact(C, :L)
W = cov_whitening(cf)
@test istriu(W)
@test W'C * W ≈ eye(d)

W = cov_whitening(C)
@test C == C0
@test istriu(W)
@test W'C * W ≈ eye(d)

W = cov_whitening(C, 0.0)
@test C == C0
@test istriu(W)
@test W'C * W ≈ eye(d)

W = cov_whitening(C, rc)
@test istriu(W)
@test W'Cr * W ≈ eye(d)

# Whitening from data

f = fit(Whitening, X)
W = f.W
@test isa(f, Whitening{Float64})
@test mean(f) === f.mean
@test istriu(W)
@test W'C * W ≈ eye(d)
@test transform(f, X) ≈ W' * (X .- f.mean)

f = fit(Whitening, X; regcoef=rc)
W = f.W
@test W'Cr * W ≈ eye(d)

f = fit(Whitening, X; mean=mv)
W = f.W
@test W'C * W ≈ eye(d)

f = fit(Whitening, X; mean=0)
Cx = (X * X') / (n - 1)
W = f.W
@test W'Cx * W ≈ eye(d)

# invsqrtm

R = invsqrtm(C)
@test C == C0
@test R ≈ inv(sqrtm(C))


function bisection(f::Function, a, b; atol = 0.001, rtol = 0.001)
    x = (a+b) / 2
    while (b-a) > atol || (b - a)/x > rtol
        if f(a)*f(x) < 0
            b = x
            x = (a+b)/2
        elseif f(x)*f(b) < 0
            a = x
            x = (a+b)/2
        elseif f(x) == 0
            return x
        end
    end
    return x
end


import PyPlot
const plt = PyPlot

function M(p::Int64,u)
    if p == 2
        if u >= 0 && u <= 2
            return 1 - abs(u-1)
        else
            return 0.0
        end
    else
        return M(p-1,u)*u/(p-1) + (p-u)/(p-1)*M(p-1,u-1)
    end
end

function main()
    @time println( M(2,1.1) )
    @time println( M(3,1.1) )
    @time println( M(4,1.1) )

    NptsPlot = 100
    x = Array{Float64}( linspace(0.0,5.0,NptsPlot) )
    y2 = Array{Float64}(NptsPlot)
    y3 = Array{Float64}(NptsPlot)
    y4 = Array{Float64}(NptsPlot)
    y5 = Array{Float64}(NptsPlot)
    for i = 1:NptsPlot
        y2[i] = M(2,x[i])
        y3[i] = M(3,x[i])
        y4[i] = M(4,x[i])
        y5[i] = M(5,x[i])
    end
    plt.clf()
    plt.plot( x, y2, linewidth=2.0, label="M2.png" )
    plt.plot( x, y3, linewidth=2.0, label="M3.png" )
    plt.plot( x, y4, linewidth=2.0, label="M4.png" )
    plt.plot( x, y5, linewidth=2.0, label="M5.png" )
    plt.grid(true)
    plt.legend()
    plt.savefig("M2.png", dpi=300)
end

main()


using ProgressBars, Plots, StatsBase, LaTeXStrings, JLD, Distributions, StatsPlots

Plots.default(titlefontsize=12, tickfontsize=10, labelfontsize=16,
    fontfamily="Computer Modern")

SampleNumber = 100000
NList = [5, 10, 100, 1000]
Data = [[sum(rand(N)) for i in ProgressBar(1:SampleNumber)] for N ∈ NList]

cd(dirname(@__FILE__))
save("../../Data/Q1/Q1-samples.jld","Samples", Data)

plots = []
colors = [:steelblue, :purple, :green, :brown]
for i ∈ 1:4
    Plt = begin
    histogram(Data[i],nbins = 60, normalize = true, c = colors[i], label = L"Histogram\ of\ Samples")
    plot!(Normal(mean(Data[i]),std(Data[i])), c = :black, lineWidth = 10,
        label = L"\mathcal{Normal}(\mu = %$(round(mean(Data[i]),digits = 2)), \sigma = %$(round(std(Data[i]),digits = 2)))")
    plot!(frame = :box, title = "N = $(NList[i]), $SampleNumber Samples",
        xlabel = L"X_N", ylabel = L"Counting\ ratio")
    end
    push!(plots,Plt)
end

plot(plots..., size = (1000,1000), plot_title = "Normalized Distribution of Samples")
savefig("../../Figs/Q1/Q1-Hist.pdf")


# [1] M. R. Teague, “Image analysis via the general theory of moments*,” Journal of the Optical Society of America, vol. 70, no. 8, p. 920, Aug. 1980.
# https://doi.org/10.1364/josa.70.000920
function measure_feature(property::RegionEllipse, t::IndexedTable, labels::AbstractArray, N::Int = 0)
    init = StepRange(typemax(Int), -1, -typemax(Int))
    ℳ₀₀ = zeros(N)
    ℳ₁₀ = zeros(N)
    ℳ₀₁ = zeros(N)
    ℳ₁₁ = zeros(N)
    ℳ₂₀ = zeros(N)
    ℳ₀₂ = zeros(N)
    # TODO Document discretization model that underpins these moment computations.
    for i in CartesianIndices(labels)
        l = labels[i]
        if  l != 0
            y, x = i.I
            ℳ₀₀[l] += 1
            ℳ₁₀[l] += x
            ℳ₀₁[l] += y
            ℳ₁₁[l] += x*y
            ℳ₂₀[l] += x^2 + 1/12
            ℳ₀₂[l] += y^2 + 1/12
        end
    end
    t = pushcol(t, :M₀₀ => ℳ₀₀, :M₁₀ => ℳ₁₀, :M₀₁ => ℳ₀₁, :M₁₁ => ℳ₁₁, :M₂₀ => ℳ₂₀, :M₀₂ => ℳ₀₂)
    fill_properties(property, t)
end

function fill_properties(property::RegionEllipse, t::IndexedTable)
    t = property.centroid ? compute_centroid(t) : t
    t = property.semi_axes ? compute_semi_axes(t) : t
    t = property.orientation ? compute_orientation(t) : t
    t = property.eccentricity ? compute_eccentricity(t) : t
end

function compute_centroid(t::IndexedTable)
    @transform t {centroid = (:M₀₁ / :M₀₀, :M₁₀ / :M₀₀)}
end

function compute_semi_axes(t::IndexedTable)
    semi_axes = select(t, (:M₀₀,  :M₁₀, :M₀₁, :M₁₁, :M₂₀, :M₀₂) => row -> compute_semi_axes(row...))
    t = pushcol(t, :semi_axes => semi_axes)
end

function compute_semi_axes(M₀₀::Real, M₁₀::Real, M₀₁::Real, M₁₁::Real, M₂₀::Real, M₀₂::Real)
    μ′₂₀ = (M₂₀ / M₀₀) - (M₁₀ / M₀₀)^2
    μ′₀₂ = (M₀₂ / M₀₀) - (M₀₁ / M₀₀)^2
    μ′₁₁ = (M₁₁ / M₀₀) - ((M₁₀ / M₀₀) * (M₀₁ / M₀₀))

    # See Equations (7) and (8) in [1].
    l₁ = sqrt((μ′₂₀ + μ′₀₂ + sqrt(4 * μ′₁₁^2 +  (μ′₂₀ - μ′₀₂)^2)) / (1 / 2))
    l₂ = sqrt((μ′₂₀ + μ′₀₂ - sqrt(4 * μ′₁₁^2 +  (μ′₂₀ - μ′₀₂)^2)) / (1 / 2))
    min(l₁, l₂), max(l₁, l₂)
end

function compute_orientation(t::IndexedTable)
    orientation = select(t, (:M₀₀,  :M₁₀, :M₀₁, :M₁₁, :M₂₀, :M₀₂) => row -> compute_orientation(row...))
    t = pushcol(t, :orientation => orientation)
end

function compute_eccentricity(t::IndexedTable)
    eccentricity = select(t, (:M₀₀,  :M₁₀, :M₀₁, :M₁₁, :M₂₀, :M₀₂) => row -> compute_eccentricity(row...))
    t = pushcol(t, :eccentricity => eccentricity)
end

function compute_eccentricity(M₀₀::Real, M₁₀::Real, M₀₁::Real, M₁₁::Real, M₂₀::Real, M₀₂::Real)
    b, a = compute_semi_axes(M₀₀, M₁₀, M₀₁, M₁₁, M₂₀, M₀₂)
    e = sqrt(1 - (b/a)^2)
end

# [1] M. R. Teague, “Image analysis via the general theory of moments*,” Journal of the Optical Society of America, vol. 70, no. 8, p. 920, Aug. 1980.
# https://doi.org/10.1364/josa.70.000920
function compute_orientation(M₀₀::Real, M₁₀::Real, M₀₁::Real, M₁₁::Real, M₂₀::Real, M₀₂::Real)
    μ′₂₀ = M₂₀ / M₀₀ - (M₁₀ / M₀₀)^2
    μ′₀₂ = M₀₂ / M₀₀ - (M₀₁ / M₀₀)^2
    μ′₁₁ = M₁₁ / M₀₀ - (M₁₀ / M₀₀) * (M₀₁ / M₀₀)

    # Ellipse tilt angle for various cases of signs of the second moments [1].
    θ = 0.0
    if μ′₂₀ - μ′₀₂ == 0 && μ′₁₁  == 0
        θ = 0.0
    elseif μ′₂₀ - μ′₀₂ == 0 && μ′₁₁  > 0
        θ = 45.0
    elseif μ′₂₀ - μ′₀₂ == 0 && μ′₁₁  < 0
        θ = -45.0
    elseif μ′₂₀ - μ′₀₂ > 0 && μ′₁₁ == 0
        θ = 0.0
    elseif μ′₂₀ - μ′₀₂ < 0 && μ′₁₁ == 0
        θ = -90.0
    elseif μ′₂₀ - μ′₀₂ > 0 && μ′₁₁ > 0
        # 0 < θ < 45
        ξ = 2*μ′₁₁ / (μ′₂₀ - μ′₀₂)
        θ = (1/2) * atand(ξ)
    elseif μ′₂₀ - μ′₀₂ > 0 && μ′₁₁ < 0
        # -45 < θ < 0
        ξ = 2*μ′₁₁ / (μ′₂₀ - μ′₀₂)
        θ = (1/2) * atand(ξ)
    elseif μ′₂₀ - μ′₀₂ < 0 && μ′₁₁ > 0
        # 45 < θ < 90
        ξ = 2*μ′₁₁ / (μ′₂₀ - μ′₀₂)
        θ = (1/2) * atand(ξ) + 90.0
    elseif μ′₂₀ - μ′₀₂ < 0 && μ′₁₁ < 0
        # -90 < θ < -45
        ξ = 2*μ′₁₁ / (μ′₂₀ - μ′₀₂)
        θ = (1/2) * atand(ξ) - 90.0
    end
    θ
end


#HW2 Problem 3
#Minhao Yan my497, Wentong Chen wc422

#Firm's Profit Function
function profit(η,c1,c2,q)
    profit1=sum(q)^(-1/η)*q[1]-c1*q[1] # Firm 1's profit
    profit2=sum(q)^(-1/η)*q[2]-c2*q[2] # Firm 2's profit
    profit=[profit1; profit2]
    return profit
end

#Function that solves q_prime
function q_prime(η,c1,c2,q)
    q_prime11=(-1/η)*sum(q)^(-1/η-1)-(1/η)*sum(q)^(-1/η-1)+(1/η^2+1/η)*q[1]*sum(q)^(-1/η-2)-c1
    q_prime12=(-1/η)*sum(q)^(-1/η-1)+(1/η^2+1/η)*sum(q)^(-1/η-2)*q[1]
    q_prime21=(-1/η)*sum(q)^(-1/η-1)+(1/η^2+1/η)*sum(q)^(-1/η-2)*q[2]
    q_prime22=(-1/η)*sum(q)^(-1/η-1)-(1/η)*sum(q)^(-1/η-1)+(1/η^2+1/η)*q[2]*sum(q)^(-1/η-2)-c2
    q_prime_m=[q_prime11 q_prime12; q_prime21 q_prime22]
    return q_prime_m
end

#Newton's method
function cournot_newton(η,c1,c2,initial_guess,tolerance)
    diff=[Inf; Inf]
    q_old=initial_guess
    q=[1e10;1e10]

    while abs.(diff)>tolerance
        q = q_old - transpose(transpose(profit(η,c1,c2,q_old))*inv(q_prime(η,c1,c2,q_old)))
        diff= q- q_old
        q_old= q
    end
 #  Print out final output
 println("The solution is $q.")
end

#   Parameters and Inputs
η=1.6
c1=0.15
c2=0.2
initial_guess=[20;20]
tolerance=[1e-5;1e-5]

#   Optimization
q_opt=cournot_newton(η,c1,c2,initial_guess,tolerance)


using Base.Test
using Clustering
using Distances

srand(34568)

X1 = randn(2, 200) .+ [0., 5.]
X2 = randn(2, 200) .+ [-5., 0.]
X3 = randn(2, 200) .+ [5., 0.]
X = hcat(X1, X2, X3)
n = size(X,2)

D = pairwise(Euclidean(), X)

R = dbscan(D, 1.0, 10)
@test isa(R, DbscanResult)
k = length(R.seeds)
# println("k = $k")
@test k == 3
@test all(R.assignments .<= k)
@test length(R.assignments) == n
@test length(R.counts) == k
for c = 1:k
    @test countnz(R.assignments .== c) == R.counts[c]
end
@test all(R.counts .>= 180)


function nlp_cvx_202_010(
    optimizer,
    objective_tol,
    primal_tol,
    dual_tol,
    termination_target = TERMINATION_TARGET_LOCAL,
    primal_target = PRIMAL_TARGET_LOCAL,
)
    # Test Goals:
    # - linear objective
    # - intersection convex quadratic constraints
    # Variants
    #   010 - one binding constraint (inflection point)
    #   011 - one binding constraint (inflection point)
    #   012 - one binding constraint (non-inflection point)
    #   013 - one binding constraint (non-inflection point)
    #   014 - intersection set

    model = Model(optimizer)

    @variable(model, x)
    @variable(model, y)
    @variable(model, z)

    @objective(model, Min, -z)
    @NLconstraint(model, x^2 + y^2 <= z)
    @NLconstraint(model, x^2 + y^2 <= -z + 1)

    optimize!(model)

    check_status(model, FEASIBLE_PROBLEM, termination_target, primal_target)
    check_objective(model, -1, tol = objective_tol)
    return check_solution([x, y, z], [0, 0, 1], tol = primal_tol)
end


# Create an image
x = LinRange(-3, 3, 150)
y = LinRange(-2, 2, 100)
# RGB values
r = (1 .+ cos.(atan.(y, x')))/2
g = (1 .+ sin.(atan.(y, x')))/2
b = exp.(-(x'.^2 .+ y.^2)/4)
data = cat(r, g, b, dims=3)
# Draw the image
imshow(data)


# This file calculates scattering from the Helmholtz equation via the adaptive spectral method.
# Δu + k^2u = 0,
# ∂u∂n|Γ = 0,
# ∂u^i/∂n = -im k d[2] e^{im k x⋅d},
# u = u^i + u^s.
# The normal derivative ∂u/∂n of the entire wave is calculated on the sound-hard boundaries.
# The scattered wave is calculated by convolving [u] with the fundamental solution.
# Then, the total wave is obtained by summing the incident and the scattered waves.

using ApproxFun, SingularIntegralEquations
include("Scatteraux.jl")

k = 50.
ω = 2π
d = (1,-1)
d = d[1]/hypot(d[1],d[2]),d[2]/hypot(d[1],d[2])
ui(x,y) = exp(im*k*(d⋅(x,y)))

# The Helmholtz Green's function, split into singular and nonsingular pieces.
g1(x,y) = besselj0(k*abs(y-x))/2
g2(x,y) = x == y ? -k^2/4 : -k*besselj1(k*abs(y-x))./abs(y-x)/2
g3(x,y) = g3neumann(x,y) # In /Scatteraux.jl
g4old(x,y) = im*k/4*hankelh1(1,k*abs(y-x))./abs(y-x).*imag(y-x)
g4(x,y) = im*k/4*besselj1(k*abs(y-x))./abs(y-x).*imag(y-x)  # For linesum
g5(x,y) = -k/2*besselj1(k*abs(y-x))./abs(y-x).*imag(y-x)  # For logkernel
g6(x,y) = k/2*abs(y-x).*(bessely1(k*abs(y-x)) - 2besselj1(k*abs(y-x)).*logabs(y-x)/π) # For Re{Cauchy}

dom = ∪(Interval,[-2.,1.],[-1.,2.])
sp = Space(dom)
cwsp,cwsp2 = CauchyWeight(sp⊗sp,0),CauchyWeight(sp⊗sp,2)
∂ui∂nΓ,⨍ = Fun(t->-im*k*d[2]*ui(real(t),imag(t)),sp),
                DefiniteLineIntegral(PiecewiseSpace(map(d->JacobiWeight(.5,.5,Ultraspherical(1,d)),dom.domains)))#DefiniteLineIntegral(dom)

@time G = GreensFun(g1,cwsp2;method=:Cholesky) + GreensFun(g2,cwsp;method=:Cholesky) + GreensFun(g3,sp⊗sp;method=:Cholesky)

@time u = ⨍[G]\-∂ui∂nΓ
println("The length of u is: ",length(u))
us(x,y) = linesum(g4,u,complex(x,y))+logkernel(g5,u,complex(x,y))+π*real(cauchy(g6,real(u),complex(x,y)))+π*im*real(cauchy(g6,imag(u),complex(x,y)))
#dom += 0im


using AngleBetweenVectors
using Test

point1 = (1.0, 0.0)
point2 = (0.0, 1.0)
point3 = (-1.0, 0.0)

@test angle(point1, point2) == angle(point2, point1)
@test angle(point1, point3) == angle(point3, point1)
@test angle(point3, point2) == angle(point2, point3)

@test angle(point1, point2) == pi/2.0
@test angle(point1, point3) == pi/1.0

point1 = (1.0, -1.0, 0.0,  0.0)
point2 = (0.0,  1.0, 0.0, -1.0)

# note: 2pi/3 !== Float64(2*BigFloat(pi)/3)
@test angle(point1, point2) == Float64(2*BigFloat(pi)/3)
@test angle(point1, point2) == angle(point2, point1)

@test (@inferred angle(0:1, -1:0)) == pi/2 # abstract vector
@test (@inferred angle(0:1, -1.0:0.0)) == pi/2 # different eltype
@test (@inferred angle(Real[0.0 1], Int16[-1 0])) == pi/2 # array
@test (@inferred angle(0:1, (-1.0:0.0)*1im)) == pi/2 # complex

@test isnan(@inferred angle(zeros(2), 1:2)) # zero point returns NaN


# # Foundations of Bilevel Programming: Example 2
# This example is from the book _Foundations of Bilevel Programming_ by Stephan
# Dempe, Chapter 3.2, Page 25. [url](https://www.springer.com/gp/book/9781402006319)
# Moving the bound on x to lower level

# Model of the problem
# First level
# ```math
# \min 3x + y,\\
# \notag s.t.\\
# x \leq 5,\\
# y \leq 8,\\
# y \geq 0,\\
# ```
# Second level
# ```math
# \min -x,\\
# \notag s.t.\\
# x + y <= 8,\\
# 4x + y >= 8,\\
# 2x + y <= 13,\\
# 2x - y <= 0,\\
# x <= 5\\
# ```

using BilevelJuMP
using Ipopt
using JuMP
using Test

model = BilevelModel(Ipopt.Optimizer, mode = BilevelJuMP.ProductMode(1e-9))

# First we need to define all of the variables in the upper and lower problems:

@variable(Upper(model), y, start = 8 / 15)
@variable(Lower(model), x, start = 3.5 * 8 / 15)

# Then we can add the objective and constraints of the upper problem:
# Upper level objective function
@objective(Upper(model), Min, 3x + y)

# Upper level constraints
@constraints(Upper(model), begin
    u1, x <= 5
    u2, y <= 8
    u3, y >= 0
end)

# Followed by the objective and constraints of the lower problem:
# Lower level objective function
@objective(Lower(model), Min, -x)

# Lower level constraints
@constraint(Lower(model), l1,  x +  y <= 8)
@constraint(Lower(model), l2, 4x +  y >= 8)
@constraint(Lower(model), l3, 2x +  y <= 13)
@constraint(Lower(model), l4, 2x - 7y <= 0)

# !!! tip
#     You can use the singular `@constraint` macro or the plural `@constraints`!

# We can also set hints for the variables associated with the problems.

# In this example, we know the duals on the lower constraints are in
# the set `[-15, 15]`:
for c in [l1, l2, l3, l4]
    BilevelJuMP.set_dual_upper_bound(c, 15)
    BilevelJuMP.set_dual_lower_bound(c, -15)
end

# While we think the primal variables are in `[-10, 6]` for `x` and `[-1, 9]`
# for `y`. These hints are optional. But supplying them (e.g., from domain
# knowledge) can be helpful for the solver.

BilevelJuMP.set_primal_lower_bound_hint(x, -10)
BilevelJuMP.set_primal_upper_bound_hint(x, 6)
BilevelJuMP.set_primal_lower_bound_hint(y, -1)
BilevelJuMP.set_primal_upper_bound_hint(y, 9)

# Now we can solve the problem and verify the solution again that reported by
# Dempe.

optimize!(model)

# Automated testing

@test objective_value(model) ≈ 3 * (3.5 * 8 / 15) + (8 / 15) atol=1e-4
@test BilevelJuMP.lower_objective_value(model) ≈ -3.5 * 8 / 15 atol=1e-4
@test objective_value(Lower(model)) ≈ -3.5 * 8 / 15 atol=1e-4
@test value(x) ≈ 3.5 * 8 / 15 atol=1e-4
@test value(y) ≈ 8 / 15 atol=1e-4
@test value(u1) ≈ 3.5 * 8 / 15 atol=1e-4
@test value(l1) ≈ 4.5 * 8 / 15 atol=1e-4
@test dual(l1) ≈ [0] atol=1e-4
@test dual(l3) ≈ [0] atol=1e-4

# TODO: why are these commented out?    #src
# @test dual(l2) #≈ [0] atol=atol       #src
# @test dual(l4) #≈ [0] atol=atol       #src


# ----------
# Geometry Module, providing Geometry convertion function
# ----------

# xyz Coordinetes -> rθz Coordinates
function xy_to_polar(ux,uy;Lx=2π,Ly=Lx,T=Float32)
  nx,ny,nz = size(ux);
  dev = CPU();
  grid = TwoDGrid(dev, nx, Lx, ny, Ly; T=T)
  Ur,Uθ = xy_to_polar(ux,uy,grid;Lx=2π,Ly=Lx,T=Float32);
  return Ur,Uθ;
end

function xy_to_polar(ux::Array,uy::Array,grid;Lx=2π,Ly=Lx,T=Float32)
#=
  Function for converting x-y vector to r-θ vector, using linear transform
    [x']  =  [cos(θ) -rsin(θ)][r']
    [y']     [sin(θ)  rcos(θ)][θ']
    So e_r =  cosθ ̂i + sinθ ̂j
       e_θ = -sinθ ̂j + cosθ ̂j
=#
  nx,ny,nz = size(ux);
  Ur = zeros(T,nx,ny,nz);
  Uθ = zeros(T,nx,ny,nz);
  for j ∈ 1:ny, i ∈ 1:nx
    r = sqrt(grid.x[i]^2+grid.y[j]^2);
    θ = atan(grid.y[j],grid.x[i]) ;
    θ = isnan(θ) ? π/2 : θ;
    sinθ = sin(θ);
    cosθ = cos(θ);
    Uθ[i,j,:] .= @. -sinθ*ux[i,j,:] + cosθ*uy[i,j,:];
    Ur[i,j,:] .= @.  cosθ*ux[i,j,:] + sinθ*uy[i,j,:];
  end
  return Ur,Uθ;
end


using Distributions
using LinearAlgebra
using Random
using Test

using EnsembleKalmanProcesses.EnsembleKalmanProcessModule
using EnsembleKalmanProcesses.ParameterDistributionStorage
@testset "EnsembleKalmanProcessModule" begin

    # Seed for pseudo-random number generator
    rng_seed = 42
    Random.seed!(rng_seed)

    ### Generate data from a linear model: a regression problem with n_par parameters
    ### and 1 observation of G(u) = A \times u, where A : R^n_par -> R^n_obs
    n_obs = 10                  # dimension of synthetic observation from G(u)
    n_par = 2                  # Number of parameteres
    u_star = [-1.0, 2.0]          # True parameters
    noise_level = 0.05            # Defining the observation noise level (std)
    Γy = noise_level^2 * Matrix(I, n_obs, n_obs) # Independent noise for synthetic observations
    noise = MvNormal(zeros(n_obs), Γy)
    C = [1 -.9; -.9 1]          # Correlation structure for linear operator
    A = rand(MvNormal(zeros(2,), C), n_obs)'    # Linear operator in R^{n_par x n_obs}

    @test size(A) == (n_obs, n_par)

    #### Define linear model
    function G(u)
        A * u
    end

    y_star = G(u_star)
    y_obs = y_star .+ rand(noise)

    @test size(y_obs) == (n_obs,)

    # sum(y-G)^2 ~ n_obs*noise_level^2
    @test isapprox(norm(y_obs .- G(u_star))^2 - n_obs * noise_level^2, 0; atol = 0.05)

    #### Define prior information on parameters
    prior_distns = [Parameterized(Normal(0.0, 0.5)), Parameterized(Normal(3.0, 0.5))]
    constraints = [[no_constraint()], [no_constraint()]]
    prior_names = ["u1", "u2"]
    prior = ParameterDistribution(prior_distns, constraints, prior_names)

    prior_mean = get_mean(prior)

    # Assuming independence of u1 and u2
    prior_cov = get_cov(prior) #convert(Array, Diagonal([sqrt(2.), sqrt(2.)]))

    ###
    ###  Calibrate: Ensemble Kalman Sampler
    ###
    N_ens = 50 # number of ensemble members (NB for @test throws, make different to N_ens)
    N_iter = 20

    initial_ensemble = EnsembleKalmanProcessModule.construct_initial_ensemble(prior, N_ens; rng_seed = rng_seed)
    @test size(initial_ensemble) == (n_par, N_ens)

    eksobj =
        EnsembleKalmanProcessModule.EnsembleKalmanProcess(initial_ensemble, y_obs, Γy, Sampler(prior_mean, prior_cov))

    g_ens = G(get_u_final(eksobj))
    @test size(g_ens) == (n_obs, N_ens)
    # as the columns of g are the data, this should throw an error
    g_ens_t = permutedims(g_ens, (2, 1))

    # EKS iterations
    for i in 1:N_iter
        params_i = get_u_final(eksobj)
        g_ens = G(params_i)
        if i == 1
            g_ens_t = permutedims(g_ens, (2, 1))
            @test_throws DimensionMismatch EnsembleKalmanProcessModule.update_ensemble!(eksobj, g_ens_t)
        end

        EnsembleKalmanProcessModule.update_ensemble!(eksobj, g_ens)
    end
    # Plot evolution of the EKS particles
    eks_final_result = vec(mean(get_u_final(eksobj), dims = 2))

    if TEST_PLOT_OUTPUT
        gr()
        p = plot(
            get_u_prior(eksobj)[1, :],
            get_u_prior(eksobj)[2, :],
            seriestype = :scatter,
            label = "Initial ensemble",
        )
        plot!(get_u_final(eksobj)[1, :], get_u_final(eksobj)[2, :], seriestype = :scatter, label = "Final ensemble")
        plot!(
            [u_star[1]],
            xaxis = "u1",
            yaxis = "u2",
            seriestype = "vline",
            linestyle = :dash,
            linecolor = :red,
            label = :none,
        )
        plot!([u_star[2]], seriestype = "hline", linestyle = :dash, linecolor = :red, label = :none)
        savefig(p, "EKS_test.png")
    end

    ###
    ###  Calibrate: Ensemble Kalman Inversion
    ###

    #N_ens = 40 # number of ensemble members
    # We will try to compare the EKI and EKS results, therefore we need comparable timesteps and iterations
    #N_iter = 7 # number of EKI iterations
    Δ_vec = eksobj.Δt[2:end]

    ekiobj = EnsembleKalmanProcessModule.EnsembleKalmanProcess(initial_ensemble, y_obs, Γy, Inversion())

    # some checks
    g_ens = G(get_u_final(ekiobj))
    @test size(g_ens) == (n_obs, N_ens)
    # as the columns of g are the data, this should throw an error
    g_ens_t = permutedims(g_ens, (2, 1))
    @test_throws DimensionMismatch find_ekp_stepsize(ekiobj, g_ens_t)
    Δ = find_ekp_stepsize(ekiobj, g_ens)
    # huge collapse for linear problem so should find timestep should < 1
    @test Δ < 1
    # NOTE We don't use this info, this is just for the test.

    # EKI iterations
    params_i_vec = []
    g_ens_vec = []
    for i in 1:N_iter
        params_i = get_u_final(ekiobj)
        push!(params_i_vec, params_i)
        g_ens = G(params_i)
        push!(g_ens_vec, g_ens)
        if i == 1
            g_ens_t = permutedims(g_ens, (2, 1))
            @test_throws DimensionMismatch EnsembleKalmanProcessModule.update_ensemble!(ekiobj, g_ens_t)
        end
        Δ = Δ_vec[i] # use the same timesteps as EKS (EKI is more stable, so this is fine to do)
        EnsembleKalmanProcessModule.update_ensemble!(ekiobj, g_ens, Δt_new = Δ)

    end
    @test isapprox(ekiobj.Δt - eksobj.Δt, zeros(N_iter + 1))

    push!(params_i_vec, get_u_final(ekiobj))

    @test get_u_prior(ekiobj) == params_i_vec[1]
    @test get_u(ekiobj) == params_i_vec
    @test get_g(ekiobj) == g_ens_vec
    @test get_g_final(ekiobj) == g_ens_vec[end]
    @test get_error(ekiobj) == ekiobj.err

    # EKI results: Test if ensemble has collapsed toward the true parameter
    # values
    eki_final_result = vec(mean(get_u_final(ekiobj), dims = 2))

    # kind of arbitrary tolerance here,
    @test norm(u_star - eki_final_result) < 0.1

    # Plot evolution of the EKI particles
    #eki_final_result = vec(mean(get_u_final(ekiobj), dims = 2))
    if TEST_PLOT_OUTPUT
        gr()
        p = plot(
            get_u_prior(ekiobj)[1, :],
            get_u_prior(ekiobj)[2, :],
            seriestype = :scatter,
            label = "Initial ensemble",
        )
        plot!(get_u_final(ekiobj)[1, :], get_u_final(ekiobj)[2, :], seriestype = :scatter, label = "Final ensemble")
        plot!(
            [u_star[1]],
            xaxis = "u1",
            yaxis = "u2",
            seriestype = "vline",
            linestyle = :dash,
            linecolor = :red,
            label = :none,
        )
        plot!([u_star[2]], seriestype = "hline", linestyle = :dash, linecolor = :red, label = :none)
        savefig(p, "EKI_test.png")
    end

    posterior_cov_inv = (A' * (Γy \ A) + 1 * Matrix(I, n_par, n_par) / prior_cov)
    ols_mean = (A' * (Γy \ A)) \ (A' * (Γy \ y_obs))
    posterior_mean = posterior_cov_inv \ ((A' * (Γy \ A)) * ols_mean + (prior_cov \ prior_mean))

    #### This tests correspond to:
    # NOTE THESE ARE VERY SENSITIVE TO THE CORRESPONDING N_iter
    # EKI provides a solution closer to the ordinary Least Squares estimate
    @test norm(ols_mean - eki_final_result) < norm(ols_mean - eks_final_result)
    # EKS provides a solution closer to the posterior mean
    @test norm(posterior_mean - eks_final_result) < norm(posterior_mean - eki_final_result)

    ##### I expect this test to make sense:
    # In words: the ensemble covariance is still a bit ill-dispersed since the
    # algorithm employed still does not include the correction term for finite-sized
    # ensembles.
    @test abs(sum(diag(posterior_cov_inv \ cov(get_u_final(eksobj), dims = 2))) - n_par) > 1e-5

    ###
    ###  Calibrate: Unscented Kalman Inversion
    ###

    N_iter = 20 # number of UKI iterations
    α_reg = 1.0
    update_freq = 0
    process = Unscented(prior_mean, prior_cov; α_reg = α_reg, update_freq = update_freq)
    ukiobj = EnsembleKalmanProcessModule.EnsembleKalmanProcess(y_star, Γy, process)

    # UKI iterations
    params_i_vec = []
    g_ens_vec = []
    for i in 1:N_iter
        params_i = get_u_final(ukiobj)
        push!(params_i_vec, params_i)
        g_ens = G(params_i)
        push!(g_ens_vec, g_ens)
        if i == 1
            g_ens_t = permutedims(g_ens, (2, 1))
            @test_throws DimensionMismatch EnsembleKalmanProcessModule.update_ensemble!(ukiobj, g_ens_t)
        end
        EnsembleKalmanProcessModule.update_ensemble!(ukiobj, g_ens)
    end
    push!(params_i_vec, get_u_final(ukiobj))

    @test get_u_prior(ukiobj) == params_i_vec[1]
    @test get_u(ukiobj) == params_i_vec
    @test get_g(ukiobj) == g_ens_vec
    @test get_g_final(ukiobj) == g_ens_vec[end]
    @test get_error(ukiobj) == ukiobj.err

    # UKI results: Test if ensemble has collapsed toward the true parameter
    # values
    uki_final_result = get_u_mean_final(ukiobj)
    @test norm(u_star - uki_final_result) < 0.5

    if TEST_PLOT_OUTPUT
        gr()
        θ_mean_arr = hcat(ukiobj.process.u_mean...)
        N_θ = length(ukiobj.process.u_mean[1])
        θθ_std_arr = zeros(Float64, (N_θ, N_iter + 1))
        for i in 1:(N_iter + 1)
            for j in 1:N_θ
                θθ_std_arr[j, i] = sqrt(ukiobj.process.uu_cov[i][j, j])
            end
        end

        ites = Array(LinRange(1, N_iter + 1, N_iter + 1))
        p = plot(ites, grid = false, θ_mean_arr[1, :], yerror = 3.0 * θθ_std_arr[1, :], label = "u1")
        plot!(ites, fill(u_star[1], N_iter + 1), linestyle = :dash, linecolor = :grey, label = :none)
        plot!(ites, grid = false, θ_mean_arr[2, :], yerror = 3.0 * θθ_std_arr[2, :], label = "u2", xaxis = "Iterations")
        plot!(ites, fill(u_star[2], N_iter + 1), linestyle = :dash, linecolor = :grey, label = :none)
        savefig(p, "UKI_test.png")
    end

    ### Generate data from G₁(u) with a sparse u
    n_obs = 1                  # Number of synthetic observations from G₁(u)
    n_par = 2                  # Number of parameteres
    u_star = [1.0, 0.0]          # True parameters
    noise_level = 1e-3            # Defining the observation noise level
    Γy = noise_level * Matrix(I, n_obs, n_obs) # Independent noise for synthetic observations
    noise = MvNormal(zeros(n_obs), Γy)

    #### Define model
    function G₁(u)
        [sqrt((u[1] - u_star[1])^2 + (u[2] - u_star[2])^2)]
    end

    y_star = G₁(u_star)
    y_obs = y_star + rand(noise)

    @test size(y_star) == (n_obs,)

    #### Define prior information on parameters
    prior_distributions = [Parameterized(Normal(0, 2)), Parameterized(Normal(0, 2))]
    constraints = [[no_constraint()], [no_constraint()]]
    prior_names = ["u1", "u2"]
    prior = ParameterDistribution(prior_distns, constraints, prior_names)

    ###
    ###  Calibrate (4): Sparse Ensemble Kalman Inversion
    ###

    N_ens = 20 # number of ensemble members
    N_iter = 5 # number of EKI iterations
    initial_ensemble = EnsembleKalmanProcessModule.construct_initial_ensemble(prior, N_ens; rng_seed = rng_seed)
    @test size(initial_ensemble) == (n_par, N_ens)

    for (threshold_eki, threshold_value, test_name) in ((false, 1e-2, "test"), (true, 1e-2, "test_thresholded"))

        # Sparse EKI parameters
        γ = 1.0
        reg = 1e-4
        uc_idx = [1, 2]

        process = SparseInversion(γ, threshold_eki, threshold_value, reg, uc_idx)

        ekiobj = EnsembleKalmanProcessModule.EnsembleKalmanProcess(initial_ensemble, y_obs, Γy, process)

        # EKI iterations
        params_i_vec = []
        g_ens_vec = []
        for i in 1:N_iter
            params_i = get_u_final(ekiobj)
            push!(params_i_vec, params_i)
            g_ens = hcat([G₁(params_i[:, i]) for i in 1:N_ens]...)
            push!(g_ens_vec, g_ens)
            if i == 1
                g_ens_t = permutedims(g_ens, (2, 1))
                @test_throws DimensionMismatch EnsembleKalmanProcessModule.update_ensemble!(ekiobj, g_ens_t)
                EnsembleKalmanProcessModule.update_ensemble!(ekiobj, g_ens)
            else
                EnsembleKalmanProcessModule.update_ensemble!(ekiobj, g_ens, Δt_new = ekiobj.Δt[1])
            end
        end
        push!(params_i_vec, get_u_final(ekiobj))

        @test get_u_prior(ekiobj) == params_i_vec[1]
        @test get_u(ekiobj) == params_i_vec
        @test get_g(ekiobj) == g_ens_vec
        @test get_g_final(ekiobj) == g_ens_vec[end]
        @test get_error(ekiobj) == ekiobj.err

        # EKI results: Test if ensemble has collapsed toward the true parameter
        # values
        eki_final_result = vec(mean(get_u_final(ekiobj), dims = 2))
        @test norm(u_star - eki_final_result) < 0.1
        @test sum(eki_final_result .> 0.05) < size(eki_final_result)[1]

        # Plot evolution of the EKI particles
        eki_final_result = vec(mean(get_u_final(ekiobj), dims = 2))

        if TEST_PLOT_OUTPUT
            gr()
            p = plot(get_u_prior(ekiobj)[1, :], get_u_prior(ekiobj)[2, :], seriestype = :scatter)
            plot!(get_u_final(ekiobj)[1, :], get_u_final(ekiobj)[2, :], seriestype = :scatter)
            plot!([u_star[1]], xaxis = "u1", yaxis = "u2", seriestype = "vline", linestyle = :dash, linecolor = :red)
            plot!([u_star[2]], seriestype = "hline", linestyle = :dash, linecolor = :red)
            savefig(p, string("SparseEKI_", test_name, ".png"))
        end
    end
end


# Pawel Zielinski
module matrixgen

using LinearAlgebra

export  blockmat

	function matcond(n::Int, c::Float64)
		# Function generates a random square matrix A of size n with
		# a given condition number c.
		# Inputs:
		#	n: size of matrix A, n>1
		#	c: condition of matrix A, c>= 1.0
		#
		# Usage: matcond(10, 100.0)
		#
		# Pawel Zielinski
        if n < 2
         error("size n should be > 1")
        end
        if c< 1.0
         error("condition number  c of a matrix  should be >= 1.0")
        end
        (U,S,V)=svd(rand(n,n))
        return U*diagm(0 =>[LinRange(1.0,c,n);])*V'
	end

  function blockmat(n::Int, l::Int, ck::Float64, outputfile::String)
		# Function generates a random block sparse matrix A of size n with
		# a given condition number ck of inner block Ak and it stores the output
		# matrix in a text file.
		# Inputs:
		#	n: size of block matrix A, n>3
		#   l: size of inner matrices Ak, n mod l =0 (n is  divisible by l), l>1
		#	ck: condition of inner matrix Ak, ck>= 1.0
		# outputfile: name of the output text file
		#
		# Usage: blockmat(100, 4 ,10.0, "A.txt")
		#
		#
		#  the output file format
	  #  n  l              <--- the size of block matrix A, the size of inner matrices Ak
		#  i1  j1   A[i1,j1] <--- a non-zero element of block matrix A
		#  i2  j2   A[i2,j2] <--- a non-zero element of block matrix A
		#  i3  j3   A[i3,j3] <--- a non-zero element of block matrix A
		#  ...
		#  ...
		#  EOF
		#

    if n < 4
     error("size n should be > 3")
    end
    if l < 2
     error("size l should be > 1")
    end
		if n%l!=0
			error("n is not divisible by l")
		end

		nb=div(n,l)
		Ak=Matrix{Float64}(undef, l, l)
		open(outputfile, "w") do f
			println(f, n," ",l)
			for k in 1:nb
				Ak=matcond(l, ck)
				for i in 1:l, j in 1:l
					println(f,(k-1)*l+i," ",(k-1)*l+j," ", Ak[i,j])
				end
				if k<nb
			  	 for i in 1:l
						println(f,(k-1)*l+i," ",k*l+i," ",0.3*rand())
				 	 end
				end
				if k>1
			   for i in 1:l
					 println(f,(k-1)*l+i," ",(k-1)*l-1," ",0.3*rand())
					 println(f,(k-1)*l+i," ",(k-1)*l," ",0.3*rand())
				 end
				end
			end
		end	 # do
	end # blockmat

end # matrixgen


### A Pluto.jl notebook ###
# v0.18.0

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el)
        el
    end
end

# ╔═╡ 150d58e7-0e73-4b36-836c-d81eef531a9c
begin
    using Pkg
	Pkg.activate(joinpath(Pkg.devdir(), "MLCourse"))
    using MLJ, MLJLinearModels, Plots, LinearAlgebra, Random, DataFrames, CSV, MLCourse
    import MLCourse: Polynomial
end

# ╔═╡ 2e9ce2a9-217e-4910-b6ce-d174f2f2668e
using PlutoUI; PlutoUI.TableOfContents()

# ╔═╡ 78bdd11d-b6f9-4ba6-8b2e-6189c4005bf1
md"# Ridge Regression (L2 Regularization)
"

# ╔═╡ 64b9cfa0-99f7-439b-b70e-f9266754ff74
md"In the following cell there is some custom code to run ridge regression and the lasso for the simple example of 1-dimensional input. In this example we penalize also the intercept β₀. For ridge regression the solution is
```math
\begin{eqnarray*}
\beta_1 &= \frac{\langle x y \rangle - \frac{\langle x\rangle \langle y\rangle}{1 + \lambda}}{\langle x^2\rangle - \frac{\langle x\rangle^2}{1 + \lambda} + \lambda}\\
\beta_0 &= \frac{\langle y \rangle - \beta_1 \langle x \rangle}{1 + \lambda}
\end{eqnarray*}
```
where ``\langle . \rangle`` denotes the average.

For the lasso, there we run a fixed point iteration that starts at the unregularized solution of linear regression and shrinks β₁ and β₀ towards zero until there is not change anymore.
"

# ╔═╡ 8bd483cc-f490-11eb-38a1-b342dd2551fd
begin
    function ridge_regression(x, y, λ)
       β₁ = (mean(x .* y) - mean(x) * mean(y)/(1 + λ))/
		    (mean(x.^2) - mean(x)^2/(1 + λ) + λ)
       β₀ = (mean(y) - β₁ * mean(x))/(1 + λ)
       (β₀ = β₀, β₁ = β₁)
    end
    function updateβ₀(x̄, ȳ, β₁, l)
        tmp = ȳ - β₁ * x̄
        abs(tmp) > l ? tmp - sign(tmp) * l : 0.
    end
    function updateβ₁(x̄, ȳ, x2, xy, β₀, l)
        tmp = (x̄ * ȳ - xy - x̄ * sign(β₀) * l)
        abs(tmp) > l ? (tmp - sign(tmp) * l)/(x̄^2 - x2) : 0
    end
    function lasso(x, y, λ)
        x̄ = mean(x)
        ȳ = mean(y)
        x2 = mean(x.^2)
        xy = mean(x .* y)
        β₁ = (x̄ * ȳ - xy)/(x̄^2 - x2)
        β₀ = ȳ - β₁ * x̄
        β₀old, β₁old = zero(β₀), zero(β₁)
        while β₀old != β₀ || β₁old != β₁
            β₀old, β₁old = β₀, β₁
            β₁ = updateβ₁(x̄, ȳ, x2, xy, β₀, λ)
            β₀ = updateβ₀(x̄, ȳ, β₁, λ)
        end
       (β₀ = β₀, β₁ = β₁)
    end
end;

# ╔═╡ 3a71618b-0d74-4480-b87b-c6ba87363b5d
md"We generate now some data to observe the effect of regularization when fitting this data."

# ╔═╡ 9e1e8284-a8c1-47a9-83d0-2d8fbd8ce005
n = 30; x = rand(n); y = 2.2x .+ .3 .+ .2randn(n);

# ╔═╡ 1009e251-59af-4f1a-9d0a-e96f4b696cad
md"λ₂ = $(@bind λ₂ Slider(0:.01:5, show_value = true))"

# ╔═╡ 50ac0b07-ffee-40c3-843e-984b3c628282
l2coefs = ridge_regression(x, y, λ₂)

# ╔═╡ 58746554-ca5a-4e8e-97e5-587a9c2aa44c
let r = λ₂ == 0 ? 6 : norm([l2coefs...]),
    ccol = plot_color(:blue, .3),
    path = hcat([[ridge_regression(x, y, l)...] for l in 0:.01:5]...)
    p1 = scatter(x, y, label = "data", xlabel = "x", ylabel = "y",
	             legend = :topleft)
    plot!(x -> l2coefs.β₀ + x * l2coefs.β₁, w = 3, label = "ridge regression")
    p2 = contour(-1:.1:3, -1:.1:3, (β₀, β₁) -> mean((β₀ .+ β₁*x .- y).^2),
                 label = "loss", title = "loss with constraints",
		         legend = :bottomright,
                 levels = 100, aspect_ratio = 1, ylims = (-1, 3), xlims = (-1, 3))
    plot!(t -> r * sin(t), t -> r * cos(t), 0:.001:2π,
          fill = (0, ccol), label = "constraint", color = ccol)
    plot!(path[1, :], path[2, :], label = "path", color = :blue, w = 3)
    scatter!([l2coefs.β₀], [l2coefs.β₁], label = "current fit", markersize = 6, color = :red)
    p3 = plot(0:.01:5, path[1, :], label = "β₀", xlabel = "λ₂", ylabel = "")
    plot!(0:.01:5, path[2, :], label = "β₁", ylims = (0, 2.4))
    scatter!([λ₂], [l2coefs.β₀], label = nothing, markersize = 6, color = :red)
    scatter!([λ₂], [l2coefs.β₁], label = nothing, markersize = 6, color = :red)
    p4 = contour(-1:.1:3, -1:.1:3, (β₀, β₁) -> mean((β₀ .+ β₁*x .- y).^2) + λ₂ * (β₀^2 + β₁^2),
                 label = "loss", title = "regularized loss",
                 levels = 100, aspect_ratio = 1, ylims = (-1, 3), xlims = (-1, 3))
    scatter!([l2coefs.β₀], [l2coefs.β₁], markersize = 6, label = nothing, color = :red)
    plot(p1, p4, p3, p2,
         layout = (2, 2), size = (700, 600), cbar = false)
end

# ╔═╡ f43a82e2-1145-426d-8e0e-5363d1c38ccf
md"# Lasso (L1 Regression)"

# ╔═╡ ff7cc2bf-2a38-46d2-8d11-529159b08c82
md"λ₁ = $(@bind λ₁ Slider(0:.01:1, show_value = true))"

# ╔═╡ 4841f9ba-f3d2-4c65-9225-bc8d0c0a9478
l1coefs = lasso(x, y, λ₁)

# ╔═╡ ed2b7969-79cd-43c8-bcdb-34dab89c2cb0
let r = λ₁ == 0 ? 10 : norm([l1coefs...], 1),
    ccol = plot_color(:blue, .3),
    path = hcat([[lasso(x, y, l)...] for l in 0:.01:1]...)
    p1 = scatter(x, y, label = "data", xlabel = "x", ylabel = "y", legend = :topleft)
    plot!(x -> l1coefs.β₀ + x * l1coefs.β₁, w = 3, label = "lasso")
    p2 = contour(-1:.1:3, -1:.1:3, (β₀, β₁) -> mean((β₀ .+ β₁*x .- y).^2),
                 label = "loss", title = "loss with constraints",
		         legend = :topright,
                 levels = 100, aspect_ratio = 1, ylims = (-1, 3), xlims = (-1, 3))
    plot!([0, r, 0, -r, 0], [r, 0, -r, 0, r],
          fill = (0, ccol), label = "constraint", color = ccol)
    plot!(path[1, :], path[2, :], label = "path", color = :blue, w = 3)
    scatter!([l1coefs.β₀], [l1coefs.β₁], label = "current fit", markersize = 6,
		     color = :red)
    p3 = plot(0:.01:1, path[1, :], label = "β₀", xlabel = "λ₁", ylabel = "")
    plot!(0:.01:1, path[2, :], label = "β₁", ylims = (0, 2.4))
    scatter!([λ₁], [l1coefs.β₀], label = nothing, markersize = 6, color = :red)
    scatter!([λ₁], [l1coefs.β₁], label = nothing, markersize = 6, color = :red)
    p4 = contour(-1:.1:3, -1:.1:3, (β₀, β₁) -> mean((β₀ .+ β₁*x .- y).^2)/2 + λ₁ * (abs(β₀) + abs(β₁)),
                 label = "loss", title = "regularized loss",
                 levels = 100, aspect_ratio = 1, ylims = (-1, 3), xlims = (-1, 3))
    scatter!([l1coefs.β₀], [l1coefs.β₁], markersize = 6, label = nothing, color = :red)
    plot(p1, p4, p3, p2,
         layout = (2, 2), size = (700, 600), cbar = false)
end

# ╔═╡ 4c3c816c-e901-4931-a27c-632b60291ad7
md"Instead of using the custom code to compute the ridge regression and the lasso we could have used some MLJ functions."

# ╔═╡ c1033416-334e-4b0e-b81e-6f9137402730
let mach = machine(RidgeRegressor(lambda = 3.82, penalize_intercept = true),
	               DataFrame(x = x), y)
	fit!(mach, verbosity = 0)
	fitted_params(mach)
end

# ╔═╡ 1dae5378-f3eb-4598-a060-445bfd8afe5e
md"Let us check that we get indeed the same result with our custom method."

# ╔═╡ 15a85810-ccbd-4aa3-98a5-fdcf68c97adb
ridge_regression(x, y, 3.82)

# ╔═╡ 0429acfe-d31e-427a-96d9-deddfa2c30f8
let mach = machine(LassoRegressor(lambda = .1,
	                              # usually the intercept is not penalized,
	                              # but here we do penalize it.
	                              penalize_intercept = true,
	                              # usually the default optimizer is quite good,
	                              # but here we decrease the tolerance to get
	                              # higher precision.
                                  solver = ISTA(tol = 1e-8)),
	            DataFrame(x = x), y)
	fit!(mach, verbosity = 0)
	fitted_params(mach)
end

# ╔═╡ 2ba5b965-1dce-4773-b6eb-fd838876674d
lasso(x, y, .1)

# ╔═╡ 6c87eb35-ddb3-44a3-b4ae-77a371e28960
md"There is also the `ElasticNetRegressor` that allows to fit with L1 and L2 penalties of different strengths. Look up the documentation to learn more about it."

# ╔═╡ ca394b88-06dc-4188-884a-50d7c180aa33
md"# Regularization Examples

## Polynomial Ridge Regression

In `MLJ` can apply ridge regression or the lasso to polynomial regression simply by replacing in the pipeline the `LinearRegressor` with a `RidgeRegressor` or a `LassoRegressor`, for example `mach = Polynomial(degree = 3) |> RidgeRegressor(lambda = 1e-3)`."

# ╔═╡ b45a9739-0f81-4c0c-a93b-434a5af91490
begin
    f(x) = .3 * sin(10x) + .7x
    function regression_data_generator(; n, seed = 3, rng = MersenneTwister(seed))
        x = rand(rng, n)
        DataFrame(x = x, y = f.(x) .+ .1*randn(rng, n))
    end
    regression_data = regression_data_generator(n = 50)
end;

# ╔═╡ 3d50111b-3a08-4a41-96ce-d77a8e37275d
md"degree = $(@bind degree Slider(0:20, show_value = true, default = 20))

$(@bind lambda Slider(-14:.1:.5, default = -4))
"

# ╔═╡ a54e3439-69b8-41c8-bfe0-4575795fb9b8
md"λ = $(lambda == -14 ? 0 : 10.0^lambda)"

# ╔═╡ bdbf0dfd-8da5-4e54-89c4-ef4d6b3796ce
let X = select(regression_data, Not(:y)), y = regression_data.y
    mach = fit!(machine(Polynomial(; degree) |> RidgeRegressor(lambda = lambda == -14 ? 0 : 10.0^lambda),
                        X, y), verbosity = 0)
    p1 = scatter(regression_data.x, y, label = "training data", ylims = (-.1, 1.1))
    plot!(f, label = "generator", c = :green, w = 2)
    grid = 0:.01:1
    pred = predict(mach, (x = grid,))
    plot!(grid, pred,
          label = "fit", w = 3, c = :red, legend = :topleft)
    annotate!([(.28, .6, "reducible error ≈ $(round(mean((pred .- f.(grid)).^2), sigdigits = 3))")])
end

# ╔═╡ fe2fe54f-0163-4f5d-9fd1-3d1aa3580875
begin
    model = Polynomial() |> RidgeRegressor()
    self_tuning_model = TunedModel(model = model,
                                   tuning =  Grid(goal = 500),
                                   resampling = CV(nfolds = 5),
                                   range = [range(model, :(polynomial.degree),
                                                  lower = 1, upper = 20),
                                            range(model, :(ridge_regressor.lambda),
                                                  lower = 1e-12, upper = 1e-3,
                                                  scale = :log)],
                                   measure = rmse)
    self_tuning_mach = machine(self_tuning_model,
                               select(regression_data, :x),
                               regression_data.y) |> fit!
end;

# ╔═╡ f5057d4a-1103-4728-becc-287d93d682ba
plot(self_tuning_mach)

# ╔═╡ bd54bfcd-f682-4b74-8a44-35463d421491
report(self_tuning_mach)

# ╔═╡ 596fd0f2-eee0-46ca-a203-e7cbac6f9788
let
    p1 = scatter(regression_data.x, regression_data.y,
                 label = "training data", ylims = (-.1, 1.1))
    plot!(f, label = "generator", c = :green, w = 2)
    grid = 0:.01:1
    pred = predict(self_tuning_mach, (x = grid,))
    plot!(grid, pred,
          label = "fit", w = 3, c = :red, legend = :topleft)
    annotate!([(.28, .6, "reducible error ≈ $(round(mean((pred .- f.(grid)).^2), sigdigits = 3))")])
end

# ╔═╡ 8e170a5a-9c46-413e-895d-796e178b69df
md"## Multiple Logistic Ridge Regression on the Spam Data

We load here the preprocessed spam data.
"

# ╔═╡ 8e542a48-ed28-4297-b2e8-d6a755a5fdf9
begin
    spam_train = CSV.read(joinpath(dirname(pathof(MLCourse)), "..", "data",
                                   "spam_preprocessed.csv"), DataFrame)
	spam_train.spam_or_ham = String.(spam_train.spam_or_ham)
    coerce!(spam_train, :spam_or_ham => OrderedFactor)
    spam_test = CSV.read(joinpath(dirname(pathof(MLCourse)), "..", "data",
                                  "spam_preprocessed_test.csv"), DataFrame)
	spam_test.spam_or_ham = String.(spam_test.spam_or_ham)
    coerce!(spam_test, :spam_or_ham => OrderedFactor)
end;

# ╔═╡ c5ef5d4e-200d-46d9-86fa-50af1896a6c3
md"The `LogisticClassifier` and the `MultinomialClassifier` have a `penalty` argument that can be used to enforce an L1 or L2 penalty. Look up the documentation to learn more about it."

# ╔═╡ d956613e-db32-488c-8ebb-fd61dfa31e59
spam_fit = fit!(machine(LogisticClassifier(penalty = :l2, lambda = 1e-5),
                        select(spam_train, Not(:spam_or_ham)),
                        spam_train.spam_or_ham));

# ╔═╡ 552f14fc-06c7-4c2a-9515-e64f28828b70
confusion_matrix(predict_mode(spam_fit, select(spam_train, Not(:spam_or_ham))),
                 spam_train.spam_or_ham)

# ╔═╡ ef701511-db7e-4dc0-8d31-ea14471943ab
confusion_matrix(predict_mode(spam_fit, select(spam_test, Not(:spam_or_ham))),
                 spam_test.spam_or_ham)

# ╔═╡ 19bdd76c-4131-422d-983e-1b29cd9edd30
md"We see that the test misclassification rate with regularization
is lower than in our original fit without regularization
(notebook \"Generalized Linear Regression\"; 48 false negatives and 48 false
positives). The misclassification rate on the training set is higher. This
indicates that unregularized logistic regression is too flexible for our spam
data set.
"

# ╔═╡ 13655a50-fbbb-46c7-bdf7-ed5644646966
md"## The Lasso Path for the Weather Data

For the Lasso it is often interesting to see the fitted parameter values for different regularization values (the Lasso path). In the following we use the package `GLMNet` to do so."

# ╔═╡ 1fa932c1-ce29-40ca-a8dc-e636aa2ecf66
weather = CSV.read(joinpath(@__DIR__, "..", "data", "weather2015-2018.csv"), DataFrame);

# ╔═╡ 470dc7f4-04a9-4253-8125-9112778021eb
import GLMNet: glmnet

# ╔═╡ ecf80b6a-1946-46fd-b1b4-bcbe91848e3c
begin
    weather_input = select(weather, Not(:LUZ_wind_peak))[1:end-5, :]
    weather_output = weather.LUZ_wind_peak[6:end]
    weather_fits = glmnet(Array(weather_input), weather_output)
end

# ╔═╡ 4652a904-5edb-463c-a046-5c5d378f7cca
let lambda = log.(weather_fits.lambda),
    col_names = names(weather_input)
    plotly()
    p = plot()
    for i in 1:size(weather_fits.betas, 1)
        plot!(lambda, weather_fits.betas[i, :], label = col_names[i])
    end
    plot!(legend = :outertopright, xlabel = "log(λ)", size = (700, 400))
    gr()
    p
end

# ╔═╡ 40bb385f-1cbd-4555-a8ab-544a67f33595
let lambda = log.(weather_fits.lambda)
    p1 = plot(lambda, 100 * weather_fits.dev_ratio, ylabel = "% variance explained")
    p2 = plot(lambda, reshape(sum(weather_fits.betas .!= 0, dims = 1), :),
              ylabel = "non-zero parameters",
              xlabel = "log(λ)")
    plot(p1, p2, layout = (2, 1), legend = false)
end

# ╔═╡ c9ed011c-8d36-4926-9ec4-84be3b4878d7
scatter(weather_input.BER_wind_peak, weather_output)

# ╔═╡ 8262b948-6d54-4348-87d1-4c762c74db30
md"# Exercises

## Conceptual

#### Exercise 1.
We review here the two formulations of regularization.
A standard approach to solve constraint optimization
problems makes use of Karush-Kuhn-Tucker (KKT) multipliers. For example, to find
the minimum of function ``f(x)`` under the constraint ``g(x) \leq s`` one can define
the loss function ``L(x, \lambda) = f(x) + \lambda(g(x) - s)`` where
``\lambda\geq0`` is a KKT multiplier.  Minimizing the loss both in ``x`` and
``\lambda`` amounts to solving the equations ``\frac{\partial L}{\partial x} = f'(x) + \lambda g'(x) = 0`` and ``\frac{\partial L}{\partial \lambda} = g(x) - s = 0``,
if the solution is on the boundary of the area defined by the inequality
constraint; otherwise one can find the solution by simply solving the
unconstrained problem, i.e. with ``\lambda = 0``. In this formulation one choses
the size ``s`` of the allowed area and finds ``\lambda`` by solving the equations.

Interestingly, the loss function ``L(x) = f(x) + \lambda
g(x)`` has exactly the same partial derivative in ``x`` as ``L(x, \lambda)`` and
therefore all critical points of ``L(x)`` have corresponding critical
points of ``L(x,\lambda)``. Because of this, regularization is often formulated
as \"adding a regularization term to the cost function\". For example, given the loss
    function of linear regression ``L(\beta) = \frac1n\sum_{i=1}^n(y_i - \beta_0 + \beta_1x_{i1} + \cdots + \beta_p x_{ip})^2`` one can define the
        L1-regularized loss function ``L_\mathrm{L1}(\beta) = L(\beta) + \lambda
\|\beta\|_1`` and choose a value for ``\lambda`` instead of the size ``s`` of the
allowed area.

1. Derive how ``\lambda`` in the second formulation depends on ``s`` in the first formulation for ridge regression with standardized one-dimensional input. *Hint:* use the analytical solution at the top of this notebook, the fact that ``\langle x \rangle = 0`` and ``\langle x^2\rangle = 0`` and note that ``\beta_0^2 + \beta_1^2 = s``, if the solution lies on the boundary.
1. Argue, why choosing ``\lambda = 0`` in the second formulation is equivalent to choosing a sufficiently large ``s`` in the first formulation.
2. Argue, why choosing ``\lambda = \infty`` in the second formulation is equivalent to choosing ``s = 0`` in the first formulation.

#### Exercise 2.
Consider a data set with as many data points as predictors ``n = p``.
Assume ``x_{ii} = 1`` and ``x_{ij} = 0`` for all ``i\neq j`` and arbitrary values
``y_i``. To simplify the problem further we perform regression without an
intercept. We would like to study L1- and L2-regularized multiple linear regression.

1. Write the mean squared error loss once with L1 regularization and once with L2 regularization for this setting and the fomulation of regularization with regularization constant ``\lambda``.
2. Show that in the case of L2 regularization the estimated coefficients take the form ``\hat \beta_j = y_j/(1 + \lambda)``.
3. Show that in the case of L1 regularization the estimated coefficients take the form ``\hat \beta_j = y_j - \lambda/2``, if ``y_j > \lambda/2``, ``\hat \beta_j = y_j + \lambda/2``, if ``y_j < -\lambda/2`` and ``\hat \beta_j = 0`` otherwise.
4. Write a brief summary on how the estimated coefficients ``\hat \beta_j`` are changed relative to the unregularized solution for both kinds of regularization.

## Applied

#### Exercise 1.
Create an artificial dataset with 20 points, 4 predictors ``X_1, X_2, X_3, X_4``
and ``Y = X_1 + \epsilon`` with ``\mathrm{Var}(\epsilon) = 0.1^2``.

1. Find with cross-validation and the lasso the best model.
2. Find with cross-validation and the ridge regression the best model.
3. Which of the two best models has the lowest reducible error?
4. Repeat the above 3 steps on an artificial data set with 20 points and 4 predictors with ``Y = 10X_1 + X_2 + .1 * X_3 + .01 * X_4 + \epsilon``.
"

# ╔═╡ e04c5e8a-15f8-44a8-845d-60acaf795813
MLCourse.list_notebooks(@__FILE__)

# ╔═╡ c48dff95-8028-4e97-8ec6-705ea2b9c72e
MLCourse.footer()

# ╔═╡ Cell order:
# ╟─2e9ce2a9-217e-4910-b6ce-d174f2f2668e
# ╠═150d58e7-0e73-4b36-836c-d81eef531a9c
# ╟─78bdd11d-b6f9-4ba6-8b2e-6189c4005bf1
# ╟─64b9cfa0-99f7-439b-b70e-f9266754ff74
# ╠═8bd483cc-f490-11eb-38a1-b342dd2551fd
# ╟─3a71618b-0d74-4480-b87b-c6ba87363b5d
# ╠═9e1e8284-a8c1-47a9-83d0-2d8fbd8ce005
# ╟─1009e251-59af-4f1a-9d0a-e96f4b696cad
# ╠═50ac0b07-ffee-40c3-843e-984b3c628282
# ╟─58746554-ca5a-4e8e-97e5-587a9c2aa44c
# ╟─f43a82e2-1145-426d-8e0e-5363d1c38ccf
# ╟─ff7cc2bf-2a38-46d2-8d11-529159b08c82
# ╠═4841f9ba-f3d2-4c65-9225-bc8d0c0a9478
# ╟─ed2b7969-79cd-43c8-bcdb-34dab89c2cb0
# ╟─4c3c816c-e901-4931-a27c-632b60291ad7
# ╠═c1033416-334e-4b0e-b81e-6f9137402730
# ╟─1dae5378-f3eb-4598-a060-445bfd8afe5e
# ╠═15a85810-ccbd-4aa3-98a5-fdcf68c97adb
# ╠═0429acfe-d31e-427a-96d9-deddfa2c30f8
# ╠═2ba5b965-1dce-4773-b6eb-fd838876674d
# ╟─6c87eb35-ddb3-44a3-b4ae-77a371e28960
# ╟─ca394b88-06dc-4188-884a-50d7c180aa33
# ╠═b45a9739-0f81-4c0c-a93b-434a5af91490
# ╟─3d50111b-3a08-4a41-96ce-d77a8e37275d
# ╟─a54e3439-69b8-41c8-bfe0-4575795fb9b8
# ╟─bdbf0dfd-8da5-4e54-89c4-ef4d6b3796ce
# ╠═fe2fe54f-0163-4f5d-9fd1-3d1aa3580875
# ╠═f5057d4a-1103-4728-becc-287d93d682ba
# ╠═bd54bfcd-f682-4b74-8a44-35463d421491
# ╟─596fd0f2-eee0-46ca-a203-e7cbac6f9788
# ╟─8e170a5a-9c46-413e-895d-796e178b69df
# ╠═8e542a48-ed28-4297-b2e8-d6a755a5fdf9
# ╟─c5ef5d4e-200d-46d9-86fa-50af1896a6c3
# ╠═d956613e-db32-488c-8ebb-fd61dfa31e59
# ╠═552f14fc-06c7-4c2a-9515-e64f28828b70
# ╠═ef701511-db7e-4dc0-8d31-ea14471943ab
# ╟─19bdd76c-4131-422d-983e-1b29cd9edd30
# ╟─13655a50-fbbb-46c7-bdf7-ed5644646966
# ╠═1fa932c1-ce29-40ca-a8dc-e636aa2ecf66
# ╠═470dc7f4-04a9-4253-8125-9112778021eb
# ╠═ecf80b6a-1946-46fd-b1b4-bcbe91848e3c
# ╟─4652a904-5edb-463c-a046-5c5d378f7cca
# ╟─40bb385f-1cbd-4555-a8ab-544a67f33595
# ╠═c9ed011c-8d36-4926-9ec4-84be3b4878d7
# ╟─8262b948-6d54-4348-87d1-4c762c74db30
# ╟─e04c5e8a-15f8-44a8-845d-60acaf795813
# ╟─c48dff95-8028-4e97-8ec6-705ea2b9c72e


# Beta distribution

export Beta

@parameterized Beta(α,β) ≃ Lebesgue(𝕀)

@kwstruct Beta(α, β)

@kwalias Beta [
    a     => α
    alpha => α
    b     => β
    beta  => β
]

TV.as(::Beta) = as𝕀

function logdensity(d::Beta{(:α, :β)}, x)
    return (d.α - 1) * log(x) + (d.β - 1) * log(1 - x) - logbeta(d.α, d.β)
end

Base.rand(rng::AbstractRNG, T::Type, μ::Beta) = rand(rng, Dists.Beta(μ.α, μ.β))

distproxy(d::Beta{(:α, :β)}) = Dists.Beta(d.α, d.β)

asparams(::Type{<:Beta}, ::Val{:α}) = asℝ₊
asparams(::Type{<:Beta}, ::Val{:β}) = asℝ₊


### A Pluto.jl notebook ###
# v0.12.4

using Markdown
using InteractiveUtils

# ╔═╡ 0d696b90-14b9-11eb-19c6-7110e9eded9e
using PotentialFlow

# ╔═╡ 40e53300-14b9-11eb-28ca-c39ecf8c05ef
using Plots

# ╔═╡ 59e4bc40-14b9-11eb-3670-cdc8fa8f0ded
begin
	X = range(-1, 1, length=101)
	Y = range(-1, 1, length=101)
	Z = [x + im*y for y in Y, x in X]
end

# ╔═╡ 27ebaaf0-14be-11eb-3b42-d3c031377d61
begin

	v2 = Vortex.Point(0.0, 2)

	ψ = streamfunction(Z,v2)

	contour(X, Y, ψ,
		color = cgrad([:red, :blue]),
		grid = false,
		legend = false,
		ratio = 1,
		xlim = (-1, 1),
		ylim = (-1, 1),)

end

# ╔═╡ 2e843420-1538-11eb-14b2-d9ab38fa091b
begin

	vs = Vortex.Point.(rand(ComplexF64, 10), π*rand(10))
    fs = Freestreams.Freestream(rand(ComplexF64))

	ψs = streamfunction(Z,(vs,fs))

	contour(X, Y, ψs,
		color = cgrad([:red, :blue]),
		grid = false,
		legend = false,
		ratio = 1,
		xlim = (0, 1),
		ylim = (0, 1),)

end

# ╔═╡ e0cf3580-1538-11eb-1ba8-1baff87ba9e1
md"""
### Flow over a cylinder
"""

# ╔═╡ Cell order:
# ╠═0d696b90-14b9-11eb-19c6-7110e9eded9e
# ╠═40e53300-14b9-11eb-28ca-c39ecf8c05ef
# ╠═59e4bc40-14b9-11eb-3670-cdc8fa8f0ded
# ╠═27ebaaf0-14be-11eb-3b42-d3c031377d61
# ╠═2e843420-1538-11eb-14b2-d9ab38fa091b
# ╟─e0cf3580-1538-11eb-1ba8-1baff87ba9e1


D = [
	-5  -9  -9   7   0;
	 7   0   4   0   9;
	 8   0   5   4  -6
]

# when data is representad as a vector
Y = [
    -1;
     0;
     5
]

X = matching_pursuit(Y, D)
@test norm(Y-D*X) < 1e-6

# when data is representad as a matrix
Y = [
    -1  2;
     0  3;
     5 -5
]

X = matching_pursuit(Y, D)
@test norm(Y-D*X) < 1e-6

# when max_iter < 0
@test_throws ArgumentError matching_pursuit(Y, D; max_iter = 0)
@test_throws ArgumentError matching_pursuit(Y, D; max_iter = -1)

# when tolerance < 0
@test_throws ArgumentError matching_pursuit(Y, D, tolerance = 0.)
@test_throws ArgumentError matching_pursuit(Y, D; tolerance = -1.0)

# when dimensions of data and atoms do not match

Y = [
     2;
    -5
]

@test_throws ArgumentError matching_pursuit(Y, D)


"""
Problema de asignación. (Vanderbei LP, pg. 227). Dado un conjunto S de m personas, un conjunto D de m tareas, y para cada iS,jD un costo cij de asignar la tarea j a la persona i, el problema de asignación consiste en asignar a cada persona una y solo una tarea, de manera tal que se minimize el costo total. Formule este problema como un problema de programación lineal con enteros. Escriba una función para resolverlo, que tome como entrada una matriz de costos cij arbitraria.

"""

export solve_with_lineal_programming;
function solve_with_lineal_programming(m::Matrix{T}) where {T<:Number}

	#Checking matrix
	checkmatrix(m);
	ml = size(m,1);

	#Create model
	local model = Model(with_optimizer(GLPK.Optimizer));

	#Variables
	@variable(model, x[1:ml,1:ml], Bin);

	#Objective
	@objective(model, Min, sum(m[i,j] * x[i,j] for i in 1:ml, j in 1:ml));

	#Constrains
	@constraint(model, [i=1:ml], sum(x[i,j] for j in 1:ml) <= 1);
	@constraint(model, [j=1:ml], sum(x[i,j] for i in 1:ml) >= 1);

	#Optimize
	optimize!(model);

	return pick_results(value.(x),m);
end

function pick_results(bm,m)
	r = Vector{eltype(m)}();
	zer0 = zero(eltype(m));
	for i in 1:length(bm)
		bm[i] != zer0 && push!(r,m[i]);
	end
	return push!(Vector{typeof(r)}(),r);
end


###############################################################################
#
# The primary module for computing the Riemann theta function.
#
# .. math::
#
#   Θ(z, ω) = Σ
    # \theta(z, \Omega) = \sum_{n \in \mathbb{Z}^g}
                        # e^{2 \pi i \left( \tfrac{1}{2} n \cdot \Omega n
                           # + n \cdot z \right)}
#
#
#  Original Authors
#  -------
#  * Chris Swierczewski (@cswiercz) - September 2012, July 2016
#  * Grady Williams (@gradyrw) - October 2012
#  * Jeremy Upsal (@jupsal) - July 2016
#
# References
# ----------
#
# .. [CRTF] B. Deconinck, M.  Heil, A. Bobenko, M. van Hoeij and M. Schmies,
#    Computing Riemann Theta Functions, Mathematics of Computation, 73, (2004),
#    1417-1442.
#
# .. [DLMF] B. Deconinck, Digital Library of Mathematics Functions - Riemann
#    Theta Functions, http://dlmf.nist.gov/21
#
# .. [SAGE] Computing Riemann theta functions in Sage with applications.
#    C. Swierczewski and B. Deconinck.Submitted for publication.  Available
#    online at
#    http://depts.washington.edu/bdecon/papers/pdfs/Swierczewski_Deconinck1.pdf
#
###############################################################################

"""
         oscillatory_part(zs::Vector{Vector{ComplexF64}},
                          Ω::Matrix{ComplexF64};
                          eps::Float64=1e-8,
                          derivs::Vector{Vector{ComplexF64}}=Vector{ComplexF64}[],
                          accuracy_radius::Float64=5.)::Vector{ComplexF64}

Return the value of the oscillatory part of the Riemann theta function for Ω and
all z in `zs` if `derivs` is empty, or the derivatives at all z in `zs` for the
given directional derivatives in `derivs`.

Parameters
----------
- `zs` : A vector of complex vectors at which to evaluate the Riemann theta function.
- `Omega` : A Riemann matrix.
- `eps` : (Default: 1e-8) The desired numerical accuracy.
- `derivs` : A vector of complex vectors giving a directional derivative.
- `accuracy_radius` : (Default: 5.) The radius from the g-dimensional origin
where the requested accuracy of the Riemann theta is guaranteed when computing
derivatives. Not used if no derivatives of theta are requested.

Returns
-------
- The value of the oscillatory part of the Riemann theta function at each point appearing in `z`.
"""
function oscillatory_part(zs::Vector{Vector{ComplexF64}},
                          Ω::Matrix{ComplexF64};
                          eps::Float64=1e-8,
                          derivs::Vector{Vector{ComplexF64}}=Vector{ComplexF64}[],
                          accuracy_radius::Float64=5.)
    # extract the requested information: the real part, inverse of the
    # imaginary part, and the cholesky decomposition of the imaginary part
    X = real.(Ω)
    Y = imag.(Ω)

    # In python version numpy.linalg.cholesky returns the lower triangular
    #  matrix, which is then transposed. Julia's cholesky returns the upper
    #  triangular matrix, hence no need to transpose.
    T = Matrix(cholesky(Y).U)

    Yinv = inv(Y)

    # compute the integer points over which we approximate the infinite sum to
    # the requested accuracy
    R = radius(eps, T, derivs, accuracy_radius)
    S = innerpoints(T, R)

    finite_sum(X, Yinv, T, zs, S, derivs)
end

"""
         exponential_part(zs::Vector{Vector{ComplexF64}},
                          Ω::Matrix{ComplexF64})::Vector{Float64}

Return the value of the exponential part of the Riemann theta function for Ω and
all z in `zs`.

Parameters
----------
- `zs` : A vector of complex vectors at which to evaluate the Riemann theta function.
- `Omega` : A Riemann matrix.

Returns
-------
The value of the exponential part of the Riemann theta function at
each point appearing in `zs`.

"""
function exponential_part(zs::Vector{Vector{ComplexF64}},
                          Ω::Matrix{ComplexF64})::Vector{Float64}
    # extract the imaginary parts of z and the inverse of the imaginary part
    # of Omega
    y = [ imag.(z) for z in zs ]
    Yinv = inv(imag.(Ω))

    # apply the quadratic form to each vector in z
    map(y -> π * dot(y, Yinv * y), y)
end

"""
     riemanntheta(zs::Vector{Vector{ComplexF64}},
                  Ω::Matrix{ComplexF64};
                  eps::Float64=1e-8,
                  derivs::Vector{Vector{ComplexF64}}=Vector{ComplexF64}[],
                  accuracy_radius::Float64=5.)::Vector{ComplexF64}

Return the value of the Riemann theta function for Ω and all z in `zs` if
`derivs` is empty, or the derivatives at all z in `zs` for the given directional
derivatives in `derivs`.

Parameters
----------
- `zs` : A vector of complex vectors at which to evaluate the Riemann theta function.
- `Omega` : A Riemann matrix.
- `eps` : (Default: 1e-8) The desired numerical accuracy.
- `derivs` : A vector of complex vectors giving a directional derivative.
- `accuracy_radius` : (Default: 5.) The radius from the g-dimensional origin
where the requested accuracy of the Riemann theta is guaranteed when computing
derivatives. Not used if no derivatives of theta are requested.

Returns
-------
The value (or derivative) of the Riemann theta function at each point in `zs`.
"""
function riemanntheta(zs::Vector{Vector{ComplexF64}},
                      Ω::Matrix{ComplexF64};
                      eps::Float64=1e-8,
                      derivs::Vector{Vector{ComplexF64}}=Vector{ComplexF64}[],
                      accuracy_radius::Float64=5.)::Vector{ComplexF64}

    u = exponential_part(zs, Ω)
    v = oscillatory_part(zs, Ω, eps=eps, derivs=derivs,
                         accuracy_radius=accuracy_radius)

    exp.(u) .* v
end