# MIT License                                                                    #src
#                                                                                #src
# Copyright (c) 2020 François Pacaud                                             #src
#                                                                                #src
# Permission is hereby granted, free of charge, to any person obtaining a copy   #src
# of this software and associated documentation files (the "Software"), to deal  #src
# in the Software without restriction, including without limitation the rights   #src
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      #src
# copies of the Software, and to permit persons to whom the Software is          #src
# furnished to do so, subject to the following conditions:                       #src
#                                                                                #src
# The above copyright notice and this permission notice shall be included in all #src
# copies or substantial portions of the Software.                                #src
#                                                                                #src
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     #src
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       #src
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    #src
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         #src
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  #src
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  #src
# SOFTWARE.                                                                      #src

# # Logistic Regression

# **Originally Contributed by**: François Pacaud

# This tutorial shows how to solve a logistic regression problem
# with JuMP. Logistic regression is a well known method in machine learning,
# useful when we want to classify binary variables with the help of
# a given set of features. To this goal,
# we find the optimal combination of features maximizing
# the (log)-likelihood onto a training set. From a modern optimization glance,
# the resulting problem is convex and differentiable. On a modern optimization
# glance, it is even conic representable.
#
# ## Formulating the logistic regression problem
#
# Suppose we have a set of training data-point $i = 1, \cdots, n$, where
# for each $i$ we have a vector of features $x_i \in \mathbb{R}^p$ and a
# categorical observation $y_i \in \{-1, 1\}$.
#
# The log-likelihood is given by
#
# ```math
# l(\theta) = \sum_{i=1}^n \log(\dfrac{1}{1 + \exp(-y_i \theta^\top x_i)})
# ```
#
# and the optimal $\theta$ minimizes the logistic loss function:
#
# ```math
# \min_{\theta}\; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)).
# ```
#
# Most of the time, instead of solving directly the previous optimization problem, we
# prefer to add a regularization term:
#
# ```math
# \min_{\theta}\; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)) + \lambda \| \theta \|
# ```
#
# with $\lambda \in \mathbb{R}_+$ a penalty and $\|.\|$ a norm function. By adding
# such a regularization term, we avoid overfitting on the training set and usually
# achieve a greater score in cross-validation.

# ## Reformulation as a conic optimization problem
#
# By introducing auxiliary variables $t_1, \cdots, t_n$ and $r$,
# the optimization problem is equivalent to
#
# ```math
# \begin{aligned}
# \min_{t, r, \theta} \;& \sum_{i=1}^n t_i + \lambda r \\
# \text{subject to } & \quad t_i \geq \log(1 + \exp(- y_i \theta^\top x_i)) \\
#                    & \quad r \geq \|\theta\|
# \end{aligned}
# ```
#
# Now, the trick is to reformulate the constraints $t_i \geq \log(1 + \exp(- y_i \theta^\top x_i))$
# with the help of the *exponential cone*
#
# ```math
# K_{exp} = \{ (x, y, z) \in \mathbb{R}^3 : \; y \exp(x / y) \leq z \} .
# ```
#
# Indeed, by passing to the exponential, we
# see that for all $i=1, \cdots, n$, the constraint $t_i \geq \log(1 + \exp(- y_i \theta^\top x_i))$
# is equivalent to
#
# ```math
# \exp(-t_i) + \exp(u_i - t_i) \leq 1
# ```
#
# with $u_i = -y_i \theta^\top x_i$. Then, by adding two auxiliary variables
# $z_{i1}$ and $z_{i2}$ such that $z_{i1} \geq \exp(u_i-t_i)$ and $z_{i2} \geq \exp(-t_i)$, we get
# the equivalent formulation
#
# ```math
# \left\{
# \begin{aligned}
# (u_i -t_i , 1, z_{i1}) & \in  K_{exp}  \\
# (-t_i , 1, z_{i2}) & \in  K_{exp}  \\
# z_{i1} + z_{i2} & \leq  1
# \end{aligned}
# \right.
# ```
#
# In this setting, the conic version of the logistic regression problems writes out
#
# ```math
# \begin{aligned}
# \min_{t, z, r, \theta}&  \; \sum_{i=1}^n t_i + \lambda r \\
# \text{subject to } & \quad  (u_i -t_i , 1, z_{i1})  \in  K_{exp}  \\
#                    & \quad  (-t_i , 1, z_{i2})  \in  K_{exp}  \\
#                    & \quad  z_{i1} + z_{i2}  \leq  1 \\
#                    & \quad u_i = -y_i x_i^\top \theta \\
#                    & \quad r \geq \|\theta\|
# \end{aligned}
# ```
#
# and thus encompasses $3n + p + 1$ variables and $3n + 1$ constraints ($u_i = -y_i \theta^\top x_i$
# is only a virtual constraint used to clarify the notation).
# Thus, if $n \gg 1$, we get a large number of variables and constraints.

# ## Fitting logistic regression with a conic solver
#
# It is now time to pass to the implementation. We choose SCS as a conic solver.
using JuMP
import Random
import SCS

Random.seed!(2713);

# We start by implementing a function to generate a fake dataset, and where
# we could tune the correlation between the feature variables. The function
# is a direct transcription of the one used in [this blog post](http://fa.bianp.net/blog/2013/numerical-optimizers-for-logistic-regression/).
function generate_dataset(n_samples=100, n_features=10; shift=0.0)
    X = randn(n_samples, n_features)
    w = randn(n_features)
    y = sign.(X * w)
    X .+= 0.8 * randn(n_samples, n_features) # add noise
    X .+= shift # shift the points in the feature space
    X = hcat(X, ones(n_samples, 1))
    return X, y
end

# We write a `softplus` function to formulate each constraint
# $t \geq \log(1 + \exp(u))$ with two exponential cones.
function softplus(model, t, u)
    z = @variable(model, [1:2], lower_bound=0.0)
    @constraint(model, sum(z) <= 1.0)
    @constraint(model, [u - t, 1, z[1]] in MOI.ExponentialCone())
    @constraint(model, [-t, 1, z[2]] in MOI.ExponentialCone())
end

# ### $\ell_2$ regularized logistic regression
# Then, with the help of the `softplus` function, we could write our
# optimization model. In the $\ell_2$ regularization case, the constraint
# $r \geq \|\theta\|_2$ rewrites as a second order cone constraint.
function build_logit_model(X, y, λ)
    n, p = size(X)
    model = Model()
    @variable(model, θ[1:p])
    @variable(model, t[1:n])
    for i in 1:n
        u = - (X[i, :]' * θ) * y[i]
        softplus(model, t[i], u)
    end
    ## Add ℓ2 regularization
    @variable(model, 0.0 <= reg)
    @constraint(model, [reg; θ] in MOI.SecondOrderCone(p+1))
    ## Define objective
    @objective(model, Min, sum(t) + λ * reg)
    return model
end

# We generate the dataset.
#
# !!! warning
#     Be careful here, for large n and p SCS could fail to converge!
#
n, p = 200, 10
X, y = generate_dataset(n, p, shift=10.0);

## We could now solve the logistic regression problem
λ = 10.0
model = build_logit_model(X, y, λ)
set_optimizer(model, SCS.Optimizer)
JuMP.optimize!(model)

#-

θ♯ = JuMP.value.(model[:θ])

# It appears that the speed of convergence is not that impacted by the correlation
# of the dataset, nor by the penalty $\lambda$.

# ### $\ell_1$ regularized logistic regression
#
# We now formulate the logistic problem with a $\ell_1$ regularization term.
# The $\ell_1$ regularization ensures sparsity in the optimal
# solution of the resulting optimization problem. Luckily, the $\ell_1$ norm
# is implemented as a set in `MathOptInterface`. Thus, we could easily formulate
# the sparse logistic regression problem with the help of a `MOI.NormOneCone`
# set.
function build_sparse_logit_model(X, y, λ)
    n, p = size(X)
    model = Model()
    @variable(model, θ[1:p])
    @variable(model, t[1:n])
    for i in 1:n
        u = - (X[i, :]' * θ) * y[i]
        softplus(model, t[i], u)
    end
    ## Add ℓ1 regularization
    @variable(model, 0.0 <= reg)
    @constraint(model, [reg; θ] in MOI.NormOneCone(p+1))
    ## Define objective
    @objective(model, Min, sum(t) + λ * reg)
    return model
end

## Auxiliary function to count non-null components:
count_nonzero(v::Vector; tol=1e-6) = sum(abs.(v) .>= tol)

## We solve the sparse logistic regression problem on the same dataset as before.
λ = 10.0
sparse_model = build_sparse_logit_model(X, y, λ)
set_optimizer(sparse_model, SCS.Optimizer)
JuMP.optimize!(sparse_model)

#-

θ♯ = JuMP.value.(sparse_model[:θ])
println("Number of non-zero components: ", count_nonzero(θ♯),
        " (out of ", p, " features)")

# ### Extensions
# A direct extension would be to consider the sparse logistic regression with
# *hard* thresholding, which, on contrary to the *soft* version using a $\ell_1$ regularization,
# adds an explicit cardinality constraint in its formulation:
#
# ```math
# \begin{aligned}
# \min_{\theta} & \; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)) + \lambda \| \theta \|_2^2 \\
# \text{subject to } & \quad \| \theta \|_0 <= k
# \end{aligned}
# ```
#
# where $k$ is the maximum number of non-zero components in the vector $\theta$,
# and $\|.\|_0$ is the $\ell_0$ pseudo-norm:
#
# ```math
# \| x\|_0 = \#\{i : \; x_i \neq 0\}
# ```
#
# The cardinality constraint $\|\theta\|_0 \leq k$ could be reformulated with
# binary variables. Thus the hard sparse regression problem could be solved
# by any solver supporting mixed integer conic problems.
#


"""
# Usage

    solve(α, dx, dt, xStart, xEnd, n, κ, FiniteDiffEx())

!!! tip
    Here, if we set ``0<\\alpha\\leq 1``, the equation is the fractional diffusion equation or subdiffusion equation, whereas for ``1<\\alpha\\leq 2``, the equation is the fractional diffusion-wave equation.

Use explicit Caputo discretization method to solve the diffusion equations.

### References

```tex
@article{Murillo2011AnED,
  title={An Explicit Difference Method for Solving Fractional Diffusion and Diffusion-Wave Equations in the Caputo Form},
  author={Joaqu{\'i}n Quintana Murillo and Santos B. Yuste},
  journal={Journal of Computational and Nonlinear Dynamics},
  year={2011},
  volume={6},
  pages={021014}
}
```

Matlab version: https://github.com/awstown/Fractional-Derivative
"""
struct FiniteDiffEx <: FractionalDiffEqAlgorithm end

function solve(α, dx, dt, xStart, xEnd, n, κ, u0t, uendt, u0, ::FiniteDiffEx)
    x = collect(0:dx:xEnd)
    t = collect(0:dt:n)
    S = κ*((dt^α)/(dx^2))
    S_bar = gamma(3-α)*S

    # Preallocate
    U = zeros(Int64(n/dt + 1), round(Int, (xEnd - xStart)/dx + 1))

    # Boundry conditions
    U[:, 1] .= u0t
    U[:, end] .= uendt
    U[1, :] = u0.(x)

    k = collect(1:length(t)-2)
    bOfK = bbcoeff(k, α)
    test = Float64[]

    for m = 1:(length(t)- 1)
        # It is from 2 to end - 1 because of the boundry conditions
        U[m+1, 2:end-1], diff = nextStep(U, m, S_bar, test, bOfK)
    end
    return U
end

function bbcoeff(k, α)
    c = zeros(length(k))
    for i=1:length(k)
        if maximum(k) < 2
            @. c = (k+1)^(1-α) - k^(1-α)
        else
            @. c = (k+1)^(2-α) - k^(2-α)
        end
    end
    return c
end

# Compute values for each time step
function nextStep(U, m, S_bar, diff, bOfK)
    row_left   = U[m, 1:end-2]
    row_right  = U[m, 3:end]
    row_center = U[m, 2:end-1]

    if m < 2
        Uspatial = row_center + S_bar*(row_left - 2*row_center + row_right)
    else
        row_below  = U[m-1, 2:end-1]
        Uspatial = 2 * row_center - row_below + S_bar*(row_left - 2*row_center + row_right)
    end

    if m > 2
        if length(diff) == 0
            diff = U[m, 2:end-1] - 2*U[m-1, 2:end-1] + U[m-2, 2:end-1]
        else
            diff = [U[m, 2:end-1] .- 2 .*U[m-1, 2:end-1] .+ U[m-2, 2:end-1]; diff]
        end
        Utemporal = bOfK[1:length(diff[:, 1])]' * diff
        Unext = Uspatial .- Utemporal
    elseif m > 1
        if length(diff) == 0
            diff = U[m, 2:end-1] .- U[m-1, 2:end-1]
        else
            diff = [U[m, 2:end-1] .- U[m-1,2:end-1]; diff]
        end
        Utemporal = bOfK[1:length(diff[:, 1])]' * diff
        Unext = Uspatial .- Utemporal
    else
        Unext = Uspatial
    end
    return Unext, diff
end


module _SpecEn
export SpecEn
using FFTW: fft
using DSP: conv
    """
        Spec, BandEn = SpecEn(Sig)

    Returns the spectral entropy estimate of the full spectrum (`Spec`)
    and the within-band entropy (`BandEn`) estimated from the data sequence (`Sig`)
    using the default  parameters:
    N-point FFT = 2*len(`Sig`) + 1, normalised band edge frequencies = [0 1],
    logarithm = base 2, normalisation = w.r.t # of spectrum/band frequency values.

        Spec, BandEn = SpecEn(Sig::AbstractArray{T,1} where T<:Real; N::Int=1 + (2*size(Sig,1)), Freqs::Tuple{Real,Real}=(0,1), Logx::Real=exp(1), Norm::Bool=true)

    Returns the spectral entropy (`Spec`) and the within-band entropy (`BandEn`)
    estimate for the data sequence (`Sig`) using the specified 'keyword' arguments:

    # Arguments:
    `N`     - Resolution of spectrum (N-point FFT), an integer > 1  \n
    `Freqs` - Normalised band edge frequencies, a 2 element tuple with values \n
              in range [0 1] where 1 corresponds to the Nyquist frequency (Fs/2).
              Note: When no band frequencies are entered, BandEn == SpecEn
    `Logx`  - Logarithm base, a positive scalar (enter 0 for natural log) \n
    `Norm`  - Normalisation of `Spec` value:\n
              [false]  no normalisation.
              [true]   normalises w.r.t # of spectrum/band frequency values - default.

    For more info, see the EntropyHub guide.

    # See also `XSpecEn`, `fft`, `MSEn`,  `XMSEn`

    # References:
        [1] G.E. Powell and I.C. Percival,
            "A spectral entropy method for distinguishing regular and
            irregular motion of Hamiltonian systems."
            Journal of Physics A: Mathematical and General
            12.11 (1979): 2053.

        [2] Tsuyoshi Inouye, et al.,
            "Quantification of EEG irregularity by use of the entropy of
            the power spectrum."
            Electroencephalography and clinical neurophysiology
            79.3 (1991): 204-210.

    """
    function SpecEn(Sig::AbstractArray{T,1} where T<:Real; N::Int=1 + (2*size(Sig,1)),
        Freqs::Tuple{Real,Real}=(0,1), Logx::Real=exp(1), Norm::Bool=true)

    (size(Sig)[1] > 4) ? nothing : error("Sig:   must be a numeric vector")
    (N > 1) ? nothing :  error("N:     must be an integer > 1")
    (0<=Freqs[1]<1 && 0<Freqs[2]<=1 && Freqs[1]<Freqs[2]) ? nothing :
        error("Freq:    must be a two element tuple with values in range [0 1].
                The values must be in increasing order.")
    (Logx>0) ? nothing :   error("Logx:     must be a positive number > 0")

    Freqs = collect(Freqs)
    Fx = Int(ceil(N/2))
    Freqs = Int.(round.(Freqs.*Fx))
    Freqs[Freqs.==0] .= 1

    if Freqs[1] > Freqs[2]
        error("Lower band frequency must come first.")
    elseif Freqs[2]-Freqs[1]<1
        error("Spectrum resoution too low to determine bandwidth.")
    elseif minimum(Freqs)<0 || maximum(Freqs)>Fx
        error("Freqs must be normalized w.r.t sampling frequency [0 1].")
    end

    Temp = conv(Sig,Sig)
    N <= size(Temp,1) ? Temp = Temp[1:N]  : Temp = vcat(Temp,zeros(N-size(Temp)[1]))

    Pt = abs.(fft(Temp))
    Pxx = Pt[1:Fx]/sum(Pt[1:Fx])
    Spec = -(transpose(Pxx)*log.(Logx, Pxx))
    Pband = (Pxx[Freqs[1]:Freqs[2]])/sum(Pxx[Freqs[1]:Freqs[2]])
    BandEn = -(transpose(Pband)*log.(Logx, Pband))

    if Norm
        Spec = Spec/(log(Logx, Fx));
        BandEn = BandEn/(log(Logx, Freqs[2]-Freqs[1]+1));
    end

    return Spec, BandEn
    end

end

"""
Copyright 2021 Matthew W. Flood, EntropyHub

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

For Terms of Use see https://github.com/MattWillFlood/EntropyHub

"""


using LinearAlgebra
using Plots
using CSV
using KernelFunctions
using Distributions
using DataFrames

include("../algo/kernels.jl")
include("../algo/regularized_Picard.jl")
include("../algo/utils.jl")

"""
    convergence_exact_solution()

Visualize the convergence of regularized Picard to exact solution (see Proposition 10)
"""
function convergence_exact_solution()

    # width
    sigma = .01
    # regularizer
    lambda =  .05
    # regularizer for positive definiteness
    epsilon = 1e-10;
    # relative objective tolerance
    tol = 1e-6

    ############# load 1 DPP samples #############
    # intensity of the generating DPP
    intensity = 100; # 50 or 100
    strg_intensity = string(intensity)

    # create an array of arrays
    indices_DPP_samples = Array{Int64,1}[];

    print("Loading DPP samples from: \n")
    i = 1; # id of the DPP sample
    file_name = "data/statspats/samples/GaussDPPsample_alpha0_00p5_rho0_"*strg_intensity*"_nb_"*string(i)*".csv"
    print(file_name, " \n")
    temp = CSV.File(file_name; header=true) |> Tables.matrix
    temp = temp[:,2:3];
    id_temp = collect(1:size(temp,1));
    push!(indices_DPP_samples,id_temp);
    total_samples = temp;

    # for approximating Fredholm determinant
    indices_Fredholm_sample = indices_DPP_samples[1]

    ############# exact solution #############

    # create full kernel matrix
    x = (total_samples)'/sigma;
    k = SqExponentialKernel();
    K = kernelmatrix(k, x) + epsilon *I ;

    # Chol decomposition
    R = cholesky(K).U;
    Rinv = inv(R);

    # Exact solution
    m = size(K,1);
    X_exact = 0.5*(real(sqrt(m^2*I + 4*m*K/lambda))-m*I);
    B_exact = Rinv'*X_exact*Rinv;

    # objective exact solution
    obj_det_exact,obj_reg_exact = Picard_objective(B_exact, indices_DPP_samples, indices_Fredholm_sample, R,lambda)
    obj_exact = obj_det_exact + obj_reg_exact;

    ############# approximate solution #############

    # initial positive definite iterate
    X = randn(size(K));
    B = X*X'+ UniformScaling(epsilon);

    use_inverse = false

    n_steps = 50;
    it_max = 1;
    err = ones(n_steps,1);
    obj = ones(n_steps,1);

    for j=1:n_steps
        B, R, obj_j, i_stop = regularized_Picard(B, K, indices_DPP_samples, indices_Fredholm_sample, lambda, it_max ,tol,use_inverse)
        err[j] = norm(B-B_exact)/norm(B_exact);
        obj[j] = obj_j[end];
    end

    #############  plotting #############

    plt_error = plot(err, yaxis=:log,legend=false,ylabel = "relative error",xlabel = "iteration")
    display(plt_error)

    plt_obj = plot(obj,legend=false,ylabel = "objective",xlabel = "iteration")
    plot!(obj_exact*ones(size(obj)))
    display(plt_obj)
end


import Base.kron

# (A ⊗ B)×vec(X) = vec(BXAᵀ)
"""`kron(A, B)`
Kronecker tensor product of A and B in linear operator form, if either
or both are linear operators. If both A and B are matrices, then
`Base.kron` is used.
"""
function kron(A::AbstractLinearOperator, B::AbstractLinearOperator)
  m, n = size(A)
  p, q = size(B)
  T = promote_type(eltype(A), eltype(B))
  function prod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), q, n)
    res .= Matrix(B * X * transpose(A))[:]
  end
  function tprod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), p, m)
    res .= Matrix(transpose(B) * X * A)[:]
  end
  function ctprod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), p, m)
    res .= Matrix(B' * X * conj(A))[:]
  end
  symm = issymmetric(A) && issymmetric(B)
  herm = ishermitian(A) && ishermitian(B)
  nrow, ncol = m * p, n * q
  Mv = Vector{T}(undef, nrow)
  Mtu = symm ? Mv : Vector{T}(undef, ncol)
  Maw = herm ? Mv : Vector{T}(undef, ncol)
  return LinearOperator{T}(nrow, ncol, symm, herm, prod!, tprod!, ctprod!, Mv, Mtu, Maw)
end

kron(A::AbstractMatrix, B::AbstractLinearOperator) = kron(LinearOperator(A), B)

kron(A::AbstractLinearOperator, B::AbstractMatrix) = kron(A, LinearOperator(B))


##  Test of fft, power spectrum, and Hilbert transform
##  using as example of the ElNino time series
##
##  Author Bernd Blasius (bernd.blasius@gmail.com)

module W

using CSV, PyPlot, FFTW, Statistics

include("../src/fft_power.jl")

function test_power_spectrum()
 # read in the data
  df = CSV.read("../data/ElNino_data.csv"; comment="#")
  t   = df[:,1]  # time (year)
  sst = df[:,2]  # el nino signal (sea surface temperature SST)
  x   = df[:,3]  # el nino temperature anomaly

  fs = 1 ./ (mean(diff(t)))  # mean sampling frequency
  f, pow = power_real(x,fs)

  figure("El Nino Power")
  clf()
  subplot(2,1,1)
  plot(t,x)
  ylabel("El Nino")
  xlabel("time")

  subplot(2,1,2)
  plot(f, log10.(pow))
  xlabel("frequency")
  ylabel("power")

end

function test_hilbert()
  # read in the El Nino data
  df = CSV.read("../data/ElNino_data.csv"; comment="#")
  t   = df[:,1]  # time (year)
  sst = df[:,2]  # el nino signal (sea surface temperature SST)
  x   = df[:,3]  # el nino temperature anomaly

  fs = 1 ./ (mean(diff(t)))  # mean sampling frequency
  h = hilbert(x,fs)

  figure("hilbert")
  clf()
  subplot(2,1,1)  # phase plane
  plot(t,x)  # El Nino
  plot(t,h,"r") # Hilber transform
  xlabel("time")
  ylabel("El Nino")

  subplot(2,1,2)  # phase plane
  plot(x,h)

end

#test_power_spectrum()
test_hilbert()

end # module


function simpson_integration(x_vec,y_vec)

    sum = 0.
    dx = x_vec[2]-x_vec[1]

    #First point
    sum += 3*y_vec[1]/8 + 3*y_vec[end]/8
    sum += 7*y_vec[2]/6 + 7*y_vec[end-1]/6
    sum += 23*y_vec[3]/24 + 23*y_vec[end-2]/24

    for idx in 4:(length(x_vec)-3)
        sum += y_vec[idx]
    end
    return sum*dx
end

function simpson_integration_2D(x_vec,y_vec,z_vec)

    sum_vector = zeros(length(y_vec))

    for idx in 1:length(sum_vector)
        sum_vector[idx] = simpson_integration(x_vec,z_vec[:,idx])
    end

    sum = simpson_integration(y_vec , sum_vector)

    return sum
end

function get_field_energy( fname::String, every::Int , tlimit::Float64)

    #Get Variables
    fid         = h5open( fname , "r")

    rmin = read_attribute(fid, "rmin")
    rmax = read_attribute(fid, "rmax")
    rnodes   = read_attribute(fid, "rnodes")

    thnodes   = read_attribute(fid, "thnodes")

    #Discretize space
    dr = (rmax - rmin)/(rnodes-1)
    r = [rmin + (i-1)*dr for i in 1:rnodes]

    dth = 2*pi/(thnodes)
    theta = [ (i-1)*dth for i in 1:thnodes]

    #Create Differential Operators
    Dr  =   Diff_Operator_2D(1 ,1, [dr,dth] , [rnodes,thnodes])
    Drr =   Diff_Operator_2D(2 ,1, [dr,dth] , [rnodes,thnodes])

    Dp  =  Diff_Operator_2D_Periodic(1 ,2, [dr,dth] , [rnodes,thnodes])
    Dpp =  Diff_Operator_2D_Periodic(2 ,2, [dr,dth] , [rnodes,thnodes])

    one_over_r  = spzeros(rnodes*thnodes,rnodes*thnodes)
    r_matrix = spzeros(rnodes*thnodes,rnodes*thnodes)

    for i in 1:rnodes*thnodes
        one_over_r[i,i] = (1/r[1 + (i-1)%rnodes])
        r_matrix[i,i] = (r[1 + (i-1)%rnodes])
    end

    #Get max time iteration
    _ , tmax , max_sim_iter  = get_time_variables(fname)
    max_plot_iter = max_sim_iter

    if(tlimit <=  tmax)
        max_plot_iter = floor(Int , max_sim_iter * tlimit/tmax )
    else
        println("ATTENTION: max time input is larger than simulation time, using tmax")
    end

    # Total volume of space
    Volume = pi*( rmax^2 - rmin^2)
    #dV = dx*dy
    #I_factor = dV / Volume

    #Create storage matrix
    Energy_matrix = zeros( rnodes , thnodes )

    #Storage vectors
    E_vector = Float64[]
    t_vector = Float64[]

    t = 0
    E = 0

    #Loop to get energy
    for i in 0:every:max_plot_iter

        #Time and fields
        t, _ , _ , ψ , dψ = get_fields(fname,i)

        #Gradient term
        Psi  = reshape(ψ, rnodes*thnodes)
        dPsi = reshape(dψ, rnodes*thnodes)

        vr = reshape( r_matrix * Drr * Psi ,  (rnodes,thnodes))
        vp = reshape( r_matrix * one_over_r * Dp * Psi , (rnodes,thnodes))
        vt = reshape( r_matrix *  dPsi , (rnodes,thnodes))

        Energy_matrix = (vr.^2 .+ vp.^2 .+ vt.^2)

        E = simpson_integration_2D(r , theta ,Energy_matrix)

        #Add to storage vectors  and normalize
        push!(t_vector, t)
        push!(E_vector, E/Volume  )

    end

    return t_vector , E_vector

end


s = """
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
"""
s2 = replace(s, "\n" => "")
a = split(s2, "")
a2 = map(x -> parse(Int, x), a)
len = length(a2)

window = 13
maxsofar = 0
for i = 1:(len-window+1)
  global maxsofar
  p = prod(a2[i:i+window-1])
  maxsofar = max(maxsofar, p)
end
println(maxsofar)


# Demonstrate NTF synthesis (synthesizeNTF)
using RSDeltaSigmaPort
j=im

#==Baseband modulator
===============================================================================#
println("\n*** 5th order, 2-level, baseband modulator")
OSR = 32

@info("Synthesizing NTF without zero-optimization..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF_noopt = synthesizeNTF(5, OSR, opt=0)
println("\tdone.")

plot = plotNTF(NTF_noopt, OSR, color=:blue)
plot.title = "5th-Order Modulator (No Zero-Optimization)"
saveimage(:png, "dsdemo1_o5_noopt.png", plot, AR=2/1, width=900)
displaygui(plot)

@info("Synthesizing NTF with optimized zeros..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF_opt = synthesizeNTF(5, OSR, opt=1)
println("\tdone.")

plot = plotNTF(NTF_opt, OSR, color=:red)
plot.title = "5th-Order Modulator (Optimized Zeros)"
saveimage(:png, "dsdemo1_o5_zopt.png", plot, AR=2/1, width=900)
displaygui(plot)

@info("Plotting NTF comparison (overlay results)")
#-------------------------------------------------------------------------------
plot = plotNTF(NTF_noopt, OSR, color=:blue)
plot = plotNTF!(plot, NTF_opt, OSR, color=:red)
plot.title = "5th-Order Modulator (Optimized Zeros - Overlay)"
saveimage(:png, "dsdemo1_o5_cmp.png", plot, AR=2/1, width=900)
displaygui(plot)

#==Bandpass modulator
===============================================================================#
println("\n*** 8th order, 2-level, bandpass modulator")
OSR = 64
order = 8
f0 = 0.125 #fs/8

function calcSTF(order, OSR, NTF, f0)
	G = _zpk(zeros(array_round(order/2)),NTF.p,1,1)
	G.k = 1/abs(evalTF(G,exp(2π*j*f0)))
	return G
end

@info("Synthesizing NTF..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF = synthesizeNTF(order, OSR, opt=2, f0=f0)
println("\tdone.")

@info("Plotting NTF")
#-------------------------------------------------------------------------------
STF = calcSTF(order, OSR, NTF, f0)
plot = plotNTF(NTF, OSR, color=:blue, f0=f0, STF=STF)
plot.title = "8th-Order Bandpass Modulator"
saveimage(:png, "dsdemo1_o8_bp.png", plot, AR=2/1, width=900)
displaygui(plot)

:END_OF_DEMO


# Largest prime factor

NUMBER = 600_851_475_143

function solve()
    n = NUMBER
    i = 2
    while n > 1
        if n % i == 0
            n /= i
        else
            i += 1
        end
    end
    return i
end

print(Int(solve()))


#=
The power set of a set is the set of all its subsets. Write a function that, given a set, generates its power set.

For example, given the set {1, 2, 3}, it should return {{}, {1}, {2}, {3}, {1, 2}, {1, 3}, {2, 3}, {1, 2, 3}}.

You may also use a list or array to represent a set.
=#

function get_powerset(set::Array{Int,1})
    powerset = [[]]
    for number in set
        push!(powerset, [number])
    end
    set_length = 1
    while set_length < length(set)
        newpowerset = copy(powerset)
        for arr_index in eachindex(powerset)
            arr = powerset[arr_index]
            for number in set
                if length(arr) == set_length && !in(number, arr) && number > arr[end]
                    new_array = union(arr, number)
                    push!(newpowerset, new_array)
                end
            end
        end
        powerset = newpowerset
        set_length = set_length + 1
    end
    return powerset
end


function issquarefree{I<:Integer}(d::I)
  # factor(d) is a dictionary whose keys are primes
  # and whose values are corresponding prime powers.
  # An integer is square free iff none of its prime
  # factors have power 2 or greater. This is equivalent
  # to its Mobius function being nonzero.
  mobius(d) != 0
end

"""

[Link](https://proofwiki.org/wiki/Definition:M%C3%B6bius_Function)
"""
function mobius{I<:Integer}(n::I)
  if n==1 return I(1) end
  powers = collect(values(factor(n)))
  return all(powers.== 1) ? I(-1)^length(powers) : I(0)
end

#  Division by monic polynomials

function onestep!{I<:Integer}(num::Array{I,1}, den::Array{I,1}, quo::Array{I,1})
  n = findlast(num .!= 0)
  d = findlast(den .!= 0)
  den[d] == 1 || error("denominator is not monic")
  k = n-d
  k >= 0 || error("numerator not divisible by denominator")
  quo[1+k] += num[n]
  for i=1:d
    num[i+k] -= num[n]*den[i]
  end
end

function divide_by_monic{I<:Integer}(num::Array{I,1}, den::Array{I,1})
  cnum = deepcopy(num)
  quo = zeros(I,length(num))
  while any(cnum .!= 0)
    onestep!(cnum, den, quo)
  end
  quo
end

function cyclotomic{I<:Integer}(n::I)
  n >= 1 || error("There is no cyclotomic of degree < 1")
  if n==1
    return [I(-1), I(1)]
  end
  num = zeros(I,n+1)
  num[n+1]=I(1)
  num[1] = I(-1)
  for d in 1:(n-1)
    if n%d == 0
      num = divide_by_monic(num, cyclotomic(d))
    end
  end
  d = findlast(num .!= 0)
  num[1:d]
end


# # Mesh with implicit geometry

# In this tutorial, we will learn
#
#    -  How to create a mesh of hexahedra.
#    -  How to create a mesh using an implicit definition of the geometry.

# The geometry is a ramp in the form of 3/4 part of a circle.

# The tutorial will produce files for mesh visualization in the
# [Paraview](https://www.paraview.org/) format (VTK). One can display this
# information by loading the file with `paraview.exe`. When the tutorial is
# executed in `mybinder.org`, the graphics file needs to be downloaded to your
# desktop, and then visualized locally.

# We shall use static arrays for the connectivities and the coordinates.
using StaticArrays

# We start with a parametric domain: a rectangular block, with dimensions of
# different physical units. The extent of the domain: the first
# coordinate is angular. The next two coordinates are in the plane generating
# section.
L, W, H = 1.5*pi, 3.0, 1.5
R = 4.0

# This is the number of element edges along each dimension.
M, N, K = 13, 4, 2

# Now we define the indexing of the vertices. The linear index converts the
# triple of indexes along the three dimensions into a single number. the
# dimensions correspond to the number of nodes along direction.
linix = LinearIndices((M+1, N+1, K+1))
# The serial-number function converts the triple of indexes into a single linear
# index.
sn(m, n, k) = linix[m, n, k]

# Now we create the connectivities of the individual hexahedral cells. The
# numbering is given by the topologically regular character of the block domain.
C = SVector{8, Int}[]
for k in 1:K, n in 1:N, m in 1:M
    c = (
        sn(m, n, k), sn(m+1, n, k), sn(m+1, n+1, k), sn(m, n+1, k),
        sn(m, n, k+1), sn(m+1, n, k+1), sn(m+1, n+1, k+1), sn(m, n+1, k+1)
        )
    push!(C, SVector{8, Int}(c))
end

# The locations of the vertices is also perfectly regular within the parametric
# domain. The nodes are distributed uniformly along each direction. the location
# in the physical space is computed from the parametric coordinates: the first
# coordinate is used as an angle, and `R` is the interior radius of the circular
# ramp.
loc = SVector{3, Float64}[]
for k in 1:K+1, n in 1:N+1, m in 1:M+1
    a = (m-1)/M*L
    x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
    push!(loc, SVector{3, Float64}(x, y, z))
end

# At this point we can create the shape collections.
using MeshCore: ShapeColl, H8, P1
# This is the collection of hexahedral shapes.
elements = ShapeColl(H8, length(C), "elements")
# And this is the collection of the vertices.
vertices = ShapeColl(P1, length(loc), "vertices")

# The geometry is attached is an attribute. Here we use vector attribute which
# actually stores three coordinates per vertex.
using MeshCore: VecAttrib
vertices.attributes["geom"] = VecAttrib(loc)
using MeshCore: IncRel
connectivities = IncRel(elements, vertices, C)

# We now write out a postprocessing file with the elements. Note that we pass
# the `connectivities` incidence relation to get the elements stored in the
# file.
using MeshSteward: vtkwrite
vtkwrite("block", connectivities)

# Now we will define another shape collection. This one will refer to geometry
# defined implicitly: through a function that returns the coordinates  of a
# vertex based on its serial number. Remember that serial numbers are stored in
# the connectivities.

vertices2 = ShapeColl(P1, prod((M+1, N+1, K+1)), "vertices2")

# The Cartesian indices will allow us to map the linear index to a Cartesian
# triple of indices.
cart = CartesianIndices((M+1, N+1, K+1))
using MeshCore: FunAttrib

# Note that this function defines the same geometry in physical space as the
# loop above. However, the geometry is not stored, it is returned as a triple of
# numbers.
access_location(i) = let
    ci  = cart[i]
    m, n, k = ci[1], ci[2], ci[3]
    a = (m-1)/M*L
    x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
    SVector{3, Float64}(x, y, z)
end

# The geometry attribute is defined by providing access to the locations of the
# vertices through the function `access_location`.
vertices2.attributes["geom"] = FunAttrib(0.0, prod((M+1, N+1, K+1)), access_location);
connectivities = IncRel(elements, vertices2, C)
vtkwrite("block2", connectivities)

# Now both data sets have been written out. Load them into
# [Paraview](https://www.paraview.org/) and it should be possible to verify that
# those two meshes sets overlap perfectly.

true


# Example of MLE estimation. The data is really drawn from the
# Logit DGP, so the model is well specified, and the MLE has
# the properties discussed in class. E.g., if you make n very
# large you should see that the estimator is very close to the
# true value of theta used to generate data

using Econometrics
include("LogitDGP.jl")
n = 30 # sample size
theta = [0, 0.5] # true theta for generating data
(y, x) = LogitDGP(n, theta) # generate the data

# now define things for estimation
model = theta -> logit(theta, y, x)
theta = zeros(size(x,2)) # start values for estimation

# Perform the estimation - Make sure that you examine
# the MLE estimation programs so that you see how this works
thetahat, objvalue, V, converged = mleresults(model, theta, "estimate logit model");


export getLaplacianMatrix, getConvectionMatrix

"""
function ddxc(n)

centered one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxc(n::Int)
    dx = spdiagm(-1=>fill(-1.0,n-1), 1=>fill(1.0,n-1))
    return dx
end

"""
function ddxfwd(n)

forward one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxfwd(n::Int)
    dx = spdiagm(0=>fill(-1.0,n),1=>fill(1.0,n-1))
    return dx
end

"""
function ddxbwd(n)

backward one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxbwd(n::Int)
    dx = spdiagm(-1=>fill(-1.0,n-1),0=>fill(1.0,n))
    return dx
end

"""
function getLaplacian(M::RegularMesh)

builds 2D/3D Laplacian for padded cell-centered vector including
ghost points and corners.

Input:

	M  - computational mesh

Output:

	Lap - 2D/3D Laplacian

"""
function getLaplacianMatrix(M::RegularMesh)
    if M.dim==2
		dx = Mesh.ddx(M.n[1]+1)/M.h[1]
		dy = Mesh.ddx(M.n[2]+1)/M.h[2]

		Dxx = kron(sparse(1.0I,M.n[2]+2, M.n[2]+2),dx'*dx)
		Dyy = kron(dy'*dy, sparse(I, M.n[1]+2, M.n[1]+2))
		return Dxx+Dyy
	elseif M.dim==3
		dx = Mesh.ddx(M.n[1]+1)/M.h[1]
		dy = Mesh.ddx(M.n[2]+1)/M.h[2]
		dz = Mesh.ddx(M.n[3]+1)/M.h[3]

		Dxx = kron(sparse(1.0I, M.n[3]+2, M.n[3]+2),kron(sparse(1.0I, M.n[2]+2, M.n[2]+2),dx'*dx))
		Dyy = kron(sparse(1.0I, M.n[3]+2, M.n[3]+2),kron(dy'*dy, sparse(1.0I, M.n[1]+2, M.n[1]+2)))
		Dzz = kron(dz'*dz, kron(sparse(1.0I, M.n[2]+2, M.n[2]+2), sparse(1.0I, M.n[1]+2, M.n[1]+2)))

		return Dxx + Dyy + Dzz
	end
end

"""
function getConvectionMatrix(M,v)

builds stationary convection operator on extended mesh

2D: C(v_1,v_2)     = Diagonal(v_1)*Dx + Diagonal(v_2)*Dy

3D: C(v_1,v_2,v_3) = Diagonal(v_1)*Dx + Diagonal(v_2)*Dy + Diagonal(v_3)*Dz

Input:

	M :: RegularMesh - original domain (not padded)
	v                - velocities discretized on cell-centers of extended mesh
	 				   or function that returns values (X -> vFun) where
					   X has x,y(,z) coordinates column wise.

Output:

	Conv - 2D Convection operator

"""
function getConvectionMatrix(M::RegularMesh,v::Array{Float64})

	n   = M.n
	h   = M.h
	dim = M.dim
	v  = reshape(v,prod(n.+2),dim)

	if dim==2
    	dxf = ddxfwd(n[1]+2)./(h[1])
    	dxb = ddxbwd(n[1]+2)./(h[1])
    	dyf = ddxfwd(n[2]+2)./(h[2])
    	dyb = ddxbwd(n[2]+2)./(h[2])

    	Dxf = kron(sparse(1.0I, n[2]+2, n[2]+2), dxf)
    	Dxb = kron(sparse(1.0I, n[2]+2, n[2]+2), dxb)
    	Dyf = kron(dyf, sparse(1.0I, n[1]+2, n[1]+2))
    	Dyb = kron(dyb, sparse(1.0I, n[1]+2, n[1]+2))
    	return Diagonal(max.(v[:,1],0))*Dxb + Diagonal(min.(v[:,1],0))*Dxf +
		       Diagonal(max.(v[:,2],0))*Dyb + Diagonal(min.(v[:,2],0))*Dyf
	elseif dim==3
		dxf = ddxfwd(n[1]+2)./(M.h[1])
		dxb = ddxbwd(n[1]+2)./(M.h[1])
		dyf = ddxfwd(n[2]+2)./(M.h[2])
		dyb = ddxbwd(n[2]+2)./(M.h[2])
		dzf = ddxfwd(n[3]+2)./(M.h[3])
		dzb = ddxbwd(n[3]+2)./(M.h[3])

		Dxf = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(sparse(1.0I, n[2]+2, n[2]+2), dxf));
		Dxb = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(sparse(1.0I, n[2]+2, n[2]+2), dxb));
		Dyf = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(dyf, sparse(1.0I, n[1]+2, n[1]+2)))
		Dyb = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(dyb, sparse(1.0I, n[1]+2, n[1]+2)))
		Dzf = kron(dzf, sparse(1.0I, prod(n[1:2].+2), prod(n[1:2].+2)))
		Dzb = kron(dzb, sparse(1.0I, prod(n[1:2].+2), prod(n[1:2].+2)))
    	return Diagonal(max.(v[:,1],0))*Dxb + Diagonal(min.(v[:,1],0))*Dxf +
               Diagonal(max.(v[:,2],0))*Dyb + Diagonal(min.(v[:,2],0))*Dyf +
               Diagonal(max.(v[:,3],0))*Dzb + Diagonal(min.(v[:,3],0))*Dzf
	end
end

function getConvectionMatrix(M::RegularMesh,v::Function)
	xc = getNodalGrid(getPaddedMesh(M))
	return getConvectionMatrix(M,v(xc))
end


using SellmeierFit
using SimpleConstants  # for micro = 1e-6
using CairoMakie

## Main section
# Load a set of measured refractive index data from https://RefractiveIndex.info.
path = "https://refractiveindex.info/tmp/data/main/SiO2/Malitson.csv"
(; λ, ε) = SellmeierFit.read(path)

# Fit the loaded data to the Sellmeier equation.
(; mdl, err) = fit_sellmeier(λ, ε)

## Analysis section
# The calculated fit parameters are the same as those shown in RefractiveIndex.info's
# Dispersion formula section.
println("B = $(mdl.str)")
println("√C (µm) = $(mdl.λres ./ micro)")
println("Error between data and fit = $err")

# Visualize the measured data and fit Sellmeier equation; note that n = √ε.
fontsize = 20
fig = Figure(; fontsize)
Axis(fig[1,1], title="Refractive Index of SiO₂", xlabel="λ (meter)", ylabel="n")
scatter!(λ, .√ε, label="Measurement", color=:black)
lines!(λ, .√mdl.(λ), label="Sellmeier equation", color=:red)
axislegend(; position=:rt)
display(fig)


using Distributions, Plots; pyplot()

alpha, beta = 8, 2
prior(lam) = pdf(Gamma(alpha, 1/beta), lam)
data = [2,1,0,0,1,0,2,2,5,2,4,0,3,2,5,0]

like(lam) = *([pdf(Poisson(lam),x) for x in data]...)
posteriorUpToK(lam) = like(lam)*prior(lam)

sig = 0.5
foldedNormalPDF(x,mu) = (1/sqrt(2*pi*sig^2))*(exp(-(x-mu)^2/2sig^2)
                                                + exp(-(x+mu)^2/2sig^2))
foldedNormalRV(mu) = abs(rand(Normal(mu,sig)))

function sampler(piProb,qProp,rvProp)
    lam = 1
    warmN, N = 10^5, 10^6
    samples = zeros(N-warmN)

    for t in 1:N
        while true
            lamTry = rvProp(lam)
            L = piProb(lamTry)/piProb(lam)
            H = min(1,L*qProp(lam,lamTry)/qProp(lamTry,lam))
            if rand() < H
                lam = lamTry
                if t > warmN
                    samples[t-warmN] = lam
                end
                break
            end
        end
    end
    return samples
end

mcmcSamples = sampler(posteriorUpToK,foldedNormalPDF,foldedNormalRV)
println("MCMC Bayes Estimate: ",mean(mcmcSamples))

stephist(mcmcSamples, bins=100,
	c=:black, normed=true, label="Histogram of MCMC samples")

lamRange = 0:0.01:10
plot!(lamRange, prior.(lamRange),
	c=:blue, label="Prior distribution")

closedFormPosterior(lam)=pdf(Gamma(alpha + sum(data),1/(beta+length(data))),lam)
plot!(lamRange, closedFormPosterior.(lamRange),
	c=:red, label="Posterior distribution",
	xlims=(0, 10), ylims=(0, 1.2),
    xlabel=L"\lambda",ylabel="Density")


using Plots
using LinearAlgebra

x = 3 # True slope
a = -2:.25:2
b = a*x + randn(length(a)) # Add noise

p1 = plot(a, x*a, label = "True line") # True relationship
scatter!(a, b, label = "Noisy data") # Noisy measurements

U, S, V = svd(reshape(collect(a), length(a), 1))
xtilde = (V*inv(diagm(S))*U'*b)[1] # Least-square fit

plot!(a, xtilde*a, label = "Regression line") # Plot fit


function getergodic(transmatrix::Array{Float64,2})
    D,V = eig(transmatrix')
    eigindex = indmin(abs.(D-1.0))
    ergodic = vec(V[:,eigindex]/sum(V[:,eigindex]))
    if any(abs.(imag(ergodic)).>1e-16)
        error("Imaginary ergodic distribution")
    else
        return real(ergodic)
    end
end

function createtransmatrix_sym(numregimes::Int)
    out = Array{SymPy.Sym}(numregimes)
    for j = 1:numregimes
        out[j] = Sym("TRANSITIONPROBABILITY_RX_RP$(j)")
    end
    return out
end

function TransitionMatrix(trans)
    # Sanity check the regime transition matrix
    numregimes = size(trans,1)
    if size(trans,2)!=size(trans,1)
        error("Regime transition matrix is not square.")
    end
    if any((abs.(sum(trans,2))-1.0).>1e-12)
        error("Rows of the regime transition matrix do not sum to 1.")
    end
    # Create a sympy version of the regime trans matrix
    syms = createtransmatrix_sym(numregimes)

    # Get the ergodic distribution of the markovian regime process
    ergodic = getergodic(trans)
    return TransitionMatrix(trans,syms,ergodic,Sym("TRANSITIONPROBABILITY"))
end


@doc """
       Newton method for local optimization (multivariable)

       Supported by:

       Ph D Kelvyn B. Sánchez
       e-mail: kelvyn.baruc@gmail.com
       GitHub: Kelvyn88
 """ ->

function Newton(f, vars, x0; error = 0.0001, maxiter = 100, iter = true)
      try
            const version = Pkg.installed("OptimJulia")
            length(vars) == 0 ? len = 1 : len = length(vars)

            length(vars) == 0 ? sep_var = 1 : sep_var = length(vars)
            const separator = "="^(9 + (sep_var + 2)*16)
            err = Inf
            count = 1
            x_num = 0
            f_num = f

            # Numeric iterations
            iter == true ? ti = time_ns() : nothing # To measure algorithm performance

            while err > error
                  x_num = x0

                  # Symbolic derivatives using Module SymPy
                  if length(vars) == 0
                        # Single variable
                        dfk = diff(f)
                  else
                        # Multivariable variable
                        dfk = Array(SymPy.Sym, 1, length(vars))
                        for i=1:length(vars)
                              dfk[i] = diff(f, vars[i])
                        end
                  end

                  f_num = f

                  # Symbolic hessian using module SymPy
                  if length(vars) == 0
                        # Single variable
                        dfk2 = diff(dfk)
                  else
                        # Multivariable variable
                        hess = hessian(f, vars)
                  end

                  # Numeric evaluations
                  if length(vars) == 0
                        # Single variable
                        dfk = dfk |> subs(vars, x_num)
                        dfk2 = dfk2 |> subs(vars,x_num)

                        # Conversion from BigInt to Float32 (avoid overflow)
                        dfk = convert(Float32, dfk)
                        dfk2 = convert(Float32, dfk2)
                  else
                        # Multivariable variable
                        for i=1:length(vars), j=1:length(vars), k=1:length(vars)
                              dfk[i] = dfk[i] |> subs(vars[k], x_num[k])
                              hess[i,j] = hess[i,j] |> subs(vars[k],x_num[k])
                        end

                        # Conversion from BigInt to Float32 (avoid overflow)
                        dfk = convert(Array{Float32,length(vars)}, dfk)
                        hess = convert(Array{Float32,length(vars)}, hess)
                  end

                  # Newton equation
                  if length(vars) == 0
                        # Single variable
                        xk = x_num - dfk/dfk2
                  else
                        # Multivariable variable
                        xk = x_num - inv(hess) * transpose(dfk)
                  end

                  err = sum(abs((xk -  x_num) ./ xk))

                  # Numeric evaluation of original function
                  for k=1:len
                        f_num = f_num |> subs(vars[k], N(xk[k]))
                  end

                  f_num = convert(Float32, f_num)

                  # Print iteration for the case of iter == true
                  if iter == true
                        if count == 1
                              # Header message and version
                              println(separator)
                              msg1 = @sprintf(" Newton Solver for Unconstrained Optimization %s ", version)
                              msg2 = @sprintf(" Ph D Kelvyn B. Sánchez ")
                              header_left = ceil((length(separator) - length(msg1))/2)
                              header_right = (length(separator) - length(msg1)) - header_left
                              msg1_left = ":"^convert(Int32,header_left)
                              msg1_right = ":"^convert(Int32,header_right)
                              msg1 = string(msg1_left, msg1, msg1_right)
                              println(msg1)

                              header_left = ceil((length(separator) - length(msg2))/2)
                              header_right = (length(separator) - length(msg2)) - header_left
                              msg2_left = ":"^convert(Int32,header_left)
                              msg2_right = ":"^convert(Int32,header_right)
                              msg2 = string(msg2_left, msg2, msg2_right)
                              println(msg2)
                        end

                        if count == 1 || ceil((count/1)/10) == (count/1)/10
                              println(separator)

                              # Header
                              msg1 = "Iter\t "

                              for i=1:len
                                    msg1 = string(msg1, vars[i], "\t\t ")
                              end

                              msg1 = string(msg1, "f(x)\t\t ","||err||\t\t ")
                              println(msg1)
                              println(separator)
                        end

                        # Custom message for iterator
                        msg2 = @sprintf("%i\t ", count)
                        for i=1:len
                              msg2 = string(msg2, @sprintf("%4.6e\t ", xk[i]))
                        end

                        msg2 = string(msg2, @sprintf("%4.6e\t ", f_num), @sprintf("%4.6e", err))
                        println(msg2)
                  end

                  # Backup of new data for next iteration
                  x0 = xk
                  x_num = xk
                  count += 1

                  if count == maxiter
                        throw(ErrorException("Not Solved!. Iteration limit reached."))
                  end
            end

            iter == true ? println(separator) : nothing
            iter == true ? tf = time_ns() : nothing # To measure algorithm performance
            iter == true ? println("elapse time: ", (tf - ti)/1.0e9, " seconds") : nothing

            return x_num,  f_num, err

      catch Msg
            println(separator)
            println(Msg)
            return NaN, NaN, NaN
      end
end


@doc """
	cgnr(A::Matrix, b::Vector, x0::Vector; prec, maxiter)

Conjugate gradient for normal equations residual method for solving the least squares problem `min norm( Ax - b )`.

- `A` is the coefficient matrix (`M-by-N`)
- `b` is the observation vector (`M`).
- `x0` is the starting vector (`N`).

The iteration stops when `norm(xnew - xold) < prec` or after at most `maxiter` iterations.
"""->
function cgnr{T<:Number}(A::AbstractMatrix{T}, b::AbstractVector{T}, x0::AbstractVector{T}; prec=LARGE_EPS, maxiter=length(x0))
	size(A,1) == length(b) || throw(DimensionMismatch())
	size(A,2) == length(x0) || throw(DimensionMismatch())
	prec >= SMALL_EPS || throw(DomainError())
	maxiter >= 1 || throw(DomainError())

	# Initialize
	x = copy(x0)
	y = A*x
	r = b - y
	z = A'*r
	p = copy(z)
	oneT = one(T)

	for iter = 1:maxiter
		ztz = norm(z)^2
		A_mul_B!(y, A, p) # y = A*p
		mu = ztz / norm(y)^2
		BLAS.axpy!(mu, p, x) # x = x + mu*p
		xdiff = mu*norm(p)

		BLAS.axpy!(-mu, y, r) # r = r - mu*y
		Ac_mul_B!(z, A, r) # z = A'*r
		tau = norm(z)^2 / ztz

		# p = z + tau*p
		scale!(p, tau)
		BLAS.axpy!(oneT, z, p)

		# Check for convergence: |xnew - xold|
		if xdiff < prec
			break
		end
	end

	return x
end

@doc """
	cgnr(T::Freq2wave, b, x0; ...) -> x

Conjugate gradient for normal equations residual method for `Freq2Wave`.
The initial point `x0` must be of the same dimension as `T`.
"""->
function cgnr(T::Freq2Wave, b::AbstractVector{Complex{Float64}}, x0::AbstractVecOrMat{Complex{Float64}}=zeros(eltype(T), wsize(T)); prec=LARGE_EPS, maxiter=max(length(x0),50))
	size(T,1) == length(b) || throw(DimensionMismatch())
	wsize(T) == size(x0) || throw(DimensionMismatch())
	prec >= SMALL_EPS || throw(DomainError())
	maxiter >= 1 || throw(DomainError())

	# Initialize
	x = copy(x0)
	y = T*x
	r = b - y
	z = T'*r
	p = copy(z)
	Cone = one(Complex{Float64})

	for iter = 1:maxiter
		ztz = vecnorm(z)^2
		A_mul_B!(y, T, p) # y = T*p
		mu = ztz / vecnorm(y)^2
		BLAS.axpy!(mu, p, x) # x = x + mu*p
		xdiff = mu*norm(p)

		BLAS.axpy!(-mu, y, r) # r = r - mu*y
		Ac_mul_B!(z, T, r) # z = T'*r
		tau = vecnorm(z)^2 / ztz

		# p = z + tau*p
		scale!(p, tau)
		BLAS.axpy!(Cone, z, p)

		# Check for convergence: |xnew - xold|
		if xdiff < prec
			break
		end
	end

	return x
end


using Mimi

@defcomp grosseconomy begin
    YGROSS  = Variable(index=[time])    # Gross output
    K       = Variable(index=[time])    # Capital
    l       = Parameter(index=[time])   # Labor
    tfp     = Parameter(index=[time])   # Total factor productivity
    s       = Parameter(index=[time])   # Savings rate
    depk    = Parameter()               # Depreciation rate on capital - Note that it has no time index
    k0      = Parameter()               # Initial level of capital
    share   = Parameter()               # Capital share

    function run_timestep(p, v, d, t)
        # Define an equation for K
        if is_first(t)
            v.K[t]  = p.k0  # Note the use of v. and p. to distinguish between variables and parameters
        else
            v.K[t]  = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5
        end

        # Define an equation for YGROSS
        v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share)
    end
end

@defcomp emissions begin
    E       = Variable(index=[time])    # Total greenhouse gas emissions
    sigma   = Parameter(index=[time])   # Emissions output ratio
    YGROSS  = Parameter(index=[time])   # Gross output - Note that YGROSS is now a parameter

    function run_timestep(p, v, d, t)
        # Define an eqation for E
        v.E[t] = p.YGROSS[t] * p.sigma[t]   # Note the p. in front of YGROSS
    end
end

function construct_model()
	m = Model()

	set_dimension!(m, :time, collect(2015:5:2110))

	# Order matters here. If the emissions component were defined first, the model would not run.
	add_comp!(m, grosseconomy)
	add_comp!(m, emissions)

	# Update parameters for the grosseconomy component
	update_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20])
	update_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20])
	update_param!(m, :grosseconomy, :s, ones(20).* 0.22)
	update_param!(m, :grosseconomy, :depk, 0.1)
	update_param!(m, :grosseconomy, :k0, 130.)
	update_param!(m, :grosseconomy, :share, 0.3)

	# Update and connect parameters for the emissions component
	update_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20])
	connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS)

	return m

end #end function

model = construct_model()

run(model)

# Show model results
model[:emissions, :E]

# Or, as a DataFrame
getdataframe(model, :emissions, :E)


using LinearAlgebra

abstract type AbstractForecasting end

export StateSpaceModel

"""

Space-State model is defined through the following equations

Let ``{x_0, x_1, \\dots, x_K }`` denote the discrete-time state process,
which is typically unknown to the observer. Let ``{y_1,y_2,... ,y_K}``
represent the observation process, which describes the measurements of the
system that are available to the observer.

These state and observation processes are related following a nonlinear
state-space model: The first equation describes how the system state ``x_k``
evolves through a nonlinear dynamical model ``f_{k−1,k}`` between successive
time steps ``t_{k−1}`` and ``t_k``.
``η = {η1,η2,... ,ηK}`` is the model noise process, which accounts for
the imperfections of the model; it is assumed to be independent and
identically distributed. At each time step, ``η_k`` is assumed
to be Gaussian with zero mean and covariance matrix Q. The second equation
models how an observation ``y_k`` at time ``t_k`` is obtained from the state
``x_k`` through a linear operator ``H``. ``ε = {ε1, ε2, . . . , εk}`` is an
independent and identically distributed process representing the observation
errors; ``ε_k`` is assumed
Gaussian with zero mean and covariance matrix ``R``. Finally, the
processes ``η`` and ``ε`` are assumed to be jointly independent and independent
of the initial (background) state x0, which is assumed to be Gaussian
with mean xb and covariance B.

```math
\\left\\{
\\begin{array}{l}
X_t = m(X_{t-1}) + \\eta_t, \\\\
Y_t = H(X_t) + \\varepsilon_t,
\\end{array}
\\right.
```

- X : hidden variables
- Y : observed variables

- `dt_integration`is the numerical time step used to solve the ODE.
- `dt_states` is the number of `dt_integration` between ``X_{t-1}`` and ``X_t``.
- `dt_obs` is the number of `dt_integration` between ``Y_{t-1}`` and ``Y_t``.

"""
struct StateSpaceModel <: AbstractForecasting

    model::Function
    dt_integration::Float64
    dt_states::Int64
    dt_obs::Int64
    params::Vector{Float64}
    var_obs::Vector{Int64}
    nb_loop_train::Int64
    nb_loop_test::Int64
    sigma2_catalog::Float64
    sigma2_obs::Float64

end


# # Solve the heat conduction equation

# Synopsis: Compute the solution of the Poisson equation of heat conduction with a
# nonzero heat source. Quadrilateral four-node elements are used.

# The problem is linear heat conduction equation posed on a bi-unit square,
# solved with Dirichlet boundary conditions around the circumference. Uniform
# nonzero heat generation rate is present. The exact solution is in this way
# manufactured and hence known. That gives us an opportunity to calculate the
# true error.

# The complete code is in the file [`tut_poisson_q4.jl`](tut_poisson_q4.jl).

# The solution will be defined  within a module in order to eliminate conflicts
# with data or functions defined elsewhere.

module tut_poisson_q4

# We'll need some functionality from linear algebra, and the mesh libraries.
# Finally we will need the `Elfel` functionality.
using LinearAlgebra
using MeshCore.Exports
using MeshSteward.Exports
using Elfel.Exports

# This is the top level function.
function run()
    # Input parameters:
    A = 1.0 # length of the side of the square
    kappa =  1.0; # thermal conductivity of the material
    Q = -6.0; # internal heat generation rate
    tempf(x, y) =(1.0 + x^2 + 2.0 * y^2); # the exact distribution of temperature
    N = 1000; # number of element edges along the sides of the square domain

    # Generate the computational mesh.
    mesh = genmesh(A, N)

    # Create the finite element space to represent the temperature solution. The
    # degrees of freedom are real numbers (`Float64`), the quadrilaterals are
    # defined by the mesh, and each of the elements has the continuity ``H
    # ^1``, i. e. both the function values and the derivatives are square
    # integrable.
    Uh = FESpace(Float64, mesh, FEH1_Q4())

    # Apply the essential boundary conditions at the circumference of the square
    # domain. We find the boundary incidence relation (`boundary(mesh)`), and
    # then the list of all vertices connected by the boundary cells. The
    # function `tempf` defines the analytical temperature variation, and hence
    # for each of the vertices `i` on the boundary (they are of manifold
    # dimension  `0`), we set the component of the field (1) to the exact value
    # of the temperature at that location.
    vl = connectedv(boundary(mesh));
    locs = geometry(mesh)
    for i in vl
        setebc!(Uh, 0, i, 1, tempf(locs[i]...))
    end

    # Number the degrees of freedom, both the unknowns and the data
    # (prescribed) degrees of freedom.
    numberdofs!(Uh)
    @show ndofs(Uh), nunknowns(Uh)

    # Assemble the conductivity matrix and the vector of the heat loads. Refer
    # to the definition of this function below.
    K, F = assembleKF(Uh, kappa, Q)

    # This is a vector to hold all degrees of freedom in the system.
    T = fill(0.0, ndofs(Uh))
    # Here we collect the data degrees of freedom (the known values).
    gathersysvec!(T, Uh)

    # The system of linear algebraic equations is solved.
    solve!(T, K, F, nunknowns(Uh))

    # The values of all the degrees of freedom can now be introduced into the
    # finite element space.
    scattersysvec!(Uh, T)

    # Here we associate the values of the finite element space with the entities
    # of the mesh as an attribute.
    makeattribute(Uh, "T", 1)

    # The correctness of the solution is checked by comparing the values at the
    # vertices.
    checkcorrectness(Uh, tempf)

    # The attribute can now be written out for visualization into a VTK file.
    vtkwrite("q4-T", baseincrel(mesh), [(name = "T",)])

    true # return success
end

# The domain is a square, meshed with quadrilateral elements. The function
# `Q4block` creates an incidence relation that defines the quadrilateral
# element shapes by the vertices connected into the shapes. This incidence
# relation is then attached to the mesh and the mesh is returned.
function genmesh(A, N)
    conn = Q4block(A, A, N, N)
    return attach!(Mesh(), conn)
end

# The `assembleKF` function constructs the left-hand side coefficient matrix,
# conductivity matrix, as a sparse matrix, and a vector of the heat loads due
# to the internal heat generation rate `Q`.

# The boundary value problem is expressed in this weak form
# ```math
# \int_{V}(\mathrm{grad}\vartheta)\; \kappa (\mathrm{grad}T
#             )^T\; \mathrm{d} V
#             -\int_{V}  \vartheta Q \; \mathrm{d} V
#              = 0
# ```
# where the test function vanishes on the boundary where the temperature is
# prescribed, ``\vartheta(x) =0``  for  ``x \in{S_1}``
# Substituting ``\vartheta = N_j `` and ``T = \sum_i N_i T_i`` we obtain the
# linear algebraic equations
# ```math
# \sum_i T_i \int_{V} \mathrm{grad}N_j \; \kappa (\mathrm{grad}N_i)^T\; \mathrm{d} V
#  -\int_{V}  N_j Q \; \mathrm{d} V  = 0 , \quad \forall j.
# ```
# The volume element is ``\mathrm{d} V``, which in our case
# becomes ``1.0\times\mathrm{d} S``, since the thickness of the two
# dimensional domain is assumed to be 1.0.

function assembleKF(Uh, kappa, Q)
    # At the top of the `assembleKF` we look at the function `integrate!` to
    # evaluate the weak-form integrals. The key to making this calculation
    # efficient is type stability. All the arguments coming in must have
    # concrete types. This is why the `integrate!` function is an inner
    # function: the function barrier allows for all arguments to be resolved to
    # concrete types.
    function integrate!(am, av, elit, qpit, kappa, Q)
        nedof = ndofsperel(elit)
        # The local assemblers are just like matrices or vectors
        ke = LocalMatrixAssembler(nedof, nedof, 0.0)
        fe = LocalVectorAssembler(nedof, 0.0)
        for el in elit # Loop over all elements
            init!(ke, eldofs(el), eldofs(el)) # zero out elementwise matrix
            init!(fe, eldofs(el)) # and vector
            for qp in qpit # Now loop over the quadrature points
                Jac, J = jacjac(el, qp) # Calculate the Jacobian matrix, Jacobian
                gradN = bfungrad(qp, Jac) # Evaluate the spatial gradients
                JxW = J * weight(qp) # elementary volume
                N = bfun(qp) # Basis function values at the quadrature point
                # This double loop evaluates the elementwise conductivity
                # matrix and the heat load vector precisely as the formula of
                # the weak form  dictates; see above.
                for i in 1:nedof
                    for j in 1:nedof
                        ke[j, i] += dot(gradN[j], gradN[i]) * (kappa * JxW)
                    end
                    fe[j] += N[j] * Q * JxW
                end
            end
            # Assemble the calculated contributions from this element
            assemble!(am, ke)
            assemble!(av, fe)
        end
        return am, av # Return the updated assemblers
    end

    # In the `assembleKF` function we first we create the element iterator. We
    # can go through all the elements that define the domain of integration
    # using this iterator. Each time a new element is accessed, some data are
    # precomputed such as the element degrees of freedom.
    elit = FEIterator(Uh)
    # This is the quadrature point iterator. We know that the elements are
    # quadrilateral, which makes the Gauss integration rule the obvious choice.
    # We also select order 2 for accuracy. Quadrature-point iterators provide
    # access to basis function values and gradients, the Jacobian matrix and
    # the Jacobian determinant, the location of the quadrature point and so
    # on.
    qpit = QPIterator(Uh, (kind = :Gauss, order = 2))
    # Next we create assemblers, one for the sparse system matrix and one for
    # the system vector.
    am = start!(SysmatAssemblerSparse(0.0), ndofs(Uh), ndofs(Uh))
    av = start!(SysvecAssembler(0.0), ndofs(Uh))
    # Now we call the integration function. The assemblers are modified inside
    # this function...
    @time integrate!(am, av, elit, qpit, kappa, Q)
    # ...so that when the integration is done, we can materialize the sparse
    # matrix and the vector and return them.
    return finish!(am), finish!(av)
end

# The linear algebraic system is solved by partitioning. The vector `T` is
# initially all zero, except in the degrees of freedom which are prescribed as
# nonzero. Therefore the product of the conductivity matrix and the vector `T`
# are the heat loads due to nonzero essential boundary conditions. To this we
# add the vector of heat loads due to the internal heat generation rate. The
# submatrix of the heat conduction matrix corresponding to the free degrees of
# freedom (unknowns), `K[1:nu, 1:nu]` is then used to solve for the unknowns `T
# [1:nu]`.
function solve!(T, K, F, nu)
    @time KT = K * T
    @time T[1:nu] = K[1:nu, 1:nu] \ (F[1:nu] - KT[1:nu])
end

# The correctness can be checked in various ways. Here we calculate the mean
# deviation of the calculated temperatures at the nodes relative to the exact
# values of the temperature.
function checkcorrectness(Uh, tempf)
    geom = geometry(Uh.mesh)
    ir = baseincrel(Uh.mesh)
    T = attribute(ir.right, "T")
    std = 0.0
    for i in 1:length(T)
        std += abs(T[i][1] - tempf(geom[i]...))
    end
    @show (std / length(T)) <= 1.0e-9
end

end # module

# The module can now be used.
using .tut_poisson_q4
tut_poisson_q4.run()


"""
    complement(g)

Produces the [graph complement](https://en.wikipedia.org/wiki/Complement_graph)
of a graph.
"""
function complement(g::Graph)
    gnv = nv(g)
    h = Graph(gnv)
    for i=1:gnv
        for j=i+1:gnv
            if !has_edge(g, i, j)
                add_edge!(h,i,j)
            end
        end
    end
    return h
end

function complement(g::DiGraph)
    gnv = nv(g)
    h = DiGraph(gnv)
    for i=1:gnv
        for j=1:gnv
            if i != j && !has_edge(g,i,j)
                add_edge!(h,i,j)
            end
        end
    end
    return h
end

"""
    reverse(g::DiGraph)

Produces a graph where all edges are reversed from the
original.
"""
function reverse(g::DiGraph)
    gnv = nv(g)
    gne = ne(g)
    h = DiGraph(gnv)
    h.fadjlist = deepcopy(g.badjlist)
    h.badjlist = deepcopy(g.fadjlist)
    h.ne = gne
    h.vertices = g.vertices

    return h
end

"""
    reverse!(g::DiGraph)

In-place reverse (modifies the original graph).
"""
function reverse!(g::DiGraph)
    g.fadjlist, g.badjlist = g.badjlist, g.fadjlist
    return g
end

doc"""
    blkdiag(g, h)

Produces a graph with $|V(g)| + |V(h)|$ vertices and $|E(g)| + |E(h)|$
edges.

Put simply, the vertices and edges from graph `h` are appended to graph `g`.
"""
function blkdiag{T<:SimpleGraph}(g::T, h::T)
    gnv = nv(g)
    r = T(gnv + nv(h))
    for e in edges(g)
        add_edge!(r, e)
    end
    for e in edges(h)
        add_edge!(r, gnv+src(e), gnv+dst(e))
    end
    return r
end

"""
    intersect(g, h)

Produces a graph with edges that are only in both graph `g` and graph `h`.

Note that this function may produce a graph with 0-degree vertices.
"""
function intersect{T<:SimpleGraph}(g::T, h::T)
    gnv = nv(g)
    hnv = nv(h)

    r = T(min(gnv, hnv))
    for e in intersect(edges(g),edges(h))
        add_edge!(r,e)
    end
    return r
end

"""
    difference(g, h)

Produces a graph with edges in graph `g` that are not in graph `h`.

Note that this function may produce a graph with 0-degree vertices.
"""
function difference{T<:SimpleGraph}(g::T, h::T)
    gnv = nv(g)
    hnv = nv(h)

    r = T(gnv)
    for e in edges(g)
        !has_edge(h, e) && add_edge!(r,e)
    end
    return r
end

"""
    symmetric_difference(g, h)

Produces a graph with edges from graph `g` that do not exist in graph `h`,
and vice versa.

Note that this function may produce a graph with 0-degree vertices.
"""
function symmetric_difference{T<:SimpleGraph}(g::T, h::T)
    gnv = nv(g)
    hnv = nv(h)

    r = T(max(gnv, hnv))
    for e in edges(g)
        !has_edge(h, e) && add_edge!(r, e)
    end
    for e in edges(h)
        !has_edge(g, e) && add_edge!(r, e)
    end
    return r
end

"""
    union(g, h)

Merges graphs `g` and `h` by taking the set union of all vertices and edges.
"""
function union{T<:SimpleGraph}(g::T, h::T)
    gnv = nv(g)
    hnv = nv(h)

    r = T(max(gnv, hnv))
    r.ne = ne(g)
    for i = 1:gnv
        r.fadjlist[i] = deepcopy(g.fadjlist[i])
        if is_directed(g)
            r.badjlist[i] = deepcopy(g.badjlist[i])
        end
    end
    for e in edges(h)
        add_edge!(r, e)
    end
    return r
end

"""
    join(g, h)

Merges graphs `g` and `h` using `blkdiag` and then adds all the edges between
 the vertices in `g` and those in `h`.
"""
function join(g::Graph, h::Graph)
    r = blkdiag(g, h)
    for i=1:nv(g)
        for j=nv(g)+1:nv(g)+nv(h)
            add_edge!(r, i, j)
        end
    end
    return r
end

"""
    crosspath(len::Integer, g::Graph)

Replicate `len` times `h` and connect each vertex with its copies in a path
"""
crosspath(len::Integer, g::Graph) = cartesian_product(PathGraph(len), g)

"""
    induced_subgraph(g, iter)

Filters graph `g` to include only the vertices present in the iterable
argument `vs`. Returns the subgraph of `g` induced by `vs`.
"""
function induced_subgraph{T<:SimpleGraph}(g::T, iter)
    n = length(iter)
    isequal(n, length(unique(iter))) || error("Vertices in subgraph list must be unique")
    isequal(n, nv(g)) && return copy(g) # if iter is not a proper subgraph

    h = T(n)
    newvid = Dict{Int, Int}()
    i=1
    for (i,v) in enumerate(iter)
        newvid[v] = i
    end

    iterset = Set(iter)
    for s in iter
        for d in out_neighbors(g, s)
            # println("s = $s, d = $d")
            if d in iterset && has_edge(g, s, d)
                newe = Edge(newvid[s], newvid[d])
                add_edge!(h, newe)
            end
        end
    end
    return h
end

# dispatch for g[[1,2,3]], g[1:3], g[Set([1,2,3])]
# these are the only allowed dispatches, everything else is slow
getindex(g::SimpleGraph, iter) = induced_subgraph(g, iter)

# The following operators allow one to use a LightGraphs.Graph as a matrix in eigensolvers for spectral ranking and partitioning.
# """Provides multiplication of a graph `g` by a vector `v` such that spectral
# graph functions in [GraphMatrices.jl](https://github.com/jpfairbanks/GraphMatrices.jl) can utilize LightGraphs natively.
# """
function *{T<:Real}(g::Graph, v::Vector{T})
    length(v) == nv(g) || error("Vector size must equal number of vertices")
    y = zeros(T, nv(g))
    for e in edges(g)
        i = src(e)
        j = dst(e)
        y[i] += v[j]
        y[j] += v[i]
    end
    return y
end

function *{T<:Real}(g::DiGraph, v::Vector{T})
    length(v) == nv(g) || error("Vector size must equal number of vertices")
    y = zeros(T, nv(g))
    for e in edges(g)
        i = src(e)
        j = dst(e)
        y[i] += v[j]
    end
    return y
end

"""sum(g,i) provides 1:indegree or 2:outdegree vectors"""
function sum(g::SimpleGraph, dim::Int)
    dim == 1 && return indegree(g, vertices(g))
    dim == 2 && return outdegree(g, vertices(g))
    error("Graphs are only two dimensional")
end

size(g::SimpleGraph) = (nv(g), nv(g))
"""size(g,i) provides 1:nv or 2:nv else 1 """
size(g::Graph,dim::Int) = (dim == 1 || dim == 2)? nv(g) : 1

"""sum(g) provides the number of edges in the graph"""
sum(g::SimpleGraph) = ne(g)

"""sparse(g) is the adjacency_matrix of g"""
sparse(g::SimpleGraph) = adjacency_matrix(g)

#arrayfunctions = (:eltype, :length, :ndims, :size, :strides, :issymmetric)
eltype(g::SimpleGraph) = Float64
length(g::SimpleGraph) = nv(g)*nv(g)
ndims(g::SimpleGraph) = 2
issymmetric(g::SimpleGraph) = !is_directed(g)

"""
    cartesian_product(g, h)

Returns the (cartesian product)[https://en.wikipedia.org/wiki/Tensor_product_of_graphs] of `g` and `h`
"""
function cartesian_product{G<:SimpleGraph}(g::G, h::G)
    z = G(nv(g)*nv(h))
    id(i, j) = (i-1)*nv(h) + j
    for (i1, i2) in edges(g)
        for j=1:nv(h)
            add_edge!(z, id(i1,j), id(i2,j))
        end
    end

    for e in edges(h)
        j1, j2 = src(e), dst(e)
        for i=1:nv(g)
            add_edge!(z, id(i,j1), id(i,j2))
        end
    end
    return z
end

"""
    tensor_product(g, h)

Returns the (tensor product)[https://en.wikipedia.org/wiki/Tensor_product_of_graphs] of `g` and `h`
"""
function tensor_product{G<:SimpleGraph}(g::G, h::G)
    z = G(nv(g)*nv(h))
    id(i, j) = (i-1)*nv(h) + j
    for (i1, i2) in edges(g)
        for (j1, j2) in edges(h)
            add_edge!(z, id(i1, j1), id(i2, j2))
        end
    end
    return z
end


"Solve Computing Eccentric Anomaly from Mean Anomally via Kepler's Equation"
module KeplerEqn

export calc_ecc_anom
export calc_ecc_anom_cpu, calc_ecc_anom_gpu
using CUDAnative

"""   ecc_anom_init_guess_danby(M, ecc)
Initial guess for eccentric anomaly given mean anomaly (M) and eccentricity (ecc)
    Based on "The Solution of Kepler's Equations - Part Three"
    Danby, J. M. A. (1987) Journal: Celestial Mechanics, Volume 40, Issue 3-4, pp. 303-312  1987CeMec..40..303D
"""
function ecc_anom_init_guess_danby(M, ecc)
    k = convert(typeof(ecc),0.85)
    if(M<zero(M)) M += 2pi end
    (M<pi) ? M + k*ecc : M - k*ecc;
end

"""   update_ecc_anom_laguerre(E, M, ecc)
Update the current guess (E) for the solution to Kepler's equation given mean anomaly (M) and eccentricity (ecc)
   Based on "An Improved Algorithm due to Laguerre for the Solution of Kepler's Equation"
   Conway, B. A.  (1986) Celestial Mechanics, Volume 39, Issue 2, pp.199-211  1986CeMec..39..199C
"""
function update_ecc_anom_laguerre(E, M, ecc)
  es = ecc*sin(E)
  ec = ecc*cos(E)
  F = (E-es)-M
  Fp = one(E)-ec
  Fpp = es
  n = 5
  root = sqrt(abs((n-1)*((n-1)*Fp*Fp-n*F*Fpp)))
  denom = Fp>zero(E) ? Fp+root : Fp-root
  return E-n*F/denom
end

"Same as update_ecc_anom_laguerre_cpu, except uses GPU math functions"
function update_ecc_anom_laguerre_gpu(E, M, ecc)
  es = ecc*CUDAnative.sin(E)
  ec = ecc*CUDAnative.cos(E)
  F = (E-es)-M
  Fp = one(E)-ec
  Fpp = es
  n = 5
  root = CUDAnative.sqrt(abs((n-1)*((n-1)*Fp*Fp-n*F*Fpp)))
  denom = Fp>zero(E) ? Fp+root : Fp-root
  return E-n*F/denom
end

"Loop to update the current estimate of the solution to Kepler's equation"
function calc_ecc_anom_itterative_laguerre(mean_anom, ecc, tol, max_its)
    M = mod(mean_anom,convert(typeof(mean_anom),2pi))
    E = ecc_anom_init_guess_danby(M,ecc)
    for i in 1:max_its
       E_old = E
       E = update_ecc_anom_laguerre(E_old, M, ecc)
       if abs(E-E_old)<convert(typeof(mean_anom),tol) break end
    end
    return E
end

"Loop to update the current estimate of the solution to Kepler's equation.  Calls GPU update"
function calc_ecc_anom_itterative_laguerre_gpu(mean_anom, ecc, tol, max_its)
    @assert zero(ecc) <= ecc < one(ecc)
    @assert tol*100 <= one(tol)
    M = mod(mean_anom,convert(typeof(mean_anom),2pi))
    E = ecc_anom_init_guess_danby(M,ecc)
    for i in 1:max_its
       E_old = E
       E = update_ecc_anom_laguerre_gpu(E_old, M, ecc)
       if abs(E-E_old)<convert(typeof(mean_anom),tol) break end
    end
    return E
end

const default_max_its_laguerre = 200
const default_ecc_anom_tol = 1e-8
"Calculate eccentric anomaly given mean anomaly and eccentricty (in radians)"
calc_ecc_anom_cpu(mean_anom, ecc) = calc_ecc_anom_itterative_laguerre(mean_anom, ecc, default_ecc_anom_tol, default_max_its_laguerre)
calc_ecc_anom_gpu(mean_anom, ecc) = calc_ecc_anom_itterative_laguerre_gpu(mean_anom, ecc, default_ecc_anom_tol, default_max_its_laguerre)
calc_ecc_anom(mean_anom, ecc) = calc_ecc_anom_cpu(mean_anom, ecc)

end # module KeplerEqn


using Revise, Random, LinearAlgebra, Optim, StatsBase,
  Statistics, Distributions, COSMO, BenchmarkTools, Plots

Random.seed!(11)

#compute the minvol portfolio given a return target, expected returns z, and Σ
function minvolw(μ, z, Σ, Σinv = Σ\I)
  K= length(z)
  some1s = ones(K)

  A = some1s' * Σinv * some1s
  B = some1s' * Σinv * z
  C = z' * Σinv * z

  λ =(μ * B - C) /(B^2 - A*C)
  γ = (μ * A - B) /(A*C -B^2)

  w = λ * Σinv * some1s .+ γ * Σinv * z

  return w
end

#same as before but acquires the solution numerically
function nminvolw(μ, z, Σ)
  K= length(z)
  some1s = ones(K)

  m = COSMO.Model()
  c1 = COSMO.Constraint(ones(1,K), -1.0, COSMO.ZeroSet)
  c2 = COSMO.Constraint(z', -μ, COSMO.ZeroSet)
  COSMO.assemble!(m, Σ, zeros(K), [c1; c2])
  res = COSMO.optimize!(m)

  return res.x

  #return w
end

function gengraphs(;N=10^5, K=15, Δ=0.001)
  Erm = 0.08 #expected market reutn
  rf = 0.02 #risk free rate
  σm = 0.1 #market standard deviation

  ivol = rand(K)*0.4 .+ 0.1 #stock ivols
  beta = rand(Normal(1,0.2), K) #stock betas

  rm = rand(Normal(Erm, σm), N) #the market returns
  noise = reduce(hcat, (σe->rand(Normal(0, σe), N)).(ivol))
  r = rm .* beta' .+ noise

  z = mean(r, dims=1) |> vec
  Σ = cov(r)

  #sanity check
  Σinv = Σ\I
  w = minvolw(0.1, z, Σ, Σinv)
  nw = nminvolw(0.1, z, Σ)
  @assert w ≈ nw

  minvolσ(μ) = minvolw(μ, z, Σ, Σinv) |> (w)->(w'*Σ*w)^0.5
  nminvolσ(μ) = nminvolw(μ, z, Σ) |> (w)->(w'*Σ*w)^0.5

  #compute the minimum variance frontier
  μs = 0.01:Δ:0.2 |> collect
  σs = similar(μs)
  Threads.@threads for i ∈ 1:length(μs)
    σs[i] = minvolσ(μs[i])
  end

  #compute global min var portfolio
  some1s = ones(K)
  A = some1s' * Σinv * some1s
  B = some1s' * Σinv * z

  wg = Σinv * some1s / A
  σg = (wg'*Σ*wg)^(0.5)
  rg = z' * wg

  #compute the tangency portfolio
  wt = Σinv * (z .- some1s .* rf) ./ (B-rf*A)
  rt = z'*wt
  σt = (wt'*Σ*wt)^(0.5)

  pth = "C:\\Users\\Clinton\\Dropbox\\AAtawork\\ChernovInvestments\\Investments\\2020\\Exams\\final"
  p = plot(σs, μs,
    yaxis = ("E(r)", (0.0,0.16), :none),
    yticks=(0.0:0.02:0.16),
    xaxis=("vol", (0.0,0.4),),
    xticks=(0.0:0.05:0.4),
    title="Minimum Variance Frontier",
    legend=:none,
    linestyle=:dot,
    linecolor=:black)

  pans = p |> deepcopy

  plot!(pans, [0.0; σt; σs[μs .> rt]], [rf; rt; μs[μs .> rt]],
    linestyle=:dash, seriestype=:path, linecolor=:black, linewidth=4,
    annotations=(σt*1.5, rt*1.2,
    Plots.text("min var frontier\n (rf asset, no borrowing)", :left, 7,)))
  plot!(pans, [0.0; σt; σt*10], [rf; rt; (rt-rf)*10],
    linestyle=:solid, seriestype=:path, linecolor=:black, linewidth=2,
    annotations=(σt*1.1, rt*1.6,
    Plots.text("min varfrontier\n (rf asset, borrowing)", :left, 7,)))

  #scatter!(pans, [0.0], [rf], markersize=6)
  scatter!(pans, [σt], [rt], markersize=6,
    annotations=(σt+0.005, rt-0.0025, Plots.text("tangency portfolio", :left, 7,)))
  scatter!(pans, [σg], [rg],markersize=6,
    annotations=(σg+0.005, rg-0.0025, Plots.text("global min var portfolio", :left, 7,)))
  savefig(p, "$pth\\problem3.pdf")
  savefig(pans, "$pth\\problemans3.pdf")

end

@time gengraphs()


function get_non_dominated_solutions_perm(population)
    ids = Int[1]
    n = length(population)

    for i in 2:n
        j = 1
        while j <= length(ids)
            jj = ids[j]
            relation = compare(population[i], population[jj])

            if relation == 2 # j dominates i
                break
            elseif relation == 1 # i dominates j
                deleteat!(ids, j)
                continue
            end

            j += 1
        end

        if j > length(ids)
            push!(ids, i)
        end

    end

    return ids
end

function get_non_dominated_solutions(population)

    mask = get_non_dominated_solutions_perm(population)
    return population[mask]

end

"""
    gen_ref_dirs(dimension, n_paritions)
Generates Das and Dennis's structured reference points. `dimension` could be
the number of objective functions in multi-objective functions.
"""
function gen_ref_dirs(dimension, n_paritions)
    return  gen_weights(dimension, n_paritions)
end

function gen_weights(a, b)
    nobj = a;
    H    = b;
    a    = zeros(nobj);
    d    = H;
    w    = [];
    produce_weight!(a, 1, d, H, nobj, w)
    return Array.(w)
end

function  produce_weight!(a, i, d, H, nobj, w)
    for k=0:d
        if i<nobj
            a[i] = k;
            d2   = d - k;
            produce_weight!(a, i+1, d2, H, nobj, w);
        else
            a[i] = d;
            push!(w, a/H)
            break;
        end
    end
end

"""
    ideal(points)

Computes the ideal point from a provided array of `Vector`s or a population or row vectors
in a `Matrix`.
"""
function ideal(points::Array{Vector{T}}) where T <: Real

    isempty(points) && isempty(points[1]) && return zeros(0)

    ideal = points[1]

    for point in points
        ideal = min.(ideal, point)
    end

    return ideal

end

"""
    nadir(points)

Computes the nadir point from a provided array of `Vector`s or a population or row vectors
in a `Matrix`.
"""
function nadir(points::Array{Vector{T}})  where T <: Real

    isempty(points) && isempty(points[1]) && return zeros(0)

    nadir = points[1]

    for point in points
        nadir = max.(nadir, point)
    end

    return nadir
end

function ideal(population::Array{xFgh_indiv})
    mask = sum_violations.(population) .== 0

    ideal(fval.(population[mask]))
end
ideal(A::Matrix) = ideal([A[i,:]  for i in 1:size(A,1)])

function nadir(population::Array{xFgh_indiv})
    mask = sum_violations.(population) .== 0

    nadir(fval.(population[mask]))
end

nadir(A::Matrix) = nadir([A[i,:]  for i in 1:size(A,1)])


"""
    $(SIGNATURES)

Discretize given percentiles (unweighted). The smallest `pctV[1]` of values in `inV` end up in bin 1, etc.
"""
function discretize_given_percentiles(inV :: AbstractVector{F1},
    pctV :: AbstractVector{F2}) where {F1, F2}

    edgeV = bin_edges_from_percentiles(inV, pctV);
    return discretize(inV, edgeV)
end

"""
    $(SIGNATURES)

Discretize given percentiles (weighted). The smallest `pctV[1]` of values in `inV` end up in bin 1, etc.
"""
function discretize_given_percentiles(inV :: AbstractVector{F1},
    wtV :: AbstractVector{F2},
    pctV :: AbstractVector{F3}) where {F1, F2, F3}

   edgeV = bin_edges_from_percentiles(inV, wtV, pctV);
   return discretize(inV, edgeV)
end

## Discretize given bounds
# Equivalent to Discretizers LinearDiscretizer
function discretize(inV :: AbstractVector{F1},
    edgeV :: AbstractVector{F2}) where {F1, F2}

    n = length(edgeV);
    outV = zeros(Int64, size(inV));
    for i1 = n : -1 : 2
        outV[inV .<= edgeV[i1]] .= i1 - 1;
    end
    outV[inV .<= edgeV[1]] .= 0;
    return outV
end

"""
Bin edges from percentiles (unweighted)
Lowest bin includes minimum of inV
"""
function bin_edges_from_percentiles(inV :: AbstractVector{F1},
    pctV :: AbstractVector{F2}) where {F1, F2}

    edgeV = quantile(inV, [zero(F2); pctV]);
    # Ensure that lowest point is inside edges
    edgeV[1] -= F1(1e-8);
    return edgeV
end

## Bin edges from percentiles (weighted)
function bin_edges_from_percentiles(inV :: AbstractVector{F1},
    wtV :: AbstractVector{F2}, pctV :: AbstractVector{F3}) where {F1, F2, F3}

    @assert check_weights(wtV)  "Invalid weights"
    # Quantile is also a built in function
    edgeV = [minimum(inV) - F1(1e-8); quantile(inV, Weights(wtV), pctV)];
    edgeV[end] += F1(1e-8);
    return edgeV
end

"""
	$(SIGNATURES)

Count how many cases occur in each bin. Unweighted.
"""
function count_bins(inV :: AbstractVector{F1},
    edgeV :: AbstractVector{F2}) where {F1, F2}

    nBins = length(edgeV) - 1;
    cntV = Vector{Int}(undef, nBins);
    for iBin = 1 : nBins
        cntV[iBin] = sum(x -> inbin(x, edgeV, iBin), inV);
    end
    return cntV
end

"""
	$(SIGNATURES)

Count how many cases occur in each bin. Weighted.
"""
function count_bins(inV :: AbstractVector{F1},
    wtV :: AbstractVector{F2},
    edgeV :: AbstractVector{F3}) where {F1, F2, F3}

    @assert check_weights(wtV)  "Invalid weights"
    nBins = length(edgeV) - 1;
    massV = Vector{Float64}(undef, nBins);
    for iBin = 1 : nBins
        massV[iBin] = mapreduce(
            (x, wt) -> (inbin(x, edgeV, iBin) ? wt : zero(F2)),
            +, inV, wtV);
    end
    return massV
end

inbin(x, edgeV, iBin) = (x > edgeV[iBin])  &&  (x <= edgeV[iBin+1]);

function check_weights(wtV :: AbstractVector{F1}) where F1
    isValid = true;
    if any_less(wtV, zero(F1))
        @warn "Negative weights"
        isValid = false;
    end
    return isValid
end

# ------------


mutable struct TrackedReal{T<:Real}
    data::T
    grad::Union{Nothing,T}
    children::Dict
    # this field is only need for printing the graph. you can safely remove it.
    name::String
end

track(x::Real,name="") = TrackedReal(x,nothing,Dict(),name)

function Base.show(io::IO, x::TrackedReal)
    t = isempty(x.name) ? "(tracked)" : "(tracked $(x.name))"
    print(io, "$(x.data) $t")
end

function accum!(x::TrackedReal)
    if isnothing(x.grad)
        x.grad = sum(accum!(v)*w for (v,w) in x.children)
    end
    x.grad
end

function gradient(f, args::Real...)
    ts = track.(args)
    y  = f(ts...)
    y.grad = 1.0
    accum!.(ts)
end

##########  RULES  #############################################################

function Base.:*(a::TrackedReal, b::TrackedReal)
    z = track(a.data * b.data, "*")
    a.children[z] = b.data  # dz/da=b
    b.children[z] = a.data  # dz/db=a
    z
end
function Base.:+(a::TrackedReal{T}, b::TrackedReal{T}) where T
    z = track(a.data + b.data, "+")
    a.children[z] = one(T)
    b.children[z] = one(T)
    z
end
function Base.sin(x::TrackedReal)
    z = track(sin(x.data), "sin")
    x.children[z] = cos(x.data)
    z
end

##########  Optimizion 2D function  ############################################

using Plots
g(x,y) = y*y + sin(x)
cscheme = cgrad(:RdYlBu_5, rev=true)
p1 = contour(-4:0.1:4, -2:0.1:2, g, fill=true, c=cscheme, xlabel="x", ylabel="y")
display(p1)

function descend(f::Function, λ::Real, args::Real...)
    Δargs = gradient(f, args...)
    args .- λ .* Δargs
end

function minimize(f::Function, args::T...; niters=20, λ=0.01) where T<:Real
    paths = ntuple(_->Vector{T}(undef,niters), length(args))
    for i in 1:niters
        args = descend(f, λ, args...)
        @info f(args...)
        for j in 1:length(args)
            paths[j][i] = args[j]
        end
    end
    paths
end

xs1, ys1 = minimize(g, 1.5, -2.4, λ=0.2, niters=34)
xs2, ys2 = minimize(g, 1.8, -2.4, λ=0.2, niters=16)

scatter!(p1, [xs1[1]], [ys1[1]], markercolor=:black, marker=:star, ms=7, label="Minimum")
scatter!(p1, [xs2[1]], [ys2[1]], markercolor=:black, marker=:star, ms=7, label=false)
scatter!(p1, [-π/2], [0], markercolor=:red, marker=:star, ms=7, label="Initial Point")
scatter!(p1, xs1[1:1], ys1[1:1], markercolor=:black, label="GD Path", xlims=(-4,4), ylims=(-2,2))

anim = @animate for i in 1:max(length(xs1), length(xs2))
    if i <= length(xs1)
        scatter!(p1, xs1[1:i], ys1[1:i], mc=:black, lw=3, xlims=(-4,4), ylims=(-2,2), label=false)
    end
    if i <= length(xs2)
        scatter!(p1, xs2[1:i], ys2[1:i], mc=:black, lw=3, label=false)
    end
    p1
end

gif(anim, "gd-path.gif", fps=15)


	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Using librarys
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	# GNU Scientific Library
	using GSL ;

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Special functions related
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	# Spherical Bessel function first kind j_n(x)
	function Sbessj( i::Int64 , x )
		return sqrt(pi*(1/(2*x)))*besselj(i+1.0/2,x) ;
	end

	# Spherical Bessel function second kind n_n(x) [Neumann]
	function Sneum( i::Int64 ,x )
		sqrt(pi*(1/(2*x)))*bessely(i+1.0/2,x) ;
	end

	# Spherical Bessel function first kind derivative ( Morse & Feshbach definition )
	function Sbessj_p( i::Int64 , x )
 		return (1/(2*i+1))*(i*Sbessj(i-1,x) - (i+1)*Sbessj(i+1,x)) ;
	end

	# Spherical Bessel second kind ( Neumann ) derivative
	function Sneum_p( i::Int64 , x )
		return Sneum(i-1,x) - ((i+1)/x)*Sneum(i,x) ;
	end

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Liquid Sphere
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	# Liquid sphere coefficient
	function Coef_LiquidSphere( n::Int64, x1::Float64 )
		return x1*Sbessj_p( n, x1 )/Sbessj( n, x1 ) ;
	end

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Prolate/Oblate auxiliar
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	function Index( M::Int64, i::Int64, j::Int64 )
		if j < i # Domain errors
			return	display("ERROR: j < i");
		end
		if i > M + 1 || j > M + 1
			return	display("ERROR: M < i-1, j-1");
		end
		return map( Int64, j + ( M + 1 )*( i - 1 ) - i*( i - 1 )/2 );
	end

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	File I/O
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	function ReadFileToArrayBF( file::AbstractString, sep::Char, elem::Int )
	#	Read 'file' (as BigFloat) with values 'sep' separated.
	#	A newline in the 'file' means that the retrieved structure will be a matrix
	# 	The 'elem' parameter selects all the entries (elem = 0) or a particular value
	# 	(elem = number)
		if elem == 0 # Return entire vector or matrix
			return map( x -> parse(BigFloat,x), readdlm( file, sep ,ASCIIString) );
		else # Return only the specified element
			return map( x -> parse(BigFloat,x), [readdlm( file, sep ,ASCIIString)[elem]] )[ 1 ] ;
		end
	end

	function ReadFileToArrayBF2( file::AbstractString, sep::Char, row::Any, col::Any )
	#	Read 'file' (as BigFloat) with values 'sep' separated.
	# 	Returned value is a matrix in general.
		return map( x -> parse(BigFloat,x), collect( readdlm( file, sep ,ASCIIString)[row,col] ) ) ;
	end

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Eigenvalues (prolate)
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	function pro_lambdamn_approx( c::Any, m::Int64, n::Int64 )
		# Transcription to Julia of the AGD (Adelman, Gumerov & Duraiswami) MATLAB code.
		# View ' pro_calculate_lambdamn_approx.m ' in the AGD source directory
		convert( Float64, c ) ;
		const N = m + n + 200 ; # Size of the matrix
		const A = zeros( Float64, N, N ) ;
		if ( mod( n - m, 2) == 0 )
			r = 0 ; # even
		else
			r = 1 ; # odd
		end
		# Fill in the A matrix
		for i = 1 : N
			if i == 1
				A[ 1, 1 ] = calculate_betar( c, m, r ) ;
				A[ 1, 2 ] = calculate_alphar( c, m, r ) ;
			elseif  i == N
				A[ N, N - 1 ] = calculate_gammar( c, m, r ) ;
				A[ N, N ] = calculate_betar( c, m, r ) ;
			else
				A[ i, i - 1 ] = calculate_gammar( c, m, r ) ;
				A[ i, i ] = calculate_betar( c, m, r ) ;
				A[ i, i + 1 ] = calculate_alphar( c, m, r ) ;
			end
			r = r + 2;
		end
		const d = eigvals!(A);
		if ( mod(n - m, 2) == 0 )
			const lambda_approx = real(d[ Int( (n - m + 2) / 2 ) ]) ;
		else
			const lambda_approx = real(d[ Int( (n - m + 1) / 2 ) ]) ;
		end
		filename = @ sprintf("data/pro_%08d_%03d_%03d_lambda_approx.txt", trunc( Int, c*1000 ), m, n ) ;
		writedlm( filename,lambda_approx );
	end

	function calculate_alphar( c::Float64, m::Int64, r::Int64 )
		return ((( 2 * m + r + 2 )*( 2 * m + r + 1 ))/
		(( 2 * m + 2 * r + 5 )*( 2 * m + 2 * r + 3 )))*( c*c );
	end

	function calculate_betar( c::Float64, m::Int64, r::Int64 )
		return ( m + r )*( m + r + 1 ) + (( 2*( m + r )*( m + r + 1 ) - 2*(m*m) - 1) /
		(( 2*m + 2*r - 1 )*( 2*m + 2 * r + 3 )))*(c*c);
	end

	function calculate_gammar( c::Float64, m::Int64, r::Int64 )
		return (( r * (r - 1) )/(( 2*m + 2*r - 3 )*( 2*m + 2*r - 1 )))*(c*c);
	end

	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
	#	Eigenvalues (oblate)
	# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

	function obl_lambdamn_approx( c::Any, m::Int64, n::Int64 )
		# Transcription to Julia of the AGD (Adelman, Gumerov & Duraiswami) MATLAB code.
		# View ' obl_calculate_lambdamn_approx.m ' in the AGD source directory
		convert( Float64, c ) ;
		const N = m + n + 200 ; # Size of the matrix
		const A = zeros( Float64, N, N ) ;
		if ( mod( n - m, 2) == 0 )
			r = 0 ; # even
		else
			r = 1 ; # odd
		end
		# Fill in the A matrix
		for i = 1 : N
			if i == 1
				A[ 1, 1 ] = calculate_betar_obl( c, m, r ) ;
				A[ 1, 2 ] = calculate_alphar_obl( c, m, r ) ;
			elseif  i == N
				A[ N, N - 1 ] = calculate_gammar_obl( c, m, r ) ;
				A[ N, N ] = calculate_betar_obl( c, m, r ) ;
			else
				A[ i, i - 1 ] = calculate_gammar_obl( c, m, r ) ;
				A[ i, i ] = calculate_betar_obl( c, m, r ) ;
				A[ i, i + 1 ] = calculate_alphar_obl( c, m, r ) ;
			end
			r = r + 2;
		end
		const d = eigvals!(A);
		if ( mod( n - m, 2 ) == 0 )
			const lambda_approx = real(d[ Int( (n - m + 2) / 2 ) ]) ;
		else
			const lambda_approx = real(d[ Int( (n - m + 1) / 2 ) ]) ;
		end
		filename = @ sprintf("data/obl_%08d_%03d_%03d_lambda_approx.txt", trunc( Int, c*1000 ), m, n ) ;
		writedlm( filename,lambda_approx );
	end

	function calculate_alphar_obl( c::Float64, m::Int64, r::Int64 )
		return ((( 2 * m + r + 2 )*( 2 * m + r + 1 ))/
		(( 2 * m + 2 * r + 5 )*( 2 * m + 2 * r + 3 )))*( -c*c );
	end

	function calculate_betar_obl( c::Float64, m::Int64, r::Int64 )
		return ( m + r )*( m + r + 1 ) + (( 2*( m + r )*( m + r + 1 ) - 2*(m*m) - 1) /
		(( 2*m + 2*r - 1 )*( 2*m + 2 * r + 3 )))*(-c*c);
	end

	function calculate_gammar_obl( c::Float64, m::Int64, r::Int64 )
		return (( r * (r - 1) )/(( 2*m + 2*r - 3 )*( 2*m + 2*r - 1 )))*(-c*c);
	end


using MinFEM
using LinearAlgebra

function semilinear(mesh::Mesh, L::AbstractMatrix, M::AbstractMatrix,
                    s::AbstractVector, BoundaryIndices::Set{Int64}=[], tol=1e-10)

  y = zeros(mesh.nnodes)

  pde = PDESystem(A=L, b=M*s, bc=zeros(mesh.nnodes), DI=BoundaryIndices)

  res = Inf
  while res > tol
    pde.A = L + asmCubicDerivativeMatrix(mesh, y)
    pde.b = -L*y + M*s - asmCubicTerm(mesh, y)
    refresh(pde)
    solve(pde)

    y += pde.state
    res = norm(pde.state)
    println(res)
  end
  return y
end

mesh = import_mesh("../meshes/semilinear.msh")

L = asmLaplacian(mesh)
M = asmMassMatrix(mesh)

# y = 3*sin(x[1]*pi)*sin(x[2]*pi)
f(x) = 3*2*pi^2*sin(x[1]*pi)*sin(x[2]*pi) + (3*sin(x[1]*pi)*sin(x[2]*pi))^3
s = evaluateMeshFunction(mesh, f)

boundary = union(mesh.Boundaries[1001].Nodes,
                 mesh.Boundaries[1002].Nodes,
                 mesh.Boundaries[1003].Nodes,
                 mesh.Boundaries[1004].Nodes);

y = semilinear(mesh, L, M, s, boundary);

vtkfile = open_vtk_file(mesh, "semilinear.vtu")
write_point_data(vtkfile, y, "y")
write_point_data(vtkfile, s, "s")
save_vtk_file(vtkfile)


const year = 365.25 * 24. * 60. * 60.

GtCO2_to_ppm(GtCO2) = GtCO2 / (2.13 * (44. /12.))
tCO2_to_ppm(tCO2) = GtCO2_to_ppm(tCO2) * 1.e-9

ppm_to_GtCO2(ppm) = ppm * (2.13 * (44. /12.))
ppm_to_tCO2(ppm) = ppm_to_GtCO2(ppm) * 1.e9