@everywhere function SimulateBattle(ArmiesAttacker::Int64, ArmiesDefender::Int64)
  while ArmiesDefender > 0 && ArmiesAttacker > 0
    DiceAttacker = rand(1:6, min(3, ArmiesAttacker))
    DiceDefender = rand(1:6, min(2, ArmiesDefender))
    sort!(DiceAttacker, rev=true)
    sort!(DiceDefender, rev=true)
    if DiceAttacker[1] > DiceDefender[1]
      ArmiesDefender -= 1
    else
    ArmiesAttacker -= 1
    end
    if ArmiesDefender >1 && ArmiesAttacker>1
      if DiceAttacker[2] > DiceDefender[2]
        ArmiesDefender -=1
      else ArmiesAttacker -=1
    end
  end
  end
  (ArmiesAttacker, ArmiesDefender)
end

@everywhere function SimulateBattleFast(ArmiesAttacker::Int64, ArmiesDefender::Int64)
  while ArmiesDefender > 2 && ArmiesAttacker > 3
    a = rand()
    if a <= 0.372
      ArmiesDefender -= 2
    elseif a > 0.372 && a <= 0.707
      ArmiesDefender -= 1
      ArmiesAttacker -= 1
    else
      ArmiesAttacker -= 2
    end
  end

  while ArmiesDefender > 0 && ArmiesAttacker > 0
    DiceAttacker = rand(1:6, min(3, ArmiesAttacker))
    DiceDefender = rand(1:6, min(2, ArmiesDefender))
    sort!(DiceAttacker, rev=true)
    sort!(DiceDefender, rev=true)
    if DiceAttacker[1] > DiceDefender[1]
      ArmiesDefender -= 1
    else
      ArmiesAttacker -= 1
    end
    if ArmiesDefender > 1 && ArmiesAttacker > 1
      if DiceAttacker[2] > DiceDefender[2]
        ArmiesDefender -= 1
      else ArmiesAttacker -= 1
    end
  end
  end
  (ArmiesAttacker, ArmiesDefender)
end

function SimulateRiskio(ArmiesAttacker, ArmiesDefender, NSim::Int64)
  result = Array{Float32}(length(ArmiesAttacker)
                          , length(ArmiesDefender))
    for (k, AA) in enumerate(ArmiesAttacker)
      for (j, AD) in enumerate(ArmiesDefender)
       battles = SharedArray(Int64,NSim,3)
        @sync @parallel for i in 1:NSim
           b = SimulateBattleFast(AA,AD)
           battles[i,1] = b[1]
           battles[i,2] = b[2]
        end
      AttackerWins = battles[:,1] .> battles[:,2]
      result[k,j] = mean(AttackerWins)
    end
  end
  result
end

d = @time SimulateRiskio(1:30,1:30,10000)

Pkg.add("Gadfly")

using Gadfly
using Colors
using Compose

draw(SVG("d:/probs.svg",15cm, 15cm),spy(d))

spy(d)


# used in shortest path calculations
# has_distances{T}(distmx::AEdgeMap{T}) =
#     issparse(distmx)? (nnz(distmx) > 0) : !isempty(distmx)

"""
    eccentricity(g, v, distmx=weights(g))

Calculates the eccentricity[ies] of a vertex `v`,
An optional matrix of edge distances may be supplied.

The eccentricity of a vertex is the maximum shortest-path distance between it
and all other vertices in the graph.
"""
function eccentricity(
    g::AGraphOrDiGraph,
    v::Int,
    distmx::AEdgeMap=weights(g)
)
    e = maximum(dijkstra_shortest_paths(g,v,distmx).dists)
    e == typemax(valtype(distmx)) && error("Infinite path length detected")

    return e
end

"""

    eccentricities(g, distmx=weights(g))
    eccentricities(g, vs, distmx=weights(g))

Returns `[eccentricity(g,v,distmx) for v in vs]`. When `vs` it is not supplied,
considers all node in the graph.

See also [`eccentricity`](@ref).

Note: the eccentricity vector returned by `eccentricity` may be eventually used as input
in some eccentricity related measures ([`periphery`](@ref), [`center`](@ref)).
"""
function eccentricities(
    g::AGraphOrDiGraph,
    vs::AbstractVector,
    distmx::AEdgeMap=weights(g)
)
    [eccentricity(g,v,distmx) for v in vs]
end

eccentricities(
    g::AGraphOrDiGraph,
    distmx::AEdgeMap=weights(g)
) = eccentricities(g, 1:nv(g), distmx)

"""
    diameter(g, distmx=weights(g))

Returns the maximum distance between any two vertices in `g`.
Distances  between two adjacent nodes are given by `distmx`.

See also [`eccentricities`](@ref), [`radius`](@ref).
"""
diameter(g::AGraphOrDiGraph, distmx::AEdgeMap = weights(g)) =
    maximum(eccentricities(g, distmx))

"""
    radius(g, distmx=weights(g))

Returns the minimum distance between any two vertices in `g`.
Distances  between two adjacent nodes are given by `distmx`.

See [`eccentricities`](@ref), [`diameter`](@ref).
"""
radius(g::AGraphOrDiGraph, distmx=weights(g)) =
    minimum(eccentricities(g, distmx))

"""
    periphery(g, distmx=weights(g))
    periphery(all_ecc)

Returns the set of all vertices whose eccentricity is equal to the graph's
diameter (that is, the set of vertices with the largest eccentricity).

Eventually a vector `all_ecc` contain the eccentricity of each node
can be passed as argument.

See [`eccentricities`](@ref).
"""
function periphery(all_e::Vector)
    diam = maximum(all_e)
    return filter((x)->all_e[x] == diam, 1:length(all_e))
end

periphery(g::AGraphOrDiGraph, distmx::AEdgeMap=weights(g)) =
    periphery(eccentricities(g, distmx))

"""
    center(g, distmx=weights(g))
    center(all_ecc)

Returns the set of all vertices whose eccentricity is equal to the graph's
radius (that is, the set of vertices with the smallest eccentricity).

Eventually a vector `all_ecc` contain the eccentricity of each node
can be passed as argument.

See [`eccentricities`](@ref).
"""
function center(all_e::Vector)
    rad = minimum(all_e)
    return filter((x)->all_e[x] == rad, 1:length(all_e))
end

center(g::AGraphOrDiGraph, distmx::AEdgeMap = weights(g)) =
    center(eccentricities(g, distmx))


export ThresholdGraph, RandomThresholdGraph, CreationSequence, ThresholdRepresentation

"""
`ThresholdGraph(w)` creates a threshold graph using weights from the
vector `w`.

`ThresholdGraph(dw)` creates a threshold graph from a dictionary
mapping vertices to weights.
"""
function ThresholdGraph(w::Array{T,1}) where {T<:Real}
    n = length(w)
    f = Dict{Int,T}()
    for k = 1:n
        f[k] = w[k]
    end
    return ThresholdGraph(f)
end

function ThresholdGraph(dw::Dict{S,T}) where {S,T<:Real}
    G = SimpleGraph{S}()
    vtcs = collect(keys(dw))
    n = length(vtcs)

    for v in vtcs
        add!(G, v)
    end

    for i = 1:n-1
        u = vtcs[i]
        wt_u = dw[u]
        for j = i+1:n
            v = vtcs[j]
            wt_v = dw[v]
            if wt_u + wt_v >= 1
                add!(G, u, v)
            end
        end
    end
    cache_save(G, :ThresholdRepresentation, dw)
    cache_save(G, :name, "Threshold graph")
    return G
end

"""
`RandomThresholdGraph(n)` creates a random threshold graph with `n`
vertices.
"""
function RandomThresholdGraph(n::Int)
    w = rand(n)
    return ThresholdGraph(w)
end

# The following code contributed by Tara Abrishami

"""
`CreationSequence(G)` creates a creation sequence for a threshold
graph `G`. This returns a pair `(seq, vtx_list)` where `seq` is the
creation sequence and `vtx_list` specifies the order in which the
vertices are added when creating `G`. If `G` is not a threshold
graph, then an error is raised.
"""
function CreationSequence(G1::SimpleGraph)
    A = Int[]
    T = eltype(G1)
    V = T[]
    G = deepcopy(G1)
    while length(vlist(G)) != 0
        r = false
        for v in vlist(G)
            if deg(G, v) == 0
                delete!(G, v)
                r = true
                unshift!(A, 0)
                unshift!(V, v)
                break
            end
            if deg(G, v) == NV(G) - 1
                delete!(G, v)
                r = true
                unshift!(A, 1)
                unshift!(V, v)
                break
            end
        end
        if !r
            error("This graph is not a threshold graph")
        end
    end
    return A, V
end

"""
`ThresholdRepresentation(G)` returns a threshold representation of
`G`. This returns a dictionary mapping vertices of `G` to `Rational`
weights. An error is raised if `G` is not a threshold graph.
"""
function ThresholdRepresentation(G::SimpleGraph)
    if cache_check(G, :ThresholdRepresentation)
        return cache_recall(G, :ThresholdRepresentation)
    end
    A, V = CreationSequence(G)

    prev::Int = 0
    prevVal::Rational = 1 // 3
    T = eltype(G)
    D = Dict{T,Rational}()
    D[V[1]] = prevVal
    small::Rational = prevVal
    large::Rational = prevVal
    for i = 2:length(A)
        if A[i] == prev
            D[V[i]] = prevVal
        elseif A[i] == 0
            D[V[i]] = (1 - large) / 2
            prevVal = (1 - large) / 2
            if (1 - large) / 2 < small
                small = (1 - large) / 2
            end
            if (1 - large) / 2 > large
                large = (1 - large) / 2
            end
        elseif A[i] == 1
            D[V[i]] = 1 - small
            prevVal = 1 - small
            if 1 - small > large
                large = 1 - small
            end
            if 1 - small < large
                small = 1 - large
            end
        end
        prev = i
    end
    cache_save(G, :ThresholdRepresentation, D)
    return D
end


#=
Challenge #252 [Easy] Sailors and monkeys and coconuts, oh my!

Description

A number of sailors (let's call it N) are stranded on an island with a huge pile
of coconuts and a monkey. During the night, each sailor (in turn) does the following
without the others knowing:
    1. He takes one N'th (e.g. if N=5, one fifth) of the coconuts in the pile and
    hides them
    2. The division leaves one coconut left over, which is given to the monkey.
In the morning, they split the remaining coconuts between them. This time the
split is even. There's nothing left over for the monkey.
Your task: Given the number of sailors (N), how many coconuts were in the pile
to begin with (lowest possible number)?

Formal inputs/outputs

Input

The input is a single number: N, the number of sailors. This number is a whole
number that is greater than or equal to 2.

Output

The output is a single number: the number of coconuts in the original pile.
Sample input/output

Input:
5
Output:
3121
Sample solution for 5 sailors: https://jsfiddle.net/722gjnze/8/

Credit

This challenge was originally suggested on /r/dailyprogrammer_ideas by
/u/TinyLebowski (prior to some changes by me). Have a cool challenge idea?
Hop on over to /r/dailyprogrammer_ideas to tell everyone about it!

( source: https://redd.it/43ouxy )
=#

how_many_coconut( N::Integer=2 ) = ( N^N - ( N - 1 ) )

function main( )
    local N::Integer = parse( Int, chomp( readline( STDIN ) ) )

    N >= 2 ? println( how_many_coconut( N ) ) : error( "Please, type a valid number: greater or equal to 2" )
end

main( )


"""
    earthmover(a, b, C)

Compute the optimal transport map
```math
argmin_{P ∈ U(a, b)} ⟨C, P⟩
```
and the dual potentials using the network simplex method.

# References

Peyré, G., & Cuturi, M.. (2018). Computational Optimal Transport. [arXiv:1803.00567](https://arxiv.org/abs/1803.00567).
"""
function earthmover(
    a::AbstractVector{<:Real},
    b::AbstractVector{<:Real},
    C::AbstractMatrix{<:Real}
)
    P = SparseArrays.spzeros(Base.promote_eltype(a, b), length(a), length(b))
    earthmover!(P, a, b, C)
end

function earthmover!(P, a, b, C)
    # find a feasible initial transport map
    initialmap!(P, a, b)

    # construct the initial graph
    n = length(a)
    m = length(b)
    graph = LightGraphs.SimpleGraph(n + m)
    for ij in findall(!iszero, P)
        i, j = Tuple(ij)

        # convert to vertices
        u = i
        v = j + n

        # add new edge
        LightGraphs.add_edge!(graph, u, v)
    end

    # compute the dual potentials
    T = Base.promote_eltype(P, C)
    f = Vector{T}(undef, n)
    g = Vector{T}(undef, m)
    dualpotentials!!(f, g, C, graph)

    while true
        # check if the dual potentials are feasible
        res = findfirstinfeasible(f, g, C)
        res === nothing && break

        # obtain vertices of the edge that will be added
        i, j = res
        u = i
        v = j + n

        # check if these vertices are already connected
        path = LightGraphs.a_star(graph, v, u)

        if !isempty(path)
            # determine the largest possible increase and which edge to remove
            e = path[1]
            θmin = P[LightGraphs.dst(e), LightGraphs.src(e) - n]
            kmin = 1
            for k in 3:2:length(path)
                e = path[k]
                θ = P[LightGraphs.dst(e), LightGraphs.src(e) - n]
                if θ < θmin
                    θmin = θ
                    kmin = k
                end
            end

            # update the transport map
            P[i, j] += θmin
            for k in 1:length(path)
                e = path[k]
                if isodd(k)
                    # try to avoid floating point errors
                    if k == kmin
                        P[LightGraphs.dst(e), LightGraphs.src(e) - n] = 0
                    else
                        P[LightGraphs.dst(e), LightGraphs.src(e) - n] -= θmin
                    end
                else
                    P[LightGraphs.src(e), LightGraphs.dst(e) - n] += θmin
                end
            end

            # remove the replaced edge
            LightGraphs.rem_edge!(graph, path[kmin])
        end

        # add the new edge
        LightGraphs.add_edge!(graph, u, v)

        # recompute the dual potentials
        dualpotentials!!(f, g, C, graph)
    end

    P, f, g
end

function findfirstinfeasible(f, g, C)
    n = length(f)
    m = length(g)

    @inbounds for j in 1:m, i in 1:n
        f[i] + g[j] > C[i, j] && return (i, j)
    end

    nothing
end

function dualpotentials!!(f, g, C, graph)
    n = length(f)
    m = length(g)

    visited = falses(n+m)
    S = Int[]
    @inbounds for v in LightGraphs.vertices(graph)
        # do not revisit processed vertices
        visited[v] && continue

        # handle first vertex in a new tree
        if v > n
            g[v - n] = 0
        else
            f[v] = 0
        end
        push!(S, v)
        visited[v] = true

        # handle all other vertices in this tree by DFS
        while !isempty(S)
            s = S[end]
            u = 0
            for t in LightGraphs.outneighbors(graph, s)
                if !visited[t]
                    u = t
                    if t > n
                        g[t - n] = C[s, t - n] - f[s]
                    else
                        f[t] = C[t, s - n] - g[s - n]
                    end
                    break
                end
            end
            if u == 0
                pop!(S)
            else
                visited[u] = true
                push!(S, u)
            end
        end
    end

    nothing
end

function initialmap!(P, a, b)
    n = length(a)
    m = length(b)
    size(P) == (n, m) ||
        throw(DimensionMismatch("dimensions of histograms and transport map do not match"))

    # initialize row and column sums
    T = eltype(P)
    r = T(a[1])
    c = T(b[1])

    i = j = 1
    while i ≤ n && j ≤ m
        # update initial map
        t = min(r, c)
        P[i, j] = t

        # update r and c
        r -= t
        c -= t

        if iszero(r)
            i += 1
            if i ≤ n
                r = T(a[i])
            end
        end

        if iszero(c)
            j += 1
            if j ≤ m
                c = T(b[j])
            end
        end
    end

    P
end


function Newton_Spectral(nlp:: AbstractNLPModel,
                         x₀ :: AbstractVector;
                         τ₀ :: Float64 = 0.0005,
                         ϵ  :: Float64 = 1e-6,
                         maxiter :: Int = 200)

    x = copy(x₀)
    iter = 0
    f, g = obj(nlp,x), grad(nlp, x)

    while (norm(g, Inf) > ϵ) && (iter <= maxiter)
        H    = Matrix(Symmetric(hess(nlp, x),:L))
        Δ, O = eigen(H)

        # Boost negative values of Δ to 1e-8
        D = Δ .+ max.((1e-8 .- Δ), 0.0)

        d = - O*diagm(1.0 ./ D)*O'*g

        # Simple Armijo backtracking
        hp0 = g'*d
        t   = 1.0
        ft  = obj(nlp, x + t*d)
        while ft > ( f + τ₀*t*hp0)
            t /= 2.0
            ft = obj(nlp, x + t*d)
        end
        x    += t*d
        f, g  = ft, grad(nlp, x)
        iter += 1
    end
    if iter > maxiter @warn "Iteration limit" end

    return x, f, norm(g), iter
end


# Linear regression (using iterative update)

using Regression
import Regression: solve, Options

# Data

d = 5
k = 3
n = 1000

W = randn(k, d+1)
X = randn(d, n)
Y = (W[:, 1:d] * X .+ W[:,d+1]) + 0.2 * randn(k, n)

# Solve

ret = solve(linearreg(X, Y; bias=1.0);
            solver=GD(),
            reg=SqrL2Reg(1.0e-3),
            options=Options(verbosity=:iter, grtol=1.0e-6 * n))

println()

# Print results

W_est = ret.sol
@printf("relative error = %.4e\n", sumabs2(W_est - W) / sumabs2(W))


function bisection(f, xmin::T, xmax::T; config = Options()) where {T <: Number}
    local x₀ = xmin
    local x₁ = xmax
    local x  = zero(T)

    x₀ < x₁ || begin x₀, x₁ = x₁, x₀ end

    local y₀ = f(x₀)
    local y₁ = f(x₁)
    local y  = zero(y₀)

    # y₀ * y₁ ≤ 0 || error("Either no or multiple real roots in [xmin,xmax]")

    for j in 1:config.max_iterations
        x = (x₀ + x₁) / 2
        y = f(x)

        # println("j = ", j, " , x₀ = ", x₀, " , x₁ = ", x₁)

        !isapprox(y, zero(y), atol=config.f_abstol) || break

        if y₀ * y > 0
            x₀ = x  # Root is in the right half of [x₀,x₁].
            y₀ = y
        else
            x₁ = x  # Root is in the left half of [x₀,x₁].
            y₁ = y
        end

        !isapprox(x₁ - x₀, zero(x), atol=config.x_abstol) || break
    end

    # println("α=", x, ", f(α)=", y, ", ftol=", config.f_abstol, ", abs(x₁-x₀)=", abs(x₁-x₀), ", xtol=", config.x_abstol)

    # i != ls.nmax || error("Max iteration number exceeded")

    return x
end

bisection(f, x::Number; kwargs...) = bisection(f, bracket_minimum(f, x)...; kwargs...)

"""
simple bisection line search
"""
mutable struct BisectionState{OBJ,OPT} <: LinesearchState where {OBJ <: UnivariateObjective, OPT <: Options}
    objective::OBJ
    config::OPT

    function BisectionState(objective, config)
        new{typeof(objective), typeof(config)}(objective, config)
    end
end

function BisectionState(objective::UnivariateObjective; config = Options())
    BisectionState(objective, config)
end

function BisectionState(objective::MultivariateObjective; config = Options())
    cache = LinesearchCache(objective.x_f)
    ls_objective = linesearch_objective(objective, cache)
    BisectionState(ls_objective, config)
end

Base.show(io::IO, ls::BisectionState) = print(io, "Bisection")

LinesearchState(algorithm::Bisection, objective; kwargs...) = BisectionState(objective; kwargs...)

function (ls::BisectionState)()
    bisection(ls.objective, 0., 1.; config = ls.config)
end


using Distributions
using Expectations
using StatsBase
using InformationGeometry
dist = LogNormal()
E = expectation(dist)
@show E(x->x^2)
dist1 = Gamma()
E1 = expectation(dist1)
@show E1(x->x^2)

crossentropy([0.2, 0.3, 0.5], [0.3, 0.4, 0.3])

lognormalpdf = x -> pdf(dist,x)

loggammapdf = x -> pdf(dist1,x)

E(x -> lognormalpdf(x))

A = rand(dist,1000)

B = rand(dist1,1000)

dist2 = Normal(1,3)

dist3 = Normal(5,2)

C = kldivergence(dist2,dist3)
KullbackLeibler(dist2,dist3)

D = kldivergence(dist1,dist)
KullbackLeibler(dist,dist3)

E1(x -> log(lognormalpdf(x)))- E1(x -> log(loggammapdf(x)))

KullbackLeibler(x->loggammapdf(x),y->lognormalpdf(y),HyperCube([-20,20]); Carlo=true, N=Int(3e6))

KullbackLeibler(x->pdf(Normal(1,3),x),y->pdf(Normal(5,2),y),HyperCube([-20,20]); Carlo=true, N=Int(3e6))
KullbackLeibler(Normal(1,3),Normal(5,2))


using DataFrames, RDatasets, Clustering

iris = data("datasets", "iris")

k = length(unique(iris[:, "Species"]))

clusters = k_means(matrix(iris[:, 2:5]), k)

df = DataFrame()
df["Cluster"] = clusters.assignments
df["Label"] = iris[:, "Species"]
head(df)

# Clustering doesn't work well using some of the numeric features
# Let's try subsets of them to see what happens
df = DataFrame()
df["AllCluster"] = k_means(matrix(iris[:, 2:5]), k).assignments
df["SepalCluster"] = k_means(matrix(iris[:, 2:3]), k).assignments
df["PetalCluster"] = k_means(matrix(iris[:, 4:5]), k).assignments
df["Label"] = iris[:, "Species"]

by(df, ["Label", "AllCluster", "SepalCluster", "PetalCluster"], nrow)

# Beyond the k-means algorithm we can also use dp-means
clusters = dp_means(matrix(iris[:, 2:5]), 5.0)


function NewEst3Ps(g0,m)
#NewEst3Ps    Calculates covariance matrix of sqrt(T)*sample average.
#
#
#  Usage:     Shat = NewEst3Ps(hhat,m)
#
#  Input:     g0          TxK matrix of moment functions
#             m           order of autoregression
#
#  Output:    Shat        covariance matrix of sum( hhat/sqrt(T) ),
#                         that is the covariance matrix of sqrt(T)*mean(hhat)
#
#
#  Note: The CLT typically say that sqrt(T)*sample average ->d N(mu,Shat),
#        where Shat is what this code estimates. Clearly, Var(sample average) =
#        Shat/T.
#
#
#  Reference: Newey and West, 1987, Econometrica
#             Newey, 1985, Journal of Econometrics
#
#
#  Paul Soderlind (Paul.Soderlind@unisg.ch), to Julia Oct 2015
#-----------------------------------------------------------------------

  T = size(g0,1)                     #g is Txq
  m = min(m,T-1)                     #number of lags

  g = g0 .- mean(g0,1)               #Normalizing to Eg=0

  S = g'g/T                          #(qxT)*(Txq)
  for s = 1:m
    Omega_s = g[s+1:T,:]'g[1:T-s,:]/T   #same as Sum[g(t)*g(t-s)',t=s+1,T]
    S       = S + (1 - s/(m+1))*(Omega_s + Omega_s')
  end

  return S

end
#-----------------------------------------------------------------------


module DatasetsSynthetic

using Random
export make_blobs, make_circles, make_moons

uniform_sample_in_zero_maxval(p, maxval) = maxval .* (1 .- rand(p))
uniform_sample_in_minval_maxval(p, minval, maxval) = (maxval-minval) .* rand(p) .+ minval
uniform_sample_in_minval_maxval(n, p, minval, maxval) = (maxval-minval) .* rand(n, p) .+ minval
normal_sample(p, mu, var) = mu .+ sqrt(var) .* randn(p)

"""
Shuffles the rows of an Array `X` and the values of a vector `y` using a randomly
generated permutation. The same permutation is used to shuffle both `X` and `y`.
"""
function shuffle_Xy(X, y; random_seed=Random.GLOBAL_RNG)
    Random.seed!(random_seed)
    perm = randperm(length(y))
    return X[perm,:], y[perm]
end

"""
make_blobs(n::Int=100;
           p::Int=2,
           centers::Int=3,
           cluster_std=1.0,
           center_box=(-10.,10.),
           element_type=Float64,
           random_seed=Random.GLOBAL_RNG,
           return_centers=false)

Generates a dataset with `n` examples of dimension `p` and returns a vector containing
as integers the membership of the different points generated.

The data is roughly grouped around several `centers`  which are created using `cluster_std`.
The data lives inside `center_box` in the case it is randomly generated.

- If `centers` is an integer the centroids are created randomly.
- If `centers` is an Array containing points the centroids are picked from `centers`.
- If `return_centers=true` the centroids of the blobs are returned.
"""
function make_blobs(n::Int=100; p::Int=2,
                    shuffle::Bool=false, centers::Int=3, cluster_std::Real=1.0, center_box=(-10.,10.),
                    element_type=Float64, random_seed=Random.GLOBAL_RNG, return_centers=false, verbose=0)

    Random.seed!(random_seed)
    X = zeros(n, p)
    y = zeros(n)

    if typeof(centers) <: Int
        n_centers = centers
        center_sample = uniform_sample_in_minval_maxval(n, p, center_box[1], center_box[2])
    else
        n_centers = length(centers)
    end

    if typeof(cluster_std) <: AbstractFloat
        cluster_std = cluster_std * randn(n_centers)
    end

    # generates div(n, n_centers) examples assigned to each center
    n_per_center = [div(n, n_centers) for x  in 1:n_centers]

    # adds the reamainding examples to each center up to n
    n_per_center = fill(div(n,n_centers), n_centers)
    n_per_center[end] += rem(n,n_centers)

    # generates the actual vectors close to each center blob
    start_ind = 1
    for (i, (n_blob, std, center)) in enumerate(zip(n_per_center, cluster_std, centers))
        ind_center = start_ind:(start_ind + n_per_center[i]-1)
        X[ind_center,:] .= center' .+ std .* randn(element_type, (n_per_center[i], p));
        y[ind_center] .= i
        if verbose>0
            println("center $i with $(n_per_center[i]) points created")
        end
        start_ind += n_per_center[i]
    end

    if shuffle
       X, y = shuffle_Xy(X, y ; random_seed=random_seed)
    end

    if return_centers
        return X, y, centers
    else
        return X, y
    end
end

"""
make_circles(n::Int=100; shuffle::Bool=true, noise::Number=0., random_seed=Random.GLOBAL_RNG, factor::Number=0.8)

Generates a dataset with `n` bi-dimensional examples. Samples are created
from two circles. One of the circles inside the other. The `noise` scalar
can be used to add noise to the generation process. The scalar `factor`
corresponds to the radius of the smallest circle.
"""
function make_circles(n::Int=100; shuffle::Bool=true, noise::Number=0., random_seed=Random.GLOBAL_RNG, factor::Number=0.8)

    Random.seed!(random_seed)

    n_out = div(n, 2)
    n_in = n - n_out

    linrange_out = Array(LinRange(0, 2 * pi, n_out))
    linrange_in  = Array(LinRange(0, 2 * pi, n_in))

    outer_circ_x = cos.(linrange_out)
    outer_circ_y = sin.(linrange_out)
    inner_circ_x = cos.(linrange_in) .* factor
    inner_circ_y = sin.(linrange_in) .* factor

    X = hcat(vcat(outer_circ_x, inner_circ_x), vcat(outer_circ_y, inner_circ_y))
    y = vcat(ones(Int,n_out), 2*ones(Int,n_in))

    if shuffle
       X, y = shuffle_Xy(X, y ; random_seed=random_seed)
    end

    X .+= noise .* rand(n, 2)

    return X, y
end

"""
make_moons(n::Int=100; shuffle::Bool=true, noise::Number=0.,
           translation::Number=0.5, factor::Number=1.0, random_seed=Random.GLOBAL_RNG)

Generates `n` examples sampling from two moons. The `noise` can be changed to add
noise to the samples.
"""
function make_moons(n::Int=100; shuffle::Bool=true, noise::Number=0.,
                   translation::Number=0.5, factor::Number=1.0, random_seed=Random.GLOBAL_RNG)

    Random.seed!(random_seed)

    n_out = div(n, 2)
    n_in = n - n_out

    linrange_out = Array(LinRange(0, pi, n_out))
    linrange_in  = Array(LinRange(0, pi, n_in))

    outer_circ_x = cos.(linrange_out)
    outer_circ_y = sin.(linrange_out)
    inner_circ_x = 1 .- cos.(linrange_in) .* factor
    inner_circ_y = 1 .- sin.(linrange_in) .* factor .- translation

    X = hcat(vcat(outer_circ_x, inner_circ_x), vcat(outer_circ_y, inner_circ_y))
    y = vcat(ones(Int,n_out), 2*ones(Int,n_in))

    if shuffle
       X, y = shuffle_Xy(X, y ; random_seed=random_seed)
    end

    X .+= noise .* rand(n, 2)

    return X, y
end

end #module


abstract type AbstractDiscreteMarkovChain <: AbstractMarkovChain end

"""
    DiscreteMarkovChain(transition_matrix)
    DiscreteMarkovChain(state_space, transition_matrix)
    DiscreteMarkovChain(continuous_markov_chain)

Creates a new discrete Markov chain object.

# Arguments
- `state_space`: The names of the states that make up the Markov chain.
- `transition_matrix`: The single step transition probability matrix.
- `continuous_markov_chain`: An instance of `ContinuousMarkovChain`.

# Examples
The following shows a basic Sunny-Cloudy-Rainy weather model.
```jldoctest DiscreteMarkovChain
using DiscreteMarkovChains
T = [
    0.9 0.1 0;
    0.5 0.2 0.3;
    0.1 0.4 0.5
]
X = DiscreteMarkovChain(["Sunny", "Cloudy", "Rainy"], T)
println(state_space(X))

# output

["Sunny", "Cloudy", "Rainy"]
```

```jldoctest DiscreteMarkovChain
println(transition_matrix(X))

# output

[0.9 0.1 0.0; 0.5 0.2 0.3; 0.1 0.4 0.5]
```

# References
1. [Wikipedia](https://en.wikipedia.org/wiki/Markov_chain#Discrete-time_Markov_chain)
2. [Dartmouth College](https://www.dartmouth.edu/~chance/teaching_aids/books_articles/probability_book/Chapter11.pdf)
"""
struct DiscreteMarkovChain <: AbstractDiscreteMarkovChain
    state_space
    transition_matrix
    function DiscreteMarkovChain(state_space, transition_matrix)
        check(state_space, transition_matrix, DiscreteMarkovChain)
        new(state_space, transition_matrix)
    end
end
"""
    required_row_sum(type)

# Arguments
- `type`: The type of Markov chain. It can be
`DiscreteMarkovChain` or `ContinuousMarkovChain`.

# Returns
The number that each row in the transition matrix should sum up to.
"""
required_row_sum(::Type{<:AbstractDiscreteMarkovChain}) = 1
required_row_sum(::Type{<:AbstractDiscreteMarkovChain}, ::Type{T}) where T = one(T)
function DiscreteMarkovChain(transition_matrix)
    return DiscreteMarkovChain(1:(size(transition_matrix)[1]), transition_matrix)
end

"""
    characteristic_matrix(::AbstractDiscreteMarkovChain)
    characteristic_matrix(::AbstractContinuousMarkovChain)

# Definitions
Many derivations and interesting ideas about Markov chains
involve the identity matrix or zero matrix somewhere along
the line. Most of the time, the identity matrix appears
more often in discrete Markov chains. Instead of the identity
matrix, the zero matrix appears in its place for continuous
Markov chains.

# Returns
The identity matrix if its argument is an instance of
`AbstractDiscreteMarkovChain`. The zero matrix if its
argument is an instance of `AbstractContinuousMarkovChain`
"""
characteristic_matrix(::AbstractDiscreteMarkovChain) = LinearAlgebra.I

"""
    periodicities(x::AbstractDiscreteMarkovChain)

A more advanced version of `communication_classes`
designed for discrete Markov chains. It is the same as
`communication_classes` but it returns periodicities as well.

# Definitions
The period, ``d_i`` of a state ``i`` is the greatest common denominator
of all integers ``n ∈ \\mathbb{N}`` for which ``p^{(n)}_{i,i} > 0``.
Written more succinctly,
```math
d_i = \\text{gcd}\\{ n ∈ \\mathbb{N} | p^{(n)}_{i,i} > 0 \\}
```
If ``d_i=1`` then state ``i`` is said to be aperiodic.

# Arguments
- `x`: some kind of discrete Markov chain.

# Returns
A tuple containing 3 arrays.
- This first array contains C arrays which store the states that communicate.
- The second array is an array of Bool where the ith value is true if the
  ith communication class is recurrent.
- The third array is the periodicity of each communication class.

# Examples
```jldoctest periodicities
using DiscreteMarkovChains
T = [
    1 0;
    0 1;
]
X = DiscreteMarkovChain(T)

periodicities(X)

# output

([[1], [2]], Any[true, true], Any[1, 1])
```

So the Markov chain has two communication classes and both are recurrent and aperiodic.

```jldoctest periodicities
T = [
    0.0 1.0 0.0;
    1.0 0.0 0.0;
    0.1 0.2 0.7;
]
X = DiscreteMarkovChain(["Sunny", "Cloudy", "Rainy"], T)

periodicities(X)

# output

([["Sunny", "Cloudy"], ["Rainy"]], Any[true, false], Any[2, 1])
```

So the Sunny and Cloudy states communicate and are recurrent with period 2.
The Rainy state is transient and aperiodic.
Note that this is not a very good weather model since once it
stops raining, it will never rain again. Also, each day after that,
the process will oscillate between Sunny and Cloudy.
"""
function periodicities(x::AbstractDiscreteMarkovChain)
    SI = state_index(x)
    T = transition_matrix(x)

    classes, recurrence = communication_classes(x)
    index_classes = [[SI[state] for state in class] for class in classes]

    periods = []
    for class in index_classes
        submatrix = T[class, class]
        push!(periods, breadth_first_search(submatrix))
    end
    return classes, recurrence, periods
end

"""
    is_regular(x)

# Definitions
A Markov chain is called a regular chain if some power of the
transition matrix has only positive elements. This is equivalent
to being ergodic and aperiodic.

# Arguments
- `x`: some kind of discrete Markov chain.

# Returns
`true` if the Markov chain, `x`, is regular.

# Examples
We will set up a matrix with 2 communication classes and
show that it is not regular.

```jldoctest is_regular
using DiscreteMarkovChains
T = [
    0 1 0;
    1 0 0;
    0 0 1;
]
X = DiscreteMarkovChain(T)

is_regular(X)

# output

false
```

Repeat the above but now all states communicate.

```jldoctest is_regular
T = [
    0.0 0.5 0.5;
    0.0 0.0 1.0;
    1.0 0.0 0.0;
]
X = DiscreteMarkovChain(T)

is_regular(X)

# output

true
```

Notice how a periodic chain is not regular even though
there is only one communication class.

```jldoctest is_regular
T = [
    0 1 0;
    0 0 1;
    1 0 0;
]
X = DiscreteMarkovChain(T)

is_regular(X)

# output

false
```
"""
function is_regular(x::AbstractDiscreteMarkovChain)
    classes, _, periods = periodicities(x)
    if length(classes) == 0
        return false
    end
    return (length(classes) == 1) & (periods[1] == 1)
end

"""
    first_passage_probabilities(x, t, i=missing, j=missing)

# Definitions
This is the probability that the process enters state ``j``
for the first time at time ``t`` given that the process started
in state ``i`` at time 0. That is, ``f^{(t)}_{i,j}``. If no `i`
or `j` is given, then it will return a matrix instead with
entries ``f^{(t)}_{i,j}`` for `i` and `j` in the state space of `x`.

# Why Do We Use A Slow Algorithm?
So that `t` can be symbolic if nessesary. That is, if symbolic math
libraries want to use this library, it will pose no hassle.

# Arguments
- `x`: some kind of Markov chain.
- `t`: the time to calculate the first passage probability.
- `i`: the state that the prcess starts in.
- `j`: the state that the process must reach for the first time.

# Returns
A scalar value or a matrix depending on whether `i` and `j` are given.

# Examples
```jldoctest first_passage_probabilities
using DiscreteMarkovChains
T = [
    0.1 0.9;
    0.3 0.7;
]
X = DiscreteMarkovChain(T)

first_passage_probabilities(X, 2)

# output

2×2 Array{Float64,2}:
 0.27  0.09
 0.21  0.27
```

If `X` has a custom state space,
then `i` and `j` must be in that state space.

```jldoctest first_passage_probabilities
T = [
    0.1 0.9;
    0.3 0.7;
]
X = DiscreteMarkovChain(["Sunny", "Rainy"], T)

first_passage_probabilities(X, 2, "Sunny", "Rainy")

# output

0.09000000000000001
```

Notice how this is the (1, 2) entry in the first example.

# References
1. [University of Windsor](https://scholar.uwindsor.ca/cgi/viewcontent.cgi?article=1125&context=major-papers)
2. [Durham University](http://maths.dur.ac.uk/stats/courses/ProbMC2H/_files/handouts/1516MarkovChains2H.pdf)
"""
function first_passage_probabilities(
    x::AbstractDiscreteMarkovChain, t, i=missing, j=missing
)
    S = state_space(x)
    T = transition_matrix(x)
    n = length(S)

    if n == 0
        return transition_matrix(x)
    end

    js = 1:n  # The columns to loop through
    calc_i_ne_j = true  # Calculate the off-diagonals
    calc_i_eq_j = true  # Calculate the diagonals
    if (i !== missing) && (j !== missing)
        i = state_index(x)[i]
        j = state_index(x)[j]
        js = [j]
        if i == j
            calc_i_ne_j = false
        else
            calc_i_eq_j = false
        end
    end

    Ft = zeros(eltype(T), n, n)  # Empty matrix

    # If i != j
    if calc_i_ne_j
        for j in js

            P0 = copy(T)
            P0[1:n, j] = zeros(eltype(T), n, 1)
            F = P0^(t-1) * T
            Ft[1:n, j] = F[1:n, j]
        end
    end

    # If i == j
    if calc_i_eq_j
        for j in js

            P_ = copy(T)
            P_[j, 1:n] = zeros(eltype(T), 1, n)

            Pnew = zeros(eltype(T), 2*n, 2*n)
            Pnew[1:n, 1:n] = T
            Pnew[(n+1):(2*n), (n+1):(2*n)] = P_
            Pnew[n+j, 1:n] = T[j, 1:n]

            P0 = copy(Pnew)
            P0[1:(2*n), j] = zeros(eltype(T), 2*n, 1)

            F = P0^(t - 1) * Pnew

            Ft[j, j] = F[n+j, j]
        end
    end

    if (i !== missing) && (j !== missing)
        return Ft[i, j]
    end
    return Ft
end


#
# Function that computes a moving average to smooth plots
#

function movingaverage( x :: Vector, n :: Int )

  if ! isodd(n)
    n = n + 1
  end
  delta = round(Int64,(n-1)/2)

  nx = length(x)
  y = similar(x)

  for i in 1:nx
    y[i] = 0.
    jmin = max(i-delta,1)
    jmax = min(i+delta,nx)
    for j in jmin:jmax
      y[i] = y[i] + x[j]
    end
    y[i] = y[i] / (jmax-jmin+1)

  end

  return y

end


# Plots power spectrum components of approximate Gaussian PSD:
using PyPlot

#nc = 4
#amp = [2.953823582534889, -0.04961614737314587, -1.6900719772140012, -0.21413545794774125]
#b =[1.589510614489265, 3.4729345651560872, 1.8698492189074047, 6.363451205766626]
#omega =[0.3329119766576696, 4.5999414581901386,1.7115890219170917, 2.205939216106191]
#amp = [2.953823582534889, -0.04961614737314587, -0.21413545794774125, -1.690719772140012]
#b =[1.589510614489265, 3.4729345651560872, 6.363451205766626, 1.8698492189074047]
#omega =[0.3329119766576696, 4.5999414581901386, 2.205939216106191,1.1115890219170917]

nc = 3
amp = [2.670143,-0.458705,-1.211438]
b = [1.500258,4.125558,1.744602]
omega = [0.346731,1.390912,1.798436]

nf = 100
f = 2*pi*logspace(-2,2,nf)
psdtot =  zeros(nf)
for i=1:nc
  psd = amp[i]*b[i]*sqrt(2./pi)*(b[i]^2+omega[i]^2+f.^2)./(b[i]^4+(omega[i]^2-f.^2).^2+2.*b[i]^2.*(omega[i]^2+f[i]^2))
  loglog(f,abs(psd))
  psdtot += psd
  loglog(f,psdtot,linestyle="dashed")
  read(STDIN,Char)
end

#plot(f,psdtot)


using ModelingToolkit, ModelingToolkitStandardLibrary, OrdinaryDiffEq
using ModelingToolkitStandardLibrary.Blocks

@parameters t

#=
Testing strategy:
The general strategy is to test systems using simple intputs where the solution
is known on closed form. For algebraic systems (without differential variables),
an integrator with a constant input is often used together with the system under test.
=#

@testset "Constant" begin
    @named c = Constant(; k=1)
    @named int = Integrator()
    @named iosys = ODESystem(connect(c.output, int.input), t, systems=[int, c])
    sys = structural_simplify(iosys)
    prob = ODEProblem(sys, Pair[int.x=>1.0], (0.0, 1.0))
    sol = solve(prob, Rodas4())
    @test sol[int.output.u][end] ≈ 2
end

@testset "Derivative" begin
    @named source = Sine(; frequency=1)
    @named int = Integrator(; k=1)
    @named der = Derivative(; k=1, T=0.001)
    @named iosys = ODESystem([
        connect(source.output, der.input),
        connect(der.output, int.input),
        ],
        t,
        systems=[int, source, der],
    )
    sys = structural_simplify(iosys)
    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))
    sol = solve(prob, Rodas4())
    @test isapprox(sol[source.output.u], sol[int.output.u], atol=1e-1)
end

@testset "PT1" begin
    pt1_func(t, k, T) = k * (1 - exp(-t / T)) # Known solution to first-order system

    k, T = 1.0, 0.1
    @named c = Constant(; k=1)
    @named pt1 = FirstOrder(; k=k, T=T)
    @named iosys = ODESystem(connect(c.output, pt1.input), t, systems=[pt1, c])
    sys = structural_simplify(iosys)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())
    @test sol[pt1.output.u] ≈ pt1_func.(sol.t, k, T) atol=1e-3
end

@testset "PT2" begin
    # Known solution to second-order system
    function pt2_func(t, k, w, d)
        y = if d==0
            -k*(-1 + cos(t*w))
        else
            d = complex(d)
            real(k*(1 + (-cosh(sqrt(-1 + d^2)*t*w) - (d*sinh(sqrt(-1 + d^2)*t*w))/sqrt(-1 + d^2))/exp(d*t*w)))
        end
    end

    k, w, d = 1.0, 1.0, 0.5
    @named c = Constant(; k=1)
    @named pt2 = SecondOrder(; k=k, w=w, d=d)
    @named iosys = ODESystem(connect(c.output, pt2.input), t, systems=[pt2, c])
    sys = structural_simplify(iosys)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())
    @test sol[pt2.output.u] ≈ pt2_func.(sol.t, k, w, d) atol=1e-3
end

@testset "StateSpace" begin
    A = [0 1;-1 -0.5]
    B = [0, 1]
    C = [0.9 1;]
    D = [0;;]
    @named ss = StateSpace(;A,B,C,D,x_start=zeros(2))
    @named c = Constant(; k=1)
    @named model = ODESystem(
        [
           connect(c.output, ss.input),
        ],
        t,
        systems=[ss, c]
    )
    sys = structural_simplify(model)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())
    # equilibrium point is at [1, 0]
    @test sol[ss.x[1]][end] ≈ 1 atol=1e-3
    @test sol[ss.x[2]][end] ≈ 0 atol=1e-3
end

"""Second order demo plant"""
function Plant(;name, x_start=zeros(2))
    @named input = RealInput()
    @named output = RealOutput()
    D = Differential(t)
    sts = @variables x1(t)=x_start[1] x2(t)=x_start[2]
    eqs= [
        D(x1) ~ x2
        D(x2) ~ -x1 - 0.5 * x2 + input.u
        output.u ~ 0.9 * x1 + x2
    ]
    compose(ODESystem(eqs, t, sts, []; name), [input, output])
end

@testset "PI" begin
    @named ref = Constant(; k=2)
    @named pi_controller = PI(k=1, T=1)
    @named plant = Plant()
    @named fb = Feedback()
    @named model = ODESystem(
        [
            connect(ref.output, fb.input1),
            connect(plant.output, fb.input2),
            connect(fb.output, pi_controller.err_input),
            connect(pi_controller.ctr_output, plant.input),
        ],
        t,
        systems=[pi_controller, plant, ref, fb]
    )
    sys = structural_simplify(model)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())
    @test sol[ref.output.u - plant.output.u][end] ≈ 0 atol=1e-3 # zero control error after 100s
end

@testset "PID" begin
    @named ref = Constant(; k=2)
    @named pid_controller = PID(k=3, Ti=0.5, Td=100)
    @named plant = Plant()
    @named fb = Feedback()
    @named model = ODESystem(
        [
            connect(ref.output, fb.input1),
            connect(plant.output, fb.input2),
            connect(fb.output, pid_controller.err_input),
            connect(pid_controller.ctr_output, plant.input),
        ],
        t,
        systems=[pid_controller, plant, ref, fb]
    )
    sys = structural_simplify(model)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())
    @test sol[ref.output.u - plant.output.u][end] ≈ 0 atol=1e-3 # zero control error after 100s
end

@test_skip begin
    k = 2
    Ti = 0.5
    Td = 0.7
    wp = 1
    wd = 1
    Ni = √(Td / Ti)
    Nd = 12
    y_max = Inf
    y_min = -Inf
    u_r = sin(t)
    u_y = 0
    function solve_with_input(; u_r, u_y,
        controller = PID(; k, Ti, Td, wp, wd, Ni, Nd, y_max, y_min, name=:controller)
    )
        @test count(ModelingToolkit.isinput, states(controller)) == 5 # 2 in PID, 1 sat, 1 I, 1 D
        @test count(ModelingToolkit.isoutput, states(controller)) == 4
        # TODO: check number of unbound inputs when available, should be 2
        @named iosys = ODESystem([controller.u_r~u_r, controller.u_y~u_y], t, systems=[controller])
        sys = structural_simplify(iosys)
        prob = ODEProblem(sys, Pair[], (0.0, 10.0))
        sol = solve(prob, Rodas4(), saveat=0:0.1:10)
        controller, sys, sol
    end

    # linearity in u_r
    controller, sys, sol1 = solve_with_input(u_r=sin(t), u_y=0)
    controller, sys, sol2 = solve_with_input(u_r=2sin(t), u_y=0)
    @test sum(abs, sol1[controller.ea]) < eps() # This is the acutator model error due to saturation
    @test 2sol1[controller.y] ≈ sol2[controller.y] rtol=1e-3 # linearity in u_r

    # linearity in u_y
    controller, sys, sol1 = solve_with_input(u_y=sin(t), u_r=0)
    controller, sys, sol2 = solve_with_input(u_y=2sin(t), u_r=0)
    @test sum(abs, sol1[controller.ea]) < eps() # This is the acutator model error due to saturation
    @test 2sol1[controller.y] ≈ sol2[controller.y] rtol=1e-3 # linearity in u_y

    # zero error
    controller, sys, sol1 = solve_with_input(u_y=sin(t), u_r=sin(t))
    @test sum(abs, sol1[controller.y]) ≈ 0 atol=sqrt(eps())

    # test saturation
    controller, sys, sol1 = solve_with_input(; u_r=10sin(t), u_y=0,
        controller = PID(; k, Ti, Td, wp, wd=0, Ni, Nd, y_max=10, y_min=-10, name=:controller)
    )
    @test extrema(sol1[controller.y]) == (-10, 10)

    # test P set-point weighting
    controller, sys, sol1 = solve_with_input(; u_r=sin(t), u_y=0,
        controller = PID(; k, Ti, Td, wp=0, wd, Ni, Nd, y_max, y_min, name=:controller)
    )
    @test sum(abs, sol1[controller.ep]) ≈ 0 atol=sqrt(eps())

    # test D set-point weighting
    controller, sys, sol1 = solve_with_input(; u_r=sin(t), u_y=0,
        controller = PID(; k, Ti, Td, wp, wd=0, Ni, Nd, y_max, y_min, name=:controller)
    )
    @test sum(abs, sol1[controller.ed]) ≈ 0 atol=sqrt(eps())

    # zero integral gain
    controller, sys, sol1 = solve_with_input(; u_r=sin(t), u_y=0,
        controller = PID(; k, Ti=false, Td, wp, wd, Ni, Nd, y_max, y_min, name=:controller)
    )
    @test isapprox(sum(abs, sol1[controller.I.y]), 0, atol=sqrt(eps()))

    # zero derivative gain
    @test_skip begin # During the resolution of the non-linear system, the evaluation of the following equation(s) resulted in a non-finite number: [5]
        controller, sys, sol1 = solve_with_input(; u_r=sin(t), u_y=0,
            controller = PID(; k, Ti, Td=false, wp, wd, Ni, Nd, y_max, y_min, name=:controller)
        )
        @test isapprox(sum(abs, sol1[controller.D.y]), 0, atol=sqrt(eps()))
    end

    # Tests below can be activated when the concept of unbound_inputs exists in MTK
    # @test isequal(Set(unbound_inputs(controller)), @nonamespace(Set([controller.u_r, controller.u_y])))
    # @test isempty(unbound_inputs(sys))
    # @test isequal(bound_inputs(sys), inputs(sys))
    # @test isequal(
    #     Set(bound_inputs(sys)),
    #     Set([controller.u_r, controller.u_y, controller.I.u, controller.D.u, controller.sat.u])
    #     )
end

@testset "LimPI" begin
    @named ref = Constant(; k=1)
    @named pi_controller_lim = LimPI(k=3, T=0.5, u_max=1.5, u_min=-1.5, Ta=0.1)
    @named pi_controller = PI(k=3, T=0.5)
    @named sat = Limiter(y_max=1.5, y_min=-1.5)
    @named plant = Plant()
    @named fb = Feedback()

    # without anti-windup measure
    sol = let
        @named model = ODESystem(
            [
                connect(ref.output, fb.input1),
                connect(plant.output, fb.input2),
                connect(fb.output, pi_controller.err_input),
                connect(pi_controller.ctr_output, sat.input),
                connect(sat.output, plant.input),
            ],
            t,
            systems=[pi_controller, plant, ref, fb, sat]
        )
        sys = structural_simplify(model)
        prob = ODEProblem(sys, Pair[], (0.0, 20.0))
        sol = solve(prob, Rodas4())
    end

    # with anti-windup measure
    sol_lim = let
        @named model = ODESystem(
            [
                connect(ref.output, fb.input1),
                connect(plant.output, fb.input2),
                connect(fb.output, pi_controller_lim.err_input),
                connect(pi_controller_lim.ctr_output, sat.input),
                connect(sat.output, plant.input),
            ],
            t,
            systems=[pi_controller_lim, plant, ref, fb, sat]
        )
        sys = structural_simplify(model)
        prob = ODEProblem(sys, Pair[], (0.0, 20.0))
        sol = solve(prob, Rodas4())
    end

    @test sol[ref.output.u - plant.output.u][end] ≈ 0 atol=1e-3 # zero control error after 100s
    @test sol_lim[ref.output.u - plant.output.u][end] ≈ 0 atol=1e-3 # zero control error after 100s

    # Plots.plot(sol; vars=[plant.output.u]) # without anti-windup measure
    # Plots.plot!(sol_lim; vars=[plant.output.u]) # with anti-windup measure
end

@testset "LimPID" begin
    @named ref = Constant(; k=1)
    @named pid_controller = LimPID(k=3, Ti=0.5, Td=100, u_max=1.5, u_min=-1.5, Ni=0.1/0.5)
    @named plant = Plant()
    @named model = ODESystem(
        [
            connect(ref.output, pid_controller.reference),
            connect(plant.output, pid_controller.measurement),
            connect(pid_controller.ctr_output, plant.input),
        ],
        t,
        systems=[pid_controller, plant, ref]
    )
    sys = structural_simplify(model)
    prob = ODEProblem(sys, Pair[], (0.0, 100.0))
    sol = solve(prob, Rodas4())

    # Plots.plot(sol, vars=[plant.output.u, plant.input.u])
    @test sol[ref.output.u - plant.output.u][end] ≈ 0 atol=1e-3 # zero control error after 100s
end


#=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=#
# Problem set 2 solutions
# Written by Tyler Ransom
# Commented by Giuseppe Grasso
#=~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~=#

using Random
using LinearAlgebra
using Statistics
using Optim
using DataFrames
using CSV
using HTTP
using GLM
using FreqTables
using ForwardDiff # for bonus at the very end
cd("/Users/peppegrass/Documents/GitHub/fall-2020/ProblemSets/PS2-optimization-intro/") # GG: sets working directory
pwd() ## GG: prints working directory
readdir() # GG: equivalent to -ls- to see elements of working directory

function allwrap()
    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # Question 1
    # Basic [# GG: NON-LINEAR] optimization in Julia
    #:::::::::::::::::::::::::::::::::::::::::::::::::::

    #=
    GG: We will use Julia’s Optim package, which is a function MINIMIZER.
    Thus, if we want to ﬁnd the maximum of f(x), we need to minimize −f(x).
    GG: Run code below in REPL to see how Optim works
    =#

    f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2 # GG: original objective function
    minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2 # GG: -(obj fct)
    startval = rand(1)   # random starting value
    #=
    GG: The Optim package provides a function called optimize(). This function requires three inputs:
    the objective function, a starting value, and an optimization algorithm.
    We will not get too deep into optimization algorithms in this course, but for now just use LBFGS().
    =#
    result = optimize(minusf, startval, BFGS()) # GG: BFGS is a gradient-based algorithm: objective function needs to be differentiable for this to work; otherwise, use different algorithm
    println(result)

    #=
    GG: Look at the measures of convergence in the result:
    3 different criteria when doing non-linear optimization. Metaphor: climing a mountain and looking at ALTITUDE & LATITUDE/LONGITUDE
    1. Look at g(x): Most intuitive and common one: is the gradient 0 [necessary condition: FOC]?
    2. Look at |f(x) - f(x')|: Since the last iteration of this procedure (with the first iteration being originally based on the starting value), how much has the FUNCTION VALUE changed (i.e. has the ALTITUDE changed) between the previous or current iteration? Is there evidence for the slope not being at zero?
    3. Look at |x - x'|: How far did I step from the last iteration to this iteration, i.e. how did the value x (current) change wrt to x' (previous)? How much did LATITUDE/LONGITUDE change
        # GG: gradient 1. is the most important and also hardest to satisfy
        Tolerance values/thresholds can be set for gradient (gtol/gval), function (ftol/fval) and value (xtol/xval); e.g. e-06 as opposed to e-08
    =#

    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # Question 2
    # Now that we’re familiar with how Optim’s optimize() function works, lets try it on some real-world data.
    # Speciﬁcally, let’s use Optim to compute OLS estimates of a simple LINEAR REGRESSION using actual data.
    # The process for passing data to Optim can be tricky, so it will be helpful to go through this example.
    # First, let’s import and set up the data. Note that you will need to put the URL all on one line when executing this code in Julia.
    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv" # GG: setting URL
    df = CSV.read(HTTP.get(url).body) # GG: reading CSV from URL
    X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1] # GG: subset of variables that will be used in regression
    y = df.married.==1 # GG: copying variable married from DataFrame df into array y
    # GG: By applying OLS to a binary choice model we are estimating a Linear Probability Model (LPM)

    # GG. A tricky thing with using Optim is that it requires something called a closure to be able to pass data into the function.
    # GG: OLS is minimizer of Sum of Squared Errors (SSR) so we write down the objective function
    function ols(beta, X, y) # GG: argument you are optimizing over has to come first in Optim; others (data) can come later
        ssr = (y.-X*beta)'*(y.-X*beta) # GG: one way of writing it (.- is called loop fusion; should be the most efficient)
        #= ssr = dot(y.-X*beta,y.-X*beta) Julia function dot() that takes dot product (an alternative) =#
        #= ssr = [sum[y[i]-X[i,:]*beta]^2 for i in 1:length(y)] (yet another alternative by comprehension; might contain mistakes; pseudo-code) =#
        #=  ssr = 0
            for i=1:length(y)
                ssr += (y[i]-X[i,:]*beta)^2
            end
        (yet another alternative using a loop; += is an operator to do iterations)
        =#
        return ssr
    end

    beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
    #= GG:
    - In exercise 1 we simply used optimize(minusf, ., .) because we had a univariate function of x
    Now here we got 3 arguments and we need to tell Julia that we are optimizing over b by using the following synthax
    optimize(b -> ols(b, X, y), ...)
    - rand(size(X,2)) are starting value which must have the same dimension as the parameter vector (or dim covariate vector including the constant)
    - LBFGS() is the algorithm chosen
    - Setting a few options through Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true)
        - tolerance value for gradient; capping number of iterations at 100k; show_trace=true prints out in the REPL what's happening at every step
    =#

    println(beta_hat_ols.minimizer) # GG: prints the output

    # GG: Since OLS has a closed form solution, we can check that this worked in a few different ways:
    # 1. by evaluating the known OLS closed form
    bols = inv(X'*X)*X'*y
    println(bols)
    # 2. by using the GLS package; very similar to R
    df.white = df.race.==1 # GG: Creating a dummy white for race==1
    bols_lm = lm(@formula(married ~ age + white + collgrad), df) # GG: like in R, once you tell where data is coming from, you can just use varnames stata-style
    println(bols_lm)
    # GG: It can be easily verified that all three solutions are the same

    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # Question 3
    # Use Optim to estimate the LOGIT LIKELIHOOD. Some things to keep in mind:
    #   To maximize the likelihood, you will need to pass Optim the negative of the likelihood function
    #   The likelihood function is included in the Lecture 4 slides
    #:::::::::::::::::::::::::::::::::::::::::::::::::::

    #=
    GG: Same principle but this time we apply it to MAXIMUM LIKELIHOOD estimation of a BINARY LOGIT model
    In previous exercise we programmed OLS; here we program logit
    =#
    function logit(alpha, X, y)

        P = exp.(X*alpha)./(1 .+ exp.(X*alpha)) # GG: choice probability. Using dots to vectorize operation, i.e. exp() applied to all elements of X*alpha; same for other operations

        loglike = -sum( (y.==1).*log.(P) .+ (y.==0).*log.(1 .- P) ) # GG: this is the LOGIT log-likelihood (times -1 coz of Optim is min fct); (y.==1) and (y.==0) are a booleans

        return loglike
    end
    alpha_hat_optim = optimize(a -> logit(a, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
    println(alpha_hat_optim.minimizer)
    # GG: as we saw in OLS, here we're optimizing wrt a, so inside optimize we use a -> logit(a, X, y)
    # GG: rand(size(X,2)) gives a vector the dimension of X of uniform 0,1 random numbers

    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # Question 4
    # Use the glm() function from the GLM package to check your answer. (Example code for how to do this is in the Lecture 3 slides.)
    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    alpha_hat_glm = glm(@formula(married ~ age + white + collgrad), df, Binomial(), LogitLink())
    println(alpha_hat_glm)

    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # Question 5
    # Use Optim to estimate a MULTINOMIAL LOGIT MODEL where the dependent variable is occupation and the covariates are the same as above.
    # Before doing this, clean the data to remove rows where occupation is missing.
    # We also need to aggregate some of the occupation categories or else we won’t be able to estimate our multinomial logit model:
    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    freqtable(df, :occupation) # note small number of obs in some occupations
    df = dropmissing(df, :occupation) # GG: dropping observations with missing occupation and overwriting df
    df[df.occupation.==8 ,:occupation] .= 7 # GG: recoding all values >7 to 7
    df[df.occupation.==9 ,:occupation] .= 7
    df[df.occupation.==10,:occupation] .= 7
    df[df.occupation.==11,:occupation] .= 7
    df[df.occupation.==12,:occupation] .= 7
    df[df.occupation.==13,:occupation] .= 7
    freqtable(df, :occupation) # problem solved
    # GG: Since we changed the number of rows of df, we also need to re-deﬁne our X and y objects:
    X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
    y = df.occupation

    #=
    GG: Hints:

    # With 7 choice alternatives, you will have K · 6 coefﬁcients, where K is the number of covariates in X.
    It may help to transform the parameter vector into a K ×6 matrix (to more easily reference the α j ’s for each j)

    # You should reset the tolerance of the gradient (g tol) to be 10 to the −5.
    This will help the estimation converge more quickly, without losing too much precision. Otherwise, it would take forever.

    # You may need to try different sets of starting values. Some candidates to consider are:
    - a vector of 0s
    - a vector of U[0,1] random numbers
    - a vector of U[−1,1] random numbers
    - the estimated values from Stata or R (see Stata example below)
        webuse nlsw88
        drop if mi(occupation)
        recode occupation (8 9 10 11 12 13 = 7)
        gen white = race==1
        mlogit occupation age white collgrad, base(7)
    In general it is a good strategy to run your model(s) through a more user-friendly interface like Stata or R before trying to implement them in Julia.
    But you might ask, “Why don’t we just use Stata or R, then?”
    The reason is because the models we will get to later in the course are much more difﬁcult to implement in those languages, because they can’t just be taken off the shelf.

    We can readapt the code from the binary logit above. We gotta readapt to the fact that we have J choices and KxJ parameters
    Functional form of choice probabilities and log-likelihood function is similar.
    So I could take loglike = -sum( (y.==1).*log.(P) .+ (y.==0).*log.(1 .- P) ) and readapt it as follows
    loglike = -sum( (y.==1).*log.(P1) .+ (y.==2).*log.(P2) .+ ...)
    or I can do it in a LOOP (which is adviced in that it allows to reduce mistakes)
    =#

    function mlogit(alpha, X, y) # GG: mlogit can oly be fed alpha as a vector, not as a matrix

        # GG: Defining K, J, N (not always a good idea to make extra allocations but in this case it helps understand what's what)
        K = size(X,2)
        J = length(unique(y))
        N = length(y)
        bigY = zeros(N,J) # GG: Initializing an NxJ matrix for the Ys. Looping over the number of occupation choices columns and creating a bunch of 1-0 columns (dummies) in the bigY matrix
        for j=1:J
            bigY[:,j] = y.==j
        end
        bigAlpha = [reshape(alpha,K,J-1) zeros(K)] # GG: reshaping alpha vector into a matrix in a way that is conformable to bigY (hence the zeroes at the end, reflecting the normalization of the parameters for the last occupation)

        # GG: need to compute numerator and denominator of choice probabilities. For MNL num changes for each of them; den doesn't.
        num = zeros(N,J)
        dem = zeros(N)
        for j=1:J
            num[:,j] = exp.(X*bigAlpha[:,j]) # GG: changes for each numerator; first one normalized at 1 by exploiting exp(0)
            dem .+= num[:,j] # GG: iterated sum of all numerators; .+= is vectorized iterative sum
        end

        # GG: and there you go, this is the choice probability;
        P = num./repeat(dem,1,J) # GG: the denominator is the same for all alternatives, but we need to make it conformable, so we repeat the denominator J times in a way that is conformable to the numerator
        # GG: repeat(dem,1,J) is a matrix the same size as the numerator (so that ./ can worl properly), repeated J times

        loglike = -sum( bigY.*log.(P) ) # GG: here we need to do a double summation (over i and j); Julia understands to do sum over the rows and over the columns
        # GG: but we could have said sum( sum( bigY.*log.(P) ; dims=2); dims=1)

        return loglike
    end

    # GG: different options for starting values; stata answers multiplied by 0-1 uniform random numbers appear to be the best starting values
    alpha_zero = zeros(6*size(X,2))
    alpha_rand = rand(6*size(X,2))
    alpha_true = [.1910213,-.0335262,.5963968,.4165052,-.1698368,-.0359784,1.30684,-.430997,.6894727,-.0104578,.5231634,-1.492475,-2.26748,-.0053001,1.391402,-.9849661,-1.398468,-.0142969,-.0176531,-1.495123,.2454891,-.0067267,-.5382892,-3.78975]
    alpha_start = alpha_true.*rand(size(alpha_true))
    println(size(alpha_true))
    alpha_hat_optim = optimize(a -> mlogit(a, X, y), alpha_start, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
    alpha_hat_mle = alpha_hat_optim.minimizer
    println(alpha_hat_mle)

    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # BONUS: how to get STANDARD ERRORS?
    # Need to adjust objective function sligthly, i.e. need to obtain the hessian of the obj fun
    #:::::::::::::::::::::::::::::::::::::::::::::::::::
    # TR: first, we need to slightly modify our objective function
    function mlogit_for_h(alpha, X, y)

        K = size(X,2)
        J = length(unique(y))
        N = length(y)
        bigY = zeros(N,J)
        for j=1:J
            bigY[:,j] = y.==j
        end
        bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

        T = promote_type(eltype(X),eltype(alpha)) # TR: this line is new; GG: becuase types matter; gotta be the same
        num   = zeros(T,N,J)                      # TR: this line is new; GG: you specify type T before specifying dimensions
        dem   = zeros(T,N)                        # TR: this line is new; GG: you specify type T before specifying dimensions
        for j=1:J
            num[:,j] = exp.(X*bigAlpha[:,j])
            dem .+= num[:,j]
        end

        P = num./repeat(dem,1,J)

        loglike = -sum( bigY.*log.(P) )

        return loglike
    end

    # GG: auodifferentiation: will get you the EXACT derivative of objective function, speeding up the optimization time because optimizer has gradient coded in (doesn't have to numerically approximate it) and it'll allow to get the numerical HESSIAN of your obj fct, which is what you need to compute STANDARD ERRORS
    # GG: need to pass a twice differentiable object (td) before the closure

    # declare that the objective function is twice differentiable
    td = TwiceDifferentiable(b -> mlogit_for_h(b, X, y), alpha_start; autodiff = :forward) # GG: uses forward mode autodifferentiation
    # run the optimizer
    alpha_hat_optim_ad = optimize(td, alpha_zero, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
    # GG: so instead of putting as closure simply b -> mlogit_for_h(b, X, y), we put td, which stands for TwiceDifferentiable(b -> mlogit_for_h(b, X, y), alpha_start; autodiff = :forward)
    # GG: everything else stays the same
    alpha_hat_mle_ad = alpha_hat_optim_ad.minimizer
    # evaluate the Hessian at the estimates
    H  = Optim.hessian!(td, alpha_hat_mle_ad) # GG: getting the Hessian of twice differentiable object evaluated at the solution
    # TR: standard errors = sqrt(diag(inv(H))) [usually it's -H but we've already multiplied the obj fun by -1]
    # GG: from Adv Metrics on efficiency of Maximum Likelihood: -inv(H) enters the variance formula (sandwich); not taking negative coz we already took the negative of the objective function
    alpha_hat_mle_ad_se = sqrt.(diag(inv(H))) # GG: Take the square root of the diagonal elements of the inverse of the Hessian at the optimum
    println([alpha_hat_mle_ad alpha_hat_mle_ad_se]) # TR: these standard errors match Stata

    return nothing
end

# GG: Rather then getting the Hessian, we could also use Bootstrap to get the standard errors (but it'll take a lot longer)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# Question 6
# Wrap all of your code above into a function and then call that function at the very bottom of your script.
# Make sure you add println() statements after obtaining each set of estimates so that you can read them.
#:::::::::::::::::::::::::::::::::::::::::::::::::::
allwrap()


# 26 in base 16 or 2
base(16, 26)
base(2, 26)

# Parse to integer
parse(Int, "1a", 16)
parse(Int, "101101", 2)


module UnivariateProblems

    ### Sources
    ###
    ### [1]  http://infinity77.net/global_optimization/test_functions_1d.html

    struct UnivariateProblem
        name::AbstractString
        f::Function
        bounds::Vector{Float64}
        minimizers::Vector{Float64}
        minima::Vector{Float64}
    end

    examples = Dict{AbstractString, UnivariateProblem}()

    f(x) = 2x^2+3x+1

    examples["Polynomial"] = UnivariateProblem("Polynomial",
                                                  f,
                                                  [-2.0, 1.0],
                                                  [-0.75,],
                                                  [f(-0.75),])

    # Problem 04 from [1]
    p04(x) = -(16x^2-24x+5)exp(-x)

    examples["Problem04"] = UnivariateProblem("Problem04",
                                                p04,
                                                [1.9, 3.9],
                                                [2.868034,],
                                                [p04(2.868034),])

    # Problem 13 from [1]
    p13(x) = -x^(2/3)-(1-x^2)^(1/3)

    examples["Problem13"] = UnivariateProblem("Problem13",
                                                p13,
                                                [0.001, 0.99],
                                                [1 ./ sqrt(2.0),],
                                                [p13(1 ./ sqrt(2.0)),])

    # Problem 18 from [1]
    p18(x) = x <= 3.0 ? (x-2.0)^2 : 2.0*log(x-2.0)+1.0

    examples["Problem18"] = UnivariateProblem("Problem18",
                                                p18,
                                                [0.0, 6.0],
                                                [2.0,],
                                                [p18(2.0),])
end


#******************************************************************************#
#                               Introduction                                   #
#******************************************************************************#

# PyPlot is package for Julia that able you to use matplotlib library.

# First step to start plotting in Julia using Pyplot is install PyPlot package.
# You should do it only once.

ENV["PYTHON"] = ""
Pkg.add("PyPlot")

# And start using PyPlot module.
using PyPlot

# Now you can working.

# To do plot use plot():

x = linspace(-2*π, 2*π, 2000) # Generating 2000 sclars from -2π to 2π
y = sin(x)
plot(x,y)

# If you need two lines in one plot:

x = linspace(-2*π, 2*π, 2000) # Generating 2000 sclars from -2π to 2π
y = sin(x)
z = cos(x)
plot(x,y)
plot(x,z)

# But it's only raw chart. Let's describe it:

plot(x,y, color = "green", linewidth = "2.0")
plot(x,z, color = "red", linewidth = "2.0", "--")
title("Sine wave")
ylabel("Amplitude")
xlabel("Time")
grid(true)
text(0, 0, "0")
legend(["sin(x)","cos(x)"], loc = "lower left", fontsize = 12, borderpad = 0.3,
    handlelength=3)

# You can do subplots using subplot(nrows,ncols, plot_number):

sinh = sinh(x)
cosh = cosh(x)

subplot(221)
    plot(x,y)
    legend(["sin(x)"], fontsize = 8,  loc = "best" )
subplot(222)
    plot(x,z)
    legend(["cos(x)"], fontsize = 8,  loc = "best"  )
subplot(223)
    plot(x,sinh)
    legend(["sinh(x)"], fontsize = 8,  loc = "best"  )
subplot(224)
    plot(x,cosh)
    legend(["cosh(x)"], fontsize = 8,  loc = "best"  )

#******************************************************************************#
#                               Scatter plots                                  #
#******************************************************************************#

n = 1024
X = rand(1,1,n)
Y = rand(1,1,n)
colors = rand(1,1,n)

scatter(X,Y, s = 50 ,c = colors)
scatter(X,Y, s = 50 ,c = colors, marker = "^", alpha = .5)

#******************************************************************************#
#                                  Bar plots                                   #
#******************************************************************************#

n = 15
X = range(0,n)
Y1 = (1-X/n) * rand(1:n)
Y2 = (1-X/n) * rand(1:n)
axes([0.025,0.025,0.95,0.95])
bar(X, +Y1, facecolor="#9999ff", edgecolor="white")
bar(X, -Y2, facecolor="#ff9999", edgecolor="white")
margins(0.05) # 5% margins

# Code below generating describe under or above each bar.

for (x,y) in zip(X,Y1)
    text(x+0.4, y+0.05, "$(round(y,2))", ha="center", va= "bottom")
end

for (x,y) in zip(X,Y2)
    text(x+0.4, -y-0.05, "$(round(y,2))",  ha="center", va= "top")
end

#******************************************************************************#
#                               Contour plots                                  #
#******************************************************************************#

f(x,y) = (1-x/2+x^5+y^3)*exp(-x^2-y^2)
n = 256
x = linspace(-3,3,n)
y = linspace(-3,3,n)
z = zeros(n,n)
for i in 1:n
    for j in 1:n
        z[i,j] = f(x[j],y[i])
    end
end
X = repmat(x',n,1)
Y = repmat(y,1,n)

contourf(X, Y, z, 8, alpha=.75, cmap="jet")

# Easier way to get contour plots is to use Gadfly package:
Pkg.add("Gadfly")

using Gadfly

plot((x,y) -> (1-x/2+x^5+y^3)*exp(-x^2-y^2), -4, 4, -4, 4)

# You can use more parameters. To get more information about Gadfly contour
# plots check this topic: https://github.com/dcjones/Gadfly.jl/issues/293

plot((x,y) -> (1-x/2+x^5+y^3)*exp(-x^2-y^2), -4, 4, -4, 4,
    Stat.contour(samples = 200, levels = linspace(-1,1,25)))

# In this example we used levels = linspace(-1,1,25) to show 25 lines from -1
# to 1 include 0. If you want to see more lines you can precise it in
# levels parameter.

#******************************************************************************#
#                                Pie charts                                    #
#******************************************************************************#

# Back to using PyPlot

using PyPlot

labels = ["PyPlot";"Plots";"Gadfly"]
colors = ["yellowgreen";"lightskyblue";"lightcoral"]
sizes = NaN*zeros(3)
explode = zeros(length(sizes))
explode[2] = 0.1 # Move slice 2 out by 0.1
sizes = [80,15,10]
font = Dict("fontname"=>"Sans","weight"=>"semibold")
axes([0.025,0.025,0.95,0.95])
pie(sizes,
    labels=labels,
    shadow=true,
    startangle=90,
    explode=explode,
    colors=colors,
    autopct="%1.1f%%",
    textprops=font
)

#******************************************************************************#
#                                   3D Plots                                   #
#******************************************************************************#

# Example no. 1

f(x) = (x[1]^2 + x[2]^2)

n = 100
x = linspace(-1, 1, n)
y = linspace(-1,1,n)

xgrid = repmat(x',n,1)
ygrid = repmat(y,1,n)

z = zeros(n,n)

for i in 1:n
    for j in 1:n
        z[i:i,j:j] = f([x[i],y[j]])
    end
end

plot_wireframe(xgrid,ygrid,z)

# Example no. 2

f(x,y) = sin(x)*cos(y)

n = 100
x = linspace(-2, 2, n)
y = linspace(-1, 1,n)

xgrid = repmat(x',n,1)
ygrid = repmat(y,1,n)

z = zeros(n,n)
for i in 1:n
    for j in 1:n
        z[i,j] = f(x[j],y[i])
    end
end

plot_surface(xgrid,ygrid,z, rstride=1, cstride=1, cmap="coolwarm",
                       linewidth=0)

# More 3D plots examples you can find there:
# http://matplotlib.org/mpl_toolkits/mplot3d/tutorial.html

#******************************************************************************#
#                               Plots package                                  #
#******************************************************************************#

# You can use simpliest package - Plots.
Pkg.add("Plots")

using Plots

x = linspace(-2*π, 2*π, 2000) # Generating 2000 scalars from -2π to 2π
y = sin(x)
z = cos(x)
plot(x,y)
plot!(x,z)

# You can find more about plots there:
# https://en.wikibooks.org/wiki/Introducing_Julia/Plotting.

# To master skill in PyPlot you should visit this link:
# http://matplotlib.org/contents.html
# If you prefer Gadfly, visit manual page:
# http://gadflyjl.org/stable/


using Test
using LinearAlgebra
using NonNegLeastSquares
using SparseArrays

#test specific
using Random
using PyCall
const pyopt = pyimport_conda("scipy.optimize", "scipy")

function test_algorithm(fh, ε=1e-5)
	# Solve A*x = b for x, subject to x >=0
	A = [ 0.53879488  0.65816267
	      0.12873446  0.98669198
	      0.24555042  0.00598804
	      0.80491791  0.32793762 ]

	b = [0.888,  0.562,  0.255,  0.077]
	x = [0.15512102, 0.69328985] # approx solution from scipy
	@test norm(fh(A,b)-x) < ε

	## A second test case
	A2 = [ -0.24  -0.82   1.35   0.36   0.35
	       -0.53  -0.20  -0.76   0.98  -0.54
	        0.22   1.25  -1.60  -1.37  -1.94
	       -0.51  -0.56  -0.08   0.96   0.46
	        0.48  -2.25   0.38   0.06  -1.29 ]
	b2 = [-1.6,  0.19,  0.17,  0.31, -1.27]
	x2 = [2.2010416, 1.19009924, 0.0, 1.55001345, 0.0]
	@test norm(fh(A2,b2)-x2) < ε

	## Test a bunch of random cases
	for i = 1:100
		m,n = rand(1:10),rand(1:10)
		A3 = randn(m,n)
		b3 = randn(m)
		x3,resid = pyopt.nnls(A3,b3)
		if resid > ε
	        @test norm(fh(A3,b3)-x3) < ε
	    else
	    	@test norm(A3*fh(A3,b3)-b3) < ε
	    end
	end
end

nnls(A,b) = nonneg_lsq(A, b; alg=:nnls)
nnls_gram(A,b) = nonneg_lsq(A'*A, A'*b; alg=:nnls, gram=true)
fnnls(A,b) = nonneg_lsq(A, b; alg=:fnnls)
fnnls_gram(A,b) = nonneg_lsq(A'*A, A'*b; alg=:fnnls, gram=true)
pivot(A,b) = nonneg_lsq(A, b; alg=:pivot)
pivot_comb(A,b) = nonneg_lsq(A, b; alg=:pivot, variant=:comb)
pivot_cache(A,b) = nonneg_lsq(A, b; alg=:pivot, variant=:cache)

algs = [nnls, nnls_gram, fnnls, fnnls_gram, pivot, pivot_comb, pivot_cache]
errs = fill(1e-5, length(algs))

for (f, ε) in zip(algs, errs)
 	print("testing ")
 	@show f
 	test_algorithm(f, ε)
 	println("done")
end

@testset "NNLS" begin include("nnls_test.jl") end
@testset "FNNLS" begin include("fnnls_test.jl") end
@testset "Pivot" begin include("pivot_test.jl") end
@testset "Sparse" begin include("sparse_test.jl") end
@testset "CoordinateDescent" begin include("cd_test.jl") end


#  function survivor_prob(evals,sp,nsvt)
#
#    n = size(evals)[1]
#    l = size(evals)[2]
#    pdt = zeros(Float64, n, l)
#
#    for i in 1:l
#      for j in 1:n
#        if evals[j,i] == 0
#          pdt[j,i] = 0
#        else
#          pdt[j,i] = evals[j,i] / sp[j,i]
#        end
#      end
#    end
#    final = sum(pdt,dims=2) / nsvt
#    return final
#
#  end
#

  function survivor_prob(evals,sp,nsvt)
    ev = sum(evals,dims=2) / nsvt
    ea = sum(sp, dims=2) / nsvt

    l     = length(ev)
    final = zeros(Float64, l)

    for i in 1:l
      if ev[i]==0
        final[i] = 0.
      else
        final[i] = ev[i] / ea[i]
      end
    end

   # final = ev ./ ea
    return final
  end


srand(1)

let
@testset "orthogonality $((m, n))" for (m, n) in ((5, 5), (10, 4), (3, 7))
    X = randorth(m, n)
    @test isorth(X)
end

@testset "singular values $((m, n))" for (m, n) in ((5, 5), (10, 4), (3, 7))
    sv = [0.01; 1.0; 0.1]
    k = min(m, n)
    X = randrealsv(sv, m, n)
    U, S, V = svd(X)
    @test S ≈ sort([sv; zeros(k-length(sv))], rev=true)
    @test size(U) == (m, k)
    @test size(V) == (n, k)
end

@testset "sparse matrices $((m, n, k))" for (m, n, k) in ((10, 5, 3), (5, 10, 4))
    X = randsparse(m, n, k, 0.3)
    @test count(diag(qrfact(X).R) .!= 0) == k
end

end # let


# D(u)_function.jl

"""
Implementation of FHIP's susceptibility formula D(u) for the Optical Absorption of Polarons.

See FHIP 1962, equation (35c):
https://link.aps.org/doi/10.1103/PhysRev.127.1004.
"""

"""
ℜD(x::Float64, y::Float64, v::Float64, w::Float64, β::Float64)

    Calculate the real part of D(u) (equation (35c) in FHIP) for a complex argument u = x + iy. v and w are the variational Polaron parameters that minimise the free energy, for the supplied α Frohlich coupling. β is the reduced thermodynamical beta = ħω/kT with k being Boltzman's constant.
"""
function ℜD(x, y, v, w, β)
    R = (v^2 - w^2) / (w^2 * v)
    P = 1 / (exp(v * β) - 1)
    w^2 / v^2 * (
        R * cos(v * x) * (exp(-v * y) + 2 * P * cosh(v * y)) - (x^2 - y^2) / β - y -
        R * (2 * P + 1)
    )
end

"""
ℑD(x::Float64, y::Float64, v::Float64, w::Float64, β::Float64)

    Calculate the imaginary part of D(u) (equation (35c) in FHIP) for a complex argument u = x + iy. v and w are the variational Polaron parameters that minimise the free energy, for the supplied α Frohlich coupling. β is the reduced thermodynamical beta = ħω/kT with k being Boltzman's constant.
"""
function ℑD(x, y, v, w, β)
    R = (v^2 - w^2) / (w^2 * v)
    P = 1 / (exp(v * β) - 1)
    w^2 / v^2 * (R * sin(v * x) * (exp(-v * y) - 2 * P * sinh(v * y)) - x * (2 * y / β - 1))
end

# Create a complex number from the real and imaginary parts of D(u).

D(x, y, v, w, β) = ℜD(x, y, v, w, β) + 1im * ℑD(x, y, v, w, β)


export charλ,
       charA,
       charB
"""
charλ(q,ν,k)

char value λ_(ν+k) for Mathieu's equation

y'' + (λ_(ν+k) - 2 q cos( 2z )) y = 0

where

q ∈ ℝ       - parameter
ν ∈ [-1,1]  - fractional part of the non-integer order
k ∈ ℤ⁺      - range of integer parts of the order

"""
function charλ(q::Real, nu_::Real; k::UnitRange=1:1) # reduced = true
    #nu = reduced ? rem(nu_+1,2)-1 : nu_;
    nu = rem(nu_+1,2)-1;

    # Set matrix size using formula from Shirts paper (1993), Eqs. (2.1)-(2.2).
    nu0 = nu + maximum(k)
    C = (8.46 + 0.444*nu0)/(1 + 0.085*nu0)
    D =  (0.24 + 0.0214*nu0)/(1 + 0.059*nu0)
    N = ceil(Int, (nu0 + 2 + C*abs(q)^D)/2) # matrix size is 2N+1

    (two, q2, nu2) = float.(promote(2, q, nu))
    d0 = (two .* (-N:N) .- nu2).^2
    d1 = q2 .* ones(eltype(q2), 2 * N)
    A = SymTridiagonal(d0, d1)
    a = eigvals(A, k)
    return a
end

"""
charA(q; k=0:4)

char value A_k for Mathieu's equation

y'' + (A_k - 2 q cos( 2z )) y = 0

where

q ∈ ℝ  - parameter
k ∈ ℤ⁺ - eigenvalue index

"""
function charA(q::Real; k::UnitRange=1:1)
    all(x -> x >= 0, k) || throw(DomainError(k, "Indices must be non-negative integers."))

    # Boolean indices of even and odd n values
    ie = map(iseven, k)
    io = map(!, ie)

    a = Array{Float64}(undef ,length(k))
    k1 = k .+ 1
    a[ie] = charλ(abs(q), 0.0; k = k1)[ie]
    if q>=0
        a[io] = charλ(q, one(q); k = k1)[io]
    else
        if 0 in k # maybe not the cleanest way to do it
            a[io] = charλ(abs(q), one(q); k = k[2]:last(k))[io[2:end]]
        else
            a[io] = charλ(abs(q), one(q); k=k)[io]
        end
    end
    return a
end

"""
charB(q,k)

char value B_k for Mathieu's equation

y'' + (B_k - 2 q cos( 2z )) y = 0

where

q ∈ ℝ  - parameter
k ∈ ℤ  - eigenvalueindex

"""
function charB(q::Real; k::UnitRange=1:1)
    all(x -> x > 0, k) || throw(DomainError(k, "Indices must be positive integers."))
    # Boolean indices of even and odd n values
    ie = map(iseven, k)
    io = map(!, ie)

    b = Array{Float64}(undef, length(k))
    b[ie] = charλ(q,0.0,k=k)[ie]
    if q>=0
        b[io] = charλ(q,1.0,k=k)[io]
    else
        b[io] = charλ(abs(q),1.0,k = (k .+ 1))[io]
    end
    return b
end


f(x, y, z) = exp.(x .* y .* z) .* sin.(π .* x) .* sin.(π .* y) .* sin.(π .* z)

function advection_dominated(;N = 50, β = 1000.0)
    # Problem: Δu + βuₓ = f
    # u = 0 on the boundaries
    # f(x, y, z) = exp(xyz) sin(πx) sin(πy) sin(πz)
    # 2nd order central differences (shows serious wiggles)

    # Total number of unknowns
    n = N^3

    # Mesh width
    h = 1.0 / (N + 1)

    # Interior points only
    xs = range(0, stop=1, length=N + 2)[2 : N + 1]

    # The Laplacian
    Δ = laplace_matrix(Float64, N, 3) ./ -h^2

    # And the dx bit.
    ∂x_1d = spdiagm(-1 => fill(-β / 2h, N - 1), 1 => fill(β / 2h, N - 1))
    ∂x = kron(speye(N^2), ∂x_1d)

    # Final matrix and rhs.
    A = Δ + ∂x
    b = reshape([f(x, y, z) for x ∈ xs, y ∈ xs, z ∈ xs], n)

    A, b
end

function laplace_matrix(::Type{T}, n, dims) where T
    D = second_order_central_diff(T, n)
    A = copy(D)

    for idx = 2 : dims
        A = kron(A, speye(n)) + kron(speye(size(A, 1)), D)
    end

    A
end

second_order_central_diff(::Type{T}, dim) where {T} = convert(
    SparseMatrixCSC{T, Int},
    SymTridiagonal(fill(2 * one(T), dim), fill(-one(T), dim - 1))
)


export cyclotomic
export jacobi, kronecker, moebius, necklace

using Primes

"""
    cyclotomic(P, n)

Calculate cylotomic polynomial of degree `n` in polynom ring `P`.
This polynom is defined to be the disisor of `x^n - 1`, which is coprime to
all `x^k - 1` with `k < n`.

If `n` is prime, `cyclotomic(P, n) = (x^n - 1) / (x - 1).

If `n` is squarefree (product of distinct primes `p1 * p2 * ... * pk`)
and `c_k(x) = cyclotomic(P, p1*...*pk)` we use the recursion formula
` c_(k+1)(x) = c_k(x^pk) / c_k(x)`.

If `v = p1*...*pk` with the distinct prime factors of `n`, we have
`cyclotomic(P, n)(x) = cyclotomic(P, v)(x^(n/v))`.
"""
function cyclotomic(::Type{P}, n::Integer) where P<:UnivariatePolynomial
    T = basetype(P)
    (n < 2 || isprime(n)) && return P(ones(T, n))
    f = factor(n)
    l = length(f)
    v = collect(keys(f))
    v1 = prod(v)
    q = P(ones(T, v[1]))
    for k = 2:l
        q = div(spread(q, v[k]), q)
    end
    n == v1 ? q : spread(q, n ÷ v1)
end

# Jacobi symbol
function jacobi(n::Integer, k::Integer)
    k > 0 && k & 1 == 1 || throw(DomainError(k, "k must be positive odd number"))
    n = mod(n, k)
    t = 1
    while n != 0
        while n & 1 == 0
            n >>= 1
            r = k & 7
            if r == 3 || r == 5
                t = -t
            end
        end
        n, k = k, n
        if n & 3 == k & 3 == 3
            t = -t
        end
        n = mod(n, k)
    end
    k == 1 ? t : 0
end

# Kronecker symbol
function kronecker(n::Integer, k::Integer)
    n&1 == 0 && k&1 == 0 && return 0
    k == 0 && return n == 1 || n == -1 ? 1 : 0
    ks = k < 0 && n < 0 ? -1 : 1
    if k < 0
        k = -k
    end
    t = trailing_zeros(k)
    k >>= t
    ks = (n&7 == 3 || n&7 == 5 ) && t&1 == 1 ? -ks : ks
    jacobi(n, k) * ks
end

# moebius function
function moebius(n::Integer)
    n > 0 || throw(ArgumentError("moebius defined for positive integers only"))
    n == 1 && return 1
    f = Primes.factor(n)
    if maximum(values(f)) == 1
        ifelse(isodd(length(f)), -1, 1)
    else
        0
    end
end

# necklace polynomial - Moreau's necklace-counting function
"""
    necklace(q, n)

Return the value of the `necklace polynomial`

Count of irreducible monic polynomials of degree n over Z/q.
"""
necklace(q::Integer, n::Integer) = _necklace(q, n) ÷ n
necklace(q::Ring, n::Integer) = _necklace(q, n) / n
function _necklace(q, n::Integer)
    f = Primes.factor(n)
    p = collect(keys(f))
    m = length(p)
    s = 0
    for x = 0:(2^m-1)
        d = n
        μ = 1
        k = 1
        while x != 0
            if x & 1 == 1
                d ÷= p[k]
                μ = -μ
            end
            x >>= 1
            k += 1
        end
        s += q^d * μ
    end
    s
end


# first example 2D

using ApproxManifoldProducts
using Test

const AMP = ApproxManifoldProducts

Λ1 = 1.0
Λ2 = 0.25
Λ3 = 5/4.0

μ1 = 1.0
μ2 = 4.0
μ3 = 2.4

@testset "basic linear 1D find mu and Lambdas Gaussian product test..." begin

Lambdas = [Λ1; Λ2]
mus = [μ1; μ2]

resid2DLinear(0.0, mus, Lambdas)

resid2DLinear(8/5, mus, Lambdas)

μ = get2DMu(mus, Lambdas)
Λ = get2DLambda(Lambdas)

@test abs(μ[1] - 8/5) < 1e-10

@test abs(Λ - 5/4) < 1e-10

end

## These work, but we are not sure about how to get the scale for product of more than 2 elements

# preS = nullspace(Lambdas*(Lambdas'))
#
# sc = sum(abs.(preS)) / abs(μ1-μ2)
#
# mu = vec(preS / sc)
# Λ = getΛ(Lambdas)
#
#
# u,s,v = svd(Lambdas*(Lambdas'))
#
# s[1,1]*u[:,2]

@testset "test 1D with 3 components on basic Euclidean line..." begin

Lambdas = [Λ1; Λ2; Λ3]
mus = [μ1; μ2; μ3]

μ = get2DMu(mus, Lambdas)
Λ = get2DLambda(Lambdas)

@test abs(μ[1] - 2.0) < 1e-10

@test abs(Λ - 10/4) < 1e-10

end

#


using FractionalCalculus
using Plots, LaTeXStrings

s="\$(x^2)'=2x\$"

tspan=collect(0:0.01:3)

result=fracdiff(x->x^2, 1, 3, 0.01, RLDiff_Matrix())

plot(tspan, result, title=s, legend=:bottomright)


function representable_ints(nbits)
    return 1:(2^(nbits-1)-1)
end

function representable_floats(nbits,ebits)
    #= returns an array of all representable positive floats exclusive 0 and ComplexInfinity.
    nbits is the total number of bits (sign,exponent,significand)
    ebits is the number of exponent bits
    =#

    # derived constants
    bias = 2^(ebits-1) - 1
    sbits = nbits-ebits-1     # number of significand bits (minus one for sign bit)

    repr_floats = []           # represented as 000...0000

    # subnormal numbers (exponent is 0)
    for f = 1:2^sbits-1
        append!(repr_floats,2.0^(1-bias)*(f/2^sbits))
    end

    # all other numbers
    for e = 1:2^ebits-2     # the exponent cannot be 11...111 this represents NaN
        for f = 0:2^sbits-1
            append!(repr_floats,2.0^(e-bias)*(1.0 + f/2^sbits))
        end
    end

    return Float64.(repr_floats)
end

function wcdp_posit(plist)
    p_am = (plist[1:end-1]+plist[2:end])/2.
    p_wda = -log10.(abs.(log10.(p_am./plist[1:end-1])))

    # extend first and last point, taking no overflow/underflow into account
    p0 = plist[1]/16    # something much smaller than minpos
    pinf = plist[end]*16    # something much bigger than maxpos

    p_wda_0 = -log10.(abs.(log10.(p0/plist[2]))) # worst-case decimal accuracy for these extreme values
    p_wda_inf = -log10.(abs.(log10.(pinf/plist[end])))

    # worst-case decimal accuracy of interpolated on minpos/maxpos
    p_wda_minpos = p_wda_0 + log10(plist[1]/p0)/log10(p0/p_am[1])*(p_wda_0-p_wda[1])
    p_wda_maxpos = p_wda_inf + log10(plist[end]/pinf)/log10(pinf/p_am[end])*(p_wda_inf-p_wda[end])

    p_wda = vcat(p_wda_0,p_wda_minpos,p_wda,p_wda_maxpos,p_wda_inf)
    p_am = vcat(p0,plist[1],p_am,plist[end],pinf)

    return p_am,p_wda,plist
end

function wcdp_float(flist)

    f_am = (flist[1:end-1]+flist[2:end])/2.
    f_wda = -log10.(abs.(log10.(f_am./flist[1:end-1])))

    # extend with zeros due to overflow
    f_wda = vcat(0.55,f_wda)        # extrapolate somehow
    f_am = vcat(flist[1],f_am)

    return f_am,f_wda
end

function wcdp_approx(flist)

    f_am = (flist[1:end-1]+flist[2:end])/2.
    f_wda = -log10.(abs.(log10.(f_am./flist[1:end-1])))

    return f_am,f_wda
end

function wc_dec_acc_int(nbits)

    # assume rounding mode down
    # somehow interpolate with 0.13 onto the smallest representable number 1...

    i_am = [1.,2.,2^(nbits-1)-1]
    i_wda = [0.13,-log10(abs(log10(2))),-log10(abs(log10((2^(nbits-1)-1)/(2^(nbits-1)-2))))]

    return i_am,i_wda
end

decprec(xe,xr) = -log10.(abs.(log10.(xe./xr)))


function disc_phantom(img, a, b, r)
    height, width = size(img)
    #find the possibility which has most surrounding pixels
    disc = zeros(Float64, (height,width))
    for x = 1:width
        for y = 1:height
            disc[y,x] = (x-a)^2 + (y-b)^2 < r^2 ? 1.0 : 0.0
        end
    end
    return disc
end


# given an 1D interval, return quadrature nodes and weights

using LinearAlgebra
using FastGaussQuadrature

function quadgrid(xmin, xmax, n)
    # get nodes and weights in reference coord [-1, 1]
    x, w = gausslegendre(n)
    # map nodes and weights to current interval
    dist = (xmax - xmin)/2
    center = (xmax + xmin)/2
    weights = dist .* w
    nodes = dist .* x .+ center
    return nodes, weights
end

function quadgrid_old(xmin, xmax, n)
    dist = (xmax - xmin)/2
    center = (xmax + xmin)/2
    if n == 1
        w = 2
        x = 0
    elseif n == 2
        w = [1, 1]
        x =  1/sqrt(3) * [1, -1]
    elseif n == 3
        w = [5/9, 8/9, 5/9]
        x = -sqrt(3/5) * [-1, 0, 1]
    elseif n == 4
        w = [(18+sqrt(30))/36, (18+sqrt(30))/36,
             (18-sqrt(30))/36, (18-sqrt(30))/36]
        x = [sqrt(3/7 - 2/7*sqrt(6/5)), -sqrt(3/7 - 2/7*sqrt(6/5)),
             sqrt(3/7 + 2/7*sqrt(6/5)), -sqrt(3/7 + 2/7*sqrt(6/5))]
    elseif n == 5
        w = [128/225, (322+13*sqrt(70))/900, (322+13*sqrt(70))/900,
                      (322-13*sqrt(70))/900, (322-13*sqrt(70))/900]
        x = [0, sqrt(5-2*sqrt(10/7))/3, -sqrt(5-2*sqrt(10/7))/3,
                sqrt(5+2*sqrt(10/7))/3, -sqrt(5+2*sqrt(10/7))/3]
    end

    w_new = dist .* w
    x_new = dist .* x .+ center
    # fx = f.(x_new)
    # int = dot(w_new, fx)

    return x_new, w_new

end


# Used "Conjugate Bayesian analysis of the Gaussian distribution" by Murphy as
# a reference.  Note that there were some typos in that document so the code
# here may not correspond exactly.

immutable NormalInverseWishart <: Distribution
    dim::Int
    zeromean::Bool
    mu::Vector{Float64}
    kappa::Float64              # This scales precision (inverse covariance)
    Lamchol::Cholesky{Float64}  # Covariance matrix (well, sqrt of one)
    nu::Float64

    function NormalInverseWishart(mu::Vector{Float64}, kappa::Real,
                                  Lamchol::Cholesky{Float64}, nu::Real)
        # Probably should put some error checking in here
        d = length(mu)
        zmean::Bool = true
        for i = 1:d
            if mu[i] != 0.
                zmean = false
                break
            end
        end
        new(d, zmean, mu, float64(kappa), Lamchol, float64(nu))
    end
end

function NormalInverseWishart(mu::Vector{Float64}, kappa::Real,
                              Lambda::Matrix{Float64}, nu::Real)
    NormalInverseWishart(mu, kappa, cholfact(Lambda), nu)

end

function insupport(::Type{NormalInverseWishart}, x::Vector{Float64}, Sig::Matrix{Float64})
    return (all(isfinite(x)) &&
           size(Sig, 1) == size(Sig, 2) &&
           isApproxSymmmetric(Sig) &&
           size(Sig, 1) == length(x) &&
           hasCholesky(Sig))
end

pdf(niw::NormalInverseWishart, x::Vector{Float64}, Sig::Matrix{Float64}) =
        exp(logpdf(niw, x, Sig))

function logpdf(niw::NormalInverseWishart, x::Vector{Float64}, Sig::Matrix{Float64})
    if !insupport(NormalInverseWishart, x, Sig)
        return -Inf
    else
        p = size(x, 1)

        nu = niw.nu
        kappa = niw.kappa
        mu = niw.mu
        Lamchol = niw.Lamchol
        hnu = 0.5 * nu
        hp = 0.5 * p

        # Normalization
        logp::Float64 = hnu * logdet(Lamchol)
        logp -= hnu * p * log(2.)
        logp -= lpgamma(p, hnu)
        logp -= hp * (log(2.*pi) - log(kappa))

        # Inverse-Wishart
        logp -= (hnu + hp + 1.) * logdet(Sig)
        logp -= 0.5 * trace(Sig \ (Lamchol[:U]' * Lamchol[:U]))

        # Normal
        z = niw.zeromean ? x : x - mu
        logp -= 0.5 * kappa * invquad(PDMat(Sig), z)

        return logp

    end
end

function rand(niw::NormalInverseWishart)
    Sig = rand(InverseWishart(niw.nu, niw.Lamchol))
    mu = rand(MvNormal(niw.mu, Sig ./ niw.kappa))
    return (mu, Sig)
end


#=
Surprisingly there are only three numbers that can be written as the sum of fourth powers of their digits:

1634 = 14 + 64 + 34 + 44
8208 = 84 + 24 + 04 + 84
9474 = 94 + 44 + 74 + 44
As 1 = 14 is not a sum it is not included.

The sum of these numbers is 1634 + 8208 + 9474 = 19316.

Find the sum of all the numbers that can be written as the sum of fifth powers of their digits.
=#
#=
Note: Calculating limit:
Checking the number 111111111.
That number has 9 digits, so the maximum sum would be 9*59049 = 531441, which doesn't even come close to 111111111.
So any 9 digit number or greater can't ever reach a big enough sum.
=#
function sumofdigitpowers(a)
  total, i = 0, 2
  while i < length(digits(i))*9^a
    current = sum(map((x)->x^a, digits(i)))
    if current == i
      total += i
    end
    i+=1
  end
  total
end
@time println(sumofdigitpowers(5))