@testset "Test Spline Basis Evaluation" begin

    # constuct specific case that we can compute by hand

    n = 5
    a = 0
    b = 1
    params = SplineParams(linspace(a,b,n),0,1)

    x = rand(10000)

    basestr = @inferred BasisMatrices.evalbase(params, x)
    base = full(basestr)

    # vector of basis functions

    function linBspline(u)

        if u >= 0 && u < 0.25

            return [1-4*u,4*u,0,0,0]

        elseif u >= 0.25 && u < 0.5

            return [0,2*(1 - 2*u),4*u-1,0,0]

        elseif u >= 0.5 && u < 0.75

            return [0,0,3 - 4*u,2*(2*u - 1),0]

        else

            return [0,0,0,4*(1 - u),4*u-3]

        end

    end

    manualbase = Array(Float64,length(x),n)

    for i in 1:length(x)

        manualbase[i,:] = linBspline(x[i])

    end

    @testset "test evalbase with linear B spline" begin
        @test  manualbase  ==  base
    end
end


@testset "Algebraic Decision Diagrams" begin
    @info "ADD"
    ADD = AlgebraicDecisionDiagrams
    @show x1 = indicator(Int,1) # equivalent to x1 = Node(1,Base.zero(Int),Base.one(Int))
    @show one(x1) - x1
    @show ¬x1
    @show apply(x -> 1-x, x1)
    @show x1 + (¬x1)
    @show x1 * (¬x1)
    @show x2 = indicator(Int,2)
    @show x1 + x2
    @show x1 * x2
    @show apply(x -> x^2, x1 + x2)
    @show ⊕(x,y) = (x + y) % 2 # sum mod 2
    @show apply(⊕,x1,x2)
    @show x3 = indicator(Int,3)

    @info "Odd Parity Function"
    @show apply(⊕,x1,apply(⊕,x2,x3))
    @show apply(max, x1 * x2, x3)

    @info "Example from Fig. 1 in Bryant's paper"
    @show apply(max, (¬x1)*x2, x1*x3)

    @info "Example in Fig. 5 of Bryant's paper"
    A3 = Node(3,zero(Int),one(Int))
    @show A4 = ADD.reduce(Node(1,Node(2,zero(Int),A3),Node(2,A3,A3)))
    # for (i,α) in enumerate(x1*x2)
    #     println(i, "\n", α)
    # end
    println()

    @info "Directed Graph ADD"

    @info "Example in Fig 1. in Bahar et al. 1997's paper"
    x0 = indicator(Int,0)
    nx0 = indicator(Int,0,true)
    x1 = indicator(Int,2)
    nx1 = indicator(Int,2,true)
    y0 = indicator(Int,1)
    ny0 = indicator(Int,1,true)
    y1 = indicator(Int,3)
    ny1 = indicator(Int,3,true)
    @show graph = Terminal(2)*nx0*nx1*ny0*y1 + Terminal(2)*nx0*nx1*y0*ny1 + Terminal(2)*nx0*x1*ny0*y1 + Terminal(2)*nx0*x1*y0*ny1 + Terminal(4)*x0*nx1*y0*y1 + Terminal(4)*x0*x1*ny0*y1
    @info "Get scope"
    # collect nonterminal nodes
    nt = filter(n -> isa(n,Node), collect(graph))
    @show scope = mapreduce(index,union,nt) # 0,1,2,3
    @test length(intersect(scope,[0,1,2,3])) == 4
    println()

    @info "Matrix examples in page 9 of Bryant's paper"
    @show f = (¬x0)*(¬y0) + x0*y0
    @show g = Terminal(4)*nx0 + Terminal(2)*x0
    @show h = f + g
    println()
    @info "Restriction"
    @show restrict(h,0,false)
    @show h | (1 => true)
    @show (h | (index(x0) => true)) + (h | (index(x0) => false))
    @test value((h | (index(x0) => true)) + (h | (index(x0) => false))) == 7
    @show (h | (index(y0) => true)) + (h | (index(y0) => false))
    println()
    @info "marginalization"
    @show marginalize(h, index(x0))
    @show marginalize(h, index(y0))
    @show A4
    @show (A4 | (2 => true)) + (A4 | (2 => false))
    @show A5 = marginalize(A4, 2)
end # testset


"random number between a and b"
function randin(a::Real, b::Real)
    -Inf < a ≤ b < Inf || DomainError()
    a + (b - a) * rand()
end

"""
Weighted random integer.
Given weights = (w1, w2, ...), returns a random integer i with probability proportional to w_i.
The weights should be 0 ≤ w_i < ∞, and do not need to sum to 1.
"""
function wrand(weights)
    wr = rand() * sum(w for w in weights)
    for (i,w) in enumerate(weights)
        @assert 0 ≤ w < Inf
        wr -= w
        if wr ≤ 0
            return i
        end
    end
end

"""
Returns a random vector x = (x1, x2, ..., xn), such that
xi ≥ 0 and sum(xi) = 1
"""
function randsimplex(n::Int)
    # https://cs.stackexchange.com/questions/3227/uniform-sampling-from-a-simplex
    @assert n > 0
    diff(sort([0; rand(n - 1); 1]))
end

"""
Returns a random vector x = x0 * v[0] + x1 * v[1] + ... + xn * v[n]
where xi ≥ 0 and sum(xi) = 1
"""
function randsimplex(V::Vector{Vector{Float64}})
    @assert allsame(length.(V))
    X = randsimplex(length(V))
    sum(x*v for (x,v) in zip(X,V))
end


#Implements Future Value calculation as per Excel
#Type = 1 means payment at start of period
function FV(Rate=0,Nper=0,Pmt=0,Pv=0,Type=0)
    Fv = (-1*Pv*(1 + Rate)^Nper - Pmt*(1 + Rate*Type)*(((1 + Rate)^Nper - 1)/Rate))
	round(Fv, 2)
end

#######################################

#Implements Present Value calculation as per Excel
#Type = 1 means payment at start of period
function PV(Rate=0,Nper=0,Pmt=0,Fv=0,Type=0)
    Pv = (-1*Fv - Pmt*(1 + Rate*Type)*(((1 + Rate)^Nper - 1)/Rate))/(1 + Rate)^Nper
	round(Pv, 2)
end

#######################################

#Implements Payment calculation as per Excel
#Type = 1 means payment at start of period
function PMT(Rate=0,Nper=0,Pv=0,Fv=0,Type=0)
    Pmt = (-1*Fv - Pv*(1 + Rate)^Nper)/((1 + Rate*Type)*((1 + Rate)^Nper - 1)/Rate)
	round(Pmt, 2)
end

#######################################

#Implements Effective Annual Rate calculation as per Excel
function EFFECT(Nominal_rate,Npery)
	Effect = ((1 + Nominal_rate/Npery)^Npery - 1)
	round(Effect, 6)
end

#######################################

#Implements Net Present Value calculation as per Excel
#Add the option of adding inital payment
function NPV(Rate,Cash_Flow,InitialPayment=0)
	npv=0
	for i in 1:length(Cash_Flow)
		npv += Cash_Flow[i] / (1+Rate)^i
	end
	npv += InitialPayment
	round(npv, 2)
end

#######################################

#Implements Internal Rate of Return calculation as per Excel
function IRR(Cash_Flow,Guess=0.1)
	const limit=0.0000001
	initial=Cash_Flow[1]
	values=Cash_Flow[2:length(Cash_Flow)]

	rate=Guess*1.01
	oldRate=Guess

	while (abs((rate-oldRate)/rate) > limit)
		newRate=rate-NPV(rate,values,initial)*((rate-oldRate)/(NPV(rate,values,initial)-NPV(oldRate,values,initial)))
		oldRate=rate
		rate=newRate
	end

	round(rate,6)
end


module HHChannels

export Stim, run_hh

using Sundials
using SIUnits.ShortUnits

immutable HHParam
    c_m       # membrane spacific capacitance
    gnabar    # Na channel cross-membrane conductivity
    gkbar     # K channel cross-membrane conductivity
    gl        # Leak conductivity
    ena       # Na channel reversal potential
    ek        # K channel reversal potential
    el        # Leak reversal potential
    q10       # temperature dependent rate coefficient
              # (= 3^((T-T₀)/10K) with T₀ = 6.3 °C)

    # constructor with default values, corresponding
    # to a resting potential of -65 mV and temperature 6.3 °C
    HHParam(;
        # default values from HH paper

        # For reversal potentials we use those computed using
        # the Nernst equation with the following values:
        #       R   8.3144598
        #       F   96485.33289
        #       nao 140   mM
        #       nai  10   mM
        #       ko    2.5 mM
        #       ki   64.4 nM
        # We don't use the default values for ena and ek taken
        # from the HH paper:
        #   ena    = 115.0mV + -65.0mV,
        #   ek     = -12.0mV + -65.0mV,
        ena    =  63.55148117386mV,
        ek     = -74.17164678272mV,

        c_m    = 0.01F*m^-2,
        gnabar = .12S*cm^-2,
        gkbar  = .036S*cm^-2,
        gl     = .0003S*cm^-2,
        el     = -54.3mV,
        q10    = 1
    ) = new(c_m, gnabar, gkbar, gl, ena, ek, el, q10)

end

immutable Stim
    t0        # start time of stimulus
    t1        # stop time of stimulus
    i_e       # stimulus current density

    Stim() = new(0s, 0s, 0A/m^2)
    Stim(t0, t1, i_e) = new(t0, t1, i_e)
end

vtrap(x,y) = x/(exp(x/y) - 1.0)

# "m" sodium activation system
function m_lims(v, q10)
    alpha = .1mV^-1 * vtrap(-(v+40mV),10mV)
    beta =  4 * exp(-(v+65mV)/18mV)
    sum = alpha + beta
    mtau = 1ms / (q10*sum)
    minf = alpha/sum
    return mtau, minf
end

# "h" sodium inactivation system
function h_lims(v, q10)
    alpha = 0.07*exp(-(v+65mV)/20mV)
    beta = 1 / (exp(-(v+35mV)/10mV) + 1)
    sum = alpha + beta
    htau = 1ms / (q10*sum)
    hinf = alpha/sum
    return htau, hinf
end

# "n" potassium activation system
function n_lims(v, q10)
    alpha = .01mV^-1 * vtrap(-(v+55mV),10mV)
    beta = .125*exp(-(v+65mV)/80mV)
    sum = alpha + beta
    ntau = 1ms / (q10*sum)
    ninf = alpha/sum
    return ntau, ninf
end

# Choose initial conditions for the system such that the gating variables
# are at steady state for the user-specified voltage v
function initial_conditions(v, q10)
    mtau, minf = m_lims(v, q10)
    htau, hinf = h_lims(v, q10)
    ntau, ninf = n_lims(v, q10)

    return (v, minf, hinf, ninf)
end

# Given time t and state (v, m, h, n),
# return (vdot, mdot, hdot, ndot)
function f(t, state; p=HHParam(), stim=Stim())
    v, m, h, n = state

    # calculate current density due to ion channels
    gna = p.gnabar*m*m*m*h
    gk = p.gkbar*n*n*n*n

    ina = gna*(v - p.ena)
    ik = gk*(v - p.ek)
    il = p.gl*(v - p.el)

    itot = ik + ina + il

    # calculate current density due to stimulus
    if t>=stim.t0 && t<stim.t1
        itot -= stim.i_e
    end

    # calculate the voltage dependent rates for the gating variables
    mtau, minf = m_lims(v, p.q10)
    htau, hinf = h_lims(v, p.q10)
    ntau, ninf = n_lims(v, p.q10)

    return (-itot/p.c_m, (minf-m)/mtau, (hinf-h)/htau, (ninf-n)/ntau)
end

function run_hh(t_end; v0=-65mV, stim=Stim(), param=HHParam(), sample_dt=0.01ms)
    v_scale = 1V
    t_scale = 1s

    v0, m0, h0, n0 = initial_conditions(v0, param.q10)
    y0 = [ v0/v_scale, m0, h0, n0 ]

    samples = collect(0s: sample_dt: t_end)

    fbis(t, y, ydot) = begin
        vdot, mdot, hdot, ndot =
            f(t*t_scale, (y[1]*v_scale, y[2], y[3], y[4]), stim=stim, p=param)

        ydot[1], ydot[2], ydot[3], ydot[4] =
            vdot*t_scale/v_scale, mdot*t_scale, hdot*t_scale, ndot*t_scale

        return Sundials.CV_SUCCESS
    end

    # Ideally would run with vector absolute tolerance to account for v_scale,
    # but this would prevent us using the nice cvode wrapper.

    res = Sundials.cvode(fbis, y0, map(t->t/t_scale, samples), abstol=1e-6, reltol=5e-10)

    # Use map here because of issues with type deduction with arrays and SIUnits.
    return samples, map(v->v*v_scale, res[:, 1])
end

end # module HHChannels


## Contains functions and objects for statistical analysis
using Statistics, StatsBase

TABLE_HEADER = ["Statistic", "N", "Mean", "Min", "25 Percentile", "Median", "75 Percentile", "Max", "Standard Deviation", "Skewness", "Kurtosis"]

"""
computes the summary statistics of a given data set
"""
function summary_data(name, data)
        n = length(data)
        μ, σ = mean_and_std(data)
        minim, q1, med, q3, maxim = StatsBase.nquantile(data, 4)
        sk = skewness(data)
        ku = kurtosis(data)
        return [name n μ minim q1 med q3 maxim σ sk ku]
end


using LinearAlgebra: Diagonal

# Loss function should return a pair of (e, ∂e/∂ŷ).
export
    mse,
    softmax, log_softmax,
    logit_cross_entropy,
    logit_binary_cross_entropy

"Mean squared error loss"
@inline mse(ŷ, y) = (0.5 * sum((y - ŷ) .^ 2), ŷ - y)

#=====

**Softmax** turns (multinomial) logit into probability:

    softmax(x)_i = exp(x_i) / Σ exp(x_i)

**Cross entropy** evaluates the expected coding length of data following distribution p when a wrong
distribution q is assumed:

    H(p, q) = -E_p[log(q)]
            = -Σ p log(q)
            = H(p) + D_KL(p || q)
            = -Σ p log(p) + Σ p log(p/q)

**Binary cross entropy** is the special case of cross entropy for 2-class classification. It is
applied on a single sigmoid output rather than a vector of size 2 as in the softmax case.

To mitigate numerical instabilities, computing by a composed function (prefixed with "logit_") is
usually preferred:

    logit_cross_entropy(ŷ, y) := cross_entropy(softmax(ŷ), y)
    logit_binary_cross_entropy(ŷ, y) := binary_cross_entropy(logistic(ŷ), y)

Softmax is invariant under translation by the same value in each coordinate. The standard logistic
function is a special case of softmax where one variable is fixed at 0.

=====#

function softmax(x::AbstractMatrix{Float64})::Matrix{Float64}
    max_x = maximum(x, dims=1)
    exp_x = exp.(x .- max_x)
    exp_x ./ sum(exp_x, dims=1)
end

function log_softmax(x::AbstractMatrix{Float64})::Matrix{Float64}
    x_ = x .- maximum(x, dims=1)
    x_ .- log.(sum(exp.(x_), dims=1))
end

#=

Derivation:

    p_i = softmax(y)_i  ⟹
    ∂p_i/∂y_j = [i = j] p_i - p_i p_j

    ∂e/∂y_j = Σ_i ∂e/∂p_i ⋅ ∂p_i/∂y_j

Or in the form of matrix calculus,

    ∂e/∂y = ∂e/∂p ⋅ ∂p/∂y

And

    e = -sum(y .* log(p)) / n  ⟹
    ∂e/∂p = -y / n * (1/p)

Also note that $∂p/∂y ∈ ℜ_{d × d}$, so if mini-batch is used it is impossible to be expressed in
matrix form. We directly write `∂e/∂p ⋅ ∂p/∂y` of one sample as

      (∂e/∂p) ⋅ (diag(p) - p ⋅ p')
    = ∇p .* p - sum(∇p .* p) * p    # a ℜ_d vector

The mini-batch case can be written as

    ... = ∇p .* p - sum(∇p .* p, dims=1) .* p    # a ℜ_{d × n} matrix

=#
function logit_cross_entropy(
    ŷ::AbstractMatrix{Float64},
    y::AbstractMatrix{Float64},
)::Tuple{Float64, Matrix{Float64}}
    n_batches = size(ŷ, 2)
    log_p = log_softmax(ŷ)
    e = -sum(y .* log_p) / n_batches

    p = softmax(ŷ)
    ∇p = @. -y / n_batches * inv(p)
    t = ∇p .* p
    ∇y = t - sum(t, dims=1) .* p

    (e, ∇y)
end

#=
    logit_binary_cross_entropy(ŷ, 0)
        = -log(1 - logistic(ŷ))
        = -log(1 / (1 + exp(ŷ)))
        = -(0 - log(1 + exp(ŷ)))
        = -(ŷ - log(1 + exp(ŷ))) + ŷ
        = -log_logistic(ŷ) + ŷ
=#
function logit_binary_cross_entropy(
    ŷ::AbstractMatrix{Float64},
    y::AbstractMatrix{Bool},
)::Tuple{Float64, Matrix{Float64}}
    @assert size(ŷ, 1) == 1
    t = -log_logistic.(ŷ)
    ny = .!y
    t[ny] .+= ŷ[ny]
    e = sum(t) / size(ŷ, 2)

    ∂t_over_∂y = -∇log_logistic.(ŷ, -t)
    ∂t_over_∂y[ny] .+= 1.0
    ∇y = ∂t_over_∂y

    (e, ∇y)
end

#=

Addendum
--------

The derivation of the gradient of softmax + cross-entropy can be found in:

- https://stats.stackexchange.com/questions/235528/backpropagation-with-softmax-cross-entropy
- https://charlee.li/how-to-compute-the-derivative-of-softmax-and-cross-entropy/
- http://denizyuret.github.io/Knet.jl/stable/softmax/#Softmax-1

Copy the LaTeX here:

$$
\begin{aligned}
\frac{\partial}{\partial z_k}\text{CE} &= \frac{\partial}{\partial z_k}\sum_{j=1}^n
\big(- y_j \log \sigma(z_j) \big) \\  &= -\sum_{j=1}^n y_j \frac{\partial}{\partial z_k} \log
\sigma(z_j) && \cdots \text{addition rule, } -y_j \text{ is constant}\\  &= -\sum_{j=1}^n y_j
\frac{1}{\sigma(z_j)} \cdot \frac{\partial}{\partial z_k} \sigma(z_j) && \cdots \text{chain rule}\\
&= -y_k \cdot \frac{\sigma(z_k)(1-\sigma(k))}{\sigma(z_k)} –  \sum_{j \neq k} y_j \cdot
\frac{-\sigma(z_j)\sigma(z_k)}{\sigma(z_j)} && \cdots \text{consier both }j = k \text{ and } j \neq
k \\  &= -y_k \cdot (1-\sigma(z_k)) + \sum_{j \neq k} y_j \sigma(z_k) \\  &= -y_k + y_k \sigma(z_k)
+  \sum_{j \neq k} y_j \sigma(z_k) \\  &= -y_k + \sigma(z_k) \sum_j y_j.
\end{aligned}
$$

=#


# Probability computations

using CommonLH, Random, Test

function prob_matrix_test()
    @testset "Probability matrix" begin
        rng = MersenneTwister(304);
        n = 4; m = 3;
        probM = rand(rng, n, m);
        probM ./= sum(probM);
        @test validate_prob_matrix(probM)

        # Prob(j)
        prob_jV = [prob_j(probM, j) for j = 1 : n];
        prob2_jV = prob_j(probM);
        @test validate_prob_vector(prob_jV)
        @test prob_jV ≈ prob2_jV
        prob3_jV = prob_j(probM, 1 : n);
        @test prob3_jV ≈ prob_jV

        prob_kV = [prob_k(probM, k) for k = 1 : m];
        prob2_kV = prob_k(probM);
        @test validate_prob_vector(prob_kV)
        @test prob_kV ≈ prob2_kV
        prob3_kV = prob_k(probM, 1 : m);
        @test prob3_kV ≈ prob_kV

        for k = 1 : m
            prob_j_kV = prob_j_k(probM, 1:n, k);
            @test isapprox(sum(prob_j_kV), 1.0)
            # Prob(j,k) for given k
            prob_jkV = prob_j_kV .* prob_k(probM, k);
            @test isapprox(prob_jkV, probM[:,k])
        end

        for j = 1 : n
            prob_k_jV = prob_k_j(probM, 1:m, j);
            @test isapprox(sum(prob_k_jV), 1.0)
            # Prob(j,k) for given j
            prob_jkV = prob_k_jV .* prob_j(probM, j);
            @test isapprox(prob_jkV, vec(probM[j,:]))
        end
    end
end

function ev_test()
	@testset "Expected values" begin
        rng = MersenneTwister(304);
        n = 4; m = 3;
        probM = rand(rng, n, m);
        probM ./= sum(probM);

        xM = rand(rng, n, m);

        evV = ev_given_j(xM, probM);
        @test size(evV) == (n,)
        for j = 1 : n
            ev = ev_given_j(xM, probM, j);
            prV = vec(probM[j,:]);
            xV = vec(xM[j,:]);
            ev2 = sum(xV .* prV) / sum(prV);
            @test isapprox(ev, ev2)
            @test isapprox(ev, evV[j])
        end

        evV = ev_given_k(xM, probM);
        @test size(evV) == (m,)
        for k = 1 : m
            ev = ev_given_k(xM, probM, k);
            prV = probM[:, k];
            xV = xM[:, k];
            ev2 = sum(xV .* prV) / sum(prV);
            @test isapprox(ev, ev2)
            @test isapprox(ev, evV[k])
        end
    end
end

function scale_test()
    @testset "Scale prob array" begin
        x = rand(4,3,2) ./ 5.0;
        x[2 : 2 : 8] .= -0.000000001;
        x ./= sum(x);
        x .+= 0.000000001;
        scale_prob_array!(x);
        @test sum(x) <= 1.0
        @test all(x .>= 0.0)
	end
end

@testset "Probabilities" begin
    ev_test()
    prob_matrix_test()
end

# -------------


# Copyright (c) 2021 Idiap Research Institute, http://www.idiap.ch/
#  Niccolò Antonello <nantonel@idiap.ch>

export shortest_distance

"""
  `shortest_distance(fst[, s=get_initial(fst); filter=arc->true, reverse=false)`

Computes the shortest distance between the state `s` to every other state.
The shortest distance between `s` and `i` is defined as the sum of the weights of all paths between these states.

Weights are required to be rigth distributive and k-closed.

If `fst` has `N` states a `N`-long vector is returned where the `i`-the element contains the shortest distance between the states `s` and `i`.

If `reversed==true` the shortest distance from the final states is computed.
In this case `s` has no effect since a new initial superstate is added to the reversed WFST.
Here weights are required to be left distributive and k-closed.

See Mohri "Semiring framework and algorithms for shortest-distance problems", Journal of Automata, Languages and Combinatorics 7(3): 321-350, 2002.

"""
function shortest_distance(fst::WFST{W}, s=get_initial(fst); filter=arc->true, reversed=false) where {W}

  if reversed
    if !isright(W)
      throw(ErrorException("Weight $W is not right distributive"))
    end
    if !ispath(W)
      throw(ErrorException("Weight $W must satisfy the \"path\" property"))
    end

    rfst = reverse(fst)
    d = no_check_shortest_distance(rfst,get_initial(rfst);filter=filter)
    return d[2:end]
  else
    if !isleft(W)
      throw(ErrorException("Weight $W is not left distributive"))
    end
    if !ispath(W)
      throw(ErrorException("Weight $W must satisfy the \"path\" property"))
    end
    return no_check_shortest_distance(fst,s;filter=filter)
  end

end

function no_check_shortest_distance(fst::WFST{W,A,D},
                                    s::D=get_initial(fst);
                                    filter=arc->true) where {W,A,D}
  Ns = size(fst,1)
  d = zeros(W,Ns) # d[q] estimate of shortest distance between q and s
  r = zeros(W,Ns) # r[q] total weight added to d[q] since last time q was extracted from S
  d[s], r[s] = one(W), one(W)
  S = Queue{D}()
  enqueue!(S,s)
  while !isempty(S)
    q = dequeue!(S)
    R = r[q]
    r[q] =  zero(W)
    for arc in fst[q]
      if filter(arc)
        n, w = get_nextstate(arc), get_weight(arc)
        dn = d[n]
        if dn != (dn+(R*w))
          d[n] = dn+(R*w)
          r[n] = r[n]+(R*w)
          if !(n in S)
            enqueue!(S,n)
          end
        end
      end
    end
  end
  d[s] = one(W)
  return d
end


##################################################
# Helper functions
##################################################

"""
    spline_catmullrom(u, y0, y1, y2, y3)

Smooth interpolation between `y1, y2`, using the Catmull-Rom spline.
The function uses `y0` and `y3` to ensure that the derivative at `y1`
and `y2` is continuous.  `u` is a parameter between `0` and `1`.

"""
spline_catmullrom(u, y0, y1, y2, y3) = y1 + 0.5u*(-y0 + y2 + u*(2y0 - 5y1 + 4y2 - y3 + u*(-y0 + 3y1 - 3y2 + y3)))

_orbitalrank(orbital_symbol) =
    startswith(string(orbital_symbol), "s") ? 1 :
    startswith(string(orbital_symbol), "p") ? 2 :
    startswith(string(orbital_symbol), "d") ? 3 :
    startswith(string(orbital_symbol), "f") ? 4 :
    error("invalid orbital symbol: $orbital_symbol")

##################################################


include("../src/plotSol.jl")
include("../src/simulate.jl")
include("../src/OrientationConversion.jl")
using Revise
using JLD
clearconsole()

m = 1.0; l = 1.0; r = 0.01 # Mass, length, and radius of bar
# Assuming bar revolves about Z axis
# I1 = [m*r^2/2 0 0; 0 m*(l^2/12 + r^2/4) 0; 0 0 m*(l^2/12 + r^2/4)]
I1 = [m*(l^2/12 + r^2/4) 0 0; 0 m*(l^2/12 + r^2/4) 0; 0 0 m*(l^2/12 + r^2/4)]

# Testing Dynamics with Revolute Joint
# System looks like this:
# ====|====     | is the joint axis.
RbI = InertialFrameAsRB()
R1 = RigidBody(m,I1,2)
x0R1 = [0.0;zeros(2);[1;zeros(3)];zeros(3);zeros(4)]
initialiseRigidBody!(R1,x0R1)

axis = [0.0 0.0 1.0][:] # Axis about which bar is revolving

rj1 = [0.0 0.0 0.0][:] # Joint Location in body frame of first body
rj2 = [0 0.0 0.0][:] # Joint location in body frame of second body

j = Joint(RbI,R1,rj1,rj2,type="Revolute",axis=axis)

# External Forces Definition
g = [0.0;0.0;0.0]

# Simulation
tEnd = 1.0
tSpan = 0.01
tSim, solFinal = simulate(tEnd,tSpan,j,g=g)

solPend  = solFinal[1]

# After adding solSim
plotErrNorm(tSim,solPend.β)
# Check if joint location has moved
jointLoc = Matrix{Float64}(undef,length(tSim),3)
ωSol = Matrix{Float64}(undef,length(tSim),3)
rEnd = Matrix{Float64}(undef,length(tSim),3)
endPos = [l/2;0;0]
for i=1:length(tSim)
    R1.dcm = quat2dcm(solFinal[1].β[i,:])
    ωSol[i,:] = angVel(solFinal[1].β[i,:],solFinal[1].βdot[i,:])
    rEnd[i,:] = solPend.r[i,:] + transpose(R1.dcm)*endPos
    jointLoc[i,:] = solFinal[1].r[i,:] + transpose(R1.dcm)*rj2
end
plotPos(tSim,jointLoc)
plotPos(tSim,solPend.r)
plotPos(tSim,rEnd)
plotQuat(tSim,solPend.β)
plotAngVel(tSim,ωSol)
# plotQuatDot(tSim,solPend.βdot)


sqrt(sum(x -> x*x, A) / length(A))


function LinearRegression(
    x :: AbstractArray{T, 1},
    y :: AbstractArray{T, 1};
    order :: Integer = 1,
) where T <: AbstractFloat

    if order < 0
        throw(ErrorException("Parameter order must be non-negative."))
    end

    N = length(x)

    bases = order + 1

    ϕ = zeros(N, bases)

    for i = 1:bases

        if i == 1
            ϕ[:, 1] .= 1.0
        else
            ϕ[:, i] .= x.^(i-1)
        end

    end

    # ϕ β = y => β = ϕ \ y

    return ϕ \ y

end

function detrend(
    x :: AbstractArray{T, 1},
    y :: AbstractArray{T, 1};
    order :: Integer = 1,
) where T <: AbstractFloat

    if order < 0
        throw(ErrorException("Parameter order must be non-negative."))
    end

    bases = order + 1

    β = LinearRegression(x, y; order=order)

    result = copy(y)
    total_mean = 0.0
    for i = 1:bases
        result .-= β[i] * x.^(i-1)
        total_mean += β[i]/ i * (x[end]^i - x[1]^i)
    end

    result .+= total_mean / (x[end] - x[1])

    return result

end


# Contains various membership function types
# ------------------------------------------

"""	 Membership function type
"""
abstract type MF end

"""	 Triangular membership function type

	 TriangularMF(l_vertex, center, r_vertex)

	 Properties
	 ----------
	 `l_vertex`, `center` and `r_vertex` are the vertices of the triangle, in order

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength
"""
mutable struct TriangularMF<:MF

	l_vertex::Real
	center::Real
	r_vertex::Real

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function TriangularMF(l_vertex::Real, center::Real, r_vertex::Real)

		if l_vertex <= center <= r_vertex

			this = new()

			this.l_vertex = l_vertex
			this.center = center
			this.r_vertex = r_vertex

			function inv(α)
				x1 = (this.center - this.l_vertex) * α + this.l_vertex
				x2 = (this.center - this.r_vertex) * α + this.r_vertex
				return x1, x2
			end
			this.eval = function eval(x)
				maximum([minimum([((x - this.l_vertex) / (this.center - this.l_vertex)), ((this.r_vertex - x) / (this.r_vertex - this.center))]), 0])
			end

			this.mean_at = function mean_at(firing_strength)

				if firing_strength != 1
					p1 = (this.center - this.l_vertex) * firing_strength + this.l_vertex
					p2 = (this.center - this.r_vertex) * firing_strength + this.r_vertex
					(p1 + p2) / 2
				elseif firing_strength == 1
					return this.center
				end

			end

			this.n_opt = function get_n(tol)
				return 0
			end

			this.cut = function cut(α)
				x1, x2 = inv(α)
				return CutMF(x1, x2, α, this)
			end

			this

		else

			error("invalid vertices")

		end

	end

end

"""
 Gaussian membership function type

	GaussianMF(center, sigma)

	 Properties
	 ----------
	 `center` is the center of the distribution
	 `sigma` determines width of the distribution

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength

"""
mutable struct GaussianMF<:MF

	center::Real
	sigma::Real

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function GaussianMF(center::Real, sigma::Real)

		this = new()

		this.center = center
		this.sigma = sigma

		function inv(α)
			x1 = this.center - this.sigma*sqrt(-2*log(α))
			x2 = this.center + this.sigma*sqrt(-2*log(α))
			return x1, x2
		end

		this.eval = function eval(x)
			exp( - 0.5 * ((x - this.center) / this.sigma) ^ 2)
		end

		this.mean_at = function mean_at(firing_strength)
			this.center
		end

		this.get_n = function get_n(tol)
			l, u = inv(0.01)
			c = this.center
			s = this.sigma
			sdiff(x) = this.eval(x)*(c^2 - 2*c*x - s^2 + x^2)/s^4
			xs = collect(l:(u-l)/100:u)
			ys = sdiff.(xs)
			ym = maximum(abs.(ys))
			sqrt((u - l)^3*ym/(12*tol))
		end

		this.cut = function cut(α)
			x1, x2 = inv(α)
			return CutMF(x1, x2, α, this)
		end

		this

	end

end

"""
	 Generalised Bell membership function type

	BellMF(a, b, c)

	 Properties
	 ----------
	 `a`, `b` and `c` the usual bell parameters with `c` being the center

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength

"""
mutable struct BellMF<:MF

	a::Real
	b::Real
	c::Real

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function BellMF(a::Real, b::Real, c::Real)

		this = new()

		this.a = a
		this.b = b
		this.c = c

		function inv(α)
			x1 = this.c - this.a*((1-α)/α)^(1/(2*b))
			x2 = this.c + this.a*((1-α)/α)^(1/(2*b))
			return x1, x2
		end
		this.eval = function eval(x)
			1 / (1 + abs((x - this.c) / this.a) ^ (2 * this.b))
		end

		this.mean_at = function mean_at(firing_strength)
			this.c
		end

		this.get_n = function get_n(tol)
			l, u = inv(α)
			return (u-l)^3/(12*tol)
		end
		this.cut = function cut(α)
			x1, x2 = inv(α)
			return CutMF(x1, x2, α, this)
		end

		this

	end

end

"""
	 Trapezoidal membership function type

	TrapezoidalMF(l_bottom_vertex, l_top_vertex, r_top_vertex, r_bottom_vertex)

	 Properties
	 ----------
	 `l_bottom_vertex`, `l_top_vertex`, `r_top_vertex` and `r_bottom_vertex` are the vertices of the trapezoid, in order

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength

"""
mutable struct TrapezoidalMF<:MF

	l_bottom_vertex::Real
	l_top_vertex::Real
	r_top_vertex::Real
	r_bottom_vertex::Real

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function TrapezoidalMF(l_bottom_vertex::Real, l_top_vertex::Real, r_top_vertex::Real, r_bottom_vertex::Real)

		if l_bottom_vertex <= l_top_vertex <= r_top_vertex <= r_bottom_vertex

			this = new()

			this.l_bottom_vertex = l_bottom_vertex
			this.l_top_vertex = l_top_vertex
			this.r_top_vertex = r_top_vertex
			this.r_bottom_vertex = r_bottom_vertex

			function inv(α)
				x1 = (this.l_top_vertex - this.l_bottom_vertex) * α + this.l_bottom_vertex
				x2 = (this.r_top_vertex - this.r_bottom_vertex) * α + this.r_bottom_vertex
				return x1, x2
			end

			this.eval = function eval(x)
				maximum([minimum([((x - this.l_bottom_vertex) / (this.l_top_vertex - this.l_bottom_vertex)), 1, ((this.r_bottom_vertex - x) / (this.r_bottom_vertex - this.r_top_vertex))]), 0])
			end

			this.mean_at = function mean_at(firing_strength)
				p1 = (this.l_top_vertex - this.l_bottom_vertex) * firing_strength + this.l_bottom_vertex
				p2 = (this.r_top_vertex - this.r_bottom_vertex) * firing_strength + this.r_bottom_vertex
				(p1 + p2) / 2
			end

			this.get_n = function get_n(tol)
				return 0
			end

			this.cut = function cut(α)
				x1, x2 = inv(α)
				return CutMF(x1, x2, α, this)
			end

			this

		else

			error("invalid vertices")

		end

	end

end

"""
	 Sigmoid membership function type

	SigmoidMF(a, c, limit)

	 Properties
	 ----------
	 `a` controls slope
	 `c` is the crossover point
	 `limit` sets the extreme limit

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength

"""
mutable struct SigmoidMF<:MF

	a::Real
	c::Real
	limit::Real

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function SigmoidMF(a::Real, c::Real, limit::Real)

		if (a > 0 && limit > c) || (a < 0 && limit < c)

			this = new()

			this.a = a
			this.c = c
			this.limit = limit

			function inv(α)
				return this.c - (1/this.a)*log((1-α)/α)
			end

			this.eval = function eval(x)
				1 / (1 + exp(-this.a * (x - this.c)))
			end

			this.mean_at = function mean_at(firing_strength)

				if firing_strength == 1
					p_firing_strength = 0.999
				elseif firing_strength == 0
					p_firing_strength = 0.001
				else
					p_firing_strength = firing_strength
				end

				p1 = -log((1 / p_firing_strength) - 1) / this.a + this.c
				p2 = this.limit
				(p1 + p2) / 2

			end

			this.get_n = function get_n(tol)
				a = this.a
				c = this.c
				if a > 0
					u = this.limit
					l = inv(0.01)
				else
					l = this.limit
					u = inv(0.01)
				end
				aux1(x) = exp(-a*(x-c))
				aux2(x) = 2*a^2*(aux1(x))^2/(aux1(x) + 1)^3
				sdiff(x) = aux2(x) - a^2*aux1(x)/(aux1(x) + 1)^2
				xs = collect(l:(u-l)/100:u)
				ys = sdiff.(xs)
				ym = maximum(abs.(ys))
				sqrt((u - l)^3*ym/(12*tol))
			end

			this.cut = function cut(α)
				x = inv(α)
				if this.a > 0
					x1 = x
					x2 = this.limit
				else
					x1 = this.limit
					x2 = x
				end
				return CutMF(x1, x2, α, this)
			end

			this

		else

			error("invalid parameters")

		end

	end
end

"""
	Cut MF

	CutMF(a, b, c)

	 Properties
	 ----------
	 `a`, `b` and `c` the usual bell parameters with `c` being the center

	 `eval` function returns membership value at a point
	 `mean_at` function returns mean value at line clipped by given firing strength

"""
mutable struct CutMF<:MF

	x1::Real
	x2::Real
	α::Real
	toCutMF::MF

	eval::Function
	mean_at::Function
	get_n::Function
	cut::Function

	function CutMF(x1::Real, x2::Real, α::Real, toCutMF::MF)

		this = new()

		this.x1 = x1
		this.x2 = x2
		this.α = α
		this.toCutMF = toCutMF

		this.eval = function eval(x)
			if x1 <= x <= x2
				return this.α
			else
				return this.toCutMF.eval(x)
			end
		end

		this.mean_at = function mean_at(firing_strength)
			if firing_strength >= α
				return this.toCutMF.mean_at(α)
			else
				return this.toCutMF.mean_at(firing_strength)
			end
		end

		this.get_n = function get_n(tol)
			return this.toCutMF.get_n(tol)
		end

		this.cut = function cut(α)
			if α >= this.α
				return this
			else
				return this.toCutMF.cut(α)
			end
		end

		this

	end

end


# Logistic regression

using Regression
import Regression: solve, Options

# Data

d = 3
n = 1000

w = randn(d+1)
X = randn(d, n)
y = sign(X'w[1:d] + w[d+1] + 0.2 * randn(n))

# Solve

ret = solve(logisticreg(X, y; bias=1.0),
            reg=SqrL2Reg(1.0e-2),
            options=Options(verbosity=:iter, grtol=1.0e-6 * n))

println()

# Print results

w_e = ret.sol

@printf("corr(truth, estimated) = %.6f\n", dot(w, w_e) / (vecnorm(w) * vecnorm(w_e)))


using LinearAlgebra
using JuMP, Clp
include("ontoSimplex.jl")

function distFunction(states1::Vector{Float64}, states2::Vector{Float64})::Array{Float64,2}
    n1= length(states1); n2= length(states2)
    dMatrix= Array{Float64}(undef, (n1,n2))
    for i= 1:n1, j= 1:n2
		dMatrix[i,j]= abs(states2[j]- states1[i])
    end
    return dMatrix
end

#	computes the Wasserstein distance
function Wasserstein(p1::Vector{Float64}, p2::Vector{Float64}, distMatrix::Array{Float64,2}, rWasserstein::Float64=1.)
	ontoSimplex!(p1); ontoSimplex!(p2)
    n1= length(p1);   n2= length(p2)

    A= kron(ones(n2)', Matrix{Float64}(I, n1, n1))
    B= kron(Matrix{Float64}(I, n2, n2), ones(n1)')

	model = Model(Clp.Optimizer)
	@variable(model, x[i=1:n1*n2] >= 0)
	@objective(model, Min, vec(distMatrix.^rWasserstein)' * x)
	@constraint(model, [A;B] * x .== [p1;p2])
	optimize!(model)
    return (distance= (objective_value(model))^(1/ rWasserstein), π= reshape(value.(x), (n1, n2)))
end

#	Sinkhorn-Knopp iteration algorithm
function Sinkhorn(p1::Vector{Float64}, p2::Vector{Float64}, distMatrix::Array{Float64,2}, rWasserstein::Float64= 1., λ::Float64= 1.)
	ontoSimplex!(p1); ontoSimplex!(p2); count= 0
	βr= Array{Float64}(undef, length(p1));
	γc= ones(size(p2))		# guess a starting value
	distMatrix.^= rWasserstein; K= exp.(-λ * distMatrix)
	while count < 1000		# Sinkhorn iteration
		γc= γc./ (p2'*γc)		# rescale
		βr= p1./ (K* γc)		# vector operation
		γc= p2./ (K'* βr)		# vector operation
		count += 1				# iteration count
	end
#	println("r=", βr, p1'*βr); println("c=", γc, p2'*γc)
	π= Diagonal(βr)*K*Diagonal(γc)
	return (distance= (sum(π.* distMatrix)) ^(1/rWasserstein), π= π)
end

# #	Sinkhorn-Knopp algorithm
# function Sinkhorn(p1::Vector{Float64}, p2::Vector{Float64}, distMatrix::Array{Float64,2}, rWasserstein::Float64= 1., λ::Float64= 1.)
# 	ontoSimplex!(p1); ontoSimplex!(p2)
# 	π= exp.(-λ * (distMatrix.^ rWasserstein))
# 	for i= 1:100
# 		π.*= (p2'./ sum(π, dims=1)) # scale to sum of column= 1
# 		π.*= (p1 ./ sum(π, dims=2)) # scale to sum of lines= 1
# 	end
# 	return (distance= sum(π.* distMatrix)^(1/ rWasserstein), π= π)
# end


using Random, LinearAlgebra, Plots; pyplot()
Random.seed!()

N = 10^5
data     = [[rand(),rand()] for _ in 1:N]
indata   = filter((x)-> (norm(x) <= 1), data)
outdata  = filter((x)-> (norm(x) > 1), data)
piApprox = 4*length(indata)/N
println("Pi Estimate: ", piApprox)

scatter(first.(indata),last.(indata), c=:blue, ms=1, msw=0)
scatter!(first.(outdata),last.(outdata), c=:red, ms=1, msw=0,
	xlims=(0,1), ylims=(0,1), legend=:none, ratio=:equal)


## Definition of Link types and methods for link, invlink and μη, the derivative of μ w.r.t. η

abstract Link

immutable IdentityLink <: Link end
immutable InverseLink <: Link end
immutable LogLink <: Link end
immutable LogitLink <: Link end

link(::IdentityLink,μ) = μ
invlink(::IdentityLink,η) = η
μη(::IdentityLink,η) = one(η)

link(::LogitLink,μ) = log(μ/(one(μ)-μ))
invlink(::LogitLink,η) = inv(one(η) + exp(-η))
μη(::LogitLink,η) = (ee = exp(-η); ee/abs2(one(η)+ee))

link(::LogLink,μ) = log(μ)
invlink(::LogLink,η) = exp(η)
μη(::LogLink,η) = exp(η)

link(::InverseLink,μ) = inv(μ)
invlink(::InverseLink,η) = inv(η)
μη(::InverseLink,η) = -inv(abs2(η))


using ApproxFun, Test
    import ApproxFun: Block, BlockBandedMatrix

## PDEs

S = JacobiWeight(1.,1.,Jacobi(1.,1.))^2
Δ = Laplacian(S)

f = Fun((x,y)->sin(π*x)*sin(π*y),S)

QR1=qr(Δ)
    ApproxFun.resizedata!(QR1,:,400)
    \(QR1,f; tolerance=1E-10)
QR1=qr(Δ)
    @time Δ[Block.(1:40), Block.(1:40)]
    @time ApproxFun.resizedata!(QR1,:,400)
    @time \(QR1,f; tolerance=1E-10)
println("Laplace Dirichlet: should be ~0.015, 0.015, 0.001")

d=ChebyshevInterval()^2
A=[Dirichlet(d); Laplacian(d)]
f=Fun((x,y)->real(exp(x+im*y)),∂(d))

QR=qr(A)
    ApproxFun.resizedata!(QR,:,150)
    \(QR,[f; 0.];tolerance=1E-10)

QR=qr(A)
    @time ApproxFun.resizedata!(QR,:,150)
    @time u=\(QR,[f; 0.];tolerance=1E-10)

println("Laplace: should be ~0.06, 0.001")

d=ChebyshevInterval()^2
u = [Neumann(d); Laplacian(d)+100I] \ [[[1,1],[1,1]],0]
@time u = [Neumann(d); Laplacian(d)+100I] \ [[[1,1],[1,1]],0]
println("Neumann Helmholtz: should be ~0.032")

# dx=Interval(0.,1.);dt=Interval(0.0,0.54)
# d=dx*dt
#
# x,y=Fun(d)
# V=x^2
#
# Dt=Derivative(d,[0,1]);Dx=Derivative(d,[1,0])
#
# ϵ=0.1
#
# u0=Fun(x->exp(-25*(x-.5)^2)*exp(-1.0im/(5*ϵ)*log(2cosh(5*(x-.5)))),dx)
# L=1im*ϵ*Dt+.5*ϵ^2*Dx^2-V⊗1
#
# PO=discretize([timedirichlet(d);L],50)
# @time PO=discretize([timedirichlet(d);L],50)
# u=PO\u0
# @time    u=PO\u0
#
# println("Schrodinger: should be ~0.013,0.015")


module SolverTests
using Base.Test;
using FactCheck;
using ValidatedNumerics;
using IntervalLinearEquations;

function interval_approx_eq(a_interval :: Interval{Float64}, b_interval :: Interval{Float64})
    isapprox(a_interval.lo, b_interval.lo) &&
    isapprox(a_interval.hi, b_interval.hi)
end

facts("Solve 2x2 system with number matrix") do
    bVector = [
        @interval(-2.0, 2.0); @interval(-2.0, 2.0);
    ] :: Array{Interval{Float64}};
    aMatrix = [
        @interval(2, 4) @interval(-2, 1);
        @interval(-1, 2) @interval(2, 4);
    ] :: Array{Interval{Float64}};

    result = IntervalLinearEquations.solve("G", eye(aMatrix), bVector, 0.1, 10, 1.0)
    @fact result --> bVector
end

facts("Solve 2x2 system with random matrix") do
    xVector = [
        @interval(-1.0, 2.0); @interval(-1.0, 2.0);
    ] :: Array{Interval{Float64}};
    aMatrix = [
        @interval(-2, 4) @interval(-3, 0);
        @interval(-1, 0) @interval(2, 5);
    ] :: Array{Interval{Float64}};
    bVector = aMatrix * xVector

    solution = IntervalLinearEquations.solve("G", aMatrix, bVector, 0.1, 10, 1.0)
    bFromSolution = aMatrix * solution
    @fact all(map(interval_approx_eq, bVector, bFromSolution)) --> true
    @fact all(map(interval_approx_eq, solution, xVector)) --> true
end

facts("Solve 2x2 system with interval matrix") do
    bVector = [
        @interval(-2.0, 2.0); @interval(-2.0, 2.0);
    ] :: Array{Interval{Float64}};
    aMatrix = [
        @interval(2, 4) @interval(-2, 1);
        @interval(-1, 2) @interval(2, 4);
    ] :: Array{Interval{Float64}};
    preciseX = [
        @interval(-0.33333333333333337, 0.33333333333333337);
        @interval(-0.33333333333333337, 0.33333333333333337);
    ] :: Array{Interval{Float64}};

    solution = IntervalLinearEquations.solve("G", aMatrix, bVector, 0.1, 10, 1.0)
    bFromSolution = aMatrix * solution
    @fact all(map(interval_approx_eq, bVector, bFromSolution)) --> true
    @fact all(map(interval_approx_eq, solution, preciseX)) --> true
end

end


using Test, SpecialFunctions
import TruncatedNormal as TruncNorm

@test TruncNorm.tnmom2(-Inf, Inf) == 1
@test TruncNorm.tnmom2(+Inf, +Inf) == Inf
@test TruncNorm.tnmom2(-Inf, -Inf) == Inf
@test TruncNorm.tnmom2(-Inf, 0) == 1
@test TruncNorm.tnmom2(0, +Inf) == 1

@test isnan(TruncNorm.tnmom2(1, 0))
@test isnan(TruncNorm.tnmom2(+Inf, 0))
@test isnan(TruncNorm.tnmom2(0, -Inf))
@test isnan(TruncNorm.tnmom2(+Inf, -Inf))
@test isnan(TruncNorm.tnmom2(NaN, NaN))
@test isnan(TruncNorm.tnmom2(0, NaN))
@test isnan(TruncNorm.tnmom2(NaN, 0))

for x = -10:10
    @test TruncNorm.tnmom2(x, +Inf) ≈ 1 + x * √(2/π) / erfcx(+x/√2)
    @test TruncNorm.tnmom2(-Inf, x) ≈ 1 - x * √(2/π) / erfcx(-x/√2)
end

@test TruncNorm.tnmom2(-Inf, Inf, 0, 1) == 1

for x = -10:10
    @test TruncNorm.tnmom2(x, +Inf, 0, 1) ≈ TruncNorm.tnmom2(x, +Inf)
    @test TruncNorm.tnmom2(-Inf, x, 0, 1) ≈ TruncNorm.tnmom2(-Inf, x)
end

@test TruncNorm.tnmom2(-1e200, 1e200) ≈ 1
@test TruncNorm.tnmom2(-1e6, -999000) ≈ 9.9800100000199999999999799599399200002e11
@test TruncNorm.tnmom2(+1e6, +Inf) ≈ 1.00000000000199999999999800000e12
@test TruncNorm.tnmom2(-Inf, -1e6) ≈ 1.00000000000199999999999800000e12

@test TruncNorm.tnmom2(-Inf, Inf, 2, 0) ≈ 4
@test TruncNorm.tnmom2(0, 20, 3, 3//2) ≈ 11.4986153820554548159603543747
@test TruncNorm.tnmom2(10, 20, 3, 3//2) ≈ 106.111726253622178129044932133
@test TruncNorm.tnmom2(-10, 20, 3, 3//2) ≈ 11.2499999999999997949173964131


using NBInclude

@nbinclude("Time Evolution(QOptics).ipynb")

using ITensors
using PyPlot

  N = 10
  cutoff = 1E-8
  tau = 0.05
  ttotal = 1.5*4

function TEvolHeis(N, cutoff,tau, ttotal)

  # Compute the number of steps to do
  Nsteps = Int(ttotal/tau)

  # Make an array of 'site' indices
  s = siteinds("S=1/2",N)

  # Make gates (1,2),(2,3),(3,4),...
  gates = ITensor[]
  for j=1:N-1
    s1 = s[j]
    s2 = s[j+1]
    hj =       op("Sz",s1) * op("Sz",s2) +
               op("Sx",s1) * op("Sx",s2) +
               op("Sy",s1) * op("Sy",s2)
    Gj = exp(-1.0im * tau/2 * hj)
    push!(gates,Gj)
  end
  # Include gates in reverse order too
  # (N,N-1),(N-1,N-2),...
  append!(gates,reverse(gates))

  c = div(N,2) # center site

  # Initialize psi to be a product state (alternating up and down)
  psi = productMPS(s, n -> n!=c ? "Up" : "Dn")

  Szc=[]
  Szc2=[]
  # Compute and print initial <Sz> value on site c
  t = 0.0
  Sz  = ITensors.expect(psi,"Sz";site_range=c:c)
  Sz2 = ITensors.expect(psi,"Sz";site_range=c+1:c+1)
  println("$t $Sz $Sz2")
  append!(Szc,Sz)
  append!(Szc2,Sz2)

  # Do the time evolution by applying the gates
  # for Nsteps steps and printing <Sz> on site c
  for step=1:Nsteps
    psi = apply(gates, psi; cutoff=cutoff)
    t += tau
    Sz  = ITensors.expect(psi,"Sz";site_range=c:c)
    Sz2 = ITensors.expect(psi,"Sz";site_range=c+1:c+1)
    println("$t $Sz $Sz2")
    append!(Szc,Sz)
    append!(Szc2,Sz2)
  end
  times = 0:tau:ttotal
  return Szc, Szc2, times
end

#Clear demostration of time-evolution
Szc1=TEvolHeis(10, 1E-8, 0.05, 1.5*4)[1];
Szc2=TEvolHeis(10, 1E-8, 0.05, 1.5*4)[2];
timess=TEvolHeis(10, 1E-8, 0.05, 1.5*4)[3];
#Data of longer time-step
Szc1t=TEvolHeis(10, 1E-8, 0.05, 1.5*8)[1];
Szc2t=TEvolHeis(10, 1E-8, 0.05, 1.5*8)[2];
timesst=TEvolHeis(10, 1E-8, 0.05, 1.5*8)[3];
#Data of bigger cut-off
Szc1f=TEvolHeis(10, 1E-4, 0.05, 1.5*4)[1];
Szc2f=TEvolHeis(10, 1E-4, 0.05, 1.5*4)[2];
timessf=TEvolHeis(10, 1E-4, 0.05, 1.5*4)[3];

#Plotting both QOptics and ITensors data of time-evolution

PyPlot.plot(timess, Szc1)
PyPlot.plot(timess, Szc2)
PyPlot.plot(tout, m5z,linestyle = ":",linewidth = 5)
PyPlot.plot(tout, m6z,linestyle = ":",linewidth = 5)
legend(["m5z(ITensors)", "m6z(ITensors)","m5z(QOptics)","m6z(QOptics)"])
xlabel("Time")
ylabel(L"$\langle S^{Z} \rangle$")

savefig( "time-evolution.pdf", bbox_inches = "tight", pad_inches = 0.1 )

#Plot of longer Time-step
PyPlot.plot(timesst, Szc1t)
PyPlot.plot(timesst, Szc2t)
title("Longer Time Scale")
xlabel("Time")
ylabel(L"$\langle  S^{Z}  \rangle$")
legend(["m5z(ITensors)","m6z(ITensors)"])
savefig( "Longer Time Scale.pdf", bbox_inches = "tight", pad_inches = 0.1 )

#Plot of bigger cut-off
PyPlot.plot(timessf, Szc1f)
PyPlot.plot(timessf, Szc2f)
title(L"Cut-Off($10^{-4}$)")
xlabel("Time")
ylabel(L"$\langle S^{Z} \rangle$")
legend(["m5z(ITensors)","m6z(ITensors)"])
savefig( "smallercut-off.pdf", bbox_inches = "tight", pad_inches = 0.1 )

TensorNetworks/Time Evolution(QO and ITensors).jl


series = "
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
"

series = filter(x -> !isspace(x), series)
max = 0
biglist=[]
for i = 1:length(series)-13
    rolling = series[i:i+13]
    products = 1
    for i in rolling
        products *= parse(Int,i)
    end
    if products > max
        global max = products
        global biglist = rolling
    end
end
println(max)
println("List = $biglist")


export SpHTypes
"""
    const SpHTypes = Union{Float64,Complex{Float64}}

The types supported by FastSphericalHarmonics (the same types as for
FastTransforms).
"""
const SpHTypes = Union{Float64,Complex{Float64}}

export sph_points
"""
    Θ, Φ = sph_points(N::Integer)
    Θ::Vector{Float64}
    Φ::Vector{Float64}

Calculate the locations of points on the sphere when using `N` points
in the θ (latitudinal) direction.

It is `length(Θ) = N` and `length(Φ) = M` where `M = 2N-1`.
"""
function sph_points(N::Int)
    @assert N > 0
    M = 2 * N - 1
    return π / N * (0.5:(N - 0.5)), 2π / M * (0:(M - 1))
end
sph_points(N::Integer) = sph_points(Int(N))

export sph_lmax
"""
    lmax = sph_lmax(N::Integer)
    lmax::Int

Calculate the maximum `l` mode that can be represented with `N`
points. It is `lmax = N - 1`.
"""
function sph_lmax(N::Int)
    @assert N > 0
    return N - 1
end
sph_lmax(N::Integer) = sph_lmax(Int(N))


## Exercise 5-4
## If you are given three sticks, you may or may not be able to arrange them in a triangle. For example, if one of the sticks is 12 inches long and the other two are one inch long, you will not be able to get the short sticks to meet in the middle. For any three lengths, there is a simple test to see if it is possible to form a triangle:

## TIP
## If any of the three lengths is greater than the sum of the other two, then you cannot form a triangle. Otherwise, you can. (If the sum of two lengths equals the third, they form what is called a “degenerate” triangle.)

## 1. Write a function named istriangle that takes three integers as arguments, and that prints either “Yes” or “No”, depending on whether you can or cannot form a triangle from sticks with the given lengths.
println("Ans 1: ")

function istriangle(a, b, c)
   return (((a + b) > c) && ((b + c) > a) && ((c + a) > b)) ? "Yes" : "No"
end

println(istriangle(13, 1, 1))
println(istriangle(4, 3, 5))

## 2. Write a function that prompts the user to input three stick lengths, converts them to integers, and uses istriangle to check whether sticks with the given lengths can form a triangle.
println("Ans 2: ")

function check_is_triangle()
    print("Enter the first side: ")
    a = parse(Int64, readline())
    print("Enter the second side: ")
    b = parse(Int64, readline())
    print("Enter the third side: ")
    c = parse(Int64, readline())

    println(istriangle(a, b, c))
end

check_is_triangle()

println("End.")


using COSMO, Test, LinearAlgebra, SparseArrays
# this is the problem
# "sdp_sigma_max_atom"
# from the Convex.jl problem set that caused errors in the
# compact_transformation

# max   σ_max(Y)
# s.t.  Y[2, 1] <= 4, Y[2, 2] >= 3, sum(Y) >= 12
# where Y in R_{3x3}
#
# which is equivalent to
#   min t
#   s.t. [t * I Y; Y' t * I]  ⪰ 0

# define x := [t; vec(Y)]
@testset "Chordal LMI-SDP - Maximum singular value problem" begin
    q = [1.0; zeros(9)]
    con1 = COSMO.Constraint([0 0 -1.0 zeros(1, 7)], 4.0, COSMO.Nonnegatives)
    con2 = COSMO.Constraint([0 0 0 0 0 1.0 zeros(1, 4)], -3.0, COSMO.Nonnegatives)
    con3 = COSMO.Constraint([0.0 ones(1, 9)], -12.0, COSMO.Nonnegatives)

    A_lmi1 = [-1.;0; -1; 0; 0; -1; 0; 0; 0; -1; 0; 0; 0; 0; -1; 0; 0; 0; 0; 0; -1]
    A_lmi2 = spzeros(21, 9)
    A_lmi2[7, 1] = A_lmi2[8, 2] = A_lmi2[9, 3] = A_lmi2[11, 4] = A_lmi2[12, 5] = A_lmi2[13, 6] = A_lmi2[16, 7] = A_lmi2[17, 8] = A_lmi2[18, 9] = -sqrt(2)
    A_lmi = [A_lmi1 A_lmi2]
    con4 = COSMO.Constraint(-A_lmi, zeros(21), COSMO.PsdConeTriangle(21))

    model = COSMO.Model()
    assemble!(model, spzeros(10, 10), q, [con1; con2; con3; con4], settings = COSMO.Settings(compact_transformation = true, decompose = true))
    res = COSMO.optimize!(model);
    Y = reshape(res.x[2:end], 3, 3)
    # check inequality constraints
    @test Y[2, 1] <= 4
    @test Y[2, 2] >= 3
    @test sum(Y) - 12.0 >= -1e-3
    # check maximum singular value
    σ_max = maximum(svd(Y).S)
    t = res.x[1]
    @test abs(σ_max - t ) <= 1e-3

end


export simple_psf
export jinc_psf

 # shift frequency to center if `shift=true`
 # and `norm` the sum to 1
function shift_and_norm(psf, shift, norm)
    if norm
        psf ./= sum(psf)
    end

    if shift
        psf = fftshift(psf)
    end
    return psf
end

"""
    simple_psf(psf_size, radius; shift=false)

Generation of an approximate 2D PSF.
`psf_size` is the output size of the PSF. The PSF will be centered
around the point [1, 1],
`radius` indicates the pupil diameter in pixel from which the PSF is generated.

# Examples
```julia-repl
julia> simple_psf([5, 5], 2)
5×5 Array{Float64,2}:
 0.36       0.104721    0.0152786    0.0152786    0.104721
 0.104721   0.0304627   0.00444444   0.00444444   0.0304627
 0.0152786  0.00444444  0.000648436  0.000648436  0.00444444
 0.0152786  0.00444444  0.000648436  0.000648436  0.00444444
 0.104721   0.0304627   0.00444444   0.00444444   0.0304627
```
"""
function simple_psf(psf_size, radius; shift=false)
    mask = rr(psf_size) .<= radius
    mask_ft = fft(mask)
    psf = abs2.(mask_ft)
    return shift_and_norm(psf, shift, true)
end

"""
    jinc_psf(psf_size, L, radius[, f]; λ=550e-9, shift=false)

Generate the normalized, incoherent 2D jinc PSF of a circular aperture.
`psf_size` is output array shape. `L` is the width of the array
expressed in meter. `radius` is the aperture radius in meter.
Keyword arguments `λ` and `f=100e-3` represent wavelength and focal
length of the lens respectively.

Reference: Mertz, J. (2019). Introduction to Optical Microscopy (2nd ed.).
"""
function jinc_psf(psf_size, L, radius, f=100e-3; λ=550e-9, shift=false)
    κ = calc_κ(λ)
    Δk⊥ = 2 * κ * radius / f
    # create real output psf

    psf = zeros(Float64, psf_size)
    # calculate each point
    for (j, x) in enumerate(fftpos(L, psf_size[2]))
        for (i, y) in enumerate(fftpos(L, psf_size[1]))
            r = sqrt(x^2 + y^2)
            psf[i, j] = jinc(π * Δk⊥ * r).^2
        end
    end
    # shift center to top left corner
    psf = ifftshift(psf)
    return shift_and_norm(psf, shift, true)
end


@doc raw"""
# Hermite's Interpolating Polynomials

Implements a two point Hermite inter-/extrapolation function which passes
through the function and its first derivative for the interval ``[0,1]``.
The polynomial is determined by four constraint equations, matching the
function and its derivative at the points ``0`` and ``1``.

Call with one of the following methods
```julia
_hermite_extrapolation!(t₀, t₁, x₀, x₁, ẋ₀, ẋ₁, t, x)
_hermite_extrapolation!(t₀, t₁, x₀, x₁, ẋ₀, ẋ₁, t, x, ẋ)
_hermite_extrapolation!(v, t₀, t₁, x₀, x₁, t, x)
_hermite_extrapolation!(v, t₀, t₁, x₀, x₁, t, x, ẋ)
```

where

* `t₀`: first  sample time $t_0$
* `t₁`: second sample time $t_1$
* `x₀`: first  solution value $x_0 = x(t_0)$
* `x₁`: second solution value $x_1 = x(t_1)$
* `ẋ₀`: first  vector field value $ẋ_0 = v(t_0, x(t_0))$
* `ẋ₁`: second vector field value $ẋ_1 = v(t_1, x(t_1))$
* `v`:  function to compute vector field with signature `v(t,x,ẋ)`
* `t`:  time $t$ to extrapolate
* `x`:  extrapolated solution value $x(t)$
* `v`:  extrapolated vector field value $ẋ(t)$

#### Derivation

The interpolation works as follows:
Start by defining the 3rd degree polynomial and its derivative by
```math
\begin{aligned}
g(x) &= a_0 + a_1 x + a_2 x^2 + a_3 x^3 , \\
g'(x) &= a_1 + 2 a_2 x + 3 a_3 x^2 ,
\end{aligned}
```
and apply the constraints
```math
\begin{aligned}
g(0) &= f_0 & & \Rightarrow & a_0 &= f_0 , \\
g(1) &= f_1 & & \Rightarrow & a_0 + a_1 + a_2 + a_3 &= f_1 , \\
g'(0) &= f'_0 & & \Rightarrow & a_1 &= f'_0 , \\
g'(1) &= f'_1 & & \Rightarrow & a_1 + 2 a_2 + 3 a_3 &= f'_1 . \\
\end{aligned}
```
Solving for ``a_0, a_1, a_2, a_3`` leads to
```math
\begin{aligned}
a_0 &= f_0 , &
a_1 &= f'_0 , &
a_2 &= - 3 f_0 + 3 f_1 - 2 f'_0 - f'_1 , &
a_3 &= 2 f_0 - 2 f_1 + f'_0 + f'_1 ,
\end{aligned}
```
so that the polynomial ``g(x)`` reads
```math
g(x) = f_0 + f'_0 x + (- 3 f_0 + 3 f_1 - 2 f'_0 - f'_1) x^2 + (2 f_0 - 2 f_1 + f'_0 + f'_1) x^3 .
```
The function and derivative values can be factored out, so that ``g(x)`` can be rewritten as
```math
g(x) = f_0 (1 - 3 x^2 + 2 x^3) + f_1 (3 x^2 - 2 x^3) + f'_0 (x - 2 x^2 + x^3) + f'_1 (- x^2 + x^3) ,
```
or in generic form as
```math
g(x) = f_0 a_0(x) + f_1 a_1(x) + f'_0 b_0(x) + f'_1 b_1(x) ,
```
with basis functions
```math
\begin{aligned}
a_0 (x) &= 1 - 3 x^2 + 2 x^3 , &
b_0 (x) &= x - 2 x^2 + x^3 , \\
a_1 (x) &= 3 x^2 - 2 x^3 , &
b_1 (x) &= - x^2 + x^3 .
\end{aligned}
```
The derivative ``g'(x)`` accordingly reads
```math
g'(x) = f_0 a'_0(x) + f_1 a'_1(x) + f'_0 b'_0(x) + f'_1 b'_1(x) ,
```
with
```math
\begin{aligned}
a'_0 (x) &= - 6 x + 6 x^2 , &
b'_0 (x) &= 1 - 4 x + 3 x^2 , \\
a'_1 (x) &= 6 x - 6 x^2 , &
b'_1 (x) &= - 2 x + 3 x^2 .
\end{aligned}
```
The basis functions ``a_0``and ``a_1`` are associated with the function
values at ``x_0`` and ``x_1``, respectively, while the basis functions
``b_0`` and ``b_1`` are associated with the derivative values at
``x_0`` and ``x_1``.
The basis functions satisfy the following relations,
```math
\begin{aligned}
a_i (x_j) &= \delta_{ij} , &
b_i (x_j) &= 0 , &
a'_i (x_j) &= 0 , &
b'_i (x_j) &= \delta_{ij} , &
i,j &= 0, 1 ,
\end{aligned}
```
where ``\delta_{ij}`` denotes the Kronecker-delta, so that
```math
\begin{aligned}
g(0) &= f_0 , &
g(1) &= f_1 , &
g'(0) &= f'_0 , &
g'(1) &= f'_1 .
\end{aligned}
```
"""
function _hermite_extrapolation! end

function _hermite_extrapolation!(t₀::TT, t₁::TT, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, ẋ₀::AbstractArray{DT}, ẋ₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}) where {DT,TT}
    local a₀::TT
    local a₁::TT
    local b₀::TT
    local b₁::TT
    local Δt = t₁ - t₀
    local s = (t - t₀) / Δt

    # Interpolate x at t
    if t == t₀
        x .= x₀
    elseif t == t₁
        x .= x₁
    else
        a₁ = 3s^2 - 2s^3
        a₀ = 1 - a₁
        b₁ = s^2*(s-1)
        b₀ = s*(1-s)+b₁
        x .= a₀ .* x₀ .+ a₁ .* x₁ .+ b₀ .* Δt .* ẋ₀ .+ b₁ .* Δt .* ẋ₁
    end

    return x
end

function _hermite_extrapolation!(t₀::TT, t₁::TT, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, ẋ₀::AbstractArray{DT}, ẋ₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}, ẋ::AbstractArray{DT}) where {DT,TT}
    local a₀::TT
    local a₁::TT
    local b₀::TT
    local b₁::TT
    local Δt = t₁ - t₀
    local s = (t - t₀) / Δt

    _hermite_extrapolation!(t₀, t₁, x₀, x₁, ẋ₀, ẋ₁, t, x)

    # Interpolate ẋ at t
    if t == t₀
        ẋ .= ẋ₀
    elseif t == t₁
        ẋ .= ẋ₁
    else
        a₁ = (6s - 6s^2) / Δt
        a₀ = - a₁
        b₁ = s*(3s-2)
        b₀ = 1-2s+b₁
        ẋ .= a₀ .* x₀ .+ a₁ .* x₁ .+ b₀ .* ẋ₀ .+ b₁ .* ẋ₁
    end

    return (x, ẋ)
end

function _get_velocities(v::Function, t₀::TT, t₁::TT, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}) where {DT,TT}
    ẋ₀ = zero(x₀)
    ẋ₁ = zero(x₁)
    v(t₀, x₀, ẋ₀)
    v(t₁, x₁, ẋ₁)
    return (ẋ₀, ẋ₁)
end

function _hermite_extrapolation!(v::Function, t₀::TT, t₁::TT, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}) where {DT,TT}
    _hermite_extrapolation!(t₀, t₁, x₀, x₁, _get_velocities(v, t₀, t₁, x₀, x₁)..., t, x)
end

function _hermite_extrapolation!(v::Function, t₀::TT, t₁::TT, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}, ẋ::AbstractArray{DT}) where {DT,TT}
    _hermite_extrapolation!(t₀, t₁, x₀, x₁, _get_velocities(v, t₀, t₁, x₀, x₁)..., t, x, ẋ)
end

struct HermiteExtrapolation{T} <: Extrapolation
    t₀::T
    t₁::T
    Δt::T

    function HermiteExtrapolation{T}(t₀, t₁) where {T}
        new(t₀, t₁, t₁-t₀)
    end
end

function HermiteExtrapolation(t₀::T, t₁::T) where {T}
    HermiteExtrapolation{T}(t₀, t₁)
end

function GeometricBase.evaluate!(int::HermiteExtrapolation{TT}, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, ẋ₀::AbstractArray{DT}, ẋ₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}) where {DT,TT}
    _hermite_extrapolation!(int.t₀, int.t₁, x₀, x₁, ẋ₀, ẋ₁, t, x)
end

function GeometricBase.evaluate!(int::HermiteExtrapolation{TT}, x₀::AbstractArray{DT}, x₁::AbstractArray{DT}, ẋ₀::AbstractArray{DT}, ẋ₁::AbstractArray{DT}, t::TT, x::AbstractArray{DT}, ẋ::AbstractArray{DT}) where {DT,TT}
    _hermite_extrapolation!(int.t₀, int.t₁, x₀, x₁, ẋ₀, ẋ₁, t, x, ẋ)
end


### Activation functions.

function activation_function_selector(activation::ASCIIString)
	if activation == "exponential"
		return activation, exponential_activation!, exponential_backward!
	elseif activation == "leaky_rectified_linear"
		return activation, leaky_rectified_linear_activation!, leaky_rectified_linear_backward!
	elseif activation == "rectified_linear"
		return activation, rectified_linear_activation!, rectified_linear_backward!
	elseif activation == "sigmoid"
		return activation, sigmoid_activation!, sigmoid_backward!
	elseif activation == "softmax"
		return activation, softmax_activation!, softmax_backward!
	elseif activation == "softplus"
		return activation, softplus_activation!, softplus_backward!
	elseif activation == "tanh"
		return activation, tanh_activation!, tanh_backward!
	# Else default to linear activations.
	else
		return "linear", linear_activation!, linear_backward!
	end
end

### Exponential activations.

function exponential_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			ACT[i] = exp(NET[i])
		end
	end
end

function exponential_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			DE_DNET[o] = DELTAS_ABOVE[o] * ACT[o]
		end
	end
end

### Linear activations.

function linear_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			ACT[i] = NET[i]
		end
	end
end

function linear_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			DE_DNET[o] = DELTAS_ABOVE[o]
		end
	end
end

### Leaky rectified linear activations.

function leaky_rectified_linear_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			if NET[i] > 0.0
				ACT[i] = NET[i]
			else
				ACT[i] = 0.01 * NET[i]
			end
		end
	end
end

function leaky_rectified_linear_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			if NET[o] > 0.0
				DE_DNET[o] = DELTAS_ABOVE[o]
			else
				DE_DNET[o] = DELTAS_ABOVE[o] * 0.01
			end
		end
	end
end

### Rectified linear activations.

function rectified_linear_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			if NET[i] > 0.0
				ACT[i] = NET[i]
			else
				ACT[i] = 0.0
			end
		end
	end
end

function rectified_linear_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			if NET[o] > 0.0
				DE_DNET[o] = DELTAS_ABOVE[o]
			else
				DE_DNET[o] = 0.0
			end
		end
	end
end

### Sigmoid activations.

function sigmoid_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			ACT[i] = 1.0 / (1.0 + exp(-NET[i]))
		end
	end
end

function sigmoid_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			DE_DNET[o] = DELTAS_ABOVE[o] * ACT[o] * (1.0 - ACT[o])
		end
	end
end

### Softmax activations.

function softmax_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for j = 1:size(NET, 2)
			# Get maximum net value (for numerical stability).
			maxnet::T = -Inf
			for i = 1:length(NET)
				if NET[i] > maxnet
					maxnet = NET[i]
				end
			end
			# Get sum of exponentials.
			expsum::T = 0.0
			for i = 1:length(NET)
				ACT[i] = exp(NET[i] - maxnet)
				expsum += ACT[i]
			end
			# Set activations.
			for i = 1:length(NET)
				ACT[i] ./= expsum
			end
		end
	end
end

function softmax_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		no = length(NET)
		for o = 1:no
			DE_DNET[o] = 0.0
			for oo = 1:no
				if o == oo
					DE_DNET[o] += DELTAS_ABOVE[oo] * ACT[o] * (1.0 - ACT[o])
				else
					DE_DNET[o] -= DELTAS_ABOVE[oo] * ACT[o] * ACT[oo]
				end
			end
		end
	end
end

### Softplus activations.

function softplus_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			ACT[i] = log(1.0 + exp(NET[i]))
		end
	end
end

function softplus_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			expx = exp(NET[o])
			DE_DNET[o] = DELTAS_ABOVE[o] * expx / (1.0 + expx)
		end
	end
end

### Tanh activations.

function tanh_activation!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T})
	@inbounds begin
		for i = 1:length(NET)
			ACT[i] = tanh(NET[i])
		end
	end
end

function tanh_backward!{T<:FloatingPoint}(NET::Vector{T}, ACT::Vector{T}, DELTAS_ABOVE::Vector{T}, DE_DNET::Vector{T})
	@inbounds begin
		for o = 1:length(NET)
			DE_DNET[o] = DELTAS_ABOVE[o] * (1.0 - ACT[o] * ACT[o])
		end
	end
end


# ### m8.1t

using Pkg, DrWatson

@quickactivate "StatisticalRethinkingTuring"
using Turing
using StatisticalRethinking
Turing.setprogress!(false)

@model ppl8_2(y) = begin

    σ ~ FlatPos(0.0) # improper prior with probability one everywhere above 0.0
    α ~ Flat() # improper prior with pobability one everywhere

    y .~ Normal(α, σ)
end

y = [-1,1];

# Sample

m8_2t = ppl8_2(y)
nchains = 4; sampler = NUTS(0.65); nsamples=2000
chns8_2t = mapreduce(c -> sample(m8_2t, sampler, nsamples), chainscat, 1:nchains)

# End of `08/m8.2t.jl`


"""
    LogNumber{S,L}

Represent a real or complex number in terms of its sign and logarithm.

This is a hack around the limitation that standard floating-point
types link the bitcount of the exponent to that of the mantissa, which
makes it impossible to represent very large or small values without
simultaneously increasing the precision of the representation. The
drawback of this hack is that values x close to 1 have a larger relative
accuracy than values far away from 1 since

    1 - exp(l*(1±ε)) / exp(l) == 1 - exp(±l*ε) ≈ ∓l*ε.

It remains to be seen whether this has any practical relevance.

A better workaround would be to create a floating-point type with
arbitrary exponent and mantissa.
"""
struct LogNumber{S,L} <: Number
    sign::S
    logabs::L
end

LogNumber(x::Number) where {S,L} = LogNumber(sign(x),logabs(x))
LogNumber{S,L}(x::Number) where {S,L} = LogNumber{S,L}(sign(x),logabs(x))
Base.AbstractFloat(x::LogNumber) = x.sign*exp(x.logabs)

Base.sign(x::LogNumber) = x.sign

"""
    logabs(x) -> log(abs(x))
"""
logabs(x::Number) = log(abs2(x))/2
logabs(x::LogNumber) = x.logabs

Base.promote_rule(::Type{LogNumber{S,L}}, ::Type{T}) where {S,L,T<:Number} =
    LogNumber{promote_type(S,typeof(sign(one(T)))), promote_type(L,typeof(logabs(one(T))))}

Base.:*(x::LogNumber,y::LogNumber) = LogNumber(x.sign * y.sign, x.logabs + y.logabs)
Base.:/(x::LogNumber,y::LogNumber) = LogNumber(x.sign * conj(y.sign), x.logabs - y.logabs)


module _SlopEn
export SlopEn
using GroupSlices
    """
        Slop = SlopEn(Sig)

    Returns the slope entropy (`Slop`) estimates for embedding dimensions
    [2, ..., m] of the data sequence (`Sig`) using the default parameters:
    embedding dimension = 2, time delay = 1,
    angular thresholds = [5 45],  logarithm = base 2

        Slop = SlopEn(Sig::AbstractArray{T,1} where T<:Real; m::Int=2, tau::Int=1, Lvls::AbstractArray{T,1} where T<:Real=[5, 45], Logx::Real=2, Norm::Bool=true)

    Returns the slope entropy (`Slop`) estimate of the data sequence (`Sig`)
    using the specified 'keyword' arguments:

    # Arguments:
    `m`     - Embedding Dimension, an integer > 1   	\n
              SlopEn returns estimates for each dimension [2,...,m]
    `tau`   - Time Delay, a positive integer    \n
    `Lvls`  - Angular thresolds, a vector of monotonically increasing
              values in the range [0 90] degrees.\n
    `Logx`  - Logarithm base, a positive scalar (enter 0 for natural log)   \n
    `Norm`  - Normalisation of SlopEn value, a boolean operator: \n
              [false]  no normalisation
              [true]   normalises w.r.t. the number of patterns found (default)

    # See also `PhasEn`, `GridEn`, `MSEn`, `CoSiEn`, `SampEn`, `ApEn`

    # References:
        [1] David Cuesta-Frau,
            "Slope Entropy: A New Time Series Complexity Estimator Based on
            Both Symbolic Patterns and Amplitude Information."
            Entropy
            21.12 (2019): 1167.

    """
    function SlopEn(Sig::AbstractArray{T,1} where T<:Real; m::Int=2, tau::Int=1,
         Lvls::AbstractArray{T,1} where T<:Real=[5, 45], Logx::Real=2, Norm::Bool=true)

    Logx == 0 ? Logx = exp(1) : nothing
    (size(Sig,1) >10) ? nothing :  error("Sig:   must be a numeric vector")
    (m > 1) ? nothing :  error("m:     must be an integer > 1")
    (tau>0) ? nothing :  error("tau:   must be an integer > 0")
    (length(Lvls)>1 && all(diff(Lvls).>0) && all(0 .< Lvls .< 90)) ? nothing :
        error("Lvls:    must be a vector of 2 or more monotonically increasing
                        values in the range [0 90] degrees")
    (Logx>0) ? nothing : error("Logx:  must be a positive number > 0")

    m = m-1;
    Tx = atand.(Sig[1+tau:end] .- Sig[1:end-tau])
    N = size(Tx,1)
    Sx = zeros(Int,N,m)
    Symbx = zeros(Int,size(Tx));
    Slop = zeros(m)
    sort!(Lvls)

    for q = 2:length(Lvls)
        Symbx[(Tx.<= Lvls[q]) .& (Tx .> Lvls[q-1])] .= q-1
        Symbx[(Tx.>=-Lvls[q]) .& (Tx .<-Lvls[q-1])] .= -(q-1)

        if q == length(Lvls)
            Symbx[Tx.> Lvls[q]] .= q
            Symbx[Tx.<-Lvls[q]] .= -q
        end
    end

    for k = 1:m
        Sx[1:N-k+1,k] = Symbx[k:N]
        Locs = groupslices(Sx[1:N-k+1,1:k])
        p = []
        [push!(p, sum(Locs.==n)) for n in unique(Locs)]

        Norm ? p ./=(N-k+1) : p./= length(p)

        if Norm && round(sum(p)) != 1
            @warn("Potential Error: Some permutations not accounted for!")
            print(round(sum(p)))
        end

        Slop[k] = -sum(p.*log.(Logx, p))
    end

    return Slop
    end

end
"""
Copyright 2021 Matthew W. Flood, EntropyHub

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

For Terms of Use see https://github.com/MattWillFlood/EntropyHub

"""


# ---
# title: 878. Nth Magical Number
# id: problem878
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Math, Binary Search
# link: <https://leetcode.com/problems/nth-magical-number/description/>
# hidden: true
# ---
#
# A positive integer is _magical_  if it is divisible by either A or B.
#
# Return the N-th magical number.  Since the answer may be very large, **return
# it modulo**`10^9 + 7`.
#
#
#
# **Example 1:**
#
#
#
#     Input: N = 1, A = 2, B = 3
#     Output: 2
#
#
# **Example 2:**
#
#
#
#     Input: N = 4, A = 2, B = 3
#     Output: 6
#
#
# **Example 3:**
#
#
#
#     Input: N = 5, A = 2, B = 4
#     Output: 10
#
#
# **Example 4:**
#
#
#
#     Input: N = 3, A = 6, B = 4
#     Output: 8
#
#
#
#
# **Note:**
#
#   1. `1 <= N <= 10^9`
#   2. `2 <= A <= 40000`
#   3. `2 <= B <= 40000`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# Single pendulums shall be solved numerically.
#
using OrdinaryDiffEq, Elliptic, Printf, DiffEqPhysics, Statistics

sol2q(sol) = [sol.u[i][j] for i in 1:length(sol.u), j in 1:length(sol.u[1])÷2]
sol2p(sol) = [sol.u[i][j] for i in 1:length(sol.u), j in length(sol.u[1])÷2+1:length(sol.u[1])]
sol2tqp(sol) = (sol.t, sol2q(sol), sol2p(sol))

# The exact solutions of single pendulums can be expressed by the Jacobian elliptic functions.
#
sn(u, k) = Jacobi.sn(u, k^2) # the Jacobian sn function

# Use PyPlot.
#
using PyPlot

colorlist = [
    "#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
    "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf",
]
cc(k) = colorlist[mod1(k, length(colorlist))]

# plot the sulution of a Hamiltonian problem
#
function plotsol(sol::ODESolution)
    local t, q, p
    t, q, p = sol2tqp(sol)
    local d = size(q)[2]
    for j in 1:d
        j_str = d > 1 ? "[$j]" : ""
        plot(t, q[:,j], color=cc(2j-1), label="q$(j_str)", lw=1)
        plot(t, p[:,j], color=cc(2j),   label="p$(j_str)", lw=1, ls="--")
    end
    grid(ls=":")
    xlabel("t")
    legend()
end

# plot the solution of a Hamiltonian problem on the 2D phase space
#
function plotsol2(sol::ODESolution)
    local t, q, p
    t, q, p = sol2tqp(sol)
    local d = size(q)[2]
    for j in 1:d
        j_str = d > 1 ? "[$j]" : ""
        plot(q[:,j], p[:,j], color=cc(j), label="(q$(j_str),p$(j_str))", lw=1)
    end
    grid(ls=":")
    xlabel("q")
    ylabel("p")
    legend()
end

# plot the energy of a Hamiltonian problem
#
function plotenergy(H, sol::ODESolution)
    local t, q, p
    t, q, p = sol2tqp(sol)
    local energy = [H(q[i,:], p[i,:], nothing) for i in 1:size(q)[1]]
    plot(t, energy, label="energy", color="red", lw=1)
    grid(ls=":")
    xlabel("t")
    legend()
    local stdenergy_str = @sprintf("%.3e", std(energy))
    title("                    std(energy) = $stdenergy_str", fontsize=10)
end

# plot the numerical and exact solutions of a single pendulum
#
# Warning: Assume q(0) = 0, p(0) = 2k.   (for the sake of laziness)
#
function plotcomparison(k, sol::ODESolution)
    local t, q, p
    t, q, p = sol2tqp(sol)
    local y = sin.(q/2)
    local y_exact = k*sn.(t, k) # the exact solution

    plot(t, y,       label="numerical", lw=1)
    plot(t, y_exact, label="exact",     lw=1, ls="--")
    grid(ls=":")
    xlabel("t")
    ylabel("y = sin(q(t)/2)")
    legend()
    local error_str = @sprintf("%.3e", maximum(abs.(y - y_exact)))
    title("maximum(abs(numerical - exact)) = $error_str", fontsize=10)
end

# plot solution and energy
#
function plotsolenergy(H, integrator, Δt, sol::ODESolution)
    local integrator_str = replace("$integrator", r"^[^.]*\." => "")

    figure(figsize=(10,8))

    subplot2grid((21,20), ( 1, 0), rowspan=10, colspan=10)
    plotsol(sol)

    subplot2grid((21,20), ( 1,10), rowspan=10, colspan=10)
    plotsol2(sol)

    subplot2grid((21,20), (11, 0), rowspan=10, colspan=10)
    plotenergy(H, sol)

    suptitle("=====    $integrator_str,   Δt = $Δt    =====")
end

# Solve a single pendulum
#
function singlependulum(k, integrator, Δt; t0 = 0.0, t1 = 100.0)
    local H(p,q,params) = p[1]^2/2 - cos(q[1]) + 1
    local q0 = [0.0]
    local p0 = [2k]
    local prob = HamiltonianProblem(H, p0, q0, (t0, t1))

    local integrator_str = replace("$integrator", r"^[^.]*\." => "")
    @printf("%-25s", "$integrator_str:")
    sol = solve(prob, integrator, dt=Δt)
    @time local sol = solve(prob, integrator, dt=Δt)

    sleep(0.1)
    figure(figsize=(10,8))

    subplot2grid((21,20), ( 1, 0), rowspan=10, colspan=10)
    plotsol(sol)

    subplot2grid((21,20), ( 1,10), rowspan=10, colspan=10)
    plotsol2(sol)

    subplot2grid((21,20), (11, 0), rowspan=10, colspan=10)
    plotenergy(H, sol)

    subplot2grid((21,20), (11,10), rowspan=10, colspan=10)
    plotcomparison(k, sol)

    suptitle("=====    $integrator_str,   Δt = $Δt    =====")
end

# Single pendulum

k = rand()
integrator = VelocityVerlet()
Δt = 0.1
singlependulum(k, integrator, Δt, t0=-20.0, t1=20.0)

# Two single pendulums

H(q,p,param) = sum(p.^2/2 .- cos.(q) .+ 1)
q0 = pi*rand(2)
p0 = zeros(2)
t0, t1 = -20.0, 20.0
prob = HamiltonianProblem(H, q0, p0, (t0, t1))

integrator = VelocityVerlet()
Δt = 0.1
sol = solve(prob, integrator, dt=Δt)
@time sol = solve(prob, integrator, dt=Δt)

sleep(0.1)
plotsolenergy(H, integrator, Δt, sol)

SymplecticIntegrators = [
    SymplecticEuler(),
    VelocityVerlet(),
    VerletLeapfrog(),
    PseudoVerletLeapfrog(),
    McAte2(),
    Ruth3(),
    McAte3(),
    CandyRoz4(),
    McAte4(),
    CalvoSanz4(),
    McAte42(),
    McAte5(),
    Yoshida6(),
    KahanLi6(),
    McAte8(),
    KahanLi8(),
    SofSpa10(),
]

k = 0.999
Δt = 0.1
for integrator in SymplecticIntegrators
    singlependulum(k, integrator, Δt)
end

k = 0.999
Δt = 0.01
for integrator in SymplecticIntegrators[1:4]
    singlependulum(k, integrator, Δt)
end

k = 0.999
Δt = 0.001
singlependulum(k, SymplecticEuler(), Δt)

k = 0.999
Δt = 0.0001
singlependulum(k, SymplecticEuler(), Δt)

using DiffEqBenchmarks
DiffEqBenchmarks.bench_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


using Distributions

f(x) = x^2 - 5 * x + 6

function get_best_solution(soln)
    best = soln[1]
    for s in soln
        if f(s) < f(best)
            best = s
        end
    end
    return best
end

function flower_pollination_algorithm(f;total_flowers=25,λ = 1.5,γ = 0.1,p = 0.8, MaxGenerations = 10)
    flowers = rand(Float64, total_flowers)
    L = Levy()

    global_best = get_best_solution(flowers)
    print("Start Global best:", global_best)
    println("Out of answers:", flowers)
    println("Best answer so far:", global_best)

    t = 1
    while t < MaxGenerations
        for i = 1:total_flowers
            new_value = nothing
            if rand() < p
                new_value = flowers[i] + γ * rand(L) * (global_best - flowers[i])
            else
                xj = rand(1:total_flowers)
                xk = rand(1:total_flowers)
                new_value = flowers[i] + rand() * (xj - xk)
            end
            if f(new_value) < f(flowers[i])
                flowers[i] = new_value
            end
        end
        global_best = get_best_solution(flowers)
        println("Global Best so far: ", global_best)
        t = t + 1
    end
    return global_best
end

global_minimum=flower_pollination_algorithm(f,total_flowers=25,λ = 1.5,γ = 0.1,p = 0.8)

println("Final Global Minimum: ", global_minimum)


function stable_principal_component_pursuit(M, sigma;
    threshold = 1e-6, solver_output=0)
    """
    This function solves the stable principal component pursuit problem.

    :param M: An arbitrary n-by-n matrix.
    :param sigma: A parameter that controls the tradeoff between nuclear norm
                  term and the L1 norm term in the objective function
    :param threshold: Singular values in the output low rank matrix below
                      threshold are set to 0. Entries in the output sparse
                      matrix below threshold are set to 0 (Float64).
    :param solver_output: Solver_output param to be passed to SCS (Int64).

    :return: This function returns two values. The first value is a tuple of 2
             n-by-n arrays that correspond to the solution of stable PCP (the
             first element in the tuple is the matrix X and the second element
             is the matrix Y). The second value is the optimal objective value
             of the optimization problem (Float64).
    """
    n = size(M)[1]
    mu = (2 * n) ^ 0.5 * sigma

    # Build stable PCP formulation using JuMP
    m = Model(SCS.Optimizer)
    set_optimizer_attribute(m, "verbose", solver_output)

    @variable(m, L[i=1:n, j=1:n])
    @variable(m, W_1[i=1:n, j=1:n])
    @variable(m, W_2[i=1:n, j=1:n])
    @variable(m, S[i=1:n, j=1:n])
    @variable(m, S_abs[i=1:n, j=1:n])
    @variable(m, error[i=1:n, j=1:n])

    @constraint(m, [W_1 L; L' W_2] in PSDCone())
    @constraint(m, [i=1:n, j=1:n], S_abs[i, j] >= S[i, j])
    @constraint(m, [i=1:n, j=1:n], S_abs[i, j] >= -S[i, j])

    @constraint(m, [i=1:n, j=1:n], (M[i, j] - L[i, j] - S[i, j])^2 <= error[i, j])

    @objective(m, Min, 0.5 * sum(W_1[i, i] for i=1:n) +
        0.5 * sum(W_2[i, i] for i=1:n) +
        n^(-0.5) * sum(S_abs[i, j] for i=1:n, j=1:n) +
        sum(error[i, j] for i=1:n, j=1:n)/(2*mu))

    # Solve stable PCP
    optimize!(m)

    L_opt = value.(L)
    S_opt = value.(S)

    L_svd = svd(L_opt)
    L_U = L_svd.U
    L_S = L_svd.S
    L_Vt = L_svd.Vt

    for i = 1:n
        if abs(L_S[i]) < threshold
            L_S[i] = 0
        end
    end

    L_opt = L_U * Diagonal(L_S) * L_Vt

    for i = 1:n
        for j = 1:n
            if abs(S_opt[i, j]) < threshold
                S_opt[i, j] = 0
            end
        end
    end

    return (L_opt, S_opt), objective_value(m)

end;


using Unitful, UnitfulAstro

import PhysicalConstants.CODATA2018: g_n

import ..SfUnits: Angle, to_angle, Acceleration

export kinetic_energy, distance, duration, acceleration, projectile_displacement, projectile_velocity,
	projectile_flight_time, projectile_apex, projectile_range, projectile_angle, projectile_angle_planetary

# TODO: rename these when the underlying compilation issues have been resolved
export projectile_range_planetary
# TODO: better names?
export projectile_peak_displacement_planetary

"""
	kinetic_energy(m::Unitful.Mass, v::Unitful.Velocity)

Compute the kinetic energy of a body with mass `m` travelling at velocity `v`.

No relativistic corrections are applied. Use `relativistic_kinetic_energy` if those are required.
"""
kinetic_energy(m::Unitful.Mass, v::Unitful.Velocity) = 0.5m*v^2 |> u"J"

"""
    distance(a::Unitful.Acceleration, t::Unitful.Time, initial_v::Unitful.Velocity = 0u"m/s")

Compute the distance travelled in time `t` by a body with initial velocity `initial_v` and uniform acceleration `a`.
"""
distance(a::Unitful.Acceleration, t::Unitful.Time, initial_v::Unitful.Velocity = 0u"m/s") = t*initial_v + a*t^2 / 2

"""
    duration(a::Unitful.Acceleration, d::Unitful.Length, initial_v::Unitful.Velocity = 0u"m/s")

Compute the time taken for a body with initial velocity `initial_v` and uniform acceleration `a` to travel a distance of `d`.
"""
function duration(a::Unitful.Acceleration, d::Unitful.Length, initial_v::Unitful.Velocity = 0u"m/s")
    if d < 0u"m"
		throw(DomainError(d, "Negative distances not allowed."))
	end

	if initial_v == 0u"m/s"
		return sqrt(2d / a)
	elseif a == 0u"m/s^2"
		if initial_v < 0u"m/s"
			throw(DomainError(d, "Negative initial velocities not allowed with zero acceleration."))
		end

		return d / initial_v
	end

    # at^2 + 2v_0t - 2d = 0
    # ax^2 + bx + c

    b = 2 * initial_v
    c = -2d

    discr = b^2 - 4*a*c

    t1 = (-b + sqrt(discr))/(2a)
    t2 = (-b - sqrt(discr))/(2a)

    # the negative results aren't wrong, per se, but as the parameters are
    # vector quantities the negative results aren't particularly interesting.

    if t1 < 0u"s"
        return t2
    elseif t2 < 0u"s"
        return t1
    else
        return min(t1, t2)
    end
end

"""
    acceleration(d::Unitful.Length, v_final::Unitful.Velocity, v_initial::Unitful.Velocity = 0u"m/s")

Given a distance `d`, initial velocity `v_initial and desired velocity `v_final` compute the required uniform acceleration
"""
acceleration(d::Unitful.Length, v_final::Unitful.Velocity, v_initial::Unitful.Velocity = 0u"m/s") = (v_final^2 - v_initial^2) / 2d

"""
    projectile_displacement(v_0::Unitful.Velocity, θ::Angle, t::Unitful.Time; g::Acceleration = g_n)

Horizontal and vertical displacement at time `t` of a projectile with initial velocity `v_0` and launch angle `θ` in a unitform gravitational acceleration `g`.
"""
projectile_displacement(v_0::Unitful.Velocity, θ::Angle, t::Unitful.Time; g::Acceleration = g_n) =
    [ v_0 * t * cos(θ), v_0 * t * sin(θ) - 0.5g * t^2 ]

"""
    projectile_displacement(v_0::Unitful.Velocity, t::Unitful.Time; g::Acceleration = g_n)

Vertical displacement at time `t` of a projectile launched vertically with initial velocity `v_0` in a unitform gravitational acceleration `g`.
"""
projectile_displacement(v_0::Unitful.Velocity, t::Unitful.Time; g::Acceleration = g_n) = projectile_displacement(v_0, 90u"°", t, g = g)[2]

"""
    projectile_velocity(v_0::Unitful.Velocity, θ::Angle, t::Unitful.Time; g::Acceleration = g_n)

Horizontal and vertical velocity at time `t` of a projectile with initial velocity `v_0` and launch angle `θ` in a unitform gravitational acceleration `g`.
"""
projectile_velocity(v_0::Unitful.Velocity, θ::Angle, t::Unitful.Time; g::Acceleration = g_n) =
    [ v_0 * cos(θ), v_0 * sin(θ) - g * t ]

"""
    projectile_velocity(v_0::Unitful.Velocity, t::Unitful.Time; g::Acceleration = g_n)

Velocity at time `t` of a projectile launched vertically with initial velocity `v_0` in a unitform gravitational acceleration `g`.
"""
projectile_velocity(v_0::Unitful.Velocity, t::Unitful.Time; g::Acceleration = g_n) = projectile_velocity(v_0, 90u"°", g, t)[2]

"""
    projectile_flight_time(v_0::Unitful.Velocity, θ::Angle = 90u"°"; g::Acceleration = g_n)

Compute total flight time for a projectile launched at angle `θ` with initial velocity `v_0` in a uniform gravitation acceleration `g`.

Flight is considered to be complete when vertical displacement returns to zero.
"""
projectile_flight_time(v_0::Unitful.Velocity, θ::Angle = 90u"°"; g::Acceleration = g_n) = 2v_0 * sin(θ) / g |> u"s"

"""
    projectile_apex(v_0::Unitful.Velocity, θ::Angle = 90u"°"; g::Acceleration = g_n)

Horizontal and vertical displacement of a projectile launched at angle `θ` with initial velocity `v_0` in a uniform gravitation acceleration `g` at the apex of its trajectory.

Default launch angle is 90° for maximum height.
"""
projectile_apex(v_0::Unitful.Velocity, θ::Angle = 90u"°"; g::Acceleration = g_n) = [ v_0^2 * sin(2θ) / 2g,  v_0^2 * sin(θ)^2 / 2g ]

"""
    projectile_range(v_0::Unitful.Velocity, θ::Angle = 45u"°"; g::Acceleration = g_n)

Compute range of a projectile launched at angle `θ` with initial velocity `v_0` in a uniform gravitation acceleration `g`.

Default launch angle is 45° for maximum range.
"""
projectile_range(v_0::Unitful.Velocity, θ::Angle = 45u"°"; g::Acceleration = g_n) = (v_0^2 * sin(2θ)) / g

"""
    projectile_range(v_0::Unitful.Velocity, y_0::Unitful.Length, θ::Angle = 45u"°"; g::Acceleration = g_n)

Compute range of a projectile launched from altitude `y_0` at angle `θ` with initial velocity `v_0` in a uniform gravitation acceleration `g`.

Range is reached when vertical displacement is zero. Default launch angle is 45° for maximum range.
"""
projectile_range(v_0::Unitful.Velocity, y_0::Unitful.Length, θ::Angle = 45u"°"; g::Acceleration = g_n) =
	(v_0 * cos(θ) / g) * (v_0 * sin(θ) + sqrt((v_0 * sin(θ))^2 + 2g * y_0))

"""
    projectile_range_planetary(v_0::Unitful.Velocity, r_planet::Unitful.Length = 1u"Rearth", θ::Angle = 45u"°"; g::Acceleration = g_n)

Compute range of a projectile launched from altitude `y_0` at angle `θ` with initial velocity `v_0` on a spherical planet with radius `r_planet` and surface gravity `g`.

NOTE: this should be called `projectile_range`, but weird internal compiler errors prevent that with this version of Julia (1.5.3).
"""
function projectile_range_planetary(v_0::Unitful.Velocity, θ::Angle = 45u"°", r_planet::Unitful.Length = 1u"Rearth"; g::Acceleration = g_n)
	v_rat2 = (v_0 / sqrt(r_planet * g))^2

	if v_rat2 > 1
		throw(DomainError("Initial velocity exceeds orbital velocity; Range undefined"))
	end

	a = v_0^2 * sin(2θ) / g
	b = sqrt(1 - (2 - v_rat2) * v_rat2 * cos(θ)^2)

	return a / b |> u"m"
end

function projectile_peak_displacement_planetary(v_0::Unitful.Velocity, θ::Angle = 45u"°", r_planet::Unitful.Length = 1u"Rearth"; g::Acceleration = g_n)
	v_rat2 = (v_0 / sqrt(r_planet * g))^2

	if v_rat2 > 1
		throw(DomainError("Initial velocity exceeds orbital velocity; Range undefined"))
	end

	a = v_0^2 * sin(θ) / g
	b = 1 - v_rat2 + sqrt(1 - (2 - v_rat2) * v_rat2 * cos(θ)^2)

	return a / b |> u"m"
end

"""
    projectile_angle(v_0::Unitful.Velocity, d::Unitful.Length; g::Acceleration = g_n)

Compute the possible launch angles for a projectile to reach a horizontal range of `d` given initial velocity `v_0` in a uniform gravitation acceleration `g`.

The shallow angle is the first result, the steep angle the second. A domain error is raised if the target is out of range.
"""
function projectile_angle(v_0::Unitful.Velocity, d::Unitful.Length; g::Acceleration = g_n)
	k = (g * d) / v_0^2

	if (k > 1)
		throw(DomainError("Target out of range; no solutions"))
	end

	return ( 0.5asin(k) * 1u"rad" |> u"°", 45u"°" + (0.5acos(k) * 1u"rad" |> u"°") )
end

"""
    projectile_angle(v_0::Unitful.Velocity, d::Unitful.Length, y::Unitful.Length; g::Acceleration = g_n)

Compute the possible launch angles for a projectile to reach a horizontal range of `d` and altitude of `y` relative to the starting point given initial velocity `v_0` in a uniform gravitation acceleration `g`.

The shallow angle is the first result, the steep angle the second. A domain error is raised if the target is out of range.
"""
function projectile_angle(v_0::Unitful.Velocity, d::Unitful.Length, y::Unitful.Length; g::Acceleration = g_n)
	det = v_0^4 - g * (g * d^2 + 2y * v_0^2)

	if (ustrip(det) < 0)
		throw(DomainError("Target out of range; no solutions"))
	end

	a = atan((v_0^2 + sqrt(det)) / (g * d)) * 1u"rad" |> u"°"
	b = atan((v_0^2 - sqrt(det)) / (g * d)) * 1u"rad" |> u"°"

	if a < b
		return (a, b)
	else
		return (b, a)
	end
end

"""
    projectile_optimum_angle(v_0::Unitful.Velocity, r_planet::Unitful.Length = 1u"Rearth"; g::Acceleration = g_n)
"""
function projectile_optimum_angle(v_0::Unitful.Velocity, r_planet::Unitful.Length = 1u"Rearth"; g::Acceleration = g_n)
	v_rat2 = (v_0 / sqrt(r_planet * g))^2

	if v_rat2 > 0
		throw(DomainError("Initial velocity exceeds orbital velocity; optimum angle undefined"))
	end

	return 0.5acos(v_rat2 / (2 - v_rat2)) * u"rad" |> u"°"
end


function eller(n)
    result = 1
    answer,hw  = factor(n)
    for i in 1:length(answer)
        if hw[i] == 1
            result *= answer[i]-1
        else
            result *= (answer[i]^hw[i] - answer[i]^(hw[i]-1))
        end
    end
    return result
end

function factor(n)
    answer = []
    hw = []
    for i in eratosphen(n)
        if n%i==0
            push!(answer,i)
            push!(hw,1)
            n=n/i
            while n%i==0
                hw[end]+=1
                n=n/i
            end
        end
    end
    return answer,hw
end

function eratosphen(n)
    ser = fill(true,n)
    ser[1] = false
    k = 2
    while k !== nothing && k<n
        ser[k^2:k:end] .= false
        k=findnext(ser, k+1)
    end
    return findall(ser)
end


using Optim, Plots
# method for contour
rosenbrock(x1,x2) = (1.0 - x1)^2 + 100.0 * (x2 - x1^2)^2
# method for optim
rosenbrock(θ) = (1.0 - θ[1])^2 + 100.0 * (θ[2] - θ[1]^2)^2
θ = zeros(2)
b1 = range(0.0,stop=1.5,length=100)
b2 = range(0.0,stop=1.5,length=100)
contour(b1,b2,(b1,b2)->rosenbrock(b1,b2),fill=true, c=:viridis)
θs = zeros(100,2)
for i = 1:100
    #global θ = optimize(rosenbrock, θ, GradientDescent(), Optim.Options(iterations=1)).minimizer
    global θ = optimize(rosenbrock, θ, Newton(), Optim.Options(iterations=1)).minimizer
    θs[i,:] = θ'
    display(scatter!(θs[1:i,1], θs[1:i,2], legend=false))
    sleep(0.1)
end
θs


# Magnetic field topology in fusion research.

using Elliptic, SpecialFunctions

"""
    getB_bottle(x, y, z, distance, a, b, I1, I2) -> Vector{Float}

Get magnetic field from a magnetic bottle.
Reference: https://en.wikipedia.org/wiki/Magnetic_mirror#Magnetic_bottles

# Arguments
- `x,y,z::Float`: location in [m].
- `distance::Float`: distance between solenoids in [m].
- `a::Float`: radius of each side coil in [m].
- `b::Float`: radius of central coil in [m].
- `I1::Float`: current in the solenoid times number of windings in side coils.
- `I2::Float`: current in the central solenoid times number of windings in the
central loop.
"""
function getB_bottle(x, y, z, distance, a, b, I1, I2)

   r = √(x^2 + y^2) # distance from z-axis

   # 1st loop
   z₁ = z + 0.5*distance
   k = √(4*r*a / (z₁^2 + (a + r)^2) )
   K, E = ellipke(k)
   Bz1 = μ₀*I1 / (2π*√(z₁^2+(a+r)^2)) * ((a^2-z₁^2-r^2)/(z₁^2+(r-a)^2)*E + K)
   Br1 = μ₀*z₁*I1/(2π*r*√(z₁^2+(a+r)^2))*((z₁^2+r^2+a^2)/(z₁^2+(r-a)^2)*E - K)
   Bx1 = Br1 * x / r
   By1 = Br1 * y / r

   # 2nd loop
   z₂ = z - 0.5*distance
   k = √(4*r*a / (z₂^2 + (a + r)^2) )
   K, E = ellipke(k)
   Bz2 = μ₀*I1 / (2π*√(z₂^2+(a+r)^2)) * ((a^2-z₂^2-r^2)/(z₂^2+(r-a)^2)*E + K)
   Br2 = μ₀*z₂*I1/(2π*r*√(z₂^2+(a+r)^2))*((z₂^2+r^2+a^2)/(z₂^2+(r-a)^2)*E - K)
   Bx2 = Br2 * x / r
   By2 = Br2 * y / r

   # central loop
   z₃ = z
   k = √(4*r*b / (z₃^2 + (b+r)^2) )
   K, E = ellipke(k)
   Bz3 = μ₀*I2 / (2π*√(z₃^2+(b+r)^2)) * ((b^2-z₃^2-r^2)/(z₃^2+(r-b)^2)*E + K)
   Br3 = μ₀*z₃*I2/(2π*r*√(z₃^2+(b+r)^2))*((z₃^2+r^2+b^2)/(z₃^2+(r-b)^2)*E - K)
   Bx3 = Br3 * x / r
   By3 = Br3 * y / r

   # total magnetic field
   if x == 0.0 && y == 0.0
      Bx = 0.0
      By = 0.0
      Bz = Bz1 + Bz2 + Bz3
   else
      Bx = Bx1 + Bx2 + Bx3
      By = By1 + By2 + By3
      Bz = Bz1 + Bz2 + Bz3
   end
   [Bx, By, Bz]
end

"""
    getB_tokamak(x, y, z, a, b, ICoils, IPlasma)

Get the magnetic field from a Tokamak topology consists of 16 coils.
Original: https://github.com/BoschSamuel/Simulation-of-a-Tokamak-Fusion-Reactor/blob/master/Simulation2.m
# Arguments
- `x,y,z::Float`: location in [m].
- `a::Float`: radius of each coil in [m].
- `b::Float`: radius of central region in [m].
- `ICoil::Float`: current in the coil times number of windings.
- `IPlasma::Float`: current of the plasma?
"""
function getB_tokamak(x, y, z, a, b, ICoils, IPlasma)

   a *= 2

   Bx, By, Bz = 0.0, 0.0, 0.0

   # magnetic field of the coils
   for i = 0:15
      θ = π/16 + i*π/8 # angle between the i-th coil and the x-axis

      if abs(sin(θ)) > 0.01
         r1_ = x/(cos(θ)-sin(θ)*tan(atan(y, x) - θ)) - a - b
         r1 = √(r1_^2 + z^2)
         z1 = ((b + a)*cos(θ) + r1_*cos(θ) - x) / sin(θ)
      else
         r1_ = x - b - a
         r1 = √(r1_^2 + z^2)
         z1 = y
      end

      k = √(4r1*a / (z1^2 + (a + r1)^2))

      K, E = ellipke(k)
      # Bz1_ is the magnetic field in the coil frame
      Bz1_ = μ₀*ICoils/(2π*√(z1^2+(a+r1)^2))*((a^2-z1^2-r1^2)/(z1^2+(r1-a)^2)*E+K)
      # Br1_ is the magnetic field in the coil frame
      Br1_ = μ₀*z1*ICoils/(2π*r1*√(z1^2+(a+r1)^2))*((z1^2+r1^2+a^2)/(z1^2+(r1-a)^2)*E-K)
      # normal coordinates
      Bx1 = -sin(θ)*Bz1_ + Br1_*r1_/r1*cos(θ)
      By1 = cos(θ)*Bz1_ + sin(θ)*Br1_*r1_/r1
      Bz1 = Br1_ * z / r1

      # add the field of a single coil to the total field
      Bx += Bx1
      By += By1
      Bz += Bz1

      if abs(Bx) < 5e-12
         Bx = 0.0
      end
      if abs(By) < 5e-12
         By = 0.0
      end
      if abs(Bz) < 5e-12
         Bz = 0.0
      end
   end

   # magnetic field of the plasma current
   σ = a/3 # parameter of the Gauss curve
   ϕ = atan(y, x)
   # distance to centre of plasma ring
   distance = √( z^2 + (x - (a + b)*cos(ϕ))^2 + (y - (a + b)*sin(ϕ))^2 )
   I2_r_plasma = IPlasma * erf(distance/(σ*√2))

   r = hypot(x, y)
   k = √(4r*(a+b)/(z^2+((a+b)+r)^2))
   K, E = ellipke(k)
   Bz_plasma = μ₀*I2_r_plasma/(2π*√(z^2+((a+b)+r)^2))*(((a+b)^2-z^2-r^2)/(z^2+(r-(a+b))^2)*E+K)
   Br_plasma = μ₀*z*I2_r_plasma/(2π*r*√(z^2+(b+r)^2))*((z^2+r^2+(a+b)^2)/(z^2+(r-(a+b))^2)*E-K)
   Bx_plasma = Br_plasma*x/r
   By_plasma = Br_plasma*y/r

   if distance > 0.0001
      Bx += Bx_plasma
      By += By_plasma
      Bz += Bz_plasma
   end
   [Bx, By, Bz]
end


function rouwenhorst(n, mu, sigma, rho)

    # grid
    width = sqrt((n-1) * sigma^2 / ( 1 - rho^2))
    grid = vcat(range( mu-width, mu + width, length=n))

    # transition matrix
    p0 = (1 + rho) / 2
    trans = [p0 1-p0; 1-p0 p0]

    if n > 2
        for i = 1:n-2
            cstr_temp = zeros(length(trans[:,1]), 1)
            trans = p0 .* [trans cstr_temp; cstr_temp' 0] + (1 - p0 ) .* [cstr_temp trans; cstr_temp' 0]  + (1 - p0 ) .*  [ cstr_temp' 0; trans cstr_temp] + p0 .* [ cstr_temp' 0; cstr_temp trans]
        end
        trans = trans ./ sum(trans,dims=2)
    end

    # ergodic distribution
    dist = ones(1,n)./n
    for i = 1:100
        dist = dist*(trans^i)
    end
    dist = vcat(dist...)

    return grid, trans, dist
end


# Julia program to check if a number is a power of two.

function power_of_two(n)
    # 0 is not a power of 2
    if(n != 0)
        if((n & (n-1)) == false)
            return true
        end
    end
    return false
end

print("Enter a number: ")
n = readline()
n = parse(Int, n)
res = power_of_two(n)
if(res)
    print("The given number is a power of 2.")
else
    print("The given number is not a power of 2.")
end

"""
Time Complexity - O(1)
Space Complexity - O(1)

SAMPLE INPUT AND OUTPUT

SAMPLE I
Enter a number: 4
The given number is a power of 2.

SAMPLE II
Enter a number: 0
The given number is not a power of 2.
"""