#=
  Aircrat position.
  From BLOG example/aircraft-position.blog

  Distributions of variable numAircraft (num:0)
  3.00000 =>    5857  (0.585700)
  4.00000 =>    3178  (0.317800)
  5.00000 =>     833  (0.083300)
  6.00000 =>     114  (0.011400)
  7.00000 =>      18  (0.001800)

  Also see ~/webppl/aircraft_position.wppl
=#

using Turing, StatsPlots, DataFrames
# using ReverseDiff, Zygote, Tracker
# Turing.setadbackend(:reversediff)
# Turing.setadbackend(:zygote)
# Turing.setadbackend(:tracker)

include("jl_utils.jl")

@model function aircraft_position()
    epsilon = 0.05

    # Poisson might get 0 which is not good for this model
    pp ~ Poisson(5)
    numAircraft ~ Dirac(pp == 0 ? 1 : pp)

    blips = tzeros(numAircraft)
    positions = tzeros(numAircraft)
    obsPos = tzeros(numAircraft)
    for b in 1:numAircraft
        blips[b] ~ flip(0.9)
        positions[b] ~ Normal(0,10)
        obsPos[b] ~ Normal(positions[b],1)
    end

    sumBlips ~ Dirac(sum(blips))

    function inRange(x,y,epsilon)
        Dirac(x > y - epsilon && x < y + epsilon)
    end

    true ~ Dirac(sumBlips == 3)

    inRange(obsPos[1],5.0,epsilon)
    numAircraftPositionLt5 ~ Dirac(sum([positions[a] > 5 ? 1 : 0 for a in 1:numAircraft]))
end

model = aircraft_position()

num_chns = 4
# chns = sample(model, Prior(), 1000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(5), 10_000)
# chns = sample(model, IS(), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, SMC(), MCMCThreads(), 10_000, num_chns)

display(chns)
# display(plot(chns))

show_var_dist_pct(chns, :numAircraft)


using SchumakerSpline
from = 0.5
to = 10
x1 = collect(range(from, stop=to, length=40))
y1 = (x1).^2
x2 = [0.5,0.75,0.8,0.93,0.9755,1.0,1.1,1.4,2.0]
y2 = sqrt.(x2)
left_spline = Schumaker(x1,y1)
right_spline = Schumaker(x2,y2)

crossover_point = get_intersection_points(left_spline,right_spline)

splice_point = crossover_point[1]
spliced = splice_splines(left_spline, right_spline, splice_point)
# Testing at splice point
abs(evaluate(spliced, splice_point) - evaluate(right_spline, splice_point))          < 100*eps()
# As this was a continuous split
abs(evaluate(spliced, splice_point- 100*eps()) - evaluate(spliced, splice_point + 100*eps()))          < 10000*eps()
# Testing in left spline territory.
abs(evaluate(spliced, splice_point-0.5) - evaluate(left_spline, splice_point-0.5))   < 100*eps()
abs(evaluate(spliced, splice_point-0.5) - evaluate(right_spline, splice_point-0.5))  > 0.1
# Testing solidly into right spline territory.
abs(evaluate(spliced, splice_point+0.5) - evaluate(left_spline, splice_point+0.5))   > 100*eps()
abs(evaluate(spliced, splice_point+0.5) - evaluate(right_spline, splice_point+0.5))  < 0.1

splice_point = 1.7
spliced = splice_splines(left_spline, right_spline, splice_point)
# Testing at splice point
abs(evaluate(spliced, splice_point) - evaluate(right_spline, splice_point))          < 100*eps()
# As this was NOT a continuous split
abs(evaluate(spliced, splice_point- 100*eps()) - evaluate(spliced, splice_point + 100*eps()))  > 0.1
# Testing in left spline territory.
abs(evaluate(spliced, splice_point-0.5) - evaluate(left_spline, splice_point-0.5))   < 100*eps()
abs(evaluate(spliced, splice_point-0.5) - evaluate(right_spline, splice_point-0.5))  > 0.1
# Testing solidly into right spline territory.
abs(evaluate(spliced, splice_point+0.5) - evaluate(left_spline, splice_point+0.5))   > 100*eps()
abs(evaluate(spliced, splice_point+0.5) - evaluate(right_spline, splice_point+0.5))  < 0.1


export gauss_legendre_weights;   # For debugging only
export distributed_kernel_trapezoidal;    # For debugging only
export distributed_kernel_gauss_legendre; # For debugging only

"
#Creates the NEP associated with example in E. Jarlebring and
#W. Michiels and K. Meerbergen, The infinite  {Arnoldi} method and an application to time-delay systems with distributed delays, Delay Systems - Methods, Applications and New Trends, 2012
"
function gallery_dep_distributed()
    # E. Jarlebring and W. Michiels and K. Meerbergen},
    # The infinite  {Arnoldi} method and an application to time-delay systems with distributed delays,
    # Delay Systems - Methods, Applications and New Trends
    # 2012
    #
    # Some correct eigenvalues:
    # -0.400236388049641 + 0.970633098237807i
    # -0.400236388049641 - 0.970633098237807i
    #  2.726146249832675 + 0.000000000000000i
    # -1.955643591177653 + 3.364550574688863i
    # -1.955643591177653 - 3.364550574688863i
    #  4.493937056300693 + 0.000000000000000i
    # -1.631513006819252 + 4.555484848248613i
    # -1.631513006819252 - 4.555484848248613i
    # -1.677320660400946 + 7.496870451838560i
    # -1.677320660400946 - 7.496870451838560i
    #
    #

    A0 = Matrix(-1.0I, 3, 3)
    A1=[2.5    2.8   -0.5
        1.8    0.3    0.3
        -2.3   -1.4    3.5];
    A2=[1.7    0.7   -0.3
        -2.4   -2.1   -0.2
        2.0    0.7    0.4];
    A3=[1.4   -1.3    0.4
        1.4    0.7    1.0
        0.6    1.6    1.7];
    idop= S -> S
    oneop = S -> one(S)
    f1 = S -> exp(-S)
    N = 10
    f2 = S -> distributed_kernel_gauss_legendre(S, N)
    #N = 1000
    #f2 = S -> distributed_kernel_trapezoidal(Matrix(S), N)
    return SPMF_NEP([A0,A1,A2,A3],[idop,oneop,f1,f2])
end

function distributed_kernel_trapezoidal(S,N0)

    fS = x -> exp(x*S) * (exp((x+0.5)^2) - exp(1/4))

    F=zero(S)

    h=1/N0;
    for i=1:(N0+1) # Trapezoidal rule
        x=float((i-1)*h-1)
        if (i==1 || i==N0+1)
            F += 0.5*fS(x)*h;
        else
            F += fS(x)*h;
        end
    end
    return F
end
"""
Computes distributed kernel matrix function using
Gauss-Legender quadrature.
"""
function distributed_kernel_gauss_legendre(S,N)
    f=x-> (exp((x+0.5)^2)-exp(1/4))
    F=zero(S);
    xv,wv=gauss_legendre_weights(N,-1,0);
    local E = one(S)

    accumulative_expm_comp=true
    for i=1:length(xv)
        if (accumulative_expm_comp)
            # An accumulative way to compute E=exp(xv[i]*S) which is
            # faster due to the fact that scaling and squaring
            # for exp((xv[i]-xv[i-1])*S) is faster than
            # exp(xv[i]*S)
            if (i==1)
                E=exp(xv[1]*S);
            else
                E=E*exp((xv[i]-xv[i-1])*S)
            end
        else
            E=exp(xv[i]*S);

        end

        fSw=E*(f(xv[i])*wv[i]);
        F=F+fSw;
    end
    return F
end

function  gauss_legendre_weights(N,a,b)

    N1=N; N2=N+1;
    xu = range(-1, stop = 1, length = N1)

    # L will be the Legendre-Gauss Vandermonde Matrix
    L=zeros(N1,N2);

    # "Derivative" of L
    Lp=zeros(N1,N2);

    # Starting values of Newton's method
    y = cos.((2 * (0:(N-1)) .+ 1) * pi / (2 * (N-1) + 2)) + (0.27/N1) * sin.(pi * xu * (N-1) / N2)
    y0 = 2

    local Lp0
    # Newton's method to decide points
    # Iterate until new points are uniformly within epsilon of old points

    while maximum(abs.(y .- y0)) > eps()

        L[:,1]=ones(size(L,1));
        Lp[:,1]=zeros(size(Lp,1));

        L[:,2]=y;
        Lp[:,2]=ones(size(Lp,1));

        # Recurrence
        for k=2:N1
            L[:,k+1]=( (2*k-1)*(y).*L[:,k]-(k-1)*L[:,k-1] )/k;
        end

        Lp0 = N2 * (L[:,N1] - y .* L[:,N2]) ./ (1 .- y.^2)

        y0=y;
        y=y0-L[:,N2]./Lp0;
    end

    # Linear map from[-1,1] to [a,b]
    x = (a * (1 .- y) + b * (1 .+ y)) / 2

    # Compute the weights
    w = (b-a) ./ ((1 .- y.^2) .* Lp0.^2) * (N2/N1)^2

    return x,w
end


# # Implementation of the quartic root finding method described in:
# # A Geometric Interpretation of the Solution of the General Quartic Polynomial
# # Author(s): William M. Faucette
# # Source: The American Mathematical Monthly , Jan., 1996, Vol. 103, No. 1 (Jan., 1996), pp. 51-57
# # Published by: The Mathematical Association of America
# # Stable URL: https://www.jstor.org/stable/2975214

const S = [-1 1 1; -1 -1 -1; 1 1 -1; 1 -1 1] # permuting signs on the roots

"""
Defined by the taking the positive square root relationship
    (-x)^(1/2) = xsign(x)*sqrt(abs(x))
"""
function xsign(x::R) where {R<:Real}

    if sign(x) == 1.0
        return im
    elseif sign(x) == -1.0
        return 1.0
    else
        error("Sign of input value not comparable to ±1.0")
        return nothing
    end

end

"""
Return all the real roots of a quartic equation represented as a
vector of real coefficients for terms of increasing degree.
Note that any higher order coefficients (degree >= 5) are ignored.

Theory described in Faucette (1996).
"""
function solve_all_quartic_roots(polycoeff::Vector{T};
    warn_scaling::Bool=true,
    leading_tol::Float64=QUARTIC_ATOL,
    coeff_tol::Float64=1.0/QUARTIC_ATOL) where {T<:Real}

    Rts = Vector{Complex{Float64}}() # object to return with real roots inside.

    if length(polycoeff) < 5
        @error "There are too few coefficients given."
        return Rts
    end

    if warn_scaling && isapprox(polycoeff[5], zero(T), atol=leading_tol)
        @warn "The leading quartic coefficient is approximately zero. Returning empty array."
        ## TODO?: fall back to solving a cubic instead?
        return Rts
    end

    a = polycoeff[1:4]./polycoeff[5]

    if warn_scaling && poor_conditioning(a, coeff_tol)
        @warn "Large coefficients in scaled polynomial. Poor conditioning may occur."
    end

    # We reduce p(z) = a1 + a2*z + a3*z^2 + a4*z^3 + z^4 via z == x - a4/4 to
    # the standard transformed form P(x) = r + q*x + p*x^2 + x^4

    p = -3*(a[4]^2)/8   + a[3]
    q =    (a[4]^3)/8   - a[4]*a[3]/2 + a[2]
    r = -3*(a[4]^4)/256 + (a[4]^2)*a[3]/16 - a[4]*a[2]/4 + a[1]

    # The "resolvent cubic of P(x)" is h(z) = z^3 - 2p*z^2 + (p^2 - 4r)z + q^2.
    # We find it's roots:
    resolvent_coeff = [q^2, p^2 - 4r, -2p, 1]
    resolvent_roots = solve_all_cubic_roots(resolvent_coeff; warn_scaling, leading_tol, coeff_tol)

    alpha = convert(T,resolvent_roots[1])  # ensure first root is Real.
    beta, gamma = resolvent_roots[2:3]

    if !isreal(beta) && !isreal(gamma) && sign(alpha) == -1.0
        A = sqrt(-alpha)
        B = sqrt(-beta)
        C = sign(q)*sqrt(-gamma)
    elseif isreal(beta) && isreal(gamma)
        A = xsign(alpha)*sqrt(abs(alpha))
        B = xsign(real(beta))*sqrt(abs(real(beta)))
        C_sign = sign(q)/(xsign(alpha)*xsign(real(beta)))
        C = C_sign*sqrt(abs(real(gamma)))
    else
        error("Unrecognised case.")
    end

    R_res = S*([A; B; C])/2
    # R_res = S*(sqrt.(-resolvent_roots))/2

    # # A, B, C = sqrt.(-resolvent_roots)
    # # x1 = (-A + B + C)/2
    # # x2 = (-A - B - C)/2
    # # x3 = ( A + B - C)/2
    # # x4 = ( A - B + C)/2

    append!(Rts, R_res .- a[4]/4)

    return Rts
end


#=
Copyright (c) 2015, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice,
  this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
=#

using HPAT
using MPI
using DocOpt

@acc hpat function kmeans(numCenter, iterNum, file_name)
    points = DataSource(Matrix{Float64},HDF5,"/points", file_name)
    D,N = size(points) # number of features, instances
    centroids = rand(D, numCenter)

    for l in 1:iterNum
        dist::Array{Array{Float64,1},1} = [ Float64[sqrt(sum((points[:,i]-centroids[:,j]).^2)) for j in 1:numCenter] for i in 1:N]
        labels::Array{Int,1} = [indmin(dist[i]) for i in 1:N]
        centroids::Array{Float64,2} = [ sum(points[j,labels.==i])/sum(labels.==i) for j in 1:D, i in 1:numCenter]
    end
    return centroids
end

function main()
    doc = """K-means clustering algorithm.

Usage:
  kmeans.jl -h | --help
  kmeans.jl [--iterations=<iterations>] [--file=<file>] [--centers=<centers>]

Options:
  -h --help                  Show this screen.
  --iterations=<iterations>  Specify number of iterations; defaults to 20.
  --file=<file>              Specify input file; defaults to HPAT's default generated data file.
  --centers=<centers>        Specify number of centers; defaults to 5.

"""
    arguments = docopt(doc)

    if (arguments["--iterations"] != nothing)
        iterations = parse(Int, arguments["--iterations"])
    else
        iterations = 20
    end

    if (arguments["--file"] != nothing)
        file_name::ASCIIString = arguments["--file"]
    else
        file_name = HPAT.getDefaultDataPath()*"kmeans_large.hdf5"
    end

    if (arguments["--centers"] != nothing)
        numCenter = parse(Int, arguments["--centers"])
    else
        numCenter = 5
    end

    srand(0)
    rank = MPI.Comm_rank(MPI.COMM_WORLD)
    pes = MPI.Comm_size(MPI.COMM_WORLD)

    if rank==0 println("iterations = ", iterations) end
    if rank==0 println("file= ", file_name) end
    if rank==0 println("centers= ", numCenter) end

    tic()
    kmeans(numCenter, 2, file_name)
    time = toq()
    if rank==0 println("SELFPRIMED ", time) end
    MPI.Barrier(MPI.COMM_WORLD)

    tic()
    centroids_out = kmeans(numCenter, iterations, file_name)
    time = toq()
    if rank==0 println("result = ", centroids_out) end
    if rank==0 println("rate = ", iterations / time, " iterations/sec") end
    if rank==0 println("SELFTIMED ", time) end

end

main()


# estimate the mean by numerical integration over uniform percentiles
estimateMean(d::ContinuousUnivariateDistribution;kwargs...) =
  meanFunOfProb(d;kwargs...,fun=(d,p)->quantile(d,p))

# estimate variance by numerical integration over uniform percentiles
function estimateVariance(d::ContinuousUnivariateDistribution; mean=missing, kwargs...)
    m = ismissing(mean) ? Distributions.mean(d) : mean
    function squaredDiff(d,p)
        t = quantile(d, p) - m
        t*t
    end
    meanFunOfProb(d;kwargs...,fun=squaredDiff)
end

"""
compute mean over a function(uniformly distributed probabilities)

used to estimate moments of logitnorm
"""
function meanFunOfProb(d::ContinuousUnivariateDistribution;relPrec = 1e-4, maxCnt=2^18, fun=(d,p)->quantile.(d,p) )
    δ = 1/32 # start with 31 points (32 intervals between 0 and 1)
    p = δ:δ:(1-δ)
    # for K=1/δ intervals, there are (K-1) points at c_i
    # The first points at δ represents interval (δ/2,3/2δ)
    # The following picture shows points and intervals for K = 4
    #---|---|---|---#
    # |---|---|---| #
    # we need to add points for δ/4 and 1-δ/4 representing the edges
    # but their weight is only half, because they represents half an inverval
    #m = sum(c_i*δ) + el*(δ/2) + er*(δ/2) = (sum(c_i) + er/2 + el/2)*δ
    s = sum(fun.(d,p))   # sum at points c_i
    el = fun(d,δ/4)  #
    er = fun(d,1-δ/4)
    m = (s + el/2 + er/2)*δ
    relErr = 1
    while 1/δ < maxCnt
        mPrev = m
        δ  = δ / 2
        # to double the number of reference points, half the interval
        # for each second point we already computed fun
        # only need to add the new points to the sum of central points
        p = δ:δ*2:(1-δ) # points at the center of current intervals
        s += sum(fun.(d,p))
        el = fun(d,δ/4)
        er = fun(d,1-δ/4)
        m = (s + el/2 + er/2)*δ
        relErr = abs(m - mPrev)/m
        #println("cnt=$(1/δ), m=$m, mPrev=$mPrev, relErr=$relErr")
        #if the estimate did not change much, can return
        relErr <= relPrec && break
    end
    relErr > relPrec && @warn "Returning meanFunOfProb results of low relative precision of $relErr"
    m
end


#=
Test function

Properties
----------

- There are 9 steady states:
    - Stable
        - (-2.39,2.35)    (2.3,-2.34) (-2.18,-2.12)   (2.06,2.12)
    - Unstable
        - (-2.27,-0.23)   (2.17,0.22) (-0.12,-2.23)   (0.33,2.22) (0.1,0.01)

Reference
---------
1. Source code from https://static-content.springer.com/esm/art%3A10.1038%2Fs41598-017-15889-2/MediaObjects/41598_2017_15889_MOESM2_ESM.rar
=#

module SampleDE

"""
Reference
---------
- `ODE.m`
"""
function ODE!(du,u,p,t)
    du[1]= -1 + 9 *u[1] - 2*u[1]^3 + 9 *u[2] - 2*u[2]^3;
    du[2]=  1 - 11*u[1] + 2*u[1]^3 + 11*u[2] - 2*u[2]^3;
end

function ODE(u,p,t)
    du = deepcopy(u)
    return ODE!(du,u,p,t)
end

u0 = [1.0, 2.0]
p = [2.0, 3.0]

"""
Reference
---------
- `exact_potential.m`
"""
function exact_potential(points, RefPointNum)
    pot = zeros(length(points), 1)
    for i in 1:length(points)
        x = points[i][1]; y = points[i][2];
        pot[i] = -5*(x^2+y^2)+(x^4+y^4)/2+x*y+x;
    end
    pot = pot .- pot[RefPointNum]
    return pot
end

"""
Reference
---------
- `Readme.pdf`
"""
ODE_features = Dict(
    "StableStates" => [ (-2.39,2.35),(2.3,-2.34), (-2.18,-2.12),   (2.06,2.12)  ],
    "UnstableStates" => [(-2.27,-0.23) ,  (2.17,0.22), (-0.12,-2.23),   (0.33,2.22), (0.1,0.01)],
    "SteadyStates" => [(-2.39,2.35),(2.3,-2.34), (-2.18,-2.12),   (2.06,2.12), (-2.27,-0.23) ,  (2.17,0.22), (-0.12,-2.23),   (0.33,2.22) ,(0.1,0.01)]
)

function gradient(dt,x)

    x = reshape(x, 2, 3)

    F=[ [(dt*(1/dt - 6*x[1, 1]^2 + 9)*(9*x[1, 1] + 9*x[2, 1] + (x[1, 1] - x[1, 2])/dt - 2*x[1, 1]^3 - 2*x[2, 1]^3 - 1))/2 + (dt*(6*x[1, 1]^2 - 11)*(11*x[2, 1] - 11*x[1, 1] + (x[2, 1] - x[2, 2])/dt + 2*x[1, 1]^3 - 2*x[2, 1]^3 + 1))/2,x[1, 1]^3 - (9*x[2, 1])/2 - (x[1, 1] - x[1, 2])/(2*dt) - (9*x[1, 1])/2 + x[2, 1]^3 + (dt*(1/dt - 6*x[1, 2]^2 + 9)*(9*x[1, 2] + 9*x[2, 2] + (x[1, 2] - x[1, 3])/dt - 2*x[1, 2]^3 - 2*x[2, 2]^3 - 1))/2 + (dt*(6*x[1, 2]^2 - 11)*(11*x[2, 2] - 11*x[1, 2] + (x[2, 2] - x[2, 3])/dt + 2*x[1, 2]^3 - 2*x[2, 2]^3 + 1))/2 + 1/2,x[1, 2]^3 - (9*x[2, 2])/2 - (x[1, 2] - x[1, 3])/(2*dt) - (9*x[1, 2])/2 + x[2, 2]^3 + 1/2 ],[
    (dt*(1/dt - 6*x[2, 1]^2 + 11)*(11*x[2, 1] - 11*x[1, 1] + (x[2, 1] - x[2, 2])/dt + 2*x[1, 1]^3 - 2*x[2, 1]^3 + 1))/2 - (dt*(6*x[2, 1]^2 - 9)*(9*x[1, 1] + 9*x[2, 1] + (x[1, 1] - x[1, 2])/dt - 2*x[1, 1]^3 - 2*x[2, 1]^3 - 1))/2,(11*x[1, 1])/2 - (11*x[2, 1])/2 - (x[2, 1] - x[2, 2])/(2*dt) - x[1, 1]^3 + x[2, 1]^3 + (dt*(1/dt - 6*x[2, 2]^2 + 11)*(11*x[2, 2] - 11*x[1, 2] + (x[2, 2] - x[2, 3])/dt + 2*x[1, 2]^3 - 2*x[2, 2]^3 + 1))/2 - (dt*(6*x[2, 2]^2 - 9)*(9*x[1, 2] + 9*x[2, 2] + (x[1, 2] - x[1, 3])/dt - 2*x[1, 2]^3 - 2*x[2, 2]^3 - 1))/2 - 1/2,(11*x[1, 2])/2 - (11*x[2, 2])/2 - (x[2, 2] - x[2, 3])/(2*dt) - x[1, 2]^3 + x[2, 2]^3 - 1/2]]
    return hcat(F...)'
end

end


using Plots
using ImplicitEquations
using ForwardDiff

a, b = 1 / √2, 1 / √2
f(x, y) = x^2 + y^2 - 1
f(xy) = f(xy[1], xy[2])
g = xy -> ForwardDiff.gradient(f, xy)

"""
x' = x-a
y' = y-b
"""
function shift(f::Function, a, b)
    t = [
        1 0 a;
        0 1 b;
        0 0 1
    ]
    function tf(xy)
        x, y = xy
        tx, ty, tz = t * [x, y, 1]
        txy = [tx, ty]
        return f(txy)
    end
    tf(x, y) = tf([x, y])
    return tf
end

"""
x'' = x'
y'' = f_x(a,b)*x' + f_y(a,b)*y'
"""
function cvt(f, a, b)
    fx, fy = g([a, b])
    t = [
        1 0 0;
        -fx / fy 1 / fy 0;
        0 0 1
    ]
    t = t / fy
    function tf(xy)
        x, y = xy
        tx, ty, tz = t * [x, y, 1]
        txy = [tx, ty]
        return f(txy)
    end
    tf(x, y) = tf([x, y])
    return tf
end

plot(aspect_ratio = :equal)
plot!(Eq(f, 0))
sf = shift(f, a, b)
plot!(Eq(sf, 0))
plot!(Eq(cvt(sf, a, b), 0))


#=
this file includes functionality for estimating linear scaling regions and
defines the `generalized_dim` function.
=#

export linear_region, linear_regions, estimate_boxsizes, linreg
#####################################################################################
# Functions and methods to deduce linear scaling regions
#####################################################################################
using Statistics
using Statistics: covm, varm
# The following function comes from a version in StatsBase that is now deleted
# StatsBase is copyrighted under the MIT License with
# Copyright (c) 2012-2016: Dahua Lin, Simon Byrne, Andreas Noack, Douglas Bates,
# John Myles White, Simon Kornblith, and other contributors.
"""
    linreg(x, y) -> a, b
Perform a linear regression to find the best coefficients so that the curve:
`z = a + b*x` has the least squared error with `y`.
"""
function linreg(x::AbstractVector, y::AbstractVector)
    # Least squares given
    # Y = a + b*X
    # where
    # b = cov(X, Y)/var(X)
    # a = mean(Y) - b*mean(X)
    if size(x) != size(y)
        throw(DimensionMismatch("x has size $(size(x)) and y has size $(size(y)), " *
            "but these must be the same size"))
    end
    mx = Statistics.mean(x)
    my = Statistics.mean(y)
    # don't need to worry about the scaling (n vs n - 1)
    # since they cancel in the ratio
    b = covm(x, mx, y, my)/varm(x, mx)
    a = my - b*mx
    return a, b
end

slope(x, y) = linreg(x, y)[2]

"""
    linear_regions(x, y; dxi::Int = 1, tol = 0.25) -> (lrs, tangents)
Identify regions where the curve `y(x)` is linear, by scanning the
`x`-axis every `dxi` indices sequentially
(e.g. at `x[1] to x[5], x[5] to x[10], x[10] to x[15]` and so on if `dxi=5`).

If the slope (calculated via linear regression) of a region of width `dxi` is
approximatelly equal to that of the previous region,
within tolerance `tol`,
then these two regions belong to the same linear region.

Return the indices of `x` that correspond to linear regions, `lrs`,
and the _correct_ `tangents` at each region
(obtained via a second linear regression at each accumulated region).
"""
function linear_regions(
        x::AbstractVector, y::AbstractVector;
        method = :sequential, dxi::Int = method == :overlap ? 3 : 1, tol = 0.25,
    )
    @assert length(x) == length(y)
    return if method == :overlap
        linear_regions_overlap(x, y, dxi, tol)
    elseif method == :sequential
        linear_regions_sequential(x, y, dxi, tol)
    end
end

function linear_regions_sequential(x, y, dxi, tol)
    maxit = length(x) ÷ dxi

    tangents = Float64[slope(view(x, 1:max(dxi, 2)), view(y, 1:max(dxi, 2)))]

    prevtang = tangents[1]
    lrs = Int[1] #start of first linear region is always 1
    lastk = 1

    # Start loop over all partitions of `x` into `dxi` intervals:
    for k in 1:maxit-1
        tang = slope(view(x, k*dxi:(k+1)*dxi), view(y, k*dxi:(k+1)*dxi))
        if isapprox(tang, prevtang, rtol=tol, atol = 0)
            # Tanget is similar with initial previous one (based on tolerance)
            continue
        else
            # Tangent is not similar.
            # Push new tangent for a new linear region
            push!(tangents, tang)

            # Set the START of a new linear region
            # which is also the END of the previous linear region
            push!(lrs, k*dxi)
            lastk = k
        end

        # Set new previous tangent (only if it was not the same as current)
        prevtang = tang
    end
    push!(lrs, length(x))
    # create new tangents that do have linear regression weighted
    tangents = Float64[]
    for i in 1:length(lrs)-1
        push!(tangents, linreg(view(x, lrs[i]:lrs[i+1]), view(y ,lrs[i]:lrs[i+1]))[2])
    end
    return lrs, tangents
end

"""
    linear_region(x, y; kwargs...) -> ((ind1, ind2), slope)
Call [`linear_regions`](@ref) and identify and return the largest linear region
and its slope. The region starts and stops at `x[ind1:ind2]`.

The keywords `dxi, tol` are propagated as-is to [`linear_regions`](@ref).
The keyword `ignore_saturation = true` ignores saturation that (sometimes) happens
at the start and end of the curve `y(x)`, where the curve flattens.
The keyword `sat = 0.01` decides what saturation is (while `abs(y[i]-y[i+1])<sat` we
are in a saturation regime).

The keyword `warning = true` prints a warning if the linear region is less than 1/3
of the available x-axis.
"""
function linear_region(x::AbstractVector, y::AbstractVector;
    dxi::Int = 1, tol::Real = 0.2, ignore_saturation = true, warning = true, sat = 0.01)

    isat = 0
    if ignore_saturation
        j = findfirst(i -> abs(y[i] - y[i-1]) > sat, length(y):-1:2)
        if !isnothing(j)
            i = (length(y):-1:2)[j]
            x, y = x[1:i], y[1:i]
        end
        k = findfirst(i -> abs(y[i+1] - y[i]) > sat, 1:length(y)-1)
        if !isnothing(k)
            x, y = x[k:end], y[k:end]
            isat = k-1
        end
    end

    lrs, tangents = linear_regions(x,y; dxi, tol)
    # Find biggest linear region:
    j = findmax(diff(lrs))[2]
    if lrs[j+1] - lrs[j] ≤ length(x)÷3 && warning
        @warn "Found linear region spans less than a 3rd of the available x-axis "*
              "and might imply inaccurate slope or insufficient data. "*
              "Recommended: plot `x` vs `y`."
    end
    return (lrs[j] + isat, lrs[j+1] + isat), tangents[j]
end

#####################################################################################
# Autotomatic estimation for proper `ε` from a Dataset
#####################################################################################
"""
    estimate_boxsizes(A::Dataset; kwargs...) → εs
Return `k` exponentially spaced values: `εs = base .^ range(lower + w, upper + z; length = k)`,
that are a good estimate for sizes ε that are used in calculating a [Fractal Dimension](@ref).
It is strongly recommended to [`standardize`](@ref) input dataset `A` before using this
function.

Let `d₋` be the minimum pair-wise distance in `A` and `d₊` the average total length of `A`
along each of the dimensions of `A`.
Then `lower = log(base, d₋)` and `upper = log(base, d₊)`.
Because by default `w=1, z=-1`, the returned sizes are an order of mangitude
larger than the minimum distance, and an order of magnitude smaller than the maximum
distance.

## Keywords
* `w = 1, z = -1, k = 20` : as explained above.
* `base = MathConstants.e` : the base used in the `log` function.
* `warning = true`: Print some warnings for bad estimates.
* `autoexpand = true`: If the final estimated range does not cover at least 2 orders of
  magnitude, it is automatically expanded by setting `w -= we` and `z -= ze`.
  You can set different default values to the keywords `we = w, ze = z`.
"""
function estimate_boxsizes(
        A::AbstractDataset;
        k::Int = 20, z = -1, w = 1, base = MathConstants.e,
        warning = true, autoexpand = true, ze = z, we = w
    )

    mi, ma = minmaxima(A)
    max_d = mean(ma - mi)
    min_d, _ = minimum_pairwise_distance(A)
    if min_d == 0 && warning
        @warn(
        "Minimum distance in the dataset is zero! Probably because of having data "*
        "with low resolution, or duplicate data points. Setting to `d₊/base^4` for now.")
        min_d = max_d/(base^4)
    end

    lower = log(base, min_d)
    upper = log(base, max_d)

    if lower ≥ upper
        error("`lower ≥ upper`. There must be something fundamentally wrong with dataset.")
    elseif lower+w ≥ upper+z && warning
        @warn(
        "Automatic boxsize determination was inappropriate: `lower+w` was found ≥ than "*
        "`upper+z`. Returning `base .^ range(lower, upper; length = k)`. "*
        "Please adjust keywords or provide a bigger dataset.")
        εs = float(base) .^ range(lower, upper; length = k)
    elseif abs(upper+z - (lower+w)) < 2 && autoexpand
        if warning
            @warn(
            "Boxsize limits do not differ by at least 2 orders of magnitude. "*
            "Setting `w-=$(we)` and `z+=$(ze)`, please adjust keywords `w, z` otherwise.")
        end
        εs = float(base) .^ range(lower+w-we, upper+z-ze; length = k)
    else
        εs = float(base) .^ range(lower+w, upper+z; length = k)
    end
    return εs
end

"""
    minimum_pairwise_distance(A::Dataset, metric = Euclidean())
Return `min_d, min_pair`: the minimum pairwise distance
of all points in the dataset, and the corresponding point pair.
"""
function minimum_pairwise_distance(A::AbstractDataset, metric = Euclidean())
    tree = KDTree(A, metric)
    min_d = eltype(A[1])(Inf)
    max_d = -min_d
    min_pair = max_pair = (0, 0)
    theiler = Theiler(0)
    for i in 1:length(A)
        inds, dists = Neighborhood.knn(tree, A[i], 1, theiler(i); sortds=false)
        ind, dist = inds[1], dists[1]
        if dist < min_d
            min_d = dist
            min_pair = (i, ind)
        end
    end
    return min_d, min_pair
end


module MatrixChainMultiply

export matrixchainmultiply

### mcm = matrix chain multiply

"""
  `matrixchainmultiply(A,B,C,D,...)`

Algorithm for **Matrix-chain multiplication** (where matrices are also
generalizations of scalars and vectors). Using *Cormen* Ed. 3, p. 371.

Arguments
---------
`A,B,C,...` are matrices, vectors, or scalars of appropriate sizes.

"""
function matrixchainmultiply(A...)
  p = mcm_makep(A...)
  m, s, n = mcm_cost_matrices(p)
  mcm_compute(s, 1, n, A...)
end
"""
  `matrixchainmultiply(fn_name::String, A...)`

Print out the optimal matrix order as an expresion and its
cost. Also use this to...

Construc a function for repeated use (Julia v0.6-?)
---------------------------------------------------
Sometimes, you will do a chain multiplication many times with matrices of the same size. Another use case for this is to do the analysis on the CPU but use the generated function on the GPU. In either case, you only need to find the optimal order and JIT compile that function once. To save this function to the global scope:

```julia
a = rand(Float32, 10000,2000)
b = rand(Float32, 2000,100)
c = rand(Float32, 100)

eval(matrixchainmultiply("mcm_abc", a,b,c))

mcm_abc(a,b,c)

# multiple-dispatch is nice; taking advantage below
using ArrayFire
aa = AFArray(a)
ab = AFArray(b)
ac = AFArray(c)

mcm_abc(aa,ab,ac)  # no new analysis; ke
```
"""
function matrixchainmultiply(fn_name::String, A...)
  matrixchainmultiply_fn(fn_name, A...)
end
function matrixchainmultiply_fn(fn_name::String, A...)
  p = mcm_makep(A...)
  m, s, n = mcm_cost_matrices(p)
  ex = parse(fn_name * "(A...) = " * mcm_print(s, 1, n))
  cost = m[1,n]
  println("Operation: $ex")
  println("Cost: $cost")
  ex
end

"Cost information tells which order to do."
function mcm_cost_matrices(p::Vector{Int})
  ∞ = typemax(Int)  # watch for overflow?
  n = length(p) - 1  # number of matrices
  @assert (n > 1) "TODO: allow to use for short lists"
  # stores costs
  m = zeros(Int, (n, n))
  # records corresponding indices
  s = zeros(Int, (n-1, n-1))  # Cormen has 2nd index going 2..n
  # zero on diagonals
  for i ∈ 1:n
    m[i,i] = 0
  end
  # l is chain length
  for l ∈ 2:n
    for i ∈ 1:(n-l+1)
      j = i + l - 1
      m[i,j] = ∞  # this should be replaced?!
      for k ∈ i:(j-1)
        q1 = m[i,k]
        q2 = m[k+1,j]
        q3 = p[i] * p[k+1] * p[j+1]  # Cormen p vec indexed from 0
        q = +(q1, q2, q3)
        if q < m[i,j]
          m[i,j] = q
          s[i,j-1] = k
        end
      end
    end
  end
  m, s, n
end

"Use this if you want to see what the operation looks like."
function mcm_print(s::Matrix, i::Int, j::Int)
  str = ""
  if i!=j
    a = mcm_print(s, i, s[i, j-1])
    b = mcm_print(s, s[i, j-1] + 1, j)
    return "($a * $b)"
  else
    return "A[$i]"
  end
end

"""
The function describing the optimal order for the input of a specific size.
"""
function mcm_optimalorder(s, i, j, mats...)
  if i!=j
    m1 = mcm_compute(s, i, s[i, j-1], mats...)
    m2 = mcm_compute(s, s[i, j-1] + 1, j, mats...)
    return :(m1 * m2)
  else
    return mats[i]
  end
end

"Do the actual matrix chain multiplication computation."
function mcm_compute(s, i, j, mats...)
  if i!=j
    m1 = mcm_compute(s, i, s[i, j-1], mats...)
    m2 = mcm_compute(s, s[i, j-1] + 1, j, mats...)
    return (m1 * m2)
  else
    return mats[i]
  end
end

"The matrix sizes."
function mcm_makep(mats...)
  # Made some assumptions below that disallow these
  lenmats = length(mats)
  # convert to matrix-style sizes
  msizes = map(msize, mats)
  # produce unique number list
  p = zeros(Int, lenmats + 1)
  for i in 1:lenmats
    firstiter::Bool = (i==1)
    if firstiter
      p[1] = msizes[1][1]
    else
      check1 = msizes[i-1][2]
      check2 = msizes[i][1]
      @assert (check1 == check2) "Matrix sizes don't match ($check1 != $check2)!"
    end
    p[i+1] = msizes[i][2]
  end
  p
end

###

"""
Matrix-style sizes. Return type looks like `(i,j)` where `i` and `j` are `Ints`.

If other packages with matrix types (e.g., OpenCl or ArrayFire) inherit from
AbstractArray, there shouldn't be issues with other packages using this one.
"""
msize(m::AbstractMatrix) = size(m)
msize(v::AbstractVector) = (size(v,1), 1)
msize(s::Number) = (1, 1)

end # module


#### Beta diversity calculation functions

## Function to calculate beta diversity statistics
"""
    betadiv(Y::Matrix)

Computes the beta diversity statistics from the community matrix `Y` based on
Legendre & De Cáceres (2013). This function was based on the `beta.div` R
function from the supplementary material of that paper, and results were also
tested against the `beta.div` function from `adespatial`.
"""
function betadiv(Y::Matrix)
    @assert !any(isnothing, Y) "Y must only contain observed sites and no nothing values"
    # S -> squared deviations from column mean
    S = (Y .- mean(Y; dims=1)) .^ 2.0
    # SStotal -> total sum of squares
    SStotal = sum(S)
    # BDtotal -> index of beta diversity, unbiased & comparable estimator of Var(Y)
    BDtotal = SStotal / (size(Y, 1) - 1)
    # SSj -> sum of squares for species j
    SSj = sum(S; dims=1)
    # SCBDj -> species contribution to beta diversity (species j, relative)
    SCBDj = SSj ./ SStotal
    # SSi -> sum of squares for site i
    SSi = sum(S; dims=2)
    # LCBD -> local contribution to beta diversity (site i, relative)
    LCBDi = SSi ./ SStotal
    # Combine results in tuple
    res = (
        S=S, SStotal=SStotal, BDtotal=BDtotal, SSj=SSj, SCBDj=SCBDj, SSi=SSi, LCBDi=LCBDi
    )
    return res
end


logistic(x) = 1 ./ (1 + exp.(-x))
@scalardiff logistic(x::Number) 1 (logistic(x) .* (1 .- logistic(x)))

function autoencoder_cost(We1, We2, Wd, b1, b2, x)
    firstLayer = logistic(We1 * x .+ b1)
    encodedInput = logistic(We2 * firstLayer .+ b2)
    reconstructedInput = logistic(Wd * encodedInput)
    cost = sum((reconstructedInput .- x) .^ 2.0)
    return cost
end

function mlp1(w1, w2, w3, x1)
    xx2 = w1 * x1
    x2 = log.(1. + exp.(xx2))
    xx3 = w2 * x2
    x3 = log.(1. + exp.(xx3))
    x4 = w3 * x3
    sum(1. ./ (1. + exp.(-x4)))
end

function mlp2(w1, w2, w3, b1, b2,  b3, x1)
    x2 = logistic(w1 * x1 .+ b1)
    x3 = logistic(w2 * x2 .+ b2)
    x4 = logistic(w3 * x3 .+ b3)
    sum(x4)
end

function rnn(Wxh, Whh, Wy, hprev, x, y)
    h = tanh.(Whh * hprev + Wxh * x)
    yhat = Why * h
    cost = sum((yhat .- y) .^ 2.0)
    return h, cost
end


using SymPy

function lagrange(order)

  is=collect(-floor(Int,order/2) : 1 : floor(Int,order/2));
  x=symbols("x");
  L=x^0;
  Ls=Array(typeof(L),(order,));

  for j = 1 : order
    ith=is[j];
    for i in is
      if i!= ith
        L=L*(x-i)/(ith-i);
      end
    end
    Ls[j]=copy(L);
    L=x^0;
  end

  return Ls;

end

function fd(order,d;t=Rational)
  Ls=lagrange(order);
  x =symbols("x");

  is=collect(-floor(Int,order/2) : 1 : floor(Int,order/2));
  D=Array(typeof(x),(order,order));
  for i = 1 : order
    for j = 1 : order
      D[i,j] =diff(Ls[j],x,d)(x=>is[i]);
    end
  end

  return Array{t}(D);

end

order=4
D=fd(order+1,2,t=Float64);


#############################################################################
#############################################################################
#
# This file implements polynomial multiplication
#
#############################################################################
#############################################################################

"""
Multiply two polynomials.
"""

function *(p1::Polynomial, p2::Polynomial)::Polynomial
    p_out = Polynomial()
    for t in p1
        p_out = p_out + (t * p2)
    end
    return p_out
end

"""
Multiply two polynomials modulo P.
"""
function *(p1::PolynomialModP, p2::PolynomialModP)::Polynomial
    @assert p1.prime == p2.prime "Primes must be the same"
    p_out = Polynomial()
    for t in p1.terms
        p_out = p_out + (t * p2.terms)
    end
    return mod(p_out, p1.prime)
end

"""
Power of a polynomial.
"""

function ^(p::Polynomial, n::Int)
    n < 0 && error("No negative power")
    out = one(p)
    binary_arr = digits(n, base=2)
    for i in 1:length(binary_arr)
        binary_arr[i] == 1 ? out = out*p : out *= 1
        p = p*p
    end
    return out
end

# I decided to keep the ^ as a general function and not restrict it to only polynomials since this also works with raising integers and other types to a power n mod p.
function ^(p, n::Int, prime::Int)
    n < 0 && error("No negative power")
    out = one(p)
    binary_arr = digits(n, base=2)
    p = mod(p, prime)
    for i in 1:length(binary_arr)
        binary_arr[i] == 1 ? out = mod(out*p, prime) : out *= 1
        p = mod(p*p, prime)
    end
    return mod(out, prime)
end

^(p::PolynomialModP, n::Int) = ^(p.terms, n, p.prime)

function CRT(poly_arr, prime_arr)

    c = 0
    max_input_deg = max(degree(poly_arr[1]), degree(poly_arr[2]))+1
    exponent_arr_1 = zeros(Int64, 1, max_input_deg)
    exponent_arr_2 = zeros(Int64, 1, max_input_deg)
    for i in poly_arr[1]
        exponent_arr_1[i.degree+1] = i.coeff
    end
    for i in poly_arr[2]
        exponent_arr_2[i.degree+1] = i.coeff
    end
    """
    The lines above actually result in a marginally slower CRT implementation as opposed to the approach where you cycle through exponent arrays 1 and 2 and check if k matches the exponent value before sending those values off to the iCRT call. I chose to keep this approach because the computational time difference was only marginal and I felt like it shows that I *really* tried to tackle the task given.
    """

    for k in max_input_deg-1:-1:0
        ak = exponent_arr_1[k+1]
        bk = exponent_arr_2[k+1]
        ck = iCRT([ak, bk], prime_arr)
        k == 0 ? c = c + ck : c = c + ck * x^k # Can probably remove the turnery, its only there because I had a strange bug with Term(1,1)
    end
    return c
end

function mult_poly_with_crt(a, b)
    height_a = maximum(coeffs(a))
    height_b = maximum(coeffs(b))
    B = 2*height_a*height_b*min(degree(a)+1, degree(b)+1)
    M = 3
    c = poly_modP_multiplication(a, b, M)
    while M < B
        nextprime(M) == M ? p = nextprime(M, 2) : p = nextprime(M)
        d = poly_modP_multiplication(a, b, p)
        c = CRT([c, d], [M, p])
        M = M*p
    end
    return c
end

function iCRT(u, m)
    v = Vector{Int}(undef, 2)
    v[1] = u[1]
    v[2] = mod((u[2] - v[1])*inverse_mod(m[1], m[2]), m[2])
    output = u[1] + v[2]*m[1]
    smod(output, m[1]) == smod(u[1], m[1]) && smod(output, m[2]) == smod(u[2], m[2]) && return smod(output, m[1]*m[2])
    error("Output modulo prime is not consistent with inputs")
end

function poly_modP_multiplication(a::Polynomial, b::Polynomial, prime::Int)
    if length(a) == 0 || length(b) == 0
        return 0
    end
    if leading(a) == a.terms[1] && leading(b) == b.terms[1]
        return mod(mod(a, prime) * mod(b, prime), prime)
    end

    if length(a.terms) == 1
        a1 = a
        a2 = Polynomial()
    else
        a1 = Polynomial(a.terms[1:(length(a.terms)÷2)])
        a2 = Polynomial(a.terms[(length(a.terms)÷2)+1:length(a.terms)])
    end

    if length(b.terms) == 1
        b1 = b
        b2 = Polynomial()
    else
        b1 = Polynomial(b.terms[1:(length(b.terms)÷2)])
        b2 = Polynomial(b.terms[(length(b.terms)÷2)+1:length(b.terms)])
    end

    return mod(poly_modP_multiplication(a1, b1, prime) + poly_modP_multiplication(a1, b2, prime) + poly_modP_multiplication(a2, b1, prime) + poly_modP_multiplication(a2, b2, prime), prime)
end

function mult_vec_el_up_to(arr::Vector, el::Int)
    @assert length(arr) > 0 "Array must contain at least 1 element."
    el == 0 ? 1 : arr[el]*mult_vec_el_up_to(arr, el-1)
end

function i_ext_euclid_alg(a,b)
    a == 0 && return b, 0, 1
    g, s, t = i_ext_euclid_alg(b % a, a)
    return g, t - (b ÷ a)*s, s
end

smod(a::Int,m::Int)::Int = mod(a,m) > m ÷ 2 ?  mod(a,m) - m : mod(a,m)

inverse_mod(a,m) = mod(i_ext_euclid_alg(a,m)[2],m);


abstract type ActivationFunction end

"""
    ReLU <: ActivationFunction

    (ReLU())(x) -> max.(x, 0)
"""
struct ReLU <: ActivationFunction end

"""
    Max <: ActivationFunction

    (Max())(x) -> max(maximum(x), 0)
"""
struct Max <: ActivationFunction end

"""
    Id <: ActivationFunction
Identity operator

    (Id())(x) -> x
"""
struct Id <: ActivationFunction end

"""
    Sigmoid <: ActivationFunction

    (Sigmoid())(x) -> 1 ./ (1 .+ exp.(-x))
"""
struct Sigmoid <: ActivationFunction end

"""
    Tanh <: ActivationFunction

    (Tanh())(x) -> tanh.(x)
"""
struct Tanh <: ActivationFunction end

"""
    GeneralAct <: ActivationFunction
Wrapper type for a general activation function.

### Usage
```julia
act = GeneralAct(tanh)

act(0) == tanh(0)           # true
act(10.0) == tanh(10.0)     # true
```
```julia
act = GeneralAct(x->tanh.(x))

julia> act(-2:2)
5-element Array{Float64,1}:
 -0.9640275800758169
 -0.7615941559557649
  0.0
  0.7615941559557649
  0.9640275800758169
```
"""
struct GeneralAct <: ActivationFunction
    f::Function
end

#=
TODO: consider writing our own interpolation scheme to avoid a dependency for this one thing.
Should only require a handful of functions.
Also NOTE: inherently not type stable unless parameterized.
=#
"""
    PiecewiseLinear <: ActivationFunction
Activation function that uses linear interpolation between supplied `knots`.
An extrapolation condition can be set for values outside the set of knots. Default is `Linear`.

    PiecewiseLinear(knots_x, knots_y, [extrapolation = Line()])

### Usage
```julia
kx = [0.0, 1.2, 1.7, 3.1]
ky = [0.0, 0.5, 1.0, 1.5]
act = PiecewiseLinear(kx, ky)

act(first(kx)) == first(ky) == 0.0
act(last(kx))  == last(ky)  == 1.5

act(1.0)    # 0.4166666666666667
act(-102)   # -42.5
```
```julia
act = PiecewiseLinear(kx, ky, Flat())

act(-102)   # 0.0
act(Inf)    # 1.5
```

### Extrapolations
- Flat()
- Line()
- constant (supply a number as the argument)
- Throw() (throws bounds error)

`PiecewiseLinear` uses [Interpolations.jl](http://juliamath.github.io/Interpolations.jl/latest/).
"""
struct PiecewiseLinear <: ActivationFunction
    f::Interpolations.Extrapolation
end

# default extrapolation is Line(). Can also do Flat() or supply a constant,
function PiecewiseLinear(knots_x::AbstractVector,
                         knots_y::AbstractVector,
                         extrapolation = Interpolations.Line())
    PiecewiseLinear(LinearInterpolation(knots_x, knots_y, extrapolation_bc = extrapolation))
end

# the type stable definitions probably don't need to go in the paper as-is
(f::ReLU)(x) = max.(x, zero(eltype(x)))
(f::Max)(x) = max(maximum(x), zero(eltype(x)))
(f::Id)(x) = x
(f::Sigmoid)(x) = @. 1 / (1 + exp(-x))
(f::Tanh)(x) = tanh.(x)
(G::GeneralAct)(x) = G.f(x)
(PL::PiecewiseLinear)(x) = PL.f(x)


"""
    correction_hull(A::IntervalMatrix{T}, t, p) where {T}

Compute the correction term for the convex hull of a point and its linear map
with an interval matrix in order to contain all trajectories of a linear system.

### Input

- `A` -- interval matrix
- `t` -- non-negative time value
- `p` -- order of the approximation

### Output

An interval matrix representing the correction term.

### Algorithm

See Theorem 3 in [1].

[1] M. Althoff, O. Stursberg, M. Buss. Reachability Analysis of Linear Systems
with Uncertain Parameters and Inputs. CDC 2007.
"""
function correction_hull(A::IntervalMatrix{T}, t, p) where {T}
    # initialize interval matrix with zero intervals
    m, n = size(A)
    F = IntervalMatrix(zeros(Interval{T}, m, n))

    A2i = A
    fac_i = 1
    t2i = t
    @inbounds for i in 2:p
        t2i *= t
        left = (one(T) / i^(i/i-1) - one(T) / i^(1/i-1)) * t2i
        itv = Interval(left, zero(T))
        A2i = A2i * A
        fac_i *= i
        F += itv * A2i * (1/fac_i)
    end
    F += _expm_remainder(A, t, p)
    return F
end


# this file depends on jacobi.jl from JuliaOptics/OpticsPolynomials that
# should be found in the same directory

export cheby1, cheby1_series, cheby1_sum, cheby2, cheby2_series, cheby2_sum

"""
    cheby1(n, x)

Compute the Chebyshev polynomial of the first kind of order n at point x.

This family of Chebyshev polynomials are a special case of the Jacobi polynomials
with α, β = -1/2.

See also: [`cheby1_series`](@ref), [`cheby1_sum`](@ref)
"""
function cheby1(n, x)
    return jacobi(n, -0.5, -0.5, x)
end

"""
    cheby1_series(ns, α, β, x)

Compute a series of Chebyshev polynomials of the first kind of orders n.
Returns an array with shape (size(x)..., length(ns)).
That is, the _final_ dimension contains the modes and the first dimension(s)
are spatial.

See also: [`cheby1`](@ref), [`cheby1_sum`](@ref)
"""
function cheby1_series(ns, x)
    return jacobi_series(ns, -0.5, -0.5, x)
end
"""
    cheby1_sum(ns, weights, x)

Compute a sum of Chebyshev polynomial of the first kind of order n weighted by weights.

See also: [`cheby1`](@ref), [`cheby1_series`](@ref)
"""
function cheby1_sum(ns, weights, x)
    return jacobi_sum(ns, weights, -0.5, -0.5, x)
end

"""
    cheby2(n, x)

Compute the Chebyshev polynomial of the second kind of order n at point x.

This family of Chebyshev polynomials are a special case of the Jacobi polynomials
with α, β = 1/2.

See also: [`cheby2_series`](@ref), [`cheby2_sum`](@ref)
"""
function cheby2(n, x)
    return jacobi(n, 0.5, 0.5, x)
end

"""
    cheby2_series(ns, α, β, x)

Compute a series of Chebyshev polynomials of the second kind of orders n.
Returns an array with shape (size(x)..., length(ns)).
That is, the _final_ dimension contains the modes and the first dimension(s)
are spatial.

See also: [`cheby1`](@ref), [`cheby1_sum`](@ref)
"""
function cheby2_series(ns, x)
    return jacobi_series(ns, 0.5, 0.5, x)
end
"""
    cheby1_sum(ns, weights, x)

Compute a sum of Chebyshev polynomial of the second kind of order n weighted by weights.

See also: [`cheby1`](@ref), [`cheby1_series`](@ref)
"""
function cheby2_sum(ns, weights, x)
    return jacobi_sum(ns, weights, 0.5, 0.5, x)
end


module SDC
export SDC, make_SDC, make_spectral_int_matrix, gauss_lobatto_nodes

using FastGaussQuadrature: gausslobatto
using InvertedIndices: Not
using Polynomials: fromroots, integrate
#import BenchmarkTools

#"""
#Solves the equation:
#    w + alpha*fI(t,w) = b
#For w given α, t, and b
#Do I need a best guess for w, as well?
#"""
#function implicit_solve(α, t, b)
#end

"""
Get n Gauss-Lobatto quadrature nodes, on interval [0-1].
"""
function gauss_lobatto_nodes(n::Int)
    nodes, weights =  gausslobatto(n)
    return 0.5*(nodes .+ 1.0)
end

"""
Get the spectral integration matrix
"""
function make_spectral_int_matrix(n::Int)
    nodes = gauss_lobatto_nodes(n) # Perhaps add option to change quadrature in the future
    S = zeros(n,n)

    for j = 1:n
        Lt = fromroots(nodes[Not(j)])
        Li = (1.0/Lt(nodes[j]))*Lt
        for i = 1:n
            S[i,j] = integrate(Li,0.0,nodes[i])
        end
    end
    return S
end

"""
Notes for improvement:

The main usage for this function will probably be something like:
for i in 1:number_of_timesteps
    SDC(...)

So I don't want to waste resources recomputing S everytime, or setting up all
the equations.

Perhaps I should have a create_SDC function? This would be provided all
parameters except the initial guess, and create an SDC function which only
needs the initial values as arguments.

From a UX perspective, I like that they will have an SDC function whose only
arguments are the initial data. That would be very easy to use. But I don't
like that they have to call a function to create a function. That seems like it
might be confusing.

Actually, I think it would be good to also have Δt as an argument to the
user-SDC function. I can imagine a user might want to use variable timestep
size (but they probably don't need to vary the number of collocation nodes,
implicit/explicit f, etc).
"""
function sdc(n, fE, fI, t, Δt, u, implicit_solver)
    nodes = gauss_lobatto_nodes(n)
    n_cor = 2*n-3
    S = make_spectral_int_matrix(n) # nodes will be computed twice; should make more efficient
    subtimes = (Δt*nodes).+ t
    subvals = Vector{typeof(u)}(undef, n)

    subvals[1] = u
    # Prediction
    for m in 1:n-1
        subvals[m+1] = begin
            u_m = subvals[m]
            t_m = subtimes[m]
            t_mp1 = subtimes[m+1]
            Δt_m = t_mp1 - t_m
            RHS = u_m + Δt_m*fE(t_m,u_m)
            implicit_solver(-Δt_m, t_m, RHS)
        end
    end

    f(t, u) = fE(t, u) + fI(t, u)
    # Correction
    for k in 1:n_cor
        # B15 - Not needed for gauss lobatto nodes, just leaves subvals as they were
        # But B15 will be needed if start point is not a node
        f_subvals = [f(subtimes[i], subvals[i]) for i in 1:n]
        u_m_k = subvals[1]
        for m in 1:n-1
            #B16
            u_m_k_next = subvals[m+1] # Should figure out better notation for this
            subvals[m+1] = begin
                u_m_kp1 = subvals[m]
                t_m = subtimes[m]
                t_mp1 = subtimes[m+1]
                Δt_m = t_mp1 - t_m
                u_mp1_k = subvals[m+1]
                ΔfE = fE(t_m, u_m_kp1) - fE(t_m, u_m_k)
                RHS = subvals[m] + Δt_m*(ΔfE - f(t_mp1, u_mp1_k))
                RHS += sum(q -> (S[m+1,q]-S[m,q])*f_subvals[q], 1:size(S)[2])
                implicit_solver(-Δt_m, fI, RHS)
            end
            u_m_k = u_m_k_next
        end
    end
    # Will be different if endpoint is a node, need to use B6
    return subvals[end]
end

"""
This function returns a function: an SDC solver which needs only the initial
conditions as the arguments. Moreover, the quadrature and spectral integration
matrix are built-in to the function, not calculated each run.
"""
function make_SDC(n, fE, fI, implicit_solver)
    nodes = gauss_lobatto_nodes(n)
    n_cor = 2*n-3
    S = make_spectral_int_matrix(n) # nodes will be computed twice; should make more efficient

    return function small_arg_list_SDC(u, t, Δt)
        subtimes = (Δt*nodes).+ t
        subvals = Vector{typeof(u)}(undef, n)
        subvals[1] = u
        # Prediction
        for m in 1:n-1
            subvals[m+1] = begin
                u_m = subvals[m]
                t_m = subtimes[m]
                t_mp1 = subtimes[m+1]
                Δt_m = t_mp1 - t_m
                RHS = u_m + Δt_m*fE(t_m,u_m)
                implicit_solver(-Δt_m, fI, RHS)
            end
        end

        f(t, u) = fE(t, u) + fI(t, u)
        # Correction
        for k in 1:n_cor
            # B15 - Not needed for gauss lobatto nodes, just leaves subvals as they were
            # But B15 will be needed if start point is not a node
            f_subvals = [f(subtimes[i], subvals[i]) for i in 1:n]
            u_m_k = subvals[1]
            for m in 1:n-1
                #B16
                u_m_k_next = subvals[m+1] # Should figure out better notation for this
                subvals[m+1] = begin
                    u_m_kp1 = subvals[m]
                    t_m = subtimes[m]
                    t_mp1 = subtimes[m+1]
                    Δt_m = t_mp1 - t_m
                    u_mp1_k = subvals[m+1]
                    ΔfE = fE(t_m, u_m_kp1) - fE(t_m, u_m_k)
                    RHS = subvals[m] + Δt_m*(ΔfE - f(t_mp1, u_mp1_k))
                    RHS += sum(q -> (S[m+1,q]-S[m,q])*f_subvals[q], 1:size(S)[2])
                    implicit_solver(-Δt_m, fI, RHS)
                end
                u_m_k = u_m_k_next
            end
        end
        # Will be different if endpoint is a node, need to use B6
        return subvals[end]
    end
end

#function graph(u_np1_saves, T::Timing;
#              graph_name::String="SDC_evolution.png",
#              display_plot::Bool=false)
#
#    # Get exact value for comparison purposes
#    u_exact = u(T.t_n + T.Δt)
#    # Compute errors
#    u_np1_saves_errors = Vector{Tuple{Int64, Float64}}(undef, length(u_np1_saves))
#    for (i, (j, u_np1)) in enumerate(u_np1_saves)
#        #u_np1_saves_errors[i] = (j, abs(u_np1-u_exact))
#        u_np1_saves_errors[i] = (j, u_np1 - u_exact)
#    end
#
#    # Plotting
#    plot = Plots.plot(
#        getindex.(u_np1_saves_errors, 1), # No of Corrections
#        getindex.(u_np1_saves_errors, 2), # Error
#        label="Approximation of u(t_n+Δt)"
#    )
#    p = length(T.Δtimes)+1
#    Plots.plot!(
#        plot,
#        title="Error Over 1 Timestep Vs No of Corrections \nλ=$λ, Δt=$(T.Δt), t_n=$(T.t_n), p=$p (Gauss-Lobatto)",
#        xlabel="# of Corrections (0 ⟹ Initial Prediction)",
#        ylabel="Error",
#        xlim=(0, 2*p),
#        #yaxis=:log,
#    )
#    # Make a line on y=0, for easier comprehension
#    Plots.hline!(plot, [0], label="")
#
#    Plots.savefig(plot, graph_name)
#
#    if display_plot
#        display(plot)
#    end
#    return nothing
#end

#"""
#Make sure collocation nodes have proper scale and order.
#"""
#function validate_collocation_nodes(c_nodes)
#    for node in c_nodes
#        if node < 0 || node > 1
#            throw(DomainError(node, "Collocation nodes must be ∈ [0,1]."))
#        end
#    end
#    for i in 1:length(c_nodes)-1
#        if c_nodes[i+1] <= c_nodes[i]
#            throw(DomainError(node, "Collocation nodes must be obey c_m+1 > c_m ∀ m = 1, ... , p."))
#        end
#    end
#end

end


"""
    omega_ratio(returns, target_return)

This function calculates the Omega ratio.

# Formula

    E[max(returns - target_return, 0)] / E[max(target_return - returns, 0)]

# Arguments
- `returns`:        Vector of asset returns.
- `target_return`:  Vector or scalar value of benchmark returns having same same frequency (e.g. daily) as the provided returns.
"""
function omega_ratio(returns, target_return)
    excess = returns .- target_return
    sum1 = sum(map(x -> max(0.0, x), excess))
    sum2 = -sum(map(x -> min(0.0, x), excess))
    sum1 / sum2
end


"""
    dominate(p, q)

Returns `1` if `p` is dominated by `q`, `-1` if otherwise, and `0` if dominance cannot be determined.
"""
function dominate(p::T, q::T) where {T <: AbstractArray}
    ret = 0
    for (i,j) in zip(p,q)
        if i < j
            ret == -1 && return 0
            ret = 1
        elseif j < i
            ret == 1 && return 0
            ret = -1
        end
    end
    return ret
end

"""
dominations(P::AbstractVector)

Returns a domination matrix of all elements in the input collection `P`.
"""
function dominations(P::AbstractVector{T}) where {T <: AbstractArray}
    l = length(P)
    D = zeros(Int8, l, l)
    for i in 1:l
        for j in (i+1):l
            D[i,j] = dominate(P[i],P[j])
            D[j,i] = -D[i,j]
        end
    end
    D
end

"""
    nondominatedsort!(R, F)

Calculate fronts for fitness values `F`, and store ranks of the individuals into `R`.
"""
function nondominatedsort!(R, P)
    n = size(P,2)
    @assert length(R) == n "Ranks must be defined for the whole population"

    Sₚ = Dict(i=>Set() for i in 1:n)
    C = zeros(Int, n)

    # construct first front
    F =[Int[]]
    for i in 1:n
        for j in i+1:n
            r = dominate(view(P,:,i), view(P,:,j)) #M[i,j]
            if r == 1
                push!(Sₚ[i], j)
                C[j] += 1
            elseif r == -1
                push!(Sₚ[j], i)
                C[i] += 1
            end
        end
        if C[i] == 0
            R[i] = 1
            push!(F[1], i)
        end
    end

    # construct rest of the fronts
    while !isempty(last(F))
        Q = Int[]
        for i in last(F)
            for j in Sₚ[i]
                C[j] -= 1
                if C[j] == 0
                    push!(Q, j)
                    R[j] = length(F) + 1
                end
            end
        end
        push!(F, Q)
    end
    isempty(last(F)) && pop!(F)

    F #, R #, Sₚ
end

"""
    crowding_distance!((C, F, fronts)

Calculate crowding distance for individuals and save the results into `C`
given the fitness values `F` and collection of `fronts`.
"""
function crowding_distance!(C::AbstractVector, F::AbstractMatrix{T}, fronts) where {T}
    for f in fronts
        cf = @view C[f]
        if length(cf) <= 2
            cf .= typemax(T)
        else
            # sort front by each objective value
            SF = F[:, f]
            d = size(SF,1)
            IX = zeros(Int, size(SF))
            IIX = zeros(Int, size(SF))
            for i in 1:d
                irow, iirow, row = view(IX,i,:), view(IIX,i,:), view(SF,i,:)
                sortperm!(irow, row)
                sortperm!(iirow, irow)
                permute!(row, irow)
            end
            nrm = SF[:,end] - SF[:,1]
            dst = (hcat(SF, fill(typemax(T), d)) - hcat(fill(typemin(T), d), SF)) ./ nrm
            dst[isnan.(dst)] .= zero(T)
            ss = sum(mapslices(v->diag(dst[:,v]) + diag(dst[:,v.+1]), IIX, dims=1), dims=1)
            cf .= vec(ss)/d
        end
    end
    C
end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _mag2flux{T<:AbstractFloat}(mag::T, zero_point::T, ABwave::T)
    if isnan(ABwave)
        return exp10(-0.4*(mag + zero_point))
    else
        return exp10(-0.4*(mag + 2.406 + 5*log10(float(ABwave))))
    end
end

"""
    mag2flux(mag[, zero_point, ABwave=number]) -> flux

### Purpose ###

Convert from magnitudes to flux expressed in erg/(s cm² Å).

### Explanation ###

This is the reverse of `flux2mag`.

### Arguments ###

* `mag`: the magnitude to be converted in flux.  It can be either a scalar or an
  array.
* `zero_point`: scalar giving the zero point level of the magnitude.  If not
 supplied then defaults to 21.1 (Code et al 1976).  Ignored if the `ABwave`
 keyword is supplied
* `ABwave` (optional numeric keyword): wavelength, scalar or array, in
 Angstroms.  If supplied, then the input `mag` is assumed to contain Oke AB
 magnitudes (Oke & Gunn 1983, ApJ, 266, 713;
 http://adsabs.harvard.edu/abs/1983ApJ...266..713O).

### Output ###

The flux.  It is of the same type, scalar or array, as `mag`.

If the `ABwave` keyword is set, then the flux is given by the expression

\$\$\\text{flux} = 10^{-0.4(\\text{mag} +2.406 + 4\\log_{10}(\\text{ABwave}))}\$\$

Otherwise the flux is given by

\$\$\\text{flux} =  10^{-0.4(\\text{mag} + \\text{zero point})}\$\$

### Example ###

``` julia
mag2flux(8.3)
# => 1.7378008287493692e-12
mag2flux(8.3, 12)
# => 7.58577575029182e-9
mag2flux(8.3, ABwave=12)
# => 3.6244115683017193e-7
```

### Notes ###

Code of this function is based on IDL Astronomy User's Library.
"""
mag2flux(mag::Real, zero_point::Real=21.1; ABwave::Real=NaN) =
    _mag2flux(promote(float(mag), float(zero_point), float(ABwave))...,)

function mag2flux{N<:Real}(mag::AbstractArray{N}, zero_point::Real=21.1;
                           ABwave::Real=NaN)
    flux = similar(mag, typeof(float(one(N))))
    for i in eachindex(mag)
        flux[i] = mag2flux(mag[i], zero_point, ABwave=ABwave)
    end
    return flux
end


"""
    density1d(x::AbstractVector, grid::AbstractVector; normalize::Bool=True)
Compute number density on a line. Normalized such that int n dx =1.
"""
function density1d(x::AbstractVector, grid::AbstractVector; normalize::Bool=true)
    bins = length(grid) -1
    # initialize array to store number
    n = zeros(bins)
    # in a cell [a,b], particles that has a<=x<b is counted.
    # for the last cell, x==b is also counted.
    for i in 1:bins
        for position in x
            if grid[i] <=position <grid[i+1]
                n[i] += 1
            end
        end
    end
    # last cell: add particles on the right boundary
    n[end] += sum(x .== grid[end])
    # use bin centers as representative locations.
    dx = diff(grid)
    bin_centers = grid[1:end-1] + dx/2
    if normalize
        n ./= dx *length(x)
    end
    return bin_centers, n
end

"""
    density1d(x::AbstractVector, xmin::Real, xmax::Real; bins::Int=100)
Compute number density on a line. Normalized such that int n dx =1.
"""
function density1d(x::AbstractVector, xmin::Real, xmax::Real; bins::Int=100, normalize::Bool=true)
    xmin < xmax ||throw(ArgumentError("xmin>=xmax"))
    grid = LinRange(xmin, xmax, bins+1)
    return density1d(x, grid, normalize=normalize)
end

"""
    density1d(x::AbstractVector; bins::Int=100, normalize::Bool=true)
Compute number density on a line.
"""
function density1d(x::AbstractVector; bins::Int=100, normalize::Bool=true)
    xmin = minimum(x)
    xmax = maximum(x)
    return density1d(x, xmin, xmax, bins=bins, normalize=normalize)
end


# # How similar do two strings look?

# ## VisualStringDistances.jl

#

# ---

# ## Let's compare strings

#

using StringDistances

# How many single-character edits are needed to turn "Julia" into "JuIia"?

StringDistances.Levenshtein()("Julia", "JuIia")

# What about "Julia" into "JuQia"?

StringDistances.Levenshtein()("Julia", "JuQia")

# We can also compare based on how many times consecutive pairs of letters appear in each string...

StringDistances.QGram(2)("Julia", "JuIia"), StringDistances.QGram(2)("Julia", "JuQia")

# ---
# ## Visual distances

#

# But none of these take into account that "Julia" and "JuIia" look pretty similar, while "Julia" and "JuQia" look pretty different.

using VisualStringDistances: VisualStringDistances
const VSD = VisualStringDistances
VSD.visual_distance("Julia",  "JuIia"), VSD.visual_distance("Julia", "JuQia")

#

# That seems better! But how do we know it does something reasonable in other cases too? And how does it work?

#

# Just need two tools:
# 1.
 A way to translate strings into images

# 2.
 A way to compare images

# ---

# ## 1. A way to translate strings into images: GNU Unifont

VSD.printglyph("GNU Unifont"; symbols=("#", "-"))

# A bitmap font!

# ---

# Unifont stores characters as bitmaps, making things quite easy for us:

VSD.Glyph("Julia")

#

# (see also FreeTypeAbstraction.jl to render bitmaps from many fonts!)

# ---

# It is low resolution, but simple and comprehensive, with 57086 supported characters, including...

chars = [VSD.get_char(k) for k in rand(collect(keys(VSD.UNIFONT_LOOKUP)), 5)];
permutedims(chars)

# Which render as:

VSD.printglyph(join(chars, " "))

# ---

VSD.printglyph("Julia vs JuIia"); VSD.printglyph("Julia vs JuQia") # hide

# ---

# ## 2. A way to compare images: Optimal transport

#

# *
 you have $a(x_1)$ amount of stuff at site $x_1$, $a(x_2)$ amount of stuff at $x_2$, ..., $a(x_n)$ stuff at $x_n$.

# *
 you want to move it around until you have $b(y_1)$ stuff at site $y_1$, $b(y_2)$ stuff at $y_2$, ..., $b(y_m)$ stuff at $y_m$

# *
 it costs $c(x_i, y_j)$ to move one unit of mass from $x_i$ to $y_j$

# ```math
# \begin{aligned}
# \operatorname{OT}(a,b) := \text{minimize} \quad & \sum\_{x,y} π(x,y)\, c(x,y)\\\\
# \text{such that} \quad & a(x) = \sum\_{y} \pi(x,y)\\\\
# & b(y) = \sum\_{x} \pi(x,y) \\\\
# & \pi(x,y) \geq 0
# \end{aligned}
# ```

# *
 We optimize to find the variables $\pi(x,y)$ (how much stuff to move from $x$ to $y$)

# ---

# ## How does optimal transport relate to our problem?

#

# -
 If we have a black pixel in the 3rd column and 2nd row of the bitmap, we can see that as $a(1) = 1$ unit of mass at site $x_1 = (2,3)$.

# -
 In this way, we can translate the bitmap representation of the string into the language of optimal transport.

# -
 $c(x,y)$ is just the distance between those points

# -
 Note: we do two modifications to this

#   -
 we solve an approximate version for speed ("entropic regularization")

#   -
 add penalties for creating/destroying stuff for the case $\sum_x a(x) \neq  \sum_y b(y)$     [1].

#

#

#

# [1]: Séjourné, T., Feydy, J., Vialard, F.-X., Trouvé, A., Peyré, G., 2019. *Sinkhorn Divergences for Unbalanced Optimal Transport*. https://arxiv.org/abs/1910.12958.

# ---

# ## What use does this have?

# Making gifs!

# ---

# ## What use does this have?

# Adding a check for new packages being added to the General registry to try to prevent the malicious impersonation another package.

# Two main concerns:

# 1. Possibly, one will make a typo, and end up at the wrong package ("typosquatting") $\leadsto$ edit distance check
# 2. Possibly, one will copy a malicious tutorial that has mimicked the appearance of the name of a popular package $\leadsto$ visual distance

#

# ---

# ## Is this the right visual distance for an automated registry check?

# I'm not sure.

# *
 Human perception is actually a bit different

#   * e.g. we mix up "p" vs "q" more than "a" vs "e"  [2], but `visual_distance` says "p" and "q" are further apart than "a" and "e"
# *
 optimal transport is a bit slow (though not prohibitively so, with entropic regularization and the low resolution font)

# *
 there are several parameters and cutoffs to tune

#
#
# Possibly a perceptually-weighted edit distance is more sensible.

#

#

# [2]: Courrieu, Pierre, Fernand Farioli, and Jonathan Grainger. *Inverse Discrimination Time as a Perceptual Distance for Alphabetic Characters*. Visual Cognition 11, no. 7 (October 2004): 901–19. https://doi.org/10.1080/13506280444000049.

# ---

# ## References & Notes

# * Package for `visual_distance`, `printglyph`, etc: VisualStringDistances.jl
# *
 Package with the underlying algorithm optimal transport algorithm: UnbalancedOptimalTransport.jl

#

#

# References:
#

# [1]: Séjourné, T., Feydy, J., Vialard, F.-X., Trouvé, A., Peyré, G., 2019. *Sinkhorn Divergences for Unbalanced Optimal Transport*. https://arxiv.org/abs/1910.12958.
#
# [2]: Courrieu, Pierre, Fernand Farioli, and Jonathan Grainger. *Inverse Discrimination Time as a Perceptual Distance for Alphabetic Characters*. Visual Cognition 11, no. 7 (October 2004): 901–19. https://doi.org/10.1080/13506280444000049.

#

# Slides made with the help of Remark.jl, Literate.jl, and Documenter.jl; gifs made with Makie.jl.

# Thanks to Stefan Karpinski for suggesting GNU Unifont.


# # Intro to AbstractGPs: one-dimensional regression
#
# ## Setup
#
# Loading the necessary packages.

using AbstractGPs
using Distributions
using StatsFuns

using Plots
default(; legend=:outertopright, size=(700, 400))

using Random
Random.seed!(42)  # setting the seed for reproducibility of this notebook
#md nothing #hide

# Load toy regression
# [dataset](https://github.com/GPflow/GPflow/blob/7705cee6723f78066981f27954130daaede55dfc/doc/sphinx/notebooks/basics/data/regression_1D.csv)
# taken from GPflow examples.

x = [
    0.8658165855998895,
    0.6661700880180962,
    0.8049218148148531,
    0.7714303440386239,
    0.14790478354654835,
    0.8666105548197428,
    0.007044577166530286,
    0.026331737288148638,
    0.17188596617099916,
    0.8897812990554013,
    0.24323574561119998,
    0.028590102134105955,
]
y = [
    1.5255314337144372,
    3.6434202968230003,
    3.010885733911661,
    3.774442382979625,
    3.3687639483798324,
    1.5506452040608503,
    3.790447985799683,
    3.8689707574953,
    3.4933565751758713,
    1.4284538820635841,
    3.8715350915692364,
    3.7045949061144983,
]
scatter(x, y; xlabel="x", ylabel="y", legend=false)

# We split the observations into train and test data.

x_train = x[1:8]
y_train = y[1:8]
x_test = x[9:end]
y_test = y[9:end]
#md nothing #hide

# We instantiate a Gaussian process with a Matern kernel. The kernel has
# fixed variance and length scale parameters of default value 1.

f = GP(Matern52Kernel())
#md nothing #hide

# We create a finite dimensional projection at the inputs of the training dataset
# observed under Gaussian noise with variance $\sigma^2 = 0.1$, and compute the
# log-likelihood of the outputs of the training dataset.

fx = f(x_train, 0.1)
logpdf(fx, y_train)

# We compute the posterior Gaussian process given the training data, and calculate the
# log-likelihood of the test dataset.

p_fx = posterior(fx, y_train)
logpdf(p_fx(x_test), y_test)

# We plot the posterior Gaussian process (its mean and a ribbon of 2 standard deviations
# around it) on a grid along with the observations.

scatter(
    x_train,
    y_train;
    xlim=(0, 1),
    xlabel="x",
    ylabel="y",
    title="posterior (default parameters)",
    label="Train Data",
)
scatter!(x_test, y_test; label="Test Data")
plot!(0:0.001:1, p_fx; label=false, ribbon_scale=2)

# ## Markov Chain Monte Carlo
#
# Previously we computed the log likelihood of the untuned kernel parameters of the GP.
# We now also perform approximate inference over said kernel parameters using different
# Markov chain Monte Carlo (MCMC) methods. I.e., we approximate the posterior distribution
# of the kernel parameters with samples from a Markov chain.
#
# We define a function which returns the log-likelihood of the data for different variance
# and inverse lengthscale parameters of the Matern kernel. We ensure that these parameters are
# positive with the softplus function
# ```math
# f(x) = \log (1 + \exp x).
# ```

function gp_loglikelihood(x, y)
    function loglikelihood(params)
        kernel =
            softplus(params[1]) * (Matern52Kernel() ∘ ScaleTransform(softplus(params[2])))
        f = GP(kernel)
        fx = f(x, 0.1)
        return logpdf(fx, y)
    end
    return loglikelihood
end

const loglik_train = gp_loglikelihood(x_train, y_train)
#md nothing #hide

# We define a Gaussian prior for the joint distribution of the two transformed kernel
# parameters. We assume that both parameters are independent with mean 0 and variance 1.

logprior(params) = logpdf(MvNormal(2, 1), params)
#md nothing #hide

# ### Hamiltonian Monte Carlo
#
# We start with a Hamiltonian Monte Carlo (HMC) sampler. More precisely, we use the
# [No-U-Turn sampler (NUTS)](http://www.jmlr.org/papers/volume15/hoffman14a/hoffman14a.pdf),
# which is provided by the Julia packages
# [AdvancedHMC.jl](https://github.com/TuringLang/AdvancedHMC.jl/) and
# [DynamicHMC.jl](https://github.com/tpapp/DynamicHMC.jl/).
#
# #### AdvancedHMC
#
# We start with performing inference with AdvancedHMC.

using AdvancedHMC
using ForwardDiff

# Set the number of samples to draw and warmup iterations.

n_samples = 2_000
n_adapts = 1_000
#md nothing #hide

# Define a Hamiltonian system of the log joint probability.

logjoint_train(params) = loglik_train(params) + logprior(params)
metric = DiagEuclideanMetric(2)
hamiltonian = Hamiltonian(metric, logjoint_train, ForwardDiff)
#md nothing #hide

# Define a leapfrog solver, with initial step size chosen heuristically.

initial_params = rand(2)
initial_ϵ = find_good_stepsize(hamiltonian, initial_params)
integrator = Leapfrog(initial_ϵ)
#md nothing #hide

# Define an HMC sampler, with the following components:
# - multinomial sampling scheme,
# - generalised No-U-Turn criteria, and
# - windowed adaption for step-size and diagonal mass matrix

proposal = NUTS{MultinomialTS,GeneralisedNoUTurn}(integrator)
adaptor = StanHMCAdaptor(MassMatrixAdaptor(metric), StepSizeAdaptor(0.8, integrator))
#md nothing #hide

# We draw samples from the posterior distribution of kernel parameters. These samples
# are in the unconstrained space $\mathbb{R}^2$.

samples, _ = sample(
    hamiltonian, proposal, initial_params, n_samples, adaptor, n_adapts; progress=false
)
#md nothing #hide

# We transform the samples back to the constrained space and compute the mean of both
# parameters:

samples_constrained = [map(softplus, p) for p in samples]
mean_samples = mean(samples_constrained)

# We plot a histogram of the samples for the two parameters.
# The vertical line in each graph indicates the mean of the samples.

histogram(
    reduce(hcat, samples_constrained)';
    xlabel="sample",
    ylabel="counts",
    layout=2,
    title=["variance" "inverse length scale"],
    legend=false,
)
vline!(mean_samples'; linewidth=2)

# We approximate the log-likelihood of the test data using the posterior Gaussian processes
# for kernels with the sampled kernel parameters. We can observe that there is a significant
# improvement over the log-likelihood of the test data with respect to the posterior
# Gaussian process with default kernel parameters of value 1.

function gp_posterior(x, y, p)
    kernel = softplus(p[1]) * (Matern52Kernel() ∘ ScaleTransform(softplus(p[2])))
    f = GP(kernel)
    return posterior(f(x, 0.1), y)
end

mean(logpdf(gp_posterior(x_train, y_train, p)(x_test), y_test) for p in samples)

# We sample 5 functions from each posterior GP given by the final 100 samples of kernel
# parameters.

plt = plot(; xlim=(0, 1), xlabel="x", ylabel="y", title="posterior (AdvancedHMC)")
for (i, p) in enumerate(samples[(end - 100):end])
    sampleplot!(
        plt,
        0:0.02:1,
        gp_posterior(x_train, y_train, p);
        samples=5,
        seriescolor="red",
        label=(i == 1 ? "samples" : nothing),
    )
end
scatter!(plt, x_train, y_train; label="Train Data", markercolor=1)
scatter!(plt, x_test, y_test; label="Test Data", markercolor=2)
plt

# #### DynamicHMC
#
# We repeat the inference with DynamicHMC. DynamicHMC requires us to
# implement the LogDensityProblems interface for `loglik_train`.

using DynamicHMC
using LogDensityProblems

## Log joint density
function LogDensityProblems.logdensity(ℓ::typeof(loglik_train), params)
    return ℓ(params) + logprior(params)
end

## The parameter space is two-dimensional
LogDensityProblems.dimension(::typeof(loglik_train)) = 2

## `loglik_train` does not allow to evaluate derivatives of
## the log-likelihood function
function LogDensityProblems.capabilities(::Type{<:typeof(loglik_train)})
    return LogDensityProblems.LogDensityOrder{0}()
end

# Now we can draw samples from the posterior distribution of kernel parameters with
# DynamicHMC. Again we use [ForwardDiff.jl](https://github.com/JuliaDiff/ForwardDiff.jl)
# to compute the derivatives of the log joint density with automatic differentiation.

samples =
    mcmc_with_warmup(
        Random.GLOBAL_RNG,
        ADgradient(:ForwardDiff, loglik_train),
        n_samples;
        reporter=NoProgressReport(),
    ).chain
#md nothing #hide

# We transform the samples back to the constrained space and compute the mean of both
# parameters:

samples_constrained = [map(softplus, p) for p in samples]
mean_samples = mean(samples_constrained)

# We plot a histogram of the samples for the two parameters.
# The vertical line in each graph indicates the mean of the samples.

histogram(
    reduce(hcat, samples_constrained)';
    xlabel="sample",
    ylabel="counts",
    layout=2,
    title=["variance" "inverse length scale"],
    legend=false,
)
vline!(mean_samples'; linewidth=2)

# Again we can observe that there is a significant improvement over the log-likelihood
# of the test data with respect to the posterior Gaussian process with default kernel
# parameters.

mean(logpdf(gp_posterior(x_train, y_train, p)(x_test), y_test) for p in samples)

# We sample a function from the posterior GP for the final 100 samples of kernel
# parameters.

plt = plot(; xlim=(0, 1), xlabel="x", ylabel="y", title="posterior (DynamicHMC)")
scatter!(plt, x_train, y_train; label="Train Data")
scatter!(plt, x_test, y_test; label="Test Data")
for p in samples[(end - 100):end]
    sampleplot!(plt, 0:0.02:1, gp_posterior(x_train, y_train, p); seriescolor="red")
end
plt

# ### Elliptical slice sampling
#
# Instead of HMC, we use
# [elliptical slice sampling](http://proceedings.mlr.press/v9/murray10a/murray10a.pdf)
# which is provided by the Julia package
# [EllipticalSliceSampling.jl](https://github.com/TuringLang/EllipticalSliceSampling.jl/).

using EllipticalSliceSampling

# We draw 2000 samples from the posterior distribution of kernel parameters.

samples = sample(ESSModel(
    MvNormal(2, 1), # Gaussian prior
    loglik_train,
), ESS(), n_samples; progress=false)
#md nothing #hide

# We transform the samples back to the constrained space and compute the mean of both
# parameters:

samples_constrained = [map(softplus, p) for p in samples]
mean_samples = mean(samples_constrained)

# We plot a histogram of the samples for the two parameters.
# The vertical line in each graph indicates the mean of the samples.

histogram(
    reduce(hcat, samples_constrained)';
    xlabel="sample",
    ylabel="counts",
    layout=2,
    title=["variance" "inverse length scale"],
)
vline!(mean_samples'; layout=2, labels="mean")

# Again we can observe that there is a significant improvement over the log-likelihood
# of the test data with respect to the posterior Gaussian process with default kernel
# parameters.

mean(logpdf(gp_posterior(x_train, y_train, p)(x_test), y_test) for p in samples)

# We sample a function from the posterior GP for the final 100 samples of kernel
# parameters.

plt = plot(;
    xlim=(0, 1), xlabel="x", ylabel="y", title="posterior (EllipticalSliceSampling)"
)
scatter!(plt, x_train, y_train; label="Train Data")
scatter!(plt, x_test, y_test; label="Test Data")
for p in samples[(end - 100):end]
    sampleplot!(plt, 0:0.02:1, gp_posterior(x_train, y_train, p); seriescolor="red")
end
plt

# ## Variational Inference
#
# Sanity check for the Evidence Lower BOund (ELBO) implemented according to
# M. K. Titsias's _Variational learning of inducing variables in sparse Gaussian processes_.

elbo(VFE(f(rand(5))), fx, y_train)

# We use the LBFGS algorithm to maximize the given ELBO. It is provided by the Julia
# package [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl).

using Optim

# We define a function which returns the negative ELBO for different variance and inverse
# lengthscale parameters of the Matern kernel and different pseudo-points. We ensure that
# the kernel parameters are positive with the softplus function
# ```math
# f(x) = \log (1 + \exp x),
# ```
# and that the pseudo-points are in the unit interval $[0,1]$ with the logistic function
# ```math
# f(x) = \frac{1}{1 + \exp{(-x)}}.
# ```

jitter = 1e-6  # "observing" the latent process with some (small) amount of jitter improves numerical stability

function objective_function(x, y)
    function negative_elbo(params)
        kernel =
            softplus(params[1]) * (Matern52Kernel() ∘ ScaleTransform(softplus(params[2])))
        f = GP(kernel)
        fx = f(x, 0.1)
        z = logistic.(params[3:end])
        approx = VFE(f(z, jitter))
        return -elbo(approx, fx, y)
    end
    return negative_elbo
end
#md nothing #hide

# We randomly initialize the kernel parameters and 5 pseudo points, and minimize the
# negative ELBO with the LBFGS algorithm and obtain the following optimal parameters:

x0 = rand(7)
opt = optimize(objective_function(x_train, y_train), x0, LBFGS())

#-

opt.minimizer

# The optimized value of the variance is

softplus(opt.minimizer[1])

# and of the inverse lengthscale is

softplus(opt.minimizer[2])

# We compute the log-likelihood of the test data for the resulting approximate
# posterior. We can observe that there is a significant improvement over the
# log-likelihood with the default kernel parameters of value 1.

opt_kernel =
    softplus(opt.minimizer[1]) *
    (Matern52Kernel() ∘ ScaleTransform(softplus(opt.minimizer[2])))
opt_f = GP(opt_kernel)
opt_fx = opt_f(x_train, 0.1)
ap = posterior(VFE(opt_f(logistic.(opt.minimizer[3:end]), jitter)), opt_fx, y_train)
logpdf(ap(x_test), y_test)

# We visualize the approximate posterior with optimized parameters.

scatter(
    x_train,
    y_train;
    xlim=(0, 1),
    xlabel="x",
    ylabel="y",
    title="posterior (VI with sparse grid)",
    label="Train Data",
)
scatter!(x_test, y_test; label="Test Data")
plot!(0:0.001:1, ap; label=false, ribbon_scale=2)
vline!(logistic.(opt.minimizer[3:end]); label="Pseudo-points")

# ## Exact Gaussian Process Inference
#
# Here we use Type-II MLE to train the hyperparameters of the Gaussian process.
# This means that our loss function is the negative log marginal likelihood.

# We re-calculate the log-likelihood of the test dataset with the
# default kernel parameters of value 1 for the sake of comparison.

logpdf(p_fx(x_test), y_test)

# We define a function which returns the negative log marginal
# likelihood for different variance and inverse lengthscale parameters
# of the Matern kernel and different pseudo-points. We ensure that the
# kernel parameters are positive with the softplus function
# ``f(x) = \log (1 + \exp x)``.

function loss_function(x, y)
    function negativelogmarginallikelihood(params)
        kernel =
            softplus(params[1]) * (Matern52Kernel() ∘ ScaleTransform(softplus(params[2])))
        f = GP(kernel)
        fx = f(x, 0.1)
        return -logpdf(fx, y)
    end
    return negativelogmarginallikelihood
end

#md nothing #hide

# We randomly initialize the kernel parameters, and minimize the
# negative log marginal likelihood with the LBFGS algorithm
# and obtain the following optimal parameters:

θ0 = randn(2)
opt = Optim.optimize(loss_function(x_train, y_train), θ0, LBFGS())

#-

opt.minimizer

# The optimized value of the variance is

softplus(opt.minimizer[1])

# and of the inverse lengthscale is

softplus(opt.minimizer[2])

# We compute the log-likelihood of the test data for the resulting optimized
# posterior. We can observe that there is a significant improvement over the
# log-likelihood with the default kernel parameters of value 1.

opt_kernel =
    softplus(opt.minimizer[1]) *
    (Matern52Kernel() ∘ ScaleTransform(softplus(opt.minimizer[2])))

opt_f = GP(opt_kernel)
opt_fx = opt_f(x_train, 0.1)
opt_p_fx = posterior(opt_fx, y_train)
logpdf(opt_p_fx(x_test), y_test)

# We visualize the posterior with optimized parameters.

scatter(
    x_train,
    y_train;
    xlim=(0, 1),
    xlabel="x",
    ylabel="y",
    title="posterior (optimized parameters)",
    label="Train Data",
)
scatter!(x_test, y_test; label="Test Data")
plot!(0:0.001:1, opt_p_fx; label=false, ribbon_scale=2)


#2-dim CAs

#Two dimensional Cellular Automaton
#TODO: Change naming to be consistent
mutable struct CA2d

    #User given values
    k::Int #Number of states
    r::Int #r-nearest neigbors

    #Internal values
    cells::Array{Int8, 3}

    function CA2d(B::Array{Int,1},
                  S::Array{Int,1},
                  init::Array{Int,2},
                  gen::Int,
                  k::Int=2,
                  r::Int=1)

        h, w = size(init)

        cells = Array{Int8}(undef, (h, w, gen)) #Syntax A(T, dims) is deprecated
        cells[:, :, 1] = Array{Int8}(init[:, :])

        for g = 2:gen
            for i = 1:h, j = 1:w
                cc = -cells[i, j, g-1]
                for p = (i-r):(i+r), q = (j-r):(j+r)

                    #Cyclic boundary conditions
                    if p < 1; p = h-p; end
                    if p > h; p = p-h; end
                    if q < 1; q = w-q; end
                    if q > w; q = q-w; end

                    cc += cells[p, q, g-1]
                end
                cells[i, j, g] = eval_rule(cc, cells[i, j, g-1], B, S)
            end #hw ij
        end #gen

        new(k, r, cells)
    end
end

#Evaluate life rules
#TODO: make this more general
function eval_rule(cc, olds, B, S, k=2)
    if cc in B || (olds == 1 && cc in S)
        return 1
    end

    return 0
end


#=
This script shows how a simple MLP net may be used
for regression. It shows how data in memory may be
used for training and evaluation, and how to obtain
the predictions from the trained net.
=#
using MXNet
using Distributions
using PyPlot

# data generating process
generate_inputs(mean, var, size) = rand(MvNormal(mean, var), size)
output(data) = sin(data[1,:]).*sin(data[2,:])./(data[1,:].*data[2,:])

# create training and evaluation data sets
mean=[0.0;0.0]
var=[1.0 0.0;0.0 1.0]
samplesize  = 5000
TrainInput = generate_inputs(mean, var, samplesize)
TrainOutput = output(TrainInput)
ValidationInput = generate_inputs(mean, var, samplesize)
ValidationOutput = output(ValidationInput)

# how to set up data providers using data in memory
batchsize = 100 # can adjust this later, but must be defined now for next line
trainprovider = mx.ArrayDataProvider(:data => TrainInput, batch_size=batchsize, shuffle=true, :label => TrainOutput)
evalprovider = mx.ArrayDataProvider(:data => ValidationInput, batch_size=batchsize, shuffle=true, :label => ValidationOutput)

# create a two hidden layer MPL: try varying num_hidden, and change tanh to relu,
# or add/remove a layer
data = mx.Variable(:data)
label = mx.Variable(:label)
net  = @mx.chain    mx.FullyConnected(data = data, num_hidden=10) =>
                    mx.Activation(act_type=:tanh) =>
                    mx.FullyConnected(num_hidden=3) =>
                    mx.Activation(act_type=:tanh) =>
                    mx.FullyConnected(num_hidden=1)

# squared error loss is appropriate for regression, don't change
cost = mx.LinearRegressionOutput(data = net, label=label)

# final model definition, don't change, except if using gpu
model = mx.FeedForward(cost, context=mx.cpu())

# set up the optimizer: select one, explore parameters, if desired
#optimizer = mx.SGD(lr=0.01, momentum=0.9, weight_decay=0.00001)
optimizer = mx.ADAM()

# train, reporting loss for training and evaluation sets
# initial training with small batch size, to get to a good neighborhood
batchsize = 100
mx.fit(model, optimizer, initializer=mx.NormalInitializer(0.0,0.1), eval_metric=mx.MSE(), trainprovider, eval_data=evalprovider, n_epoch = 500)
# more training with the full sample
batchsize = samplesize
mx.fit(model, optimizer, eval_metric=mx.MSE(), trainprovider, eval_data=evalprovider, n_epoch = 500)

# obtain predictions
plotprovider = mx.ArrayDataProvider(:data => ValidationInput, :label => ValidationOutput)
fit = mx.predict(model, plotprovider)
plot(ValidationOutput,fit',".")
xlabel("true")
ylabel("predicted")
title("outputs: true versus predicted. 45º line is what we hope for")


function mult(M, V) #funkcja mnozaca macierze M V
    R = zeros(size(M,1), size(V,2))
    for i = 1:size(M,1)
        for j = 1:size(V,2)
            for k = 1:size(M,1)
                R[i,j] += M[i,k] * V[k,j]
            end
        end
    end
    return R
end

function strassen(A, B, size, stop) #algorytm Strassena, mnozy macierze A, B o rozmiarze size
   if size <= stop
      return mult(A,B)
   end
   half = div(size,2)
   a11 = A[1:half, 1:half]
   a12 = A[1:half, half+1:size]
   a21 = A[half+1:size, 1:half]
   a22 = A[half+1:size, half+1:size]

   b11 = B[1:half, 1:half]
   b12 = B[1:half, half+1:size]
   b21 = B[half+1:size, 1:half]
   b22 = B[half+1:size, half+1:size]

   m1 = strassen(a11 + a22, b11 + b22, half, stop)
   m2 = strassen(a21 + a22, b11, half, stop)
   m3 = strassen(a11, b12 - b22, half, stop)
   m4 = strassen(a22, b21 - b11, half, stop)
   m5 = strassen(a11 + a12, b22, half, stop)
   m6 = strassen(a21 - a11, b11 + b12, half, stop)
   m7 = strassen(a12 - a22, b21 + b22, half, stop)

   c11 = m1 + m4 - m5 + m7
   c12 = m3 + m5
   c21 = m2 + m4
   c22 = m1 - m2 + m3 + m6

   res = vcat(hcat(c11, c12), hcat(c21, c22))
   return res
end
function s_mult(A,B,stop)
   edge1 = size(A, 1)
   edge2 = size(A, 2)
   pow2 = 2^Integer(ceil(max(log(2, edge2),
                             log(2, edge1))))
   if edge1 < pow2 || edge2 < pow2
      zero_ = zeros(pow2-edge1, edge2)
      A = vcat(A, zero_)
      zero_ = zeros(size(A, 1), pow2-edge2)
      A = hcat(A, zero_)
      zero_ = zeros(edge2, pow2-edge1)
      B = hcat(B, zero_)
      zero_ = zeros(pow2-edge2, size(B,2))
      B = vcat(B, zero_)
   end

   res = strassen(A,B, pow2, stop)
   return res[1:edge1, 1:edge1]
end

function err(M)
   r = size(M, 1)
   c = size(M, 2)
   sum = Float64(0)
   for i in 1:r
      for j in 1:c
         k = M[r,c]
         sum += k*k
      end
   end
    return sum
end

function bestStrass(A, B)
   return s_mult(A, B, 64)
end

function compare()
    M_size = 64
    for i in 1:10
        A = rand(-100.0:0.001:100.0, M_size, M_size)
        B = rand(-100.0:0.001:100.0, M_size, M_size)
        C = rand(-100.0:0.001:100.0, M_size, M_size)
        S_result = test_str(A,B,C, 64)
        N_result = test_normal(A,B,C)
        println(err(S_result))
        println(err(N_result))
        println("Blad wzgledny:", abs(err(S_result) - err(N_result)) / err(N_result) )
    end
end
#compare()
function memory_usage()
    M_size = 512
    A = rand(-100.0:0.001:100.0, M_size, M_size)
    B = rand(-100.0:0.001:100.0, M_size, M_size)
    @timev mult(A,B)
    @timev s_mult(A,B, 64)
end
#memory_usage()

II rok/I semestr/ANM/Pracownia 1/prog/program.jl


# This code generates the 2l+1 dimensional matrix Fourier coefficients
# for a function defined over the rotation and rotation/reflection groups
#
# The routines in this file are based upon:
#
#   FFTs on the Rotation Group
#   Peter J. Kostelec and Daniel N. Rockmore
#   J. Fourier. Anal. Appl., Vol. 14, Issue. 2, p. 145-179, 2008

import Base.fft

function w(B::Int, k::Int)
    s = 0.0

    for j in 0:B-1
        s += sin((2j + 1) * (2k + 1) * (π / (4B))) / (2j + 1)
    end

    s *= (2/B) * sin(π * (2k + 1) / (4B))
end

function S₁(f::Function, B::Int, k::Int, j₂::Int)
    β = π * (2k + 1) / (4B)
    γ = (2π * j₂) / (2B)

    A = [exp(-im * ((2π * j₁) / (2B)) * (B - 1)) * f(O3((2π * j₁) / (2B), β, γ)) for j₁ in 0:2B-1]
    A′ = ifft(A)
end

function S₂(f::Function, B::Int, k::Int)
    S = [S₁(f, B, k, j₂) for j₂ in 0:2B-1]
    A = Vector{Vector{Complex128}}(2B)
    A′ = Vector{Vector{Complex128}}(2B)

    for M′ in -(B-1):B
        A[M′ + B] = [exp(-im * ((2π * j₂) / (2B)) * (B - 1)) * S[j₂ + 1][M′ + B] for j₂ in 0:2B-1]
        A′[M′ + B] = ifft(A[M′ + B])
    end

    return hcat(A′...)
end

function fast_fourier_coeffs(f::Function, B::Int)
    S = [S₂(f, B, k) for k in 0:2B-1]
    f̂ = [zeros(Complex128, 2l + 1, 2l + 1) for l in 0:B-1]

    for l in 0:B-1
        for M in -l:l, M′ in -l:l
            for k in 0:2B-1
                β = π * (2k + 1) / (4B)
                f̂[l + 1][M′ + l + 1, M + l + 1] += wigner_d̃(l, M, M′, β) * w(B, k) * S[k + 1][M′ + B, M + B] / sqrt(2 / (2l + 1))
            end
        end
    end

    return f̂
end

fft(f::Function, ::Type{O3}, L::Int) = fast_fourier_coeffs(f, L)


# Algorithm 3.2
# Newton's interpolatory divided difference.

function dividedDiff(x, y)
	n = length(x)
	out = zeros(n, n)
	for i ∈ 1:n
		out[i, 1] = y[i]
	end
	for i ∈ 2:n
		for j ∈ 2:i
			out[i, j] = (out[i, j - 1] - out[i - 1, j - 1]) / (x[i] - x[i - j + 1])
		end
	end
	return out
end

println(dividedDiff([1, 1.3, 1.6, 1.9, 2.2], [-0.76, -0.54, -0.57, -0.57, 0.3]))


# Source: https://julialang.org/blog/2016/02/iteration/#writing_multidimensional_algorithms_with_cartesianindex_iterators
# compute the "moving average" over a 3-by-3-by-... block around each element
# Origin code:
function boxcar3(A::AbstractArray)
    out = similar(A)
    R = CartesianIndices(A)
    Ifirst, Ilast = first(R), last(R)
    I1 = oneunit(Ifirst)
    for I in R
        n, s = 0, zero(eltype(out))
        for J in max(Ifirst, I-I1):min(Ilast, I+I1)
            s += A[J]
            n += 1
        end
        out[I] = s/n
    end
    out
end
#--------------------------------------------------------------
"""
    move_average(A, N)

compute the moving average over a (2N+1)-by-(2N+1)-by-... block around each element
"""
function move_average(A::AbstractArray, N::Integer)
    out = similar(A)
    R = CartesianIndices(A)
    Ifirst, Ilast = first(R), last(R)
    I1 = oneunit(Ifirst)
    for I in R
        n, s = 0, zero(eltype(out))
        for J in max(Ifirst, I-N*I1):min(Ilast, I+N*I1)
            s += A[J]
            n += 1
        end
        out[I] = s/n
    end
    out
end


# PAU 2016-Modelo A-Pregunta 3

# Física Enrique García @FiQuiPedia www.fiquipedia.es
# Código por Samuel Gómez @profesor_s www.ingetec.com.es
# Licencia Creative Commons
# http://creativecommons.org/licenses/by/4.0/deed.es_ES

# Una carga puntual, q = 3 μC, se encuentra situada en
# el origen de coordenadas, tal y como se muestra en la figura. Una
# segunda carga q1 = 1 μC se encuentra inicialmente en el punto P1(1,0)
# m y, recorriendo la espiral de la figura, llega al punto P2(0,2) m.
# Determine:
# a. La diferencia de potencial entre los puntos P1 y P2.
# b. El trabajo realizado para llevar la carga q1 del punto P1 al P2.
# Datos: Constante de la Ley de Coulomb; K = 9·109 N m2 C-2

using Distances     # https://github.com/JuliaStats/Distances.jl

const k = 9.00e9            # N m^2 C^-2

type carga
    punto::Vector{Float64}  # (x,y). Metros
    carga::Float16          # Culombios
end

q = carga([0,0], 3e-6)
q1 = carga([1,0], 1e-6)
destino = [0, 2]

# Potencial V
function potencial(q, r)
    return (k*q)/r
end

# Distancia entre dos puntos
function dist(a, b)
    return evaluate(Euclidean(), a, b)
end

### Resolución

# a. La diferencia de potencial entre los puntos P1 y P2.
v1 = potencial(q.carga, dist(q.punto, q1.punto))
v2 = potencial(q.carga, dist(q.punto, destino))
v = v2 - v1
println("a) La diferencia de potencial es ",round(v,2)," voltios")

# b. El trabajo realizado para llevar la carga q1 del punto P1 al P2.
w = -q1.carga * (v2-v1)
println("b) El trabajo realizado es ",round(w, 4), " julios")