# (c) David Barber, University College London 2015
#= How this works:

GENERAL AUTODIFF THEORY:

According to the general autodiff theory (see my online notes http://web4.cs.ucl.ac.uk/staff/D.Barber/publications/ParameterTying.pdf) the total derivative t[n] at a node n in the graph is related to the children c of node n by
 \sum_c t[c] df[c]/df[n]

or, in the notation used in the code

grad_n = \sum_c grad_c* df[c]/df[n]

RETURNING A TOTAL DERIVATIVE FUNCTION:

Consider a node that computes h=sigma(W*x).  Since the output of this node h is a vector, we need to calculate all the derivatives from these elements of this child h to its parents W and x. For example,

dh[i]/dW[a,b] = dsigma(W[a,:]'*x)*x[b]*delta[i,a]

where delta[i,a] is the Kronecker delta function and dsigma is the derivative of the transfer function sigma. However, when we use this as part of the reverse recursion, the Kronecker delta gives a simplification, namely

\sum_i grad_c[i]dh[i]/dW[a,b] = grad_c[a]dsigma(W[a,:]'*x)*x[b]

What this means is that we can exploit the mathematical simplication that results from the Kronecker delta and return instead a derivative as a function of the child total derivative grad_c. For example, for the logistic sigmoid, dsigma=sigma*(1-sigma) and we use

(grad_c.*f_c.*(1.-f_c))*x'   for the derivative wrt W
W'*(grad_c.*f_c.*(1.-f_c))   for the derivative wrt x

Here f_c is the value of the child node (which is sigma(W*x).

The derivative can then be returned directly as a function of the node value, the parent node values and the child node value -- no additional storage is required. This is more efficient than defining two nodes, first h=W*x, then sigma(h); here we would have to store the value h and also sigma(h), which will cost roughly twice as much in terms of storage.

REUSING INFORMATION BY STORING AUXILIARY VALUES:

Some computational savings can be made in the reverse pass by reusing results computed during the forward pass.

For example, for the logistic sigmoid, we calculate f=sigma(W*h) on the forward pass. The derivative pass requires us to calculate dsigma(W*x) but since this is equal to sigma(W*h)*(1-sigma(W*h)) this is just a simple function of the forward pass result, namely f.*(1-f) so that we can simply reuse the forward calculation values, without having to compute the sigmoid function again in the reverse pass -- this saves on computation.

The code therefore allows during the forward pass to store auxilliary information at a node that might be useful for speeding up the computation in the backward pass. The deriviate can then be a function of the forward pass value (f_c) or the auxilliary information (faux_c) stored at the forward pass node.

Note that the functions all have two versions -- a standard one which returns a value, and an inplace version that updates an existing value. (The CUDA versions only require the inplace routines).  The standard one is only required during compilation. All subsequent uses of the function are inplace.

The gradients are defined to update preexisting gradients -- inplace. It's also important to bear in mind that (at each backward pass, after iniitalisating gradients to zero) we must add gradients at a node. An explanation for this is that we can have situations in which a variable x has two directed paths to a variable f. For example, f=sum([x x]).

For future, it would be interesting to map any function f and reduce it with sum

function Fsumf(f,x...)
    return sum(f(collect(x)))
end
Then
Dsumf(derivativeIDX,s,a,t,grad,reset,D,f,x...)=t.*D[f](x[derivativeIDX])
We could even then find the symbolic derivative df of f. We wouldn't want to have to do this each time we call the function though.
We could I think do this with a Dict.
    D=Dict()
    D["x^2"]=(x)->2x
    We can then evaluate using
    D["x^2"](3.2)
    We can then make the Dict and the function f both nodes in the graph
    For the GPU though we would need to have compiled kernels for each function f and its associated derivative df
=#

# --------------------------------------------------------------------------------------
# Functions and their Derivatives:

# functions are defined as F(x)=(self,aux). Here self is the function value and aux is the auxiliary value (it can be empty []) that might be useful to speed up the return pass calculation.

Derivative=Dict() # mappings of functions to their derivatives
Inplace=Dict() # Inplace version of function

#TODO:
# sqrt, power fn, trig functions,
# diagm(A)+
# diag(A) for both CPU and GPU (note that diagm(A) is implemented for both GPU and CPU)
# A/s where s is a 1x1 array and A is an array
# A.*B', A'.*B, A'.*B' for CPU and GPU; currently these are computed using A.*trans(B), etc which is wasteful
# A./B', A'./B, A'./B' for CPU and GPU
# array subindexing (see old code at end) Not sure if this is really necessary
# The KL loss methods are expensive if one only wants to maximise the corresponding likelihood. Could make versions with a flag to drop the entropy term
# Some of the GPU demos are very slow -- DemoLSTMsimple, for example. Need to investigate why.

f=[
   "Fnx", "Ftanh",
   "Fstanh", "Fsigmoid", "Frectlin",
   "FAhadamardprodB",    "FAhadamarddivB",
   "Fsum", "Fmean",
   "FsumSquare",    "FmeanSquare",
   "FAX", "FAXplusBias",
   "Fxpy", "Fxmy",
   "FmeanSquareLoss",
   "FBinaryKullbackLeiblerLoss", "FBinaryKullbackLeiblerLossXsigmoidY",
   "FsigmoidAXplusBias",
   "FrectlinAXplusBias",
   "FstanhAXplusBias",
   "Fsoftmax",   "FKLsoftmax",
   "Fexp", "Flog",
   "Ftranspose",
   "FAtransposeX",
   "FAXtranspose",
   "FAtransposeXtranspose",
   "FXtransposePY",
   "FXPYtranspose",
   "FXtransposePYtranspose",
   "FXtransposeMY",
   "FXMYtranspose",
   "FXtransposeMYtranspose",
   "FmeanAbs","Fabs",
   "Fvcat", "Fhcat",
   "Fdiagm", "FdiagAmultX", "FAmultdiagX",
   "FabsAXplusBias",
   "Felu",
   "Fabs",
   "Fkinklin",
   "FkinklinAXplusBias",
   "Falex",
   "Fgetcol",
   "Fgetrow"
   ]

for fn in f
    include("functions/"*fn*".jl");  println(fn);
end

include("TrainingAlgorithms.jl")

## Negative function: f(x)=-x
#Fnegative(x)=(-x,[])
#Dnegative=Array(Function,1)
#Dnegative[1]=dnegative1(x,self,aux,t)=-t.*ones(size(x))
#Dnegative[1]=dnegative1(x::Float32,self,aux,t)=-t
#Derivative[Fnegative]=Dnegative # Define dictionary lookup
#export Fnegative
#export dnegative1 # need for source code execution
#ADnegative(n)=ADnode(Fnegative,n)
#export ADnegative

## KL Loss: f(p,q)=KL(p,q)
#FKLLoss(p::Array{Float32,2},q::Array{Float32,2})=begin DN=prod(size(p));(sum(p.*(log(p)-log(q)))/DN,DN); end
#DKLLoss=Array(Function,2)
#DKLLoss[1]=DKLLoss1(p::Array{Float32,2},q::Array{Float32,2},self,aux,t)=t.*(1+log(p)-log(q))/aux
#DKLLoss[2]=DKLLoss2(p::Array{Float32,2},q::Array{Float32,2},self,aux,t)=t.*(p./q)./aux
#Derivative[FKLLoss]=DKLLoss
#export FKLLoss,DKLLoss
#export DKLLoss1,DKLLoss2 # need for source code execution
#ADKLLoss(np,nq)=ADnode(FKLLoss,[np nq])
#export ADKLLoss
#
# Multinomial Logistic Loss: -sum(log(p_c)) where p\propto exp(x) and c is a bit array of the same size as x
#FMultLogisticLoss(c::BitArray{2},x::Array{Float32,2})=begin DN=prod(size(x));logZ=logsumexp(x);aux=(logZ,DN); return (#(-sum(x[c])+sum(logZ))/DN,aux); end
#DMultLogisticLoss=Array(Function,2)
#DMultLogisticLoss[1]=DMultLogisticLoss1(c::BitArray{2},x::Array{Float32,2},self,aux,t)=nan # not needed
#DMultLogisticLoss[2]=DMultLogisticLoss2(c::BitArray{2},x::Array{Float32,2},self,aux,t)=begin p=zeros(size(c)); p[c]=1.#; return t.*(exp(x.-aux[1])-p)./aux[2]; end
#Derivative[FMultLogisticLoss]=DMultLogisticLoss
#export FMultLogisticLoss,DMultLogisticLoss
#export DMultLogisticLoss1,DMultLogisticLoss2 # need for source code execution
#ADMultLogisticLoss(nc,nx)=ADnode(FMultLogisticLoss,[nc nx])
#export ADMultLogisticLoss
#
##Logistic Loss: -sum(log(sigma(c.*x)))/prod(size(x)) c[i] is +1 or -1 class variable
#FLogisticLoss(c::Array{Float32,2},x::Array{Float32,2})=begin DN=prod(size(x)); aux=(1./(1.+exp(-c.*x)),DN); return (-s#um(log(aux[1]))/DN,aux); end
#DLogisticLoss=Array(Function,2)
#DLogisticLoss[1]=DLogisticLoss1(c::Array{Float32,2},x::Array{Float32,2},self,aux,t)=-t.*(1.-aux[1]).*x/aux[2]
#DLogisticLoss[2]=DLogisticLoss2(c::Array{Float32,2},x::Array{Float32,2},self,aux,t)=-t.*(1.-aux[1]).*c/aux[2]
#
#FLogisticLoss(c::Array{Float32},x::Array{Float32})=begin DN=prod(size(x)); aux=(1./(1.+exp(-c.*x)),DN); return (-sum(l#og(aux[1]))/DN,aux); end
#DLogisticLoss=Array(Function,2)
#DLogisticLoss[1]=DLogisticLoss1(c::Array{Float32,1},x::Array{Float32,2},self,aux,t)=-t.*(1.-aux[1]).*x/aux[2]
#DLogisticLoss[2]=DLogisticLoss2(c::Array{Float32,1},x::Array{Float32,2},self,aux,t)=-t.*(1.-aux[1]).*c/aux[2]
#Derivative[FLogisticLoss]=DLogisticLoss
#export FLogisticLoss,DLogisticLoss
#export DLogisticLoss1,DLogisticLoss2 # need for source code execution
#ADLogisticLoss(nc,nx)=ADnode(FLogisticLoss,[nc nx])
#export ADLogisticLoss

## Array subindexing:
#Fgetcol(x,col)=begin if col>0 tmp=zeros(size(x,1),1); tmp[:]=x[:,col]; return (tmp, zeros(size(x))) else retur#n (zeros(size(x,1)), zeros(size(x))); end; end
#Dgetcol=Array(Function,2)
#Dgetcol[1]=dgetcol1(x,col,self,aux,t)=begin if col>0 aux[:,col]=t; return aux; else return zeros(size(x)); end#; end
#Dgetcol[2]=dgetcol2(x,col,self,aux,t)=[] # can't differentiate wrt col index
#Derivative[Fgetcol]=Dgetcol
#export Fgetcol
#export dgetcol1,dgetcol2 # need for source code execution
#ADgetcol(n,col)=ADnode(Fgetcol,[n col])
#export ADgetcol

#    md=CuModule("gaxpy2.ptx",false)
#    gaxpy2=CuFunction(md,"gaxpy2")
#    function gaxpy2!(alpha::CudaArray,B::CudaArray,C::CudaArray)
#        launch(gaxpy2,size(B,1),size(B,2),(alpha,B,C))
#    end
#    export gaxpy2!

#    md=CuModule("gaxpy3.ptx",false)
#    gaxpy3=CuFunction(md,"gaxpy3")
#    function gaxpy3!(alpha::CudaArray,B::CudaArray,C::CudaArray)
#        launch(gaxpy3,(size(B,1),size(B,2)),1,(length(B),alpha,B,C))
#    end
#    export gaxpy3!

#    md=CuModule("gaxpy4.ptx",false)
#    gaxpy4=CuFunction(md,"gaxpy4")
#    function gaxpy4!(alpha::CudaArray,B::CudaArray,C::CudaArray)
#        launch(gaxpy4,(size(B,1),size(B,2)),1024,(length(B),alpha,B,C))
#    end
#    export gaxpy4!
#    export gaxpy4

#    md=CuModule("serialsum.ptx",false)
#    serialsumkernel=CuFunction(md,"serialsum")
#    function serialsum(A::CudaArray)
#        gy=CudaArray(ones(1,1))
#        launch(serialsumkernel,1,1,(length(A),A,gy))
#        return gy
#    end
#    export serialsum


# Question 1: Modifisere binærsøk
## Binærsøk er en enkel, men viktig algoritme med mange bruksområder. Les om den i appendiks C («Binærsøk») i pensumheftet.
## Under vises julia-koden for funksjonen bisect_left som er en variant av binærsøk. Denne har noen forskjeller sammenliknet med Bisect-algoritmen fra pensumheftet. Dersom det finnes flere forekomster av v i tabellen, returneres en indeks slik at om v settes inn på denne indeksen, havner elementet direkte til venstre for de andre forekomstene. Dersom v ikke finnes i A er indeksen som returneres slik at v kan settes inn på denne indeksen i sortert rekkefølge.

function bisect_left(A, p, r, v)
    i = p
    if p < r
       q = floor(Int, (p+r)/2)  # floor() er en innebygd funksjon som runder ned. ceil() runder opp.
       if v <= A[q]
           i = bisect_left(A, p, q, v)
       else
           i = bisect_left(A, q+1, r, v)
       end
    end
    return i
end

#Implementer funksjonen => bisect_right(A, p, r, v)

function bisect_right(A, p, r, v)
    i = p
    if p < r
        q = floor(Int, (p+r)/2)
        if v < A[q]
            i = bisect_right(A, p, q, v)
       	else
            i = bisect_right(A, q+1, r, v)
        end
    end
    return i
end

A = [0,1,1,1,2,3,4,5,6,7,8,9]

V = 1
#@info "V" V
#@info "index where we want to insert V"  bisect_left(A,1,length(A)+1,V)
#@info "The list is still" A

@info "Bisect_left" bisect_left(A,1,length(A)+1,V) "should be 2"
@info "Bisect_right" bisect_right(A,1,length(A)+1,V) "Should be 5"


# Hinge loss function

export SqrHingeLoss

"""
**Squared Hinge loss**

  SqrHingeLoss(y, μ=1.0)

Returns the function
```math
f(x) = μ⋅∑_i \\max\\{0, 1 - y_i ⋅ x_i\\}^2,
```
where `y` is an array and `μ` is a positive parameter.
"""
struct SqrHingeLoss{R <: Real, T <: AbstractArray{R}} <: ProximableFunction
  y::T
  mu::R
  function SqrHingeLoss{R, T}(y::T, mu::R) where {R <: Real, T <: AbstractArray{R}}
    if mu <= 0
      error("parameter mu must be positive")
    else
      new(y, mu)
    end
  end
end

is_separable(f::SqrHingeLoss) = true
is_convex(f::SqrHingeLoss) = true
is_smooth(f::SqrHingeLoss) = true

SqrHingeLoss(b::T, mu::R=1.0) where {R <: Real, T <: AbstractArray{R}} = SqrHingeLoss{R, T}(b, mu)

(f::SqrHingeLoss)(x::T) where {R <: Real, T <: AbstractArray{R}} = f.mu*sum(max.(zero(R), (one(R) .- f.y.*x)).^2)

function gradient!(y::AbstractArray{R}, f::SqrHingeLoss{R, T}, x::AbstractArray{R}) where {R <: Real, T}
	sum = zero(R)
	for i in eachindex(x)
		zz = 1-f.y[i]*x[i]
		z = max(zero(R), zz)
		y[i] = z .> 0 ? -2*f.mu*f.y[i]*zz : 0
		sum += z^2
	end
	return f.mu*sum
end

function prox!(z::AbstractArray{R}, f::SqrHingeLoss{R, T}, x::AbstractArray{R}, gamma::R=one(R)) where {R, T}
    v = zero(R)
    for k in eachindex(x)
        if f.y[k]*x[k] >= 1
            z[k] = x[k]
        else
            z[k] = (x[k] + 2*f.mu*gamma*f.y[k])/(1+2*f.mu*gamma*f.y[k]^2)
            v += (1-f.y[k]*z[k])^2
        end
    end
    return f.mu*v
end

fun_name(f::SqrHingeLoss) = "squared hinge loss"
fun_dom(f::SqrHingeLoss) = "AbstractArray{Real}"
fun_expr(f::SqrHingeLoss) = "x ↦ μ * sum( max(0,1-b*x_i)^2, i=1,...,n )"
fun_params(f::SqrHingeLoss) = "b = $(typeof(f.y)), μ = $(f.mu)"

function prox_naive(f::SqrHingeLoss{R, T}, x::AbstractArray{R}, gamma::R=one(R)) where {R, T}
    flag = f.y.*x .<= 1
    z = copy(x)
    z[flag] = (x[flag] .+ 2 .* f.mu.*gamma.*f.y[flag])./(1. + 2 .* f.mu.*gamma.*f.y[flag].^2)
    return z, f.mu*sum(max.(0.0, 1 .- f.y.*z).^2)
end


# coding: utf-8

# In[1]:

#this is a test on how recursion with memory reduces time complexity

# In[2]:

#this is the fibonacci recursion function without memory
#it is basically algorithm 101 for any coding language
function fib(n)

    if n==1
        return 1
    elseif n==2
        return 1
    elseif n<=0
        printstyled("Invalid input",color=:red)
        return
    else
        return fib(n-1)+fib(n-2)

    end
end

# In[3]:

#i need a global dictionary to do the memorization
#or i can change function mmz(n) into mmz(n,memoization)
global memoization=Dict(1=>1,2=>1)

# In[4]:

#mmz(n) is recursion with memory
#everytime we do the calculation, we store it in the dictionary
#i denote the key as the n th fibonacci number
#the value as the number itself
#if we can find the key in dictionary
#we simply return the value
#if not, we compute and update the dictionary then return the value
function mmz(n)

    if n<=0
        printstyled("Invalid input",color=:red)
        return
    end

    if !(n in keys(memoization))
        global memoization[n]=mmz(n-1)+mmz(n-2)
    end

    return memoization[n]
end

# In[5]:

#using ijulia inline magic
#equivalent to %timeit in ipython
#0.000093 seconds
@time begin
    fib(20)
end

# In[6]:

#0.019746 seconds (5.60 k allocations: 335.035 KiB)
#it seems to be slower
#but if we compute mmz(30) now
#its much faster than fib(30)
@time begin
    mmz(20)
end

# In[7]:

#0.021588 seconds
@time begin
    fib(30)
end

# In[8]:

#0.000023 seconds (52 allocations: 832 bytes)
@time begin
    mmz(30)
end


"""
    perm_rows(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix

Apply a permutation to the rows of a given Matrix.
Used to create corresponding permutation matrices with `A = identity`.
"""
function perm_rows(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix
    if !isperm(p)
        throw("p is not a permutation.")
        return nothing
    end
    B = copy(A)
    for i in 1:length(p)
        B[i,:] = A[p[i],:]
    end
    return B
end

"""
    perm_cols(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix

Apply a permutation to the columns of a given Matrix.
Used to create corresponding inverse permutation matrices with `A = identity`.
"""
function perm_cols(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix
    if !isperm(p)
        throw("p is not a permutation.")
        return nothing
    end
    B = copy(A)
    for i in 1:length(p)
        B[:,i] = A[:,p[i]]
    end
    return B
end

"""
    apply_permutation(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix

Applies a given permutation-vector `p` to the given matrix `A` using the corresponding permutation-matrices.
"""
function apply_permutation(A::AbstractMatrix, p::Vector{Int64})::AbstractMatrix
    if !isperm(p)
        throw("p is not a permutation.")
        return nothing
    end
    n = length(p)
    if size(A, 1) != n || size(A,2) != n
        throw("Only square matrices allowed with the same number of rows/columns as the permutation.")
        return nothing
    end
    I = id(n)
    P = perm_rows(I, p)
    P_inv = perm_cols(I,p)
    return P * A * P_inv
end

"""
    print_permutation_cycles(labelled_list::Bijection{String, T}, permutations::Vector{Vector{Int64}}, filename_identifier::String; filename_prefix="permutation_cycles_") where T

Applies all given permutations to all given labelled elements and prints out the permutation cycles using the labels.
The output is redirected and appended to a file `filename_prefix * filename_identifier * ".txt"`, e.g. `"permutation_cycles_1.txt"`
"""
function print_permutation_cycles(labelled_list::Bijection{String, T}, permutations::Vector{Vector{Int64}}, filename_identifier::String; filename_prefix="permutation_cycles_") where T
    # construct the filename
    filename = filename_prefix * filename_identifier * ".txt"
    n = length(permutations[1])
    # start output in "append" mode
    open(filename, "a") do io
        for k in sort(String.(keys(labelled_list)))
            for i in 1:length(permutations)
                el_prev = labelled_list[k]
                found = String[]
                cond = true
                I = id(n)
                p = permutations[i]
                P = perm_rows(I, p)
                P_inv = perm_cols(I,p)

                print(io, "p[$i]: " * k)
                push!(found, k)
                while cond
                    el = P * el_prev * P_inv
                    lbl = labelled_list(el)
                    print(io, " → " * lbl)
                    if lbl in found
                        cond = false
                    end
                    push!(found, lbl)
                    el_prev = el
                end
                println(io,"")
            end
        end
    end
end


# ---
# title: 1103. Distribute Candies to People
# id: problem1103
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/distribute-candies-to-people/description/>
# hidden: true
# ---
#
# We distribute some number of `candies`, to a row of **`n = num_people`**
# people in the following way:
#
# We then give 1 candy to the first person, 2 candies to the second person, and
# so on until we give `n` candies to the last person.
#
# Then, we go back to the start of the row, giving `n + 1` candies to the first
# person, `n + 2` candies to the second person, and so on until we give `2 * n`
# candies to the last person.
#
# This process repeats (with us giving one more candy each time, and moving to
# the start of the row after we reach the end) until we run out of candies.  The
# last person will receive all of our remaining candies (not necessarily one
# more than the previous gift).
#
# Return an array (of length `num_people` and sum `candies`) that represents the
# final distribution of candies.
#
#
#
# **Example 1:**
#
#
#
#     Input: candies = 7, num_people = 4
#     Output: [1,2,3,1]
#     Explanation:
#     On the first turn, ans[0] += 1, and the array is [1,0,0,0].
#     On the second turn, ans[1] += 2, and the array is [1,2,0,0].
#     On the third turn, ans[2] += 3, and the array is [1,2,3,0].
#     On the fourth turn, ans[3] += 1 (because there is only one candy left), and the final array is [1,2,3,1].
#
#
# **Example 2:**
#
#
#
#     Input: candies = 10, num_people = 3
#     Output: [5,2,3]
#     Explanation:
#     On the first turn, ans[0] += 1, and the array is [1,0,0].
#     On the second turn, ans[1] += 2, and the array is [1,2,0].
#     On the third turn, ans[2] += 3, and the array is [1,2,3].
#     On the fourth turn, ans[0] += 4, and the final array is [5,2,3].
#
#
#
#
# **Constraints:**
#
#   * 1 <= candies <= 10^9
#   * 1 <= num_people <= 1000
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# This file includes node dynamics

abstract type AbstractNode end

"""
    $TYPEDEF

Lorenz dyanmics

# Fields

    $TYPEDFIELDS
"""
@kwdef mutable struct Lorenz <: AbstractNode
    σ::Float64 = 10.0
    β::Float64 = 8 / 3
    ρ::Float64 = 35.0
    x::Vector{Float64} = rand(3)
    t::Float64 = 0.0
end

function (node::Lorenz)(dx, x, u, t)
    dx[1] = node.σ * (x[2] - x[1])
    dx[2] = x[1] * (node.ρ - x[3]) - x[2]
    dx[3] = x[1] * x[2] - node.β * x[3]
end

dimension(::Type{Lorenz}) = 3

"""
    $TYPEDEF

Rossler dyanmics

# Fields

    $TYPEDFIELDS
"""
@kwdef mutable struct Rossler <: AbstractNode
    a::Float64 = 0.2
    b::Float64 = 0.2
    c::Float64 = 5.7
    x::Vector{Float64} = rand(3)
    t::Float64 = 0.0
end

function (node::Rossler)(dx, x, u, t)
    dx[1] = -x[2] - x[1]
    dx[2] = x[1] + node.a * x[2]
    dx[3] = node.b + x[3] * (x[1] - node.c)
end

dimension(::Type{Rossler}) = 3

# Piecewise Chua diode
@kwdef struct Diode
    a::Float64 = -8 / 7
    b::Float64 = -5 / 7
end

function (diode::Diode)(x)
    if x > 1
        diode.b * x + diode.a - diode.b
    elseif abs(x) ≤ 1
        diode.a * x
    else
        diode.b * x - diode.a + diode.b
    end
end

@kwdef mutable struct Chua <: AbstractNode
    diode::Diode = Diode()
    α::Float64 = 9.0
    β::Float64 = 100 / 7
    x::Vector{Float64} = 1e-3 * rand(3)
    t::Float64 = 0.00
end

function (ds::Chua)(dx::AbstractVector, x::AbstractVector, u, t)
    dx[1] = ds.α * (x[2] - x[1] - ds.diode(x[1]))
    dx[2] = x[1] - x[2] + x[3]
    dx[3] = -ds.β * x[2]
end


"""
    Const <: Kernel

Constant kernel
```math
k(x,x') = σ²
```
with signal standard deviation ``σ``.
"""
mutable struct Const{T} <: Kernel where {T<:Real}
    "Signal variance"
    σ2::T
    "Priors for kernel parameters"
    priors::Array
end

"""
Constant kernel function

    Const(lσ::T)

# Arguments
  - `lσ::Real`: signal standard deviation (given on log scale)
"""
Const(lσ::T) where T = Const{T}(exp(2 * lσ), [])

function set_params!(cons::Const, hyp::AbstractVector)
    length(hyp) == 1 || throw(ArgumentError("Constant kernel has one parameter, received $(length(hyp))."))
    cons.σ2 = exp(2.0*hyp[1])
end

get_params(cons::Const{T}) where T = T[log(cons.σ2)/2.0]
get_param_names(cons::Const) = [:lσ]
num_params(cons::Const) = 1

cov(cons::Const) = cons.σ2
function cov(cons::Const, x::AbstractVector, y::AbstractVector)
    return cov(cons)
end

@inline dk_dlσ(cons::Const) = 2.0*cov(cons)
@inline function dKij_dθp(cons::Const, X1, X2, i::Int, j::Int, p::Int, dim::Int)
    if p == 1
        return dk_dlσ(cons)
    else
        return NaN
    end
end


### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ 065a1e67-6b63-43df-9d6d-303af08d8434
begin
	using Pkg
	Pkg.activate(joinpath(Pkg.devdir(), "MLCourse"))
    using PlutoUI
    PlutoUI.TableOfContents()
end

# ╔═╡ 034f7cc2-eacd-11eb-29e2-810a573295cf
using LinearAlgebra

# ╔═╡ 034f7c90-eacd-11eb-1b40-8722209689fa
using DataFrames # load the DataFrames package

# ╔═╡ 034f7d1a-eacd-11eb-1d48-591daf597cd6
using Random

# ╔═╡ 034f7d3a-eacd-11eb-1144-65f9acfcf24c
begin
	using Plots
	plotly()
end

# ╔═╡ 034f7d58-eacd-11eb-3189-bff16ec27739
using Distributions

# ╔═╡ 034f7d76-eacd-11eb-24cc-fd8d53d736c2
using Statistics

# ╔═╡ d6e5fe02-21a5-486c-a237-878be1d95439
begin
    using MLCourse
    MLCourse.list_notebooks(@__FILE__)
end

# ╔═╡ f3508747-da29-47c9-a98e-22ea15caaf2f
Markdown.parse("Hi and welcome to an introduction to the Julia programming language.

$(haskey(ENV, "html_export") ? "This page was created with an interactive [Pluto notebook](https://plutojl.org/). If you want to run it locally on your machine, follow the instructions [here](https://github.com/jbrea/MLCourse)." :
"This is an interactive Pluto notebook. You can create new cells by clicking any \"+\" above or below existing cells. In these cells you can write code and run it by clicking the play button or [Shift] + [Enter] on your keyboard (or [Ctrl] + [Enter]).
The output gets displayed above the cell. Have a look at [this website](https://plutojl.org/) if you want to learn more about Pluto notebooks.

To get help, please open the Live docs at the bottom right of this page and click on the code you want to get help for. For example, click on the ÷ symbol in the 8th cell below.")

If you want to learn more about Julia visit [julialang.org](https://julialang.org).
In the following sections you find links to different chapters of the [manual](https://docs.julialang.org/en/v1/). There is also this [cheat sheet](https://juliadocs.github.io/Julia-Cheat-Sheet/).")

# ╔═╡ d92c0729-c7ab-42cc-b713-30f00e237833
md"# Mathematical Operations

Here is the [link](https://docs.julialang.org/en/v1/manual/mathematical-operations/) to the respective section in the Julia manual."

# ╔═╡ b8630ee9-d2c0-4cd0-b3f7-77f66e997a80

# ╔═╡ 034f7b46-eacd-11eb-1aa2-f380d08206cc
1 + 2

# ╔═╡ 034f7b64-eacd-11eb-1618-6de6def66405
2 * 3

# ╔═╡ 034f7b6e-eacd-11eb-1733-650ad4da2730
3 ^ 2

# ╔═╡ 034f7b78-eacd-11eb-003a-5b5801c19d80
3 == 9 / 3 # test for equality

# ╔═╡ 034f7b6e-eacd-11eb-184a-d504fab31681
4 / 3 # compute the fraction as a floating point number

# ╔═╡ fb3677a9-1610-49dd-96e4-3bd7badc58db
4 // 3 # leave the fraction as a fraction

# ╔═╡ 884f568b-079c-47d6-a597-34b67925f195
4//3 * 3//2 * 1//7 # simplify fractions

# ╔═╡ 034f7b84-eacd-11eb-2836-11cebedb05b9
5 ÷ 2 # Write this symbol with \div [TAB]. Try it in a new cell!

# ╔═╡ 52efdece-3c21-43d4-a826-3b96e3a23e60
md"# Variables and Simple Functions

See also [here](https://docs.julialang.org/en/v1/manual/variables/)."

# ╔═╡ 034f7ba0-eacd-11eb-3fc1-9dca0e072755
x = 2

# ╔═╡ 034f7ba0-eacd-11eb-23ae-cd327563ec60
y = 4.5

# ╔═╡ 034f7ba0-eacd-11eb-3347-49deb13761d9
z = x * y

# ╔═╡ 121b3dbb-f86c-44b8-8b0f-f6b4d0b9992f
md"In the following cells we apply simple built-in functions to the variables defined above. If you change the values of these variables, the output of the cells below will also change, thanks to the reactivity of this notebook. Try it by assigning another value to `x` in the cell above."

# ╔═╡ 034f7baa-eacd-11eb-33f2-5f619e4a7832
sin(x)

# ╔═╡ 034f7bb2-eacd-11eb-006b-7b935e35af66
exp(x)

# ╔═╡ 034f7bb2-eacd-11eb-214b-cbfc83965c26
log(x)

# ╔═╡ ec1dd4c4-d61d-4fdb-b576-3fe3b059eb3a
typeof(2)

# ╔═╡ 1a3387ee-b59d-49d7-9cca-2a8bd04eaaad
typeof(2.0)

# ╔═╡ 034f7b96-eacd-11eb-0c81-ad33210de120
typeof(4//3)

# ╔═╡ 034f7b96-eacd-11eb-2ee1-cd8e6813a726
typeof(2f0)

# ╔═╡ 0f4e3f45-71dd-41b2-a0b3-c55a04ace61f
md"In the following cells you see 3 different ways to define custom functions. See also [here](https://docs.julialang.org/en/v1/manual/functions/), and, in particular, [here](https://docs.julialang.org/en/v1/manual/functions/#man-anonymous-functions) for anonymous functions."

# ╔═╡ 034f7bbe-eacd-11eb-2e95-a93ad2d5e867
myfunc(x, y) = 2 * x + 3 - y # creates a new function called `myfunc`

# ╔═╡ 034f7bbe-eacd-11eb-2c36-d3949367ce5a
myfunc(x, y)

# ╔═╡ e0ec7ba3-4858-4572-994d-953caa98080b
anonfunc = x -> x^2 # creates an anonymous function called `anonfunc`

# ╔═╡ 04da39a1-e3bc-4086-b555-6cea2cf1689d
anonfunc(4)

# ╔═╡ 034f7bc8-eacd-11eb-0396-5b743b505b2a
function longfunc(x, y)
    tmp = x + y
    tmp ^ 2 - 0.25 # the result of the last line is the output
end

# ╔═╡ 034f7bc8-eacd-11eb-31f9-c119d1dfb8a5
longfunc(x, y)

# ╔═╡ ce3c7687-c7a9-4b5d-9a55-51514ed90fd9
md"We will sometimes use the pipe operator to chain functions."

# ╔═╡ 034f7bd2-eacd-11eb-28f3-b707012e1db0
x |> sin |> exp |> log10

# ╔═╡ 034f7bd2-eacd-11eb-1789-7380a89d5339
log10(exp(sin(x)))

# ╔═╡ 034f7cd6-eacd-11eb-0f8c-11179c3ab480
md" # Strings and Markdown

See also [here](https://docs.julialang.org/en/v1/manual/strings/)
and [here](https://docs.julialang.org/en/v1/stdlib/Markdown/).
"

# ╔═╡ 034f7cd6-eacd-11eb-0f8c-11179c3ab48f
s1 = "Hi! "

# ╔═╡ 034f7ce0-eacd-11eb-3c49-e39b8d3e917f
s2 = "Welcome to Machine Learning"

# ╔═╡ 034f7ce0-eacd-11eb-18a1-bd2d1849b9bc
s1 * s2

# ╔═╡ 034f7cec-eacd-11eb-3262-bdc0f2e13a3a
s1 ^ 3 * s2

# ╔═╡ 034f7cec-eacd-11eb-27da-9bb861643df2
split(s2)

# ╔═╡ 832ce046-ab34-4623-9d02-793dc5748208
md"Sometimes we want to paste something computed into a string. This is called [interpolation](https://docs.julialang.org/en/v1/manual/strings/#string-interpolation)."

# ╔═╡ 91b0de0c-221c-468e-9077-5233aedeb4ca
answer = 42

# ╔═╡ 0c7ae3ec-a40f-4e3e-85d8-3f79f091e3b4
"The answer is $answer." # we interpolate with the dollar sign

# ╔═╡ 732ae082-b90f-4108-9872-36077fb2c54c
md"Whenever you see just text here, it is actually the output of a markdown cell.
Markdown cells can be written in two ways: either as a markdown string `md\"this is a markdown string\"` (note the trailing `md`) or as `Markdown.parse(\"this is a markdown string\")`.
You can look at the code of this cell by toggling its visibility with the \"eye\"
button on the left. For you to take notes, it may be useful to know some of the
markdown features. We look at some in the cell below.
"

# ╔═╡ 6bfb60b4-964e-46ec-adc2-ec9e9a0f8158
md"#### This is a fourth level header

[This is a link](https://docs.julialang.org/en/v1/)

- here we have a list
- of multiple items
   1. with different levels.
   1. The number you put here doesn't matter.
   0. The markdown interpreter knows how to count.

Below you see a Julia code block:

```julia
my_fancy_function(x) = x^x # here I define my function
my_fancy_function(3) == 27
```

and here is the same thing in python:

```python
def my_fancy_function(x): # here I define my function
    return x**x

my_fancy_function(3) == 27
```

This is a formula ``E = mc^2``.

And below we have some nicely colored text boxes.

!!! note

    Sometimes I just note random stuff.

!!! tip

    It is useful to know markdown.

!!! warning \"Don't forget the exercises!\"

    No pain, no gain!

!!! danger \"Machine Learning Can Be Addictive\"

    Rumours say that in the past there were students that liked their first
    Machine Learning course so much that they decided to pursue a career in
    Machine Learning.
"

# ╔═╡ 0efb1f1a-8051-4843-aa68-1f4ad8618f92
md"# Vectors, Matrices and Arrays

See also [here](https://docs.julialang.org/en/v1/manual/arrays/)."

# ╔═╡ 034f7bdc-eacd-11eb-1999-9b62efdb4128
v = [1, 2, 3] # this is a vector

# ╔═╡ 034f7cba-eacd-11eb-2f1a-836ccc185dd9
push!(v, 12)

# ╔═╡ 034f7cba-eacd-11eb-3901-2bc3d0ff0e39
pop!(v)

# ╔═╡ 50d83a01-f4c5-45c1-9c81-ed71d8313642
md"If you wonder about the ! in the function names: have a look [here](https://docs.julialang.org/en/v1/manual/style-guide/#bang-convention)."

# ╔═╡ 034f7bfa-eacd-11eb-1839-371285e2c6ed
w = [1 2 3] # this is a matrix with 1 row and 3 columns

# ╔═╡ 034f7be4-eacd-11eb-1616-2df92f244e73
v[1] # access to the first element of the vector

# ╔═╡ 034f7bf0-eacd-11eb-0b3f-69f172657051
v[2] = 4 # assign value 4 to the second element of the vector

# ╔═╡ 034f7bf0-eacd-11eb-012e-997132cd41c6
v # here we check that the second element is indeed 4 now

# ╔═╡ 034f7bfa-eacd-11eb-2bf8-c795453a8959
m = [1 2 3
     4 5 6
     7 8 9] # this is a 3 x 3 matrix.

# ╔═╡ 034f7c04-eacd-11eb-3448-832bd75c3356
m[3, 2] # accessing the element in the 3 row, second column

# ╔═╡ 9c34e4ef-4eba-459f-a7ff-b20708155e65
m' # transpose the matrix

# ╔═╡ e576bf44-9cc1-4053-b40f-b82c8c39c89d
md"Any function can be applied element-wise to arrays using the [dot syntax](https://docs.julialang.org/en/v1/manual/functions/#man-vectorized)."

# ╔═╡ 034f7c0e-eacd-11eb-114f-9dbad3e74dd6
sin.(v)

# ╔═╡ 034f7c0e-eacd-11eb-1b73-27539a68f26d
exp.(m)

# ╔═╡ 034f7c16-eacd-11eb-0d0b-518b28558fda
v .+ 3

# ╔═╡ 034f7c16-eacd-11eb-2cb0-8b1198bfcb78
myfunc.(v, v)

# ╔═╡ 034f7ccc-eacd-11eb-1613-cbdf2386b60b
m * v  # matrix vector multiplication

# ╔═╡ 034f7ccc-eacd-11eb-2ab4-599e0ed86304
v' * v # inner product

# ╔═╡ 034f7cd6-eacd-11eb-37bb-f1930f2f2772
eigvals(m) # compute the Eigen values

# ╔═╡ c997a07f-bc89-4c29-9945-62046b6889d2
md"To concatenate vectors and matrices we have the following syntax (see also [here](https://docs.julialang.org/en/v1/manual/arrays/#man-array-concatenation))."

# ╔═╡ 07fd5baf-ae0b-4e74-97a6-5a6e14644122
[[1, 2, 3]; [77, 88, 99]] # the semicolon concatenates the two vectors

# ╔═╡ 11422ca4-c096-4167-aa53-de312615348e
md"The same can be achieved with"

# ╔═╡ 0ace8c81-e481-4323-9b20-116de412330a
vcat([1, 2, 3], [77, 88, 99])

# ╔═╡ 5d09ccbf-5a51-41f7-a7ef-84cf2cc5fa1f
[[1, 2, 3] [77, 88, 99]] # alternatively you can use hcat([1, 2, 3], [77, 88, 99])

# ╔═╡ 9fd6e8c1-7b84-4d76-abef-9e3da4a56dc7
md"Sometimes it is easier to construct a vector or matrix with [comprehension](https://docs.julialang.org/en/v1/manual/arrays/#man-comprehensions)."

# ╔═╡ e33c0eea-4f5c-4fc8-800d-fa16335eedd0
[i^2 for i in [3, 4, 9, 7]]

# ╔═╡ 315e5a3b-dc28-4c3b-8fc2-3442cb7590be
[i^2 for i in [3, 4, 9, 7] if isodd(i)]

# ╔═╡ 45deab46-9a8c-4d22-9460-980a3900396f
["$i, $j" for i in 1:5, j in 1:5] # see string interpolation above for the $

# ╔═╡ b91eda3c-95cb-4372-9c6b-03fa6a6b02b2
md"# Ranges

See also [here](https://docs.julialang.org/en/v1/base/math/#Base.::)."

# ╔═╡ 034f7c16-eacd-11eb-3877-6ba06a933698
collect(1:10) # 1:10 is a range; the function `collect` turns it into a vector

# ╔═╡ 034f7c22-eacd-11eb-3ab7-b30813d11e94
r1 = 1:.1:10 # a range from 1 to 10 with step-size 0.1

# ╔═╡ 034f7c2c-eacd-11eb-1aae-97b030af71e9
r3 = 10:-.1:1 # a range from 10 to 1 with step-size -0.1

# ╔═╡ 034f7c2c-eacd-11eb-1f56-052874c5c6ab
collect(r3)

# ╔═╡ 034f7c36-eacd-11eb-0188-cd307154555c
exp.(r1) # apply exp to the range

# ╔═╡ 034f7c40-eacd-11eb-26d9-956b65235a10
v[2:3] # you can use ranges to access multiple elements of a vector or matrix

# ╔═╡ 4237f5df-208d-44c4-90d7-db0efbe7b710
v[[1, 3]] # a vector of indices can also be used

# ╔═╡ 034f7c40-eacd-11eb-1beb-bb9e9bb325db
v[2:end]

# ╔═╡ 034f7c4a-eacd-11eb-0de4-537297318c85
v[begin:2]

# ╔═╡ 034f7c4a-eacd-11eb-03ed-bfc7b532501b
m[:, 2] # the colon : alone is a short-hand for begin:end

# ╔═╡ 034f7c54-eacd-11eb-2918-ef8f2befddc1
m[1:2, 2:3]

# ╔═╡ 034f7c54-eacd-11eb-2fe9-5330d3cd3cd5
function is_larger_than_10(x)
	result = ""
    for i in 1:10 # ranges are useful to construct for loops
        if x < i
            return result * " is smaller than 10"
        else
            result *= "|"
        end
    end
    result * "... is larger than 10"
end

# ╔═╡ 034f7c5e-eacd-11eb-24c6-d7e7213a7a5e
is_larger_than_10(4)

# ╔═╡ 034f7c5e-eacd-11eb-1ff7-55f09fb796fe
is_larger_than_10(78)

# ╔═╡ d9ad1d6d-a3b5-4a83-bda9-05ee43ded9d6
md"# Tuples

See also [here](https://docs.julialang.org/en/v1/manual/functions/#Tuples)."

# ╔═╡ 034f7c68-eacd-11eb-1e11-d77379c677c8
t = (2, 3, 4., "bla")

# ╔═╡ 034f7c68-eacd-11eb-12e6-51022c54f6ed
typeof(t)

# ╔═╡ 034f7c72-eacd-11eb-28a5-d7d6f3b9f75a
t[1]

# ╔═╡ 034f7c72-eacd-11eb-1cc2-91eaa2de5e25
t[4]

# ╔═╡ 034f7c7c-eacd-11eb-1aaa-297c951dd8cd
nt = (year = 1789,
	  event = "French Revolution",
	  slogan = "liberté, égalité, fraternité")

# ╔═╡ 034f7c88-eacd-11eb-28f2-c1c7e05279c6
typeof(nt)

# ╔═╡ 034f7c88-eacd-11eb-14fe-b3d49ff91622
nt.year

# ╔═╡ 034f7c90-eacd-11eb-0d1c-5dbd7b62d55c
nt.slogan

# ╔═╡ df8588cb-400b-4423-9766-e6cac2c9717a
md"# DataFrames

DataFrames are tables with named columns.
We will constantly use DataFrames to organize data.
See also [the documentation of the DataFrames.jl package](https://dataframes.juliadata.org/stable/)."

# ╔═╡ 034f7c9a-eacd-11eb-2e85-29c5bffbc39a
df = DataFrame(year = [1789, 1863],
               event = ["French Revolution", "Foundation of Red Cross"],
               slogan = ["liberté, égalité, fraternité", missing])

# ╔═╡ 034f7cae-eacd-11eb-39dc-ab39e97cc716
push!(df, [1789,
           "George Washington becomes first president of the USA",
           "Deeds, not Words."]) # append a new row to the already created dataframe

# ╔═╡ 034f7c9a-eacd-11eb-3aee-0bab7ab5d502
df.year

# ╔═╡ 034f7ca4-eacd-11eb-2f3b-c1dd9be6ad84
df[:, 1] # we can also access columns like for matrices

# ╔═╡ 034f7ca4-eacd-11eb-054e-1f930c99d304
dropmissing(df) # removes all rows with missing entries.

# ╔═╡ 034f7cae-eacd-11eb-38c0-57df4ba09992
df[1, :] # show the first row

# ╔═╡ 034f7cc2-eacd-11eb-0f0f-ff16d1ac454b
df[df.year .== 1789, [:year, :slogan]] # select all rows where year == 1789 and columns year and slogan

# ╔═╡ 034f7cf4-eacd-11eb-302f-55b164df3b90
md"# Random Numbers

See also [here](https://docs.julialang.org/en/v1/stdlib/Random/).
"

# ╔═╡ 034f7cf4-eacd-11eb-302f-55b164df3b9f
rand() # a sample from the uniform distribution over the interval [0, 1).

# ╔═╡ 034f7cf4-eacd-11eb-003c-133f34cca508
rand() # different number than in the cell above.

# ╔═╡ 034f7cfe-eacd-11eb-2514-4b6fdbc42e8f
rand(10)

# ╔═╡ 034f7d08-eacd-11eb-2694-51c1baa997e6
rand(3, 3)

# ╔═╡ 034f7d12-eacd-11eb-1673-c94969427bbf
randn() # a sample from the normal distribution with mean 0 and variance 1

# ╔═╡ 034f7d12-eacd-11eb-261c-a5de2f06f420
rand((:bla, "bli", 3, 1.2))

# ╔═╡ 034f7d1a-eacd-11eb-1e34-21d5eef62940
Random.seed!(123) # sets the seed of the random number generator to 123

# ╔═╡ 034f7d26-eacd-11eb-066c-538e5b614536
rand()

# ╔═╡ 034f7d30-eacd-11eb-081f-970c4daee184
Random.seed!(123) # resets the seed

# ╔═╡ 034f7d30-eacd-11eb-0b09-7b3b9ac47570
rand() # same result as when `rand()` was called last time.

# ╔═╡ 034f7d3a-eacd-11eb-3fbe-6dc4ab1efd50
md"# Plotting

See also the [documentation of the Plots.jl package](http://docs.juliaplots.org/latest/)
"

# ╔═╡ 034f7d3a-eacd-11eb-3fbe-6dc4ab1efd56
plot(1:4, rand(4))

# ╔═╡ 034f7d3a-eacd-11eb-1ef3-4f89b8e5aee9
scatter(rand(100), rand(100))

# ╔═╡ 034f7d44-eacd-11eb-0ab2-2f0d2caed9d6
with_terminal(plotattr) # get some help on plotting attributes

# ╔═╡ 034f7d4c-eacd-11eb-3960-d10d1c16075d
with_terminal(plotattr, :Series)

# ╔═╡ 034f7d4c-eacd-11eb-3841-97ca37e34c9f
with_terminal(plotattr, :Series, "label")

# ╔═╡ 034f7d58-eacd-11eb-123f-9b563a7ae27e
scatter(rand(100), rand(100), label = "my data", xlabel = "X1", ylabel = "X2")

# ╔═╡ 4ca1cd98-a24b-491f-b9ac-28a7daf96d50
md"To compose figures with multiple elements, you can use the `!` version of the plotting functions"

# ╔═╡ 65e73d27-3709-4894-87ab-432343d314c6
begin # we use begin to write a cell with multiple lines
	scatter(1:50, sqrt.(1:50) .+ randn(50), label = "data points")
	plot!(sqrt, color = :red, label = "square root function")
end

# ╔═╡ 9f69c2ab-2940-40ce-b164-07bfa6cc1697
md"Instead of `begin`-`end`-blocks we will also use `let`-`end`-blocks. Variables computed in `let`-blocks are local to that cell, whereas variables defined in `begin`-blocks are global and can be accessed by other cells. (With the `;` at the end of the cell we suppress the output)."

# ╔═╡ e7ed9e83-7f17-4d9a-a49b-e87bc0747035
begin
	a_global_variable = 17
	another_global_variable = "Hi"
end;

# ╔═╡ 0d66592a-d098-4ab5-9323-503db8fc73f6
a_global_variable * 3

# ╔═╡ a9a3f29e-4931-4464-9bc5-af6598815f24
"$another_global_variable there!"

# ╔═╡ 6e65247f-b285-49e5-a16e-31b4541225bf
let
	a_local_variable = 17
	another_local_variable = "Hi"
end;

# ╔═╡ fcd77620-f18a-499b-87c4-fffaa876e59f
a_local_variable * 3

# ╔═╡ 034f7d62-eacd-11eb-3840-51c0622eb8a0
md"# Distributions

See also the [documentation of the Distributions.jl package](https://juliastats.org/Distributions.jl/stable/)
"

# ╔═╡ 034f7d62-eacd-11eb-3840-51c0622eb8a9
d1 = Normal(3, 2.5)

# ╔═╡ 034f7d6c-eacd-11eb-0eb6-277bf88c13d3
rand(d1) # draw a sample from this normal distribution

# ╔═╡ 034f7d6c-eacd-11eb-2792-d773a2c77cb6
rand(d1, 10) # draw 10 samples from this distriution

# ╔═╡ 034f7d6c-eacd-11eb-0dc0-3bd2b5a24d38
mean(rand(d1, 10^5))

# ╔═╡ 034f7d76-eacd-11eb-141d-0f90eef73f08
std(rand(d1, 10^5)) # standard deviation

# ╔═╡ 034f7d7e-eacd-11eb-0945-2dda6be01349
d2 = Bernoulli(.7)

# ╔═╡ 034f7d7e-eacd-11eb-2a1b-933e1231c220
rand(d2, 10)

# ╔═╡ 034f7d8a-eacd-11eb-1dc0-d99d481ab6fa
md"# Other Packages

Here are some other popular Julia packages. They are not relevant for this course, but maybe you want to explore them at some point.
* [Symbolics](https://github.com/JuliaSymbolics/Symbolics.jl)
* [DifferentialEquations](https://github.com/SciML/DifferentialEquations.jl)
* [Optim](https://github.com/JuliaNLSolvers/Optim.jl)
* [Images](https://github.com/JuliaImages/Images.jl)
* [Turing](https://github.com/TuringLang/Turing.jl)
"

# ╔═╡ 4a03cfae-9876-4cf0-a498-d750853191cb
md"""# Exercises

1. Create a data frame with 3 columns named A, B and C.
   1. Column A contains 5 random numbers sampled from a Bernoulli distribution with rate 0.3, column B contains 5 random numbers from the uniform distribution over the interval [0, 1), and column C contains 5 samples from the set `(:hip, :hop)`.
   2. Create a vector whose i'th element contains the sum of the i'th entries of columns A and B of the data frame created in 1.
   3. Select all rows with `:hop` in column C and display the resulting data frame.
1.
   1. Use comprehension (see "Vectors, Matrices, Arrays") to create a vector where with all numbers of the form ``x^y`` with ``x=1, \ldots, 10``, ``y = 2, \ldots, 7`` and ``y > x``.
   2. Compute the sum of the square root of this numbers.
2.
   1. Write a function that returns the smallest entry of a vector (without using the built-in function `minimum`, `argmin` or `findmin`).
   2. Test your function on a vector of 10 randomly sampled integers in the range 1 to 100.
3.
   1. Plot the `cos` function on the interval 0 to 4π. Hint: type `\pi + [Tab]` to enter the symbol π. To learn how to place custom tick labels on the x-axis, type `xticks` in a cell and open the \"Live docs\" at the bottom-right.
   2. Add a scatter plot with 100 points whose ``x`` and ``y`` coordinates are randomly sampled from the interval ``[0, 1)`` on top of the figure with the cosine.
"""

# ╔═╡ 0314376e-ff8c-4ad0-8a4b-f94f04f31f2c
MLCourse.footer()

# ╔═╡ Cell order:
# ╟─f3508747-da29-47c9-a98e-22ea15caaf2f
# ╟─d92c0729-c7ab-42cc-b713-30f00e237833
# ╠═b8630ee9-d2c0-4cd0-b3f7-77f66e997a80
# ╠═034f7b46-eacd-11eb-1aa2-f380d08206cc
# ╠═034f7b64-eacd-11eb-1618-6de6def66405
# ╠═034f7b6e-eacd-11eb-1733-650ad4da2730
# ╠═034f7b78-eacd-11eb-003a-5b5801c19d80
# ╠═034f7b6e-eacd-11eb-184a-d504fab31681
# ╠═fb3677a9-1610-49dd-96e4-3bd7badc58db
# ╠═884f568b-079c-47d6-a597-34b67925f195
# ╠═034f7b84-eacd-11eb-2836-11cebedb05b9
# ╟─52efdece-3c21-43d4-a826-3b96e3a23e60
# ╠═034f7ba0-eacd-11eb-3fc1-9dca0e072755
# ╠═034f7ba0-eacd-11eb-23ae-cd327563ec60
# ╠═034f7ba0-eacd-11eb-3347-49deb13761d9
# ╟─121b3dbb-f86c-44b8-8b0f-f6b4d0b9992f
# ╠═034f7baa-eacd-11eb-33f2-5f619e4a7832
# ╠═034f7bb2-eacd-11eb-006b-7b935e35af66
# ╠═034f7bb2-eacd-11eb-214b-cbfc83965c26
# ╠═ec1dd4c4-d61d-4fdb-b576-3fe3b059eb3a
# ╠═1a3387ee-b59d-49d7-9cca-2a8bd04eaaad
# ╠═034f7b96-eacd-11eb-0c81-ad33210de120
# ╠═034f7b96-eacd-11eb-2ee1-cd8e6813a726
# ╟─0f4e3f45-71dd-41b2-a0b3-c55a04ace61f
# ╠═034f7bbe-eacd-11eb-2e95-a93ad2d5e867
# ╠═034f7bbe-eacd-11eb-2c36-d3949367ce5a
# ╠═e0ec7ba3-4858-4572-994d-953caa98080b
# ╠═04da39a1-e3bc-4086-b555-6cea2cf1689d
# ╠═034f7bc8-eacd-11eb-0396-5b743b505b2a
# ╠═034f7bc8-eacd-11eb-31f9-c119d1dfb8a5
# ╟─ce3c7687-c7a9-4b5d-9a55-51514ed90fd9
# ╠═034f7bd2-eacd-11eb-28f3-b707012e1db0
# ╠═034f7bd2-eacd-11eb-1789-7380a89d5339
# ╟─034f7cd6-eacd-11eb-0f8c-11179c3ab480
# ╠═034f7cd6-eacd-11eb-0f8c-11179c3ab48f
# ╠═034f7ce0-eacd-11eb-3c49-e39b8d3e917f
# ╠═034f7ce0-eacd-11eb-18a1-bd2d1849b9bc
# ╠═034f7cec-eacd-11eb-3262-bdc0f2e13a3a
# ╠═034f7cec-eacd-11eb-27da-9bb861643df2
# ╟─832ce046-ab34-4623-9d02-793dc5748208
# ╠═91b0de0c-221c-468e-9077-5233aedeb4ca
# ╠═0c7ae3ec-a40f-4e3e-85d8-3f79f091e3b4
# ╟─732ae082-b90f-4108-9872-36077fb2c54c
# ╠═6bfb60b4-964e-46ec-adc2-ec9e9a0f8158
# ╟─0efb1f1a-8051-4843-aa68-1f4ad8618f92
# ╠═034f7bdc-eacd-11eb-1999-9b62efdb4128
# ╠═034f7cba-eacd-11eb-2f1a-836ccc185dd9
# ╠═034f7cba-eacd-11eb-3901-2bc3d0ff0e39
# ╟─50d83a01-f4c5-45c1-9c81-ed71d8313642
# ╠═034f7bfa-eacd-11eb-1839-371285e2c6ed
# ╠═034f7be4-eacd-11eb-1616-2df92f244e73
# ╠═034f7bf0-eacd-11eb-0b3f-69f172657051
# ╠═034f7bf0-eacd-11eb-012e-997132cd41c6
# ╠═034f7bfa-eacd-11eb-2bf8-c795453a8959
# ╠═034f7c04-eacd-11eb-3448-832bd75c3356
# ╠═9c34e4ef-4eba-459f-a7ff-b20708155e65
# ╟─e576bf44-9cc1-4053-b40f-b82c8c39c89d
# ╠═034f7c0e-eacd-11eb-114f-9dbad3e74dd6
# ╠═034f7c0e-eacd-11eb-1b73-27539a68f26d
# ╠═034f7c16-eacd-11eb-0d0b-518b28558fda
# ╠═034f7c16-eacd-11eb-2cb0-8b1198bfcb78
# ╠═034f7ccc-eacd-11eb-1613-cbdf2386b60b
# ╠═034f7ccc-eacd-11eb-2ab4-599e0ed86304
# ╠═034f7cc2-eacd-11eb-29e2-810a573295cf
# ╠═034f7cd6-eacd-11eb-37bb-f1930f2f2772
# ╟─c997a07f-bc89-4c29-9945-62046b6889d2
# ╠═07fd5baf-ae0b-4e74-97a6-5a6e14644122
# ╟─11422ca4-c096-4167-aa53-de312615348e
# ╠═0ace8c81-e481-4323-9b20-116de412330a
# ╠═5d09ccbf-5a51-41f7-a7ef-84cf2cc5fa1f
# ╟─9fd6e8c1-7b84-4d76-abef-9e3da4a56dc7
# ╠═e33c0eea-4f5c-4fc8-800d-fa16335eedd0
# ╠═315e5a3b-dc28-4c3b-8fc2-3442cb7590be
# ╠═45deab46-9a8c-4d22-9460-980a3900396f
# ╟─b91eda3c-95cb-4372-9c6b-03fa6a6b02b2
# ╠═034f7c16-eacd-11eb-3877-6ba06a933698
# ╠═034f7c22-eacd-11eb-3ab7-b30813d11e94
# ╠═034f7c2c-eacd-11eb-1aae-97b030af71e9
# ╠═034f7c2c-eacd-11eb-1f56-052874c5c6ab
# ╠═034f7c36-eacd-11eb-0188-cd307154555c
# ╠═034f7c40-eacd-11eb-26d9-956b65235a10
# ╠═4237f5df-208d-44c4-90d7-db0efbe7b710
# ╠═034f7c40-eacd-11eb-1beb-bb9e9bb325db
# ╠═034f7c4a-eacd-11eb-0de4-537297318c85
# ╠═034f7c4a-eacd-11eb-03ed-bfc7b532501b
# ╠═034f7c54-eacd-11eb-2918-ef8f2befddc1
# ╠═034f7c54-eacd-11eb-2fe9-5330d3cd3cd5
# ╠═034f7c5e-eacd-11eb-24c6-d7e7213a7a5e
# ╠═034f7c5e-eacd-11eb-1ff7-55f09fb796fe
# ╟─d9ad1d6d-a3b5-4a83-bda9-05ee43ded9d6
# ╠═034f7c68-eacd-11eb-1e11-d77379c677c8
# ╠═034f7c68-eacd-11eb-12e6-51022c54f6ed
# ╠═034f7c72-eacd-11eb-28a5-d7d6f3b9f75a
# ╠═034f7c72-eacd-11eb-1cc2-91eaa2de5e25
# ╠═034f7c7c-eacd-11eb-1aaa-297c951dd8cd
# ╠═034f7c88-eacd-11eb-28f2-c1c7e05279c6
# ╠═034f7c88-eacd-11eb-14fe-b3d49ff91622
# ╠═034f7c90-eacd-11eb-0d1c-5dbd7b62d55c
# ╟─df8588cb-400b-4423-9766-e6cac2c9717a
# ╠═034f7c90-eacd-11eb-1b40-8722209689fa
# ╠═034f7c9a-eacd-11eb-2e85-29c5bffbc39a
# ╠═034f7cae-eacd-11eb-39dc-ab39e97cc716
# ╠═034f7c9a-eacd-11eb-3aee-0bab7ab5d502
# ╠═034f7ca4-eacd-11eb-2f3b-c1dd9be6ad84
# ╠═034f7ca4-eacd-11eb-054e-1f930c99d304
# ╠═034f7cae-eacd-11eb-38c0-57df4ba09992
# ╠═034f7cc2-eacd-11eb-0f0f-ff16d1ac454b
# ╟─034f7cf4-eacd-11eb-302f-55b164df3b90
# ╠═034f7cf4-eacd-11eb-302f-55b164df3b9f
# ╠═034f7cf4-eacd-11eb-003c-133f34cca508
# ╠═034f7cfe-eacd-11eb-2514-4b6fdbc42e8f
# ╠═034f7d08-eacd-11eb-2694-51c1baa997e6
# ╠═034f7d12-eacd-11eb-1673-c94969427bbf
# ╠═034f7d12-eacd-11eb-261c-a5de2f06f420
# ╠═034f7d1a-eacd-11eb-1d48-591daf597cd6
# ╠═034f7d1a-eacd-11eb-1e34-21d5eef62940
# ╠═034f7d26-eacd-11eb-066c-538e5b614536
# ╠═034f7d30-eacd-11eb-081f-970c4daee184
# ╠═034f7d30-eacd-11eb-0b09-7b3b9ac47570
# ╟─034f7d3a-eacd-11eb-3fbe-6dc4ab1efd50
# ╠═034f7d3a-eacd-11eb-1144-65f9acfcf24c
# ╠═034f7d3a-eacd-11eb-3fbe-6dc4ab1efd56
# ╠═034f7d3a-eacd-11eb-1ef3-4f89b8e5aee9
# ╠═034f7d44-eacd-11eb-0ab2-2f0d2caed9d6
# ╠═034f7d4c-eacd-11eb-3960-d10d1c16075d
# ╠═034f7d4c-eacd-11eb-3841-97ca37e34c9f
# ╠═034f7d58-eacd-11eb-123f-9b563a7ae27e
# ╟─4ca1cd98-a24b-491f-b9ac-28a7daf96d50
# ╠═65e73d27-3709-4894-87ab-432343d314c6
# ╟─9f69c2ab-2940-40ce-b164-07bfa6cc1697
# ╠═e7ed9e83-7f17-4d9a-a49b-e87bc0747035
# ╠═0d66592a-d098-4ab5-9323-503db8fc73f6
# ╠═a9a3f29e-4931-4464-9bc5-af6598815f24
# ╠═6e65247f-b285-49e5-a16e-31b4541225bf
# ╠═fcd77620-f18a-499b-87c4-fffaa876e59f
# ╟─034f7d62-eacd-11eb-3840-51c0622eb8a0
# ╠═034f7d58-eacd-11eb-3189-bff16ec27739
# ╠═034f7d62-eacd-11eb-3840-51c0622eb8a9
# ╠═034f7d6c-eacd-11eb-0eb6-277bf88c13d3
# ╠═034f7d6c-eacd-11eb-2792-d773a2c77cb6
# ╠═034f7d6c-eacd-11eb-0dc0-3bd2b5a24d38
# ╠═034f7d76-eacd-11eb-24cc-fd8d53d736c2
# ╠═034f7d76-eacd-11eb-141d-0f90eef73f08
# ╠═034f7d7e-eacd-11eb-0945-2dda6be01349
# ╠═034f7d7e-eacd-11eb-2a1b-933e1231c220
# ╟─034f7d8a-eacd-11eb-1dc0-d99d481ab6fa
# ╟─4a03cfae-9876-4cf0-a498-d750853191cb
# ╟─d6e5fe02-21a5-486c-a237-878be1d95439
# ╟─065a1e67-6b63-43df-9d6d-303af08d8434
# ╟─0314376e-ff8c-4ad0-8a4b-f94f04f31f2c


"""
    ECON 602 Problem Set 1 Question 4
"""

using Roots, Plots

function adequilibrium(π, γ, ωA, ωB, p_0)
    # solve for equilibrium price and consumption
    # p_0 is initial guess for numerical solver
    println("\n AD equilibrium:")
     @show π, γ, ωA, ωB;

    f(p, π, γ, ωA, ωB) = π*(ωA[1] + p*ωB[1]) + 1/(1 + p*(p*π/(1-π))^(-1/γ))*(ωA[2] + p*ωB[2]) - ωA[1] - ωA[2]
    cA(p, π, γ, ωA, ωB) = [π*(ωA[1] + p*ωB[1]), 1/(1 + p*(p*π/(1 - π))^(-1/γ))*(ωA[2] + p*ωB[2])]
    cB(p, π, γ, ωA, ωB) = [(1 - π)/p * (ωA[1] + p*ωB[1]), 1/(p + (p*π/(1 - π))^(1/γ))*(ωA[2] + p*ωB[2])]

    # It never hurts to plot and see what you're working with
    plot(p -> f(p, π, γ, ωA, ωB), 0, 4)
    plot!(x -> 0, 0, 4)

    @show p_star = fzero(p -> f(p, π, γ, ωA, ωB), p_0);
    #=
     The first argument, p -> f(p, π, γ, ωA, ωB), defines and 'anonymous'
     function of only p. It is a 'wrapper' for the function f that assigns
     the values to all other parameters as they are currently defined.
    =#

    @show cA = cA(p_star, π, γ, ωA, ωB);
    @show cB = cB(p_star, π, γ, ωA, ωB);
    return (p_star = p_star, cA = cA, cB = cB)
end

function adequilibrium_alt(π, γ, ωA, ωB, p_min, p_max)
    println("\n AD equilibrium (alternative function):")
    # solve for equilibrium price and consumption
    # alternative syntax and solution algorithm using bounds
    @show π, γ, ωA, ωB;

    f(p, π, γ, ωA, ωB) = π*(ωA[1] + p*ωB[1]) + 1/(1 + p*(p*π/(1-π))^(-1/γ))*(ωA[2] + p*ωB[2]) - ωA[1] - ωA[2]
    cA(p, π, γ, ωA, ωB) = [π*(ωA[1] + p*ωB[1]), 1/(1 + p*(p*π/(1 - π))^(-1/γ))*(ωA[2] + p*ωB[2])]
    cB(p, π, γ, ωA, ωB) = [(1 - π)/p * (ωA[1] + p*ωB[1]), 1/(p + (p*π/(1 - π))^(1/γ))*(ωA[2] + p*ωB[2])]

    # It never hurts to plot and see what you're working with
    plot(p -> f(p, π, γ, ωA, ωB), p_min, p_max)
    plot!(x -> 0, p_min, p_max)

    # Alternative to anonymous function
    f(p) = f(p, π, γ, ωA, ωB)
    #=
     Note that this is a distinct function from the first
     function I defined, as it has a different number of
     arguments
    =#
    @show p_star = fzero(f, p_min, p_max);
    #=
     This provides an allowable range for solutions between
     0.01 and 4, rather than a single initial guess. This
     can be useful to prevent the algorithm from searching
     for negatives prices for example.
    =#

    @show cA = cA(p_star, π, γ, ωA, ωB);
    @show cB = cB(p_star, π, γ, ωA, ωB);
    return (p_star = p_star, cA = cA, cB = cB);
end

# q4e
ωA = [2, 5];
ωB = [5, 2];
γ = 2;
π = 0.5;
q4e = adequilibrium(π, γ, ωA, ωB, 1.1);
q4e_alt = adequilibrium_alt(π, γ, ωA, ωB, 0.01, 10.);

# q4f
ωA = [2, 5];
ωB = [5, 2];
γ = 2;
# π = 0.25
π = 0.25;
q4f_π025 = adequilibrium(π, γ, ωA, ωB, 1.1);
# π = 0.75
π = 0.75;
q4f_π075 = adequilibrium(π, γ, ωA, ωB, 1.1);


__precompile__()
module BarycentricLagrange
# Barycentric Lagrange Interpolation.
# Berrut, J.-P., & Trefethen, L. N. (2004).
# SIAM Review, 46(3), 501–517. doi:10.1137/S36144502417716

export bclag_interp_eval
export bclag_interp_matrix
export bclag_interp_weights

function bclag_interp_eval(x, f, xi, w)
    # fi = bclag_interp_eval(x, f, xi, w)
    # xi scalar, not in x
    tmp = w ./ (xi - x)
    fi = sum(f.*tmp)/sum(tmp)
    return fi
end
bclag_interp_eval(x, f, xi) = bclag_interp_eval(x, f, xi, bclag_interp_weights(x))

function bclag_interp_matrix(x, xi, w)
    n = length(x)
    N = length(xi)
    @assert length(w)==n

    B = zeros(N, n)
    denom = zeros(N)
    exact = zeros(Int64, N)
    for j=1:n
        for k=1:N
            xdiff = xi[k]-x[j]
            if xdiff != 0
                temp = w[j]/xdiff
                B[k,j] = temp
                denom[k] += temp
            else
                exact[k] = j
            end
        end
    end

    B ./= denom
    for jj=1:N
        if exact[jj] != 0
            B[jj,:] = 0.0
            B[jj + N*(exact[jj]-1)] = 1.0
        end
    end
    return B
end
bclag_interp_matrix(x, xi) = bclag_interp_matrix(x, xi, bclag_interp_weights(x))

function bclag_interp_weights(x)
    n = length(x);
    w = zeros(n);
    for j=1:n
        w[j] = 1/prod(x[j]-x[1:n.!=j]);
    end
    return w
end

end # module