function [OUT, W] = CrossWeightAverage(DATA,WEIGHT)
% =======================================================================
% Compute the weighted average of DATA, computing the weights form the
% matrix WEIGHT. Note that if there are NaNs in DATA the weights are
% rescaled. WEIGHTS cannot contain NaN
% =======================================================================
% [average, weight] = CrossWeightAverage(DATA,WEIGHT)
% -----------------------------------------------------------------------
% INPUT
%	- DATA: panel of time series DATA(T,N) of T observation x N series
%       (NaN are accepted and weights rescaled)
%   - WEIGHT: time series of WEIGHTS (TxN). You can also provide a (1xN)
%       matrix in this case the code would automatically compute fixed
%       weights
% -----------------------------------------------------------------------
% OUTPUT
%	- average : weighted average vector (T x 1) [double]
%	- weight  : matrix of weights (T x N) [double]
% -----------------------------------------------------------------------
% EXAMPLE 1: foreign variables as in GVAR (example with fixed weights)
%   x = [1 NaN  3 ;
%         2   3  4 ;
%         3   4  5 ;
%         4   5  6];
%    w = [0  .4 .6];
%    [average, weight] = CrossWeightAverage(x,w)
% -----------------------------------------------------------------------
% EXAMPLE 2: weighted average (with time-varying weights)
%    x = [ 1 NaN   3 ;
%          2   3   4 ;
%          3   4   5 ;
%          4   5   6];
%    w = [.4  .2  .4 ;
%         .3  .4  .3 ;
%         .4  .3  .3 ;
%         .5  .3  .2];
%    [average, weight] = CrossWeightAverage(x,w)
% =======================================================================
% VAR Toolbox 3.0
% Ambrogio Cesa-Bianchi
% ambrogiocesabianchi@gmail.com
% March 2012. Updated November 2020
% -----------------------------------------------------------------------

%% Preliminaries
% Check inputs
[r1, c1] = size(DATA);
[r2, c2] = size(WEIGHT);
if r2==1 && c2==c1
    % Create a matrix of fixed weights
    aux = [];
    for ii=1:r1
        aux = [aux; WEIGHT];
    end
    WEIGHT = aux;
end

% Re-Check inputs
[r1, c1] = size(DATA);
[r2, c2] = size(WEIGHT);
if r1~=r2 || c1~=c2
    disp(' ')
    disp('The matrices of data and weights are not conformable.')
    return
end

%% Cross section weihgted average
[nobs, nvars] = size(DATA);
nans = isnan(DATA);
WEIGHT(nans) = NaN;

OUT(1:nobs,1) = NaN;
W(1:nobs,1:nvars) = NaN;

for ii=1:nobs
    aux_weight = WEIGHT(ii,:)./nansum(WEIGHT(ii,:));
    W(ii,:) = aux_weight;
    aux_weight(nans(ii,:)) = 0;
    DATA(ii,nans(ii,:)) = 0;
    if sum(nans(ii,:))==nvars
        OUT(ii,1) = NaN;
    else
        OUT(ii,1) = DATA(ii,:) * aux_weight';
    end
end


clear all, close all, clc

% Define domain
dx = 0.001;
L = pi;
x = (-1+dx:dx:1)*L;
n = length(x);   nquart = floor(n/4);

% Define hat function
f = 0*x;
f(nquart:2*nquart) = 4*(1:nquart+1)/n;
f(2*nquart+1:3*nquart) = 1-4*(0:nquart-1)/n;
plot(x,f,'-k','LineWidth',1.5), hold on

% Compute Fourier series
CC = jet(20);
A0 = sum(f.*ones(size(x)))*dx;
fFS = A0/2;
for k=1:20
    A(k) = sum(f.*cos(pi*k*x/L))*dx; % Inner product
    B(k) = sum(f.*sin(pi*k*x/L))*dx;
    fFS = fFS + A(k)*cos(k*pi*x/L) + B(k)*sin(k*pi*x/L);
    plot(x,fFS,'-','Color',CC(k,:),'LineWidth',1.2)
end

%% Plot amplitudes
figure
clear ERR
clear A
fFS = A0/2;
A(1) = A0/2;
ERR(1) = norm(f-fFS);
kmax = 100;
for k=1:kmax
    A(k+1) = sum(f.*cos(pi*k*x/L))*dx;
    B(k+1) = sum(f.*sin(pi*k*x/L))*dx;
%     plot(x,B(k)*sin(2*k*pi*x/L),'k-','LineWidth',1.2);
    fFS = fFS + A(k+1)*cos(k*pi*x/L) + B(k+1)*sin(k*pi*x/L);
    ERR(k+1) = norm(f-fFS)/norm(f);
end
thresh = median(ERR)*sqrt(kmax)*4/sqrt(3);
r = max(find(ERR>thresh));
r = 7;
subplot(2,1,1)
semilogy(0:1:kmax,A,'k','LineWidth',1.5)
hold on
semilogy(r,A(r+1),'bo','LineWidth',1.5)
xlim([0 kmax])
ylim([10^(-7) 1])
subplot(2,1,2)
semilogy(0:1:kmax,ERR,'k','LineWidth',1.5)
hold on
semilogy(r,ERR(r+1),'bo','LineWidth',1.5)


function isMonge=isMongeMatrix(A)
%%ISMONGEMATRIX Determine whether a matrix is a Monge matrix. An n1Xn2
%          matrix is a Monge matrix if A(i1,j1)+A(i2,j2)<=A(i1,j2)+A(i2,j1)
%          for all 1<=i1<i2<=n1 and 1<=j1<j2<=n2. Monge matrices have
%          special properties in some discrete optimization problems.
%
%INPUTS: A An n1Xn2 matrix.
%
%OUTPUTS: isMonge This is true if A is a Monge matrix and false otherwise.
%
%EXAMPLE:
% A=[35, 25, 42, 88;
%    84, 63, 75, 96;
%    51,  9,  4, 25;
%    93, 16,  6, 18];
% isMonge=isMongeMatrix(A)
%A is a Monge matrix so isMonge will be 1.
%
%February 2018 David F. Crouse, Naval Research Laboratory, Washington D.C.
%(UNCLASSIFIED) DISTRIBUTION STATEMENT A. Approved for public release.

n1=size(A,1);
n2=size(A,2);

isMonge=true;
for i2=2:n1
    for i1=1:(i2-1)
        for j2=2:n2
            for j1=1:(j2-1)
                if(A(i1,j1)+A(i2,j2)>A(i1,j2)+A(i2,j1))
                   isMonge=false;
                   return;
                end
            end
        end
    end
end

end

%LICENSE:
%
%The source code is in the public domain and not licensed or under
%copyright. The information and software may be used freely by the public.
%As required by 17 U.S.C. 403, third parties producing copyrighted works
%consisting predominantly of the material produced by U.S. government
%agencies must provide notice with such work(s) identifying the U.S.
%Government material incorporated and stating that such material is not
%subject to copyright protection.
%
%Derived works shall not identify themselves in a manner that implies an
%endorsement by or an affiliation with the Naval Research Laboratory.
%
%RECIPIENT BEARS ALL RISK RELATING TO QUALITY AND PERFORMANCE OF THE
%SOFTWARE AND ANY RELATED MATERIALS, AND AGREES TO INDEMNIFY THE NAVAL
%RESEARCH LABORATORY FOR ALL THIRD-PARTY CLAIMS RESULTING FROM THE ACTIONS
%OF RECIPIENT IN THE USE OF THE SOFTWARE.


function a = bivand2 ( n, alpha, beta )

%*****************************************************************************80
%
%% BIVAND2 returns a bidimensional Vandermonde2 matrix.
%
%  Discussion:
%
%    N = 3, ALPHA = ( 1, 2, 3 ), BETA = ( 10, 20, 30 )
%
%    (x,y)   | (1,10) (2,10) (3,10) (1,20) (2,20) (3,20) (1,30) (2,30) (3,30)
%    --------+---------------------------------------------------------------
%    1       |     1      1      1      1      1      1      1      1      1
%    x       |     1      2      3      1      2      1      1      2      3
%    x^2     |     1      4      9      1      4      1      1      4      9
%       y    |    10     10     10     20     20     20     30     30     30
%    x  y    |    10     20     30     20     40     60     30     60     90
%    x^2y    |    10     40     90     20     80    180     30    120    270
%       y^2  |   100    100    100    400    400    400    900    900    900
%    x  y^2  |   100    200    300    400    800   1200    900   1800   2700
%    x^2y^2  |   100    400    900    400   1600   3600    900   3600   8100
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    01 May 2014
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer N, the order of the data vectors.
%
%    Input, real ALPHA(N), BETA(N), the values that define A.
%
%    Output, real A(N^2,N^2), the matrix.
%
  nn = n * n;
  a = zeros ( nn, nn );

  i = 0
  for iy = 1 : n
    for ix = 1 : n
      i = i + 1;
      j = 0;
      for jy = 1 : n
        for jx = 1 : n
          j = j + 1;
          a(i,j) = alpha(jx) ^ ( ix - 1 ) * beta(jy) ^ ( iy - 1 );
        end
      end
    end
  end

  return
end


function x = u1_to_sphere_unit_2d ( u )

%*****************************************************************************80
%
%% U1_TO_SPHERE_UNIT_2D maps a point in the unit interval onto the unit sphere in 2D.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    06 July 2004
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, real U, a point in the unit interval.
%
%    Output, real X(2), the corresponding point on the circle.
%
  angle = 2.0 * pi * u(1);

  x(1) = cos ( angle );
  x(2) = sin ( angle );

  return
end


function [ M ] = simGaussian( M, sigma )
%SIMGAUSSIAN Calculates Gaussian similarity on matrix
%   simGaussian(M, sigma) returns a matrix of the same size as
%   the distance matrix M, which contains similarity values
%   that are computed by using a Gaussian similarity function
%   with parameter sigma.
%
%   Author: Ingo Buerk
%   Year  : 2011/2012
%   Bachelor Thesis

M = exp(-M.^2 ./ (2*sigma^2));

end


function [J, grad] = costFunction(theta, X, y)
%COSTFUNCTION Compute cost and gradient for logistic regression
%   J = COSTFUNCTION(theta, X, y) computes the cost of using theta as the
%   parameter for logistic regression and the gradient of the cost
%   w.r.t. to the parameters.

% Initialize some useful values
m = length(y); % number of training examples

% You need to return the following variables correctly
% J = 0;
% grad = zeros(size(theta));

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the cost of a particular choice of theta.
%               You should set J to the cost.
%               Compute the partial derivatives and set grad to the partial
%               derivatives of the cost w.r.t. each parameter in theta
%
% Note: grad should have the same dimensions as theta
%

temp1 = -1 * (y .* log(sigmoid(X * theta)));
temp2 = (1 - y) .* log(1 - sigmoid(X * theta));

J = sum(temp1 - temp2) / m;

grad = (X' * (sigmoid(X * theta) - y)) * (1/m);

% =============================================================

end


function [ d ] = edist( x , y )
%edist euclidean distance between two vectors

d = sqrt( sum( (x-y).^2,2 ) );


function a = r8vec_even ( n, alo, ahi )

%*****************************************************************************80
%
%% R8VEC_EVEN returns N real values, evenly spaced between ALO and AHI.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    24 January 2004
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer N, the number of values.
%
%    Input, real ALO, AHI, the low and high values.
%
%    Output, real A(N), N evenly spaced values.
%    Normally, A(1) = ALO and A(N) = AHI.
%    However, if N = 1, then A(1) = 0.5*(ALO+AHI).
%
  if ( n == 1 )

    a(1) = 0.5 * ( alo + ahi );

  else

    a(1:n) = ( (n-1:-1:0) * alo + (0:n-1) * ahi ) / ( n - 1 );

  end

  return
end


function price = Price_Exchange_Option_Margrabe_2D( S0_1, S0_2, T, rho, sigma_1, sigma_2, q_1, q_2)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% About: calcuates Price of 2D Exchange option, (S_1(T) - S_2(T)), using Magrabe Formula Under 2D Diffusion
% Author: Justin Lars Kirkby
%
% -----------------
% Params
% -----------------
% S0_1    = initial asset price of first asset, e.g. S0_1 = 100
% S0_2    = initial asset price of second asset, e.g. S0_2 = 100
% T       = time to maturity in years (e.g. T=1)
% rho     = instantaneous correlation between S_1 and S_2
% sigma_1 = volatility of first asset (annualized), e.g. sigma = 0.2
% sigma_2 = volatility of second asset (annualized), e.g. sigma = 0.2
% q_1     = dividend yield of first asset, e.g. q_1 = 0.02
% q_2     = dividend yield of second asset, e.g. q_2 = 0.02
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

sig = sqrt(sigma_1^2 - 2*rho*sigma_1*sigma_2 + sigma_2^2);
d1 = (log(S0_1/S0_2) + 0.5*sig^2*T)/(sig*sqrt(T));
d2 = d1 - sig*sqrt(T);
price = exp(-q_1*T)*S0_1*normcdf(d1) - exp(-q_2*T)*S0_2*normcdf(d2);

end


function [mu sigma2] = estimateGaussian(X)
%ESTIMATEGAUSSIAN This function estimates the parameters of a
%Gaussian distribution using the data in X
%   [mu sigma2] = estimateGaussian(X),
%   The input X is the dataset with each n-dimensional data point in one row
%   The output is an n-dimensional vector mu, the mean of the data set
%   and the variances sigma^2, an n x 1 vector
%

% Useful variables
[m, n] = size(X);

% You should return these values correctly
mu = zeros(n, 1);
sigma2 = zeros(n, 1);

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the mean of the data and the variances
%               In particular, mu(i) should contain the mean of
%               the data for the i-th feature and sigma2(i)
%               should contain variance of the i-th feature.
%

mu = sum(X) ./ m
sigma2 = sum((X - (mu' * ones(1, m))').^2) ./ m

% =============================================================

end


function trans = createRotation(varargin)
%CREATEROTATION Create the 3*3 matrix of a rotation.
%
%   TRANS = createRotation(THETA);
%   Returns the rotation corresponding to angle THETA (in radians)
%   The returned matrix has the form :
%   [cos(theta) -sin(theta)  0]
%   [sin(theta)  cos(theta)  0]
%   [0           0           1]
%
%   TRANS = createRotation(POINT, THETA);
%   TRANS = createRotation(X0, Y0, THETA);
%   Also specifies origin of rotation. The result is similar as performing
%   translation(-X0, -Y0), rotation(THETA), and translation(X0, Y0).
%
%   Example
%     % apply a rotation on a polygon
%     poly = [0 0; 30 0;30 10;10 10;10 20;0 20];
%     trans = createRotation([10 20], pi/6);
%     polyT = transformPoint(poly, trans);
%     % display the original and the rotated polygons
%     figure; hold on; axis equal; axis([-10 40 -10 40]);
%     drawPolygon(poly, 'k');
%     drawPolygon(polyT, 'b');
%
%   See also:
%   transforms2d, transformPoint, createRotation90, createTranslation
%

%   ---------
%   author : David Legland
%   INRA - TPV URPOI - BIA IMASTE
%   created the 06/04/2004.
%

%   HISTORY
%   22/04/2009: rename as createRotation

% default values
cx = 0;
cy = 0;
theta = 0;

% get input values
if length(varargin)==1
    % only angle
    theta = varargin{1};
elseif length(varargin)==2
    % origin point (as array) and angle
    var = varargin{1};
    cx = var(1);
    cy = var(2);
    theta = varargin{2};
elseif length(varargin)==3
    % origin (x and y) and angle
    cx = varargin{1};
    cy = varargin{2};
    theta = varargin{3};
end

% compute coefs
cot = cos(theta);
sit = sin(theta);
tx =  cy*sit - cx*cot + cx;
ty = -cy*cot - cx*sit + cy;

% create transformation matrix
trans = [cot -sit tx; sit cot ty; 0 0 1];


function [ i, j ] = i4_to_pascal ( k )

%*****************************************************************************80
%
%% I4_TO_PASCAL converts a linear index to Pascal triangle coordinates.
%
%  Discussion:
%
%    We describe the grid points in Pascal's triangle in two ways:
%
%    As a linear index K:
%
%                     1
%                   2   3
%                 4   5   6
%               7   8   9   10
%
%    As elements (I,J) of Pascal's triangle:
%
%                     0,0
%                  1,0   0,1
%               2,0   1,1    0,2
%            3,0   2,1   1,2    0,3
%
%  Example:
%
%     K  I  J
%
%     1  0  0
%     2  1  0
%     3  0  1
%     4  2  0
%     5  1  1
%     6  0  2
%     7  3  0
%     8  2  1
%     9  1  2
%    10  0  3
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    14 April 2015
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer K, the linear index of the (I,J) element.
%    1 <= K.
%
%    Output, integer I, J, the Pascal indices.
%
  if ( k <= 0 )
    fprintf ( 1, '\n' );
    fprintf ( 1, 'I4_TO_PASCAL - Fatal error!\n' );
    fprintf ( 1, '  K must be positive.\n' );
    error ( 'I4_TO_PASCAL - Fatal error!' );
  end

  d = i4_to_pascal_degree ( k );

  j = k - ( d * ( d + 1 ) ) / 2 - 1;
  i = d - j;

  return
end


%GENDATL Generation of Lithuanian classes
%
%  A = GENDATL(N,S)
%
% INPUT
%  N  Number of objects per class (optional; default: [50 50])
%  S  Standard deviation for the data generation (optional; default: 1)
%
% OUTPUT
%  A  Dataset
%
% DESCRIPTION
% Generation of Lithuanian classes, a 2-dimensional, 2-class dataset A
% of N objects according to the definition given by Raudys.
% The data is uniformly distributed along two sausages and is superimposed
% by a normal distribution with standard deviation S in all directions.
% Class priors are P(1) = P(2) = 0.5.
%
% SEE ALSO (PRTools Guide)
% DATASETS, PRDATASETS

% Copyright: M. Skurichina, R.P.W. Duin, duin@ph.tn.tudelft.nl
% Faculty of Applied Sciences, Delft University of Technology
% P.O. Box 5046, 2600 GA Delft, The Netherlands

% $Id: gendatl.m,v 1.3 2007/06/19 11:44:14 duin Exp $

function a = gendatl(N,s)
		if nargin < 1,
		prwarning(3,'Class cardinalities are not specified, assuming [50 50].');
		N = [50 50];
	end
	if nargin < 2,
		prwarning(4,'Standard deviation for the data generation is not specified, assuming 1.');
		s = 1;
	end
	if (length(N) == 1),
		N(2) = N(1);
	end;

	u1	= 2*pi/3*(rand(N(1),1)-0.5*ones(N(1),1));
	u2	= 2*pi/3*(rand(N(2),1)-0.5*ones(N(2),1));
	a 	= [[10*cos(u1) + s*randn(N(1),1) 10*sin(u1) + s*randn(N(1),1)]; ...
				[6.2*cos(u2) + s*randn(N(2),1) 6.2*sin(u2) + s*randn(N(2),1)]];
	lab = genlab(N);
	a 	= prdataset(a,lab,'name','Lithuanian Classes');
	a   = setlablist(a,[1 2]');
	a   = setprior(a,[0.5 0.5]);
return;


function An = slnormalize(A, p, d)
%SLNORMALIZE Normalize the sub-arrays
%
% $ Syntax $
%   - An = slnormalize(A)
%   - An = slnormalize(A, p)
%   - An = slnormalize(A, [], d)
%   - An = slnormalize(A, p, d)
%
% $ Arguments $
%   - A:        the input array
%   - p:        the order of the norm (default = 2)
%   - d:        the dimension of subarrays (default = 1, column vectors)
%   - An:       the normalized array
%
% $ Description $
%   - An = slnormalize(A) normalizes the column vectors of A by 2nd-order.
%
%   - An = slnormalize(A, p) normalizes the column vectors of a by
%     pth-order.
%
%   - An = slnormalize(A, [], d) normalizes the subarrays of A along
%     dimensions specified by d by 2nd-order.
%
%   - An = slnormalize(A, p, d) normalizes the subarrays of A along
%     dimensions specified by d by pth-order.
%
% $ Remarks $
%   # Normalize an array by pth order means dividing the elements of
%     the array by its p-th norm.
%   # p can be inf or -inf. If p = inf, then the Lp norm is simply the
%     maximum magnitude value; while if p = -inf, then the Lp norm is the
%     minimum magnitude value.
%
% $ History $
%   - Created by Dahua Lin on Nov 19th, 2005
%   - Modified by Dahua Lin on Sep 10th, 2006
%       - replace slmul by slmulvec to increase efficiency
%

%% parse and verify input arguments
if nargin < 2 || isempty(p)
    p = 2;
end
if nargin < 3 || isempty(d)
    d = 1;
end
if ~isscalar(p)
    error('sltoolbox:notscalar', 'p should be a scalar');
end
if p == 0
    error('sltoolbox:zeroerr', 'p should not be zero');
end

%% compute
nrms = slnorm(A, p, d);
nrms = slenforce(nrms, 'positive');
An = slmulvec(A, 1 ./ nrms);


function [ mO ] = ImageConvFrequencyDomain( mI, mH, convShape )
% ----------------------------------------------------------------------------------------------- %
% [ mO ] = ImageConvFrequencyDomain( mI, mH, convShape )
% Applies Image Convolution in the Frequency Domain.
% Input:
%   - mI                -   Input Image.
%                           Structure: Matrix (numRows x numCols).
%                           Type: 'Single' / 'Double'.
%                           Range: (-inf, inf).
%   - mH                -   Filtering Kernel.
%                           Structure: Matrix.
%                           Type: 'Single' / 'Double'.
%                           Range: (-inf, inf).
%   - convShape         -   Convolution Shape.
%                           Sets the convolution shape.
%                           Structure: Scalar.
%                           Type: 'Single' / 'Double'.
%                           Range: {1, 2, 3}.
% Output:
%   - mI                -   Output Image.
%                           Structure: Matrix.
%                           Type: 'Single' / 'Double'.
%                           Range: (-inf, inf).
% References:
%   1.  MATLAB's 'conv2()' - https://www.mathworks.com/help/matlab/ref/conv2.html.
% Remarks:
%   1.  For the transformation the (0, 0) of the image and the kernel is
%       the top left corner of teh matrix (i = 1, j = 1). Hence it can be
%       thought that a shifting is needed (To center the kernel).
%   1.  The function supports single channel image only.
% TODO:
%   1.
%   Release Notes:
%   -   1.0.000     29/04/2021  Royi Avital     RoyiAvital@yahoo.com
%       *   First release version.
% ----------------------------------------------------------------------------------------------- %

CONV_SHAPE_FULL     = 1;
CONV_SHAPE_SAME     = 2;
CONV_SHAPE_VALID    = 3;

numRows     = size(mI, 1);
numCols     = size(mI, 2);

numRowsKernel = size(mH, 1);
numColsKernel = size(mH, 2);

switch(convShape)
    case(CONV_SHAPE_FULL)
        numRowsFft  = numRows + numRowsKernel - 1;
        numColsFft  = numCols + numColsKernel - 1;
        firstRowIdx = 1;
        firstColIdx = 1;
        lastRowIdx  = numRowsFft;
        lastColdIdx = numColsFft;
    case(CONV_SHAPE_SAME)
        numRowsFft  = numRows + numRowsKernel;
        numColsFft  = numCols + numColsKernel;
        firstRowIdx = ceil((numRowsKernel + 1) / 2);
        firstColIdx = ceil((numColsKernel + 1) / 2);
        lastRowIdx  = firstRowIdx + numRows - 1;
        lastColdIdx = firstColIdx + numCols - 1;
    case(CONV_SHAPE_VALID)
        numRowsFft = numRows;
        numColsFft = numCols;
        firstRowIdx = numRowsKernel;
        firstColIdx = numColsKernel;
        lastRowIdx  = numRowsFft;
        lastColdIdx = numColsFft;
end

mO = ifft2(fft2(mI, numRowsFft, numColsFft) .* fft2(mH, numRowsFft, numColsFft), 'symmetric');
mO = mO(firstRowIdx:lastRowIdx, firstColIdx:lastColdIdx);

end


%
% Written by M. Harper Langston - 5/10/00
% harper@cims.nyu.edu
%
% Using Lemma 10.5, form the orthogonal matrix, Q, the block matrix of
% matrices of eigenvectors of a TST block matrix. Then use this to form Gamma
% See Iselres page 246 or Lemma 10.5 for more info for Q.
% Gamma will be transformed into a plain tridiagonal matrix, using the method
% in Iserles, page 247 and 249.
function [Gam] = Form_Gamma(m,N)
for j = 1:m
   for k = 1:m
      Q(j,k) = sqrt(2/(m+1))*sin(pi*j*k/(m+1));
   end
end
% Now, form Gamma.  As per Iserles, page 247, we can rearrange columns into
% rows of the vectors, so Gamma will be tridiagonal.  The only difference is
% that in the problem, Gamma*t = c, solving for t, c must be converted and
% then t will have to be converted back.  See Iselres, pages 247 and 249
% for more informtion.
if N==9 | N==10
	S = (diag((-10/3)*ones(m,1)) + diag((2/3)*ones(m-1,1),1) + diag((2/3)*ones(m-1,1),-1));
   T = (diag((2/3)*ones(m,1)) + diag((1/6)*ones(m-1,1),1) + diag((1/6)*ones(m-1,1),-1));
elseif N==5
   S = (diag((-4)*ones(m,1)) + diag((1)*ones(m-1,1),1) + diag((1)*ones(m-1,1),-1));
   T = diag((1)*ones(m,1));
elseif N==101 % N = 101 is called by Modified_Right.m only, to  speed up
              % the modified 9=point scheme
   S = (diag((2/3)*ones(m,1)) + diag((1/12)*ones(m-1,1),1) + diag((1/12)*ones(m-1,1),-1));
   T = (diag((1/12)*ones(m,1)));
else
   error('Must either use 5 or 9-point scheme (N=5 or N=9 in poisson.m)');
end
% inv(Q) = Q, so don't need to invert Q to diagonalize S and T
GS = diag(Q*S*Q);
GT = diag(Q*T*Q);
Gam = sparse(zeros(size(Q)));
for j = 1:m
   temp = (j-1)*m +1;
   Gam(temp:j*m,temp:j*m) = diag(GS(j)*ones(m,1)) ...
	        + diag((GT(j))*ones(m-1,1),1) + diag((GT(j))*ones(m-1,1),-1);
end
Gam = sparse(Gam);
%
% Written by M. Harper Langston - 5/10/00
% harper@cims.nyu.edu
%


function chebyshev_polynomial_test01 ( )

%*****************************************************************************80
%
%% CHEBYSHEV_POLYNOMIAL_TEST01 tests T_PROJECT_COEFFICIENTS_DATA.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    22 April 2012
%
%  Author:
%
%    John Burkardt
%
  fprintf ( 1, '\n' );
  fprintf ( 1, 'CHEBYSHEV_POLYNOMIAL_TEST01:\n' );
  fprintf ( 1, '  T_PROJECT_COEFFICIENTS_DATA estimates the Chebyshev polynomial\n' );
  fprintf ( 1, '  coefficients for a function given as data (x,fx).\n' );
  fprintf ( 1, '\n' );
  fprintf ( 1, '  Here, we use fx = f(x) = x^2 for the data.\n' );
  fprintf ( 1, '\n' );
  fprintf ( 1, '  Since T(0,x) = 1 and T(2,x) = 2*x^2 - 1, the correct expansion is\n' );
  fprintf ( 1, '  f(x) = 1/2 T(0,x) + 0 T(1,x) + 1/2 T(2,x) + 0 * all other polys.\n' );
%
%  Data in [0,1];
%
  a = 0.0;
  b = 1.0;
  m = 20;
  seed = 123456789;
  [ x, seed ] = r8vec_uniform_01 ( m, seed );
  d = x.^2;

  r8vec2_print ( m, x, d, '  Data ( X, D ):' );

  n = 4;
  c = t_project_coefficients_data ( a, b, m, n, x, d );

  r8vec_print ( n, c, '  Coefficients of Chebyshev expansion of degree 4.' );
%
%  Compare Chebyshev expansion and original function.
%
  d2 = t_project_value ( m, n, x, c );

  fprintf ( 1, '\n' );
  fprintf ( 1, '   I      X(I)     Data(I)      Chebyshev(X(I))\n' );
  fprintf ( 1, '\n' );
  for i = 1 : m
    fprintf ( 1, '  %2d  %12g  %12g  %12g\n', i, x(i), d(i), d2(i) );
  end

  return
end


function f = local ( m, x )

%*****************************************************************************80
%
%% LOCAL computes the local function.
%
%  Discussion:
%
%    This function has a local minimizer:
%
%      X* = ( 0.2858054412..., 0.2793263206...), F(X*) = 5.9225...
%
%    and a global minimizer:
%
%      X* = ( -21.02667179..., -36.75997872...), F(X*) = 0.
%
%    Suggested starting point for local minimizer:
%
%      X = ( 1, 1 ), F(X) = 3.33 * 10^6.
%
%    Suggested starting point for global minimizer:
%
%      X = ( -15, -35), F(X) = 1.49 * 10^8.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    04 January 2012
%
%  Author:
%
%    John Burkardt
%
%  Reference:
%
%    David Himmelblau,
%    Applied Nonlinear Programming,
%    McGraw Hill, 1972,
%    ISBN13: 978-0070289215,
%    LC: T57.8.H55.
%
%  Parameters:
%
%    Input, integer M, the number of variables.
%
%    Input, real X(M), the argument of the function.
%
%    Output, real F, the value of the function at X.
%
  f = ( x(1)^2 + 12.0 * x(2) - 1.0 )^2 ...
    + ( 49.0 * x(1)^2 + 49.0 * x(2)^2 + 84.0 * x(1) + 2324.0 * x(2) - 681.0 )^2;

  return
end


function [J, grad] = lrCostFunction(theta, X, y, lambda)
%LRCOSTFUNCTION Compute cost and gradient for logistic regression with %regularization
%   J = LRCOSTFUNCTION(theta, X, y, lambda) computes the cost of using
%   theta as the parameter for regularized logistic regression and the
%   gradient of the cost w.r.t. to the parameters.

% Initialize some useful values
m = length(y); % number of training examples

% You need to return the following variables correctly
J = 0;
grad = zeros(size(theta));

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the cost of a particular choice of theta.
%               You should set J to the cost.
%               Compute the partial derivatives and set grad to the partial
%               derivatives of the cost w.r.t. each parameter in theta
%
% Hint: The computation of the cost function and gradients can be
%       efficiently vectorized. For example, consider the computation
%
%           sigmoid(X * theta)
%
%       Each row of the resulting matrix will contain the value of the
%       prediction for that example. You can make use of this to vectorize
%       the cost function and gradient computations.
%
% Hint: When computing the gradient of the regularized cost function,
%       there're many possible vectorized solutions, but one solution
%       looks like:
%           grad = (unregularized gradient for logistic regression)
%           temp = theta;
%           temp(1) = 0;   % because we don't add anything for j = 0
%           grad = grad + YOUR_CODE_HERE (using the temp variable)
%

% Cost function
h = sigmoid(X * theta);
theta1 = [0; theta(2:size(theta)) ];

cost = (1 / m) * sum( (-y .* log(h)) - ((1 - y) .* log(1 - h)) );
reg_cost = (lambda / (2 * m)) .* (theta1' * theta);

J = cost + reg_cost;

grad = (1 / m) * X' * (h - y);
reg_grad = (lambda / m) .* theta1;

grad = grad + reg_grad;

% =============================================================

grad = grad(:);

end


function [ X,i ] = gmm_sample(nbSamples,Priors,Mu,Sigma )
%GMM_SAMPLE Samples points from a Gaussian Mixture Model
%
%   input -----------------------------------------------------------------
%
%       o nbSamples : (1 x 1), number of samples to return
%
%       o Priors    : (1 x K), gmm weights
%
%       o Mu        : (D x K), gmm means
%
%       o Sigma     : (D x D x K), gmm covariance
%
%   output ----------------------------------------------------------------
%
%       o X         : (D x N), N samples of dimension D
%
%       o i         : (N x 1), index of Gaussian center from which the data
%                              points were sampled form.

D = size(Mu,1);
X = zeros(D,nbSamples);

i = discretesample(Priors', nbSamples);

if D == 1

    for v=1:nbSamples

        X(v) = normrnd(Mu(i(v)), sqrt(Sigma(i(v))) );

    end
else

    for v = 1:nbSamples
        j = i(v);
        X(:,v) = mvnrnd(Mu(:,j),Sigma(:,:,j));
    end

end

end


function pSum=polySum(p1,p2)
%%POLYSUM Add together two vectors representing 1D polynomials as power
%         series. The format of the polynomial vectors is the same as that
%         used in Matlab's polyval function: The coefficients are ordered
%         y(z)=p(end)+p(end-1)*z+p(end-2)*z^2+...
%
%INPUTS: p1,p2 Coefficients for the two power series. The number of terms
%              in the different series can be different. Thus p1 and p2 do
%              not have the same length. p1(1) is the coefficient of the
%              highest order term. p(end) is the additive constant term. p1
%              and p2 can be row or column vectors.
%
%OUTPUTS: pSum A column vector holding the sum of p1 and p2. The first
%              element corresponds to the highest term, as used in Matlab's
%              polyval function. Note that the vector is not shortened when
%              the sum of high order terms equals zero (terms cancel).
%
%Polynomials are summed by adding corresponding terms.
%
%As an example, consider adding
%x^2-3*x+2
%to
%x^5+3*x^3-4*x^2+1
%This is done as
% p1=[1;-3;2];
% p2=[1;0;3;-4;0;1];
% pSum=polySum(p1,p2)
%One will see that the result is pSum'=[1,0,3,-3,-3,3], which corresponds
%to x^5+3*x^3-3*x^2-3*x+3.
%
%August 2015 David F. Crouse, Naval Research Laboratory, Washington D.C.
%(UNCLASSIFIED) DISTRIBUTION STATEMENT A. Approved for public release.

pSum=[zeros(length(p1)-length(p2),1);p2(:)]+[zeros(length(p2)-length(p1),1);p1(:)];

end

%LICENSE:
%
%The source code is in the public domain and not licensed or under
%copyright. The information and software may be used freely by the public.
%As required by 17 U.S.C. 403, third parties producing copyrighted works
%consisting predominantly of the material produced by U.S. government
%agencies must provide notice with such work(s) identifying the U.S.
%Government material incorporated and stating that such material is not
%subject to copyright protection.
%
%Derived works shall not identify themselves in a manner that implies an
%endorsement by or an affiliation with the Naval Research Laboratory.
%
%RECIPIENT BEARS ALL RISK RELATING TO QUALITY AND PERFORMANCE OF THE
%SOFTWARE AND ANY RELATED MATERIALS, AND AGREES TO INDEMNIFY THE NAVAL
%RESEARCH LABORATORY FOR ALL THIRD-PARTY CLAIMS RESULTING FROM THE ACTIONS
%OF RECIPIENT IN THE USE OF THE SOFTWARE.


function [mu sigma2] = estimateGaussian(X)
%ESTIMATEGAUSSIAN This function estimates the parameters of a
%Gaussian distribution using the data in X
%   [mu sigma2] = estimateGaussian(X),
%   The input X is the dataset with each n-dimensional data point in one row
%   The output is an n-dimensional vector mu, the mean of the data set
%   and the variances sigma^2, an n x 1 vector
%

% Useful variables
[m, n] = size(X);

% You should return these values correctly
mu = zeros(n, 1);
sigma2 = zeros(n, 1);

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the mean of the data and the variances
%               In particular, mu(i) should contain the mean of
%               the data for the i-th feature and sigma2(i)
%               should contain variance of the i-th feature.
%

mu = mean(X);
sigma2 = var(X, 1);

% =============================================================

end


function p = predict(Theta1, Theta2, X)
%PREDICT Predict the label of an input given a trained neural network
%   p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the
%   trained weights of a neural network (Theta1, Theta2)

% Useful values
m = size(X, 1);
num_labels = size(Theta2, 1);

% You need to return the following variables correctly
p = zeros(size(X, 1), 1);

% ====================== YOUR CODE HERE ======================
% Instructions: Complete the following code to make predictions using
%               your learned neural network. You should set p to a
%               vector containing labels between 1 to num_labels.
%
% Hint: The max function might come in useful. In particular, the max
%       function can also return the index of the max element, for more
%       information see 'help max'. If your examples are in rows, then, you
%       can use max(A, [], 2) to obtain the max for each row.
%

temp1 = [ones(m, 1) X];
temp2 = [ones(m, 1) sigmoid(temp1 * Theta1')];
temp3 = sigmoid(temp2 * Theta2');
[maxTemp3, maxTemp3_2] = max(temp3');
p = maxTemp3_2';

% =========================================================================

end


function y = sumlogs(x,dim)
%SUMLOGS Sum of elements where numbers are represented by their logarithms.
%
%  Description
%    C=SUMLOGS(A) computes C=log(sum(exp(A))) in such a fashion
%    that it works even when elements have large magnitude.
%
%    C=SUMLOGS(A,DIM) sums along the dimension DIM.
%
%    See also SUM
%
% Copyright (c) 2013 Aki Vehtari

% This software is distributed under the GNU General Public
% License (version 3 or later); please refer to the file
% License.txt, included with the software, for details.

if nargin<2
  dim=find(size(x)>1,1);
end
maxx=max(x(:));
y=maxx+log(sum(exp(x-maxx),dim));


function call_price=bs_european_call(S, K, r, sigma, time)

%--------------------------------------------------------------------------
%
% DESCRIPTION:
%
% European put option using Black-Scholes' formula
%
%
% Reference:
%
% John Hull, "Options, Futures and other Derivative Securities",
% Prentice-Hall, second edition, 1993.
%
%--------------------------------------------------------------------------
%
% INPUTS:
%
%   S:      spot price
%   K:      strike price
%   r:      interest rate
%   sigma:  volatility
%   time:   time to maturity
%
%--------------------------------------------------------------------------
%
% OUTPUT:
%
% call_price: price of a call option
%
%--------------------------------------------------------------------------
%
% Author:  Paolo Z., February 2012
%
%--------------------------------------------------------------------------

time_sqrt = sqrt(time);

d1 = (log(S/K)+r*time)/(sigma*time_sqrt)+0.5*sigma*time_sqrt;
d2 = d1-(sigma*time_sqrt);

call_price  = S*normcdf(d1)-K*exp(-r*time)*normcdf(d2);


function P = xyz2polar(X)
% XYZ2POLAR Transformation of cartesian coordinates to polar coordinates.
% ------------------------------------------------------------------------------
% DESCRIPTION/NOTES
% Transformation of cartesian coordinates [x, y, z] to polar coordinates
% [distance, horizontal angle, vertical angle].
% ------------------------------------------------------------------------------
% INPUT
% X
%   Matrix of cartesian coordinates x, y and z. Each point in a row.
% ------------------------------------------------------------------------------
% OUTPUT
% P
%   Matrix of polar coordinates distace, horizontal angle and vertical angle.
%   Each point in a row.
%   Horizontal angle is defined mathematically positive starting from x-axis.
%   Vertical   angle is defined as angle between point and zenith.
% ------------------------------------------------------------------------------
% EXAMPLES
% Transformation of randomly generated points into polar coordinates and back.
%   X1   = [rand(10,1)*10 rand(10,1)*5 rand(10,1)];
%   P    = xyz2polar(X1);
%   X2   = polar2xyz(P);
%   Diff = X1-X2;
% ------------------------------------------------------------------------------
% philipp.glira@gmail.com
% ------------------------------------------------------------------------------

% Input parsing ----------------------------------------------------------------

p = inputParser;
p.addRequired('X', @(x) isreal(x) && size(x,2) == 3);
p.parse(X);
p = p.Results;
% Clear required inputs to avoid confusion
clear X

% Distance ---------------------------------------------------------------------

d = sqrt(p.X(:,1).^2 + p.X(:,2).^2 + p.X(:,3).^2);

% Vertical angle ---------------------------------------------------------------

va = acosg(p.X(:,3) ./ d); % always positive [0,200]

% Horizontal angle -------------------------------------------------------------

% Compute all 4 solutions for the horizontal angle
hSol(:,1) = acosg( p.X(:,1) ./ (d.*sing(va)) ); % sol1: always positive [   0,200]
hSol(:,2) = 400 - hSol(:,1);                    % sol2: always positive [ 200,400]
hSol(:,3) = asing( p.X(:,2) ./ (d.*sing(va)) ); % sol3: can be negative [-100,100]
hSol(:,4) = 200 - hSol(:,3);                    % sol4: always positive [ 100,300]

% Eliminate complex part (occure if argument of acosg/asing is not in range
% [-1, 1]); should be the case just for few points
hSol = real(hSol);

% If some values of sol3 are negativ, transform them into positive range
idxNeg = hSol(:,3) < 0;
hSol(idxNeg,3) = hSol(idxNeg,3) + 400; % now positive

% Compute the differences between the solutions
% Pairs for comparisons: 1,3 1,4 2,3 2,4
diff = [hSol(:,1)-hSol(:,3), hSol(:,1)-hSol(:,4), hSol(:,2)-hSol(:,3), hSol(:,2)-hSol(:,4)];
% Find for each point the pair with the lowest difference
[~, idx] = min(abs(diff), [], 2);

% If pair 1 (1,3) take column 1 of hSol (i.e. sol1)
% If pair 2 (1,4) take column 1 of hSol (i.e. sol1)
% If pair 3 (2,3) take column 2 of hSol (i.e. sol2)
% If pair 4 (2,4) take column 2 of hSol (i.e. sol2)
idx(idx == 2) = 1;
idx(idx == 3) = 2;
idx(idx == 4) = 2;

% Convert subscripts to linear indices to access the elements
idxLin = sub2ind(size(hSol), [1:size(hSol,1)]', idx);

% Get values
ha = hSol(idxLin);

% Output -----------------------------------------------------------------------

P = [d ha va];

end


function p = norm_density(x, mu, sigma)
% Compute density of normal distribution function
%
% file:      	norm_density.m, (c) Matthew Roughan, Tue Jul 21 2009
% directory:   /home/mroughan/src/matlab/NUMERICAL_ROUTINES/
% created: 	Tue Jul 21 2009
% author:  	Matthew Roughan
% email:   	matthew.roughan@adelaide.edu.au
%
% Calculate normal density in 1D.
%
% INPUTS:
%   x      = a vector of points at which to calculate the normal CDF
%   mu     = mean of the normal
%   sigma  = std dev. of the normal
%
%
p = exp( -(x-mu).^2 / (2*sigma^2) ) / (sigma * sqrt(2*pi) );


function [J, grad] = cofiCostFunc(params, Y, R, num_users, num_movies, ...
                                  num_features, lambda)
%COFICOSTFUNC Collaborative filtering cost function
%   [J, grad] = COFICOSTFUNC(params, Y, R, num_users, num_movies, ...
%   num_features, lambda) returns the cost and gradient for the
%   collaborative filtering problem.
%

% Unfold the U and W matrices from params
X = reshape(params(1:num_movies*num_features), num_movies, num_features);
Theta = reshape(params(num_movies*num_features+1:end), ...
                num_users, num_features);

% You need to return the following values correctly
J = 0;
X_grad = zeros(size(X));
Theta_grad = zeros(size(Theta));

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the cost function and gradient for collaborative
%               filtering. Concretely, you should first implement the cost
%               function (without regularization) and make sure it is
%               matches our costs. After that, you should implement the
%               gradient and use the checkCostFunction routine to check
%               that the gradient is correct. Finally, you should implement
%               regularization.
%
% Notes: X - num_movies  x num_features matrix of movie features
%        Theta - num_users  x num_features matrix of user features
%        Y - num_movies x num_users matrix of user ratings of movies
%        R - num_movies x num_users matrix, where R(i, j) = 1 if the
%            i-th movie was rated by the j-th user
%
% You should set the following variables correctly:
%
%        X_grad - num_movies x num_features matrix, containing the
%                 partial derivatives w.r.t. to each element of X
%        Theta_grad - num_users x num_features matrix, containing the
%                     partial derivatives w.r.t. to each element of Theta
%

% X.shape=[nm, n]; Theta.shape=[nu, n]
% Y.shape=[nm, nu];
J = 1.0 / 2 * sum(sum(R.*(X*Theta'-Y).^2));
J = J + lambda/2*sum(sum(Theta.^2)) + lambda/2*sum(sum(X.^2));

% X_grad.shape=[nm, n]; Theta.grad.shape=[nu, n]
X_grad = R.*(X*Theta'-Y)*Theta;
X_grad = X_grad + lambda*X;
Theta_grad = (R.*(X*Theta'-Y))'*X;
Theta_grad = Theta_grad + lambda*Theta;

% =============================================================

grad = [X_grad(:); Theta_grad(:)];

end


function [ loglik ] = LogLikelihood_gmm(X,Priors,Mu,Sigma,w)
%LOGLIKELIHOOD_GMM
%
%   input------------------------------------------------------------------
%
%       o X : (D x N), set of N data points of dimension  D
%
%       o w : (1 x N), set of weights if the data points are weighted
%
%   output ----------------------------------------------------------------
%
%       o loglik : (N x 1) , loglikelihood
%

N = size(X,2);
K = size(Priors,2);

Pxi = zeros(N,K);
for i=1:K
    Pxi(:,i) = gaussPDF(X, Mu(:,i), Sigma(:,:,i));
end

if exist('w','var') && ~isempty(w)
    %               (N x K)
    %             (N x 1)
    F            = (repmat(w(:),1,K) .* Pxi) * Priors';
else

    %Compute the log likelihood
    F            = Pxi*Priors';

end

F(F<realmin) = realmin;
F            = Pxi*Priors';
loglik       = sum(log(F));

end


function [C, sigma] = dataset3Params(X, y, Xval, yval)
%DATASET3PARAMS returns your choice of C and sigma for Part 3 of the exercise
%where you select the optimal (C, sigma) learning parameters to use for SVM
%with RBF kernel
%   [C, sigma] = DATASET3PARAMS(X, y, Xval, yval) returns your choice of C and
%   sigma. You should complete this function to return the optimal C and
%   sigma based on a cross-validation set.
%

% You need to return the following variables correctly.
C = 1;
sigma = 0.3;

% ====================== YOUR CODE HERE ======================
% Instructions: Fill in this function to return the optimal C and sigma
%               learning parameters found using the cross validation set.
%               You can use svmPredict to predict the labels on the cross
%               validation set. For example,
%                   predictions = svmPredict(model, Xval);
%               will return the predictions on the cross validation set.
%
%  Note: You can compute the prediction error using
%        mean(double(predictions ~= yval))
%

params = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30];

min_error = 0;

for c=params
    for s=params
        model = svmTrain(X, y, c, @(x1, x2) gaussianKernel(x1, x2, s)); % 模仿ex6的调用方式
        predictions = svmPredict(model, Xval); % 查看svmPredict
        error = mean(double(predictions ~= yval));

        if min_error == 0 || error < min_error
            min_error = error;
            C = c;
            sigma = s;
        end
    end
end

% =========================================================================

end


function [theta, J_history] = gradientDescentMulti(X, y, theta, alpha, num_iters)
%GRADIENTDESCENTMULTI Performs gradient descent to learn theta
%   theta = GRADIENTDESCENTMULTI(x, y, theta, alpha, num_iters) updates theta by
%   taking num_iters gradient steps with learning rate alpha

% Initialize some useful values
m = length(y); % number of training examples
J_history = zeros(num_iters, 1);

for iter = 1:num_iters

    % ====================== YOUR CODE HERE ======================
    % Instructions: Perform a single gradient step on the parameter vector
    %               theta.
    %
    % Hint: While debugging, it can be useful to print out the values
    %       of the cost function (computeCostMulti) and gradient here.
    %

    S = (1 / m) * (X' * (X * theta - y));
    theta = theta - alpha .* S;

    % ============================================================

    % Save the cost J in every iteration
    J_history(iter) = computeCostMulti(X, y, theta);

end

end


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%   Gradient Boosting Regression 1D Example  %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%               1) Load 1D Regression Datasets               %%
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% (1a) (simple example) Generate Data from sine function
clear all; close all; clc;
nbSamples = 200;
epsilon   = 0.1;
x_limits  = [0, 100];

% Generate True function and data
X         = linspace(x_limits(1),x_limits(2),nbSamples)';
y_true    = sin(X*0.05);
y         = y_true + normrnd(0,epsilon,1,nbSamples)';

% Plot data
options             = [];
options.points_size = 10;
options.title       = 'noisy sinusoidal data';

if exist('h1','var') && isvalid(h1), delete(h1);end
h1      = ml_plot_data([X(:),y(:)],options); hold on;

% Plot True function and Data
plot(X,y_true,'--k','LineWidth',2);
legend({'data','true function'})

%% (1b) (complex example) Generate Data from a sinc function
clear all; close all; clc;
% Set parameters for sinc function data
nbSamples = 200;
epsilon   = 0.1;
y_offset  = 0.5;
x_limits  = [-5, 5];

% Generate True function and data
X = linspace(x_limits(1),x_limits(2),nbSamples) ;
y_true = sinc(X) + y_offset ;
y = y_true + normrnd(0,epsilon,1,nbSamples);
X = X';
y = y';

% Plot data
options             = [];
options.points_size = 15;
options.title       = 'noisy sinc data';

if exist('h1','var') && isvalid(h1), delete(h1);end
h1      = ml_plot_data([X(:),y(:)],options); hold on;

% Plot True function and Data
plot(X,y_true,'--k','LineWidth',2);
legend({'data','true function'})

%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                 2)  GRADIENT BOOSTING REGRESSION                     %%
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Gradient Boosting with decision stumps weak learners
% Options
options.nbWeakLearners = 200;

% Train Gradient Boosting Model
model = fitensemble(X,y,'LSBoost',options.nbWeakLearners,'Tree');

% Compute prediction

Y = predict(model,X);

% Plot data
options.title       = [options.title ' : Prediction'];

if exist('hl','var') && isvalid(hl), delete(hl);end
hl      = ml_plot_data([X(:),y(:)],options); hold on;

% Plot prediction from Gradient Boosting

plot(X,Y,'k','LineWidth',2);
legend({'data', 'predicted function'});

% Plot Loss function of Training Model

if exist('hl2','var') && isvalid(hl2), delete(hl2);end
figure
hl2 = plot(model.FitInfo);
xlabel('Number of weak learners','FontSize',14);
ylabel('Loss (MSE)','FontSize',14);
title('Cumulative Loss of the train data','FontSize',18);


function cdf = empirical_discrete_cdf ( x, a, b, c )

%*****************************************************************************80
%
%% EMPIRICAL_DISCRETE_CDF evaluates the Empirical Discrete CDF.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    09 October 2004
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, real X, the argument of the CDF.
%
%    Input, integer A, the number of values.
%    0 < A.
%
%    Input, real B(A), the weights of each value.
%    0 <= B(1:A) and at least one value is nonzero.
%
%    Input, real C(A), the values.
%    The values must be distinct and in ascending order.
%
%    Output, real CDF, the value of the CDF.
%
  cdf = 0.0;

  bsum = sum ( b(1:a) );

  for i = 1 : a

    if ( x < c(i) )
      return
    end

    cdf = cdf + b(i) / bsum;

  end

  return
end


function h = orientedGaussianKernel(size, sigma, theta)
%ORIENTEDGAUSSIANKERNEL Oriented Gaussian kernel for directional filtering
%
%   H = orientedGaussianKernel(SIZE, SIGMA, THETA)
%   THETA is in degrees.
%
%   Example
%     orientedGaussianKernel
%
%   See also
%     imDirectionalFilter, orientedLaplacianKernel, imfilter

% ------
% Author: David Legland
% e-mail: david.legland@grignon.inra.fr
% Created: 2012-12-04,    using Matlab 7.9.0.529 (R2009b)
% Copyright 2012 INRA - Cepia Software Platform.

% ensure size is a vector
if length(size) == 1
    size = [size size];
end

% ensure sigma is a vector
if length(sigma) == 1
    sigma = [sigma sigma];
end

% compute reference for each axis
nx = round((size(1) - 1) / 2);
ny = round((size(2) - 1) / 2);

% create 3D grid
[x, y] = meshgrid(-nx:nx, -ny:ny);

% precompute angles
cot = cosd(-theta);
sit = sind(-theta);

% Apply rotation by -theta
x2 = x * cot - y * sit;
y2 = x * sit + y * cot;

% compute gaussian function for each point of the grid
sx2 = 2 * sigma(1) ^ 2;
sy2 = 2 * sigma(2) ^ 2;
h = exp(-(x2.^2 / sx2)) .* exp(-(y2.^2 / sy2));

% normalize intensities
h = h / (sum(h(:)));


function [ v_weight, predecessor ] = bellman_ford ( v_num, e_num, source, ...
  e, e_weight )

%*****************************************************************************80
%
%  Purpose:
%
%    BELLMAN_FORD finds shortest paths from a given vertex of a weighted directed graph.
%
%  Discussion:
%
%    The Bellman-Ford algorithm is used.
%
%    Each edge of the graph has a weight, which may be negative.  However,
%    it should not be the case that there is any negative loop, that is,
%    a circuit whose total weight is negative.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    12 November 2014
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer V_NUM, the number of vertices.
%
%    Input, integer E_NUM, the number of edges.
%
%    Input, integer SOURCE, the vertex from which distances will
%    be calculated.
%
%    Input, integer E(2,E_NUM), the edges, given as pairs of
%    vertex indices.
%
%    Input, real E_WEIGHT(E_NUM), the weight of each edge.
%
%    Output, real V_WEIGHT(V_NUM), the weight of each node,
%    that is, its minimum distance from SOURCE.
%
%    Output, integer PREDECESSOR(V_NUM), a list of predecessors,
%    which can be used to recover the shortest path from any node back to SOURCE.
%
  r8_big = 1.0E+30;
%
%  Step 1: initialize the graph.
%
  v_weight(1:v_num) = r8_big;
  v_weight(source) = 0.0;

  predecessor(1:v_num) = -1;
%
%  Step 2: Relax edges repeatedly.
%
  for i = 1 : v_num - 1
    for j = 1 : e_num
      u = e(2,j);
      v = e(1,j);
      t = v_weight(u) + e_weight(j);
      if ( t < v_weight(v) )
        v_weight(v) = t;
        predecessor(v) = u;
      end
    end
  end
%
%  Step 3: check for negative-weight cycles
%
  for j = 1 : e_num
    u = e(2,j);
    v = e(1,j);
    if ( v_weight(u) + e_weight(j) < v_weight(v) )
      fprintf ( 1, '\n' );
      fprintf ( 1, 'BELLMAN_FORD - Fatal error!\n' );
      fprintf ( 1, '  Graph contains a cycle with negative weight.\n' );
      error ( 'BELLMAN_FORD - Fatal error!' );
    end
  end

  return
end


function [ mO ] = CompressImageSvd( mI, energyThr, blockRadius )
% ----------------------------------------------------------------------------------------------- %
% [ mO ] = CompressImageSvd( mI, energyThr, blockRadius )
%   Compresses the image using SVD.
% Input:
%   - mI            -   Input Image.
%                       Structure: Image Matrix (Signle Channel or RGB).
%                       Type: 'Single' / 'Double'.
%                       Range: [0, 1].
%   - energyThr     -   Energy Threshold.
%                       Sets the threshold for Singular Value kept energy.
%                       Structure: Scalar.
%                       Type: 'Single' / 'Double'.
%                       Range: [0, 1].
%   - blockRadius   -   Block Radius.
%                       Sets the block radius for the compression.
%                       Structure: Scalar.
%                       Type: 'Single' / 'Double'.
%                       Range: {1, 2, ...}.
% Output:
%   - mO            -   Output Image.
%                       Structure: Image Matrix (Signle Channel or RGB).
%                       Type: 'Single' / 'Double'.
%                       Range: [0, 1].
% References
%   1.  SVD Wikipedia - https://en.wikipedia.org/wiki/Singular_value_decomposition.
% Remarks:
%   1.  a
% TODO:
%   1.  U.
% Release Notes:
%   -   1.0.000     01/09/2017  Royi
%       *   First release version.
% ----------------------------------------------------------------------------------------------- %

FALSE   = 0;
TRUE    = 1;

OFF     = 0;
ON      = 1;

numRows = size(mI, 1);
numCols = size(mI, 2);
numChan = size(mI, 3); %<! Num Channels

vImageDim = [numRows, numCols];

blockLength = (2 * blockRadius) + 1;
vBlockDim   = [blockLength, blockLength];

mO = zeros([numRows, numCols, numChan]);

for ii = 1:numChan

    mII     = mI(:, :, ii);
    dcLevel = mean(mII(:)); %<! Extracting DC Level
    mII     = mII - dcLevel;

    % Decomposing the image into blocks. Each block becomes a vector in the
    % Columns Images.
    mColImage   = im2col(mII, vBlockDim, 'distinct');

    % The SVD Step
    [mU, mS, mV] = svd(mColImage);

    vSingularValues = diag(mS);

    vSingularValueEnergy = cumsum(vSingularValues) / sum(vSingularValues);
    lastIdx = find(vSingularValueEnergy >= energyThr, 1, 'first');

    vSingularValues(lastIdx + 1:end) = 0;
    % mS isn't necessarily square matrix. Hence only work on its main
    % diagonal.
    mS(1:length(vSingularValues), 1:length(vSingularValues)) = diag(vSingularValues);

    % Reconstruction of the image using "Less Energy".
    mColImage = mU * mS * mV.';

    % Restorig the original structure and the DC Level
    mO(:, :, ii) = col2im(mColImage, vBlockDim, vImageDim, 'distinct') + dcLevel;

end

end


% this m-file calculates the real roots of the given polynomial using
% newton raphson technique.this m-file calls the functions in the two m-files named as syn_division and
% derivate.
% coeff_function is the 1xn matrix conatining the coeff of the polynomial.
% Keerthi Venkateswara Rao
function [final_roots,functionvalue] = newton(coeff_function,initial_guess,error_tolerance,max_iterations)
iterations=0;
max_no_roots=size(coeff_function,2);
final_roots=0;
functionvalue=0;
for no_roots=1:max_no_roots-1
    fun_root_new=initial_guess;
    flag=1;
    coeff_der_function=derivate(coeff_function);
    order_fun=size(coeff_function,2);
    order_der_fun=size(coeff_der_function,2);
    while flag==1
        fun_root_old=fun_root_new;
        fx=0;
        dfx=0;
        nonzero=1;
        while nonzero==1
            powers=order_fun-1;
            for index=1:order_fun
                fx=fx+coeff_function(index)*fun_root_old^powers;
                powers=powers-1;
            end
            powers=order_der_fun-1;
            for index=1:order_der_fun
                dfx=dfx+coeff_der_function(index)*fun_root_old^powers;
                powers=powers-1;
            end
            if dfx==0
                fun_root_old=fun_root_old+1;
            else
                nonzero=0;
            end
        end
        iterations = iterations + 1;
        fun_root_new = fun_root_old - fx/dfx;
        if iterations >= max_iterations
            flag=0;
        elseif  abs(fun_root_new-fun_root_old)<=error_tolerance
            flag=0;
            final_roots(no_roots)=fun_root_new;
            functionvalue(no_roots)=fx;
        end
    end
    coeff_function=syn_division(coeff_function,fun_root_new);
 end


function variance = bradford_variance ( a, b, c )

%*****************************************************************************80
%
%% BRADFORD_VARIANCE returns the variance of the Bradford PDF.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    03 September 2004
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, real A, B, C, the parameters of the PDF.
%    A < B,
%    0.0 < C.
%
%    Output, real VARIANCE, the variance of the PDF.
%
  variance = ( b - a )^2 * ...
    ( c * ( log ( c + 1.0 ) - 2.0 ) + 2.0 * log ( c + 1.0 ) ) ...
    / ( 2.0 * c * ( log ( c + 1.0 ) )^2 );

  return
end


function p = triangle_circumcenter ( n, t )

%*****************************************************************************80
%
%% TRIANGLE_CIRCUMCENTER computes the circumcenter of a triangle in ND.
%
%  Discussion:
%
%    Three ND points A, B and C lie on a circle.
%
%    The circumcenter P has the formula
%
%      P = ( Area ( PBC ) * A + Area ( APC) * B + Area ( ABP ) * C )
%        / ( Area ( PBC )     + Area ( APC )    + Area ( ABP ) )
%
%    The details of the formula rely on information supplied
%    by Oscar Lanzi III.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    28 October 2010
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer N, the spatial dimension.
%
%    Input, real T(N,3), the triangle vertices.
%
%    Output, real P(N,1), the circumcenter of the triangle.
%
  a = r8vec_normsq_affine ( n, t(1:n,2), t(1:n,3) );
  b = r8vec_normsq_affine ( n, t(1:n,3), t(1:n,1) );
  c = r8vec_normsq_affine ( n, t(1:n,1), t(1:n,2) );

  pbc = a * ( - a + b + c );
  apc = b * (   a - b + c );
  abp = c * (   a + b - c );

  p = zeros ( n, 1 );

  p(1:n,1) = ( pbc * t(1:n,1) + apc * t(1:n,2) + abp * t(1:n,3) ) ...
           / ( pbc            + apc            + abp );

  return
end


function [logval val]=mndensity(x,mu,sigma,k)

% function [logval val]=mndensity(x,mu,sigma,k)
% computes the density of the multivariate normal distribution
% uses the log of the density (a.2.1), split into several parts for computation ease
% inputs:  - vector 'x': the argument of the density function
%          - vector 'mu': the mean of the distribution
%          - matrix 'sigma': the covariance of the distribution
%          - integer 'k': the dimension of the argument vector
% outputs: - scalar 'logval': the log of the density value
%          - scalar 'val': the density value

temp(1,1)=-(k/2)*log(2*pi);
temp(2,1)=-0.5*log(det(sigma));
temp(3,1)=-0.5*(x-mu)'*(sigma\(x-mu));

% temp(3,1)=-0.5*(x-mu)'*inv(sigma)*(x-mu);

logval=sum(temp);
val=exp(logval);


function n2 = dist2(x, c)
%DIST2	Calculates squared distance between two sets of points.
%
%	Description
%	D = DIST2(X, C) takes two matrices of vectors and calculates the
%	squared Euclidean distance between them.  Both matrices must be of
%	the same column dimension.  If X has M rows and N columns, and C has
%	L rows and N columns, then the result has M rows and L columns.  The
%	I, Jth entry is the  squared distance from the Ith row of X to the
%	Jth row of C.
%   D(i,j) = norm(X(i,:)-C(j,:)).^2;
%
%	See also
%	GMMACTIV, KMEANS, RBFFWD
%

%	Copyright (c) Christopher M Bishop, Ian T Nabney (1996, 1997)

%	Dec 6, 2011: Modified by Da Kuang, with performance improvement.

[ndata, dimx] = size(x);
[ncentres, dimc] = size(c);
if dimx ~= dimc
	error('Data dimension does not match dimension of centres')
end

tempx = full(sum(x.^2, 2));
tempc = full(sum(c.^2, 2)');

n2 = tempx(:, ones(1,ncentres)) + ...
  		tempc(ones(1,ndata), :) - ...
  		2.*(x*(c'));


function wval = lagrange_factor ( npol, xpol, nval, xval )

%*****************************************************************************80
%
%% LAGRANGE_FACTOR evaluates the polynomial Lagrange factor at a point.
%
%  Formula:
%
%    W(X) = Product ( 1 <= I <= NPOL ) ( X - XPOL(I) )
%
%  Discussion:
%
%    Suppose F(X) is at least N times continuously differentiable in the
%    interval [A,B].  Pick NPOL distinct points XPOL(I) in [A,B] and compute
%    the interpolating polynomial P(X) of order NPOL ( and degree NPOL-1)
%    which passes through all the points ( XPOL(I), F(XPOL(I)) ).
%    Then in the interval [A,B], the maximum error
%
%      abs ( F(X) - P(X) )
%
%    is bounded by:
%
%      C * FNMAX * W(X)
%
%    where
%
%      C is a constant,
%      FNMAX is the maximum value of the NPOL-th derivative of F in [A,B],
%      W(X) is the Lagrange factor.
%
%    Thus, the value of W(X) is useful as part of an estimated bound
%    for the interpolation error.
%
%    Note that the Chebyshev abscissas have the property that they minimize
%    the value of W(X) over the interval [A,B].  Hence, if the abscissas may
%    be chosen arbitrarily, the Chebyshev abscissas have this advantage over
%    other choices.
%
%    For a set of points XPOL(I), 1 <= I <= NPOL, the IPOL-th Lagrange basis
%    polynomial L(IPOL)(X), has the property:
%
%      L(IPOL)( XPOL(J) ) = delta ( IPOL, J )
%
%    and may be expressed as:
%
%      L(IPOL)(X) = W(X) / ( ( X - XPOL(IPOL) ) * W'(XPOL(IPOL)) )
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    26 April 2005
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, integer NPOL, the number of abscissas.
%    NPOL must be at least 1.
%
%    Input, real XPOL(NPOL), the abscissas, which should
%    be distinct.
%
%    Input, integer NVAL, the number of evaluation points.
%
%    Input, real XVA(NVAL), the points at which the Lagrange
%    factor is to be evaluated.
%
%    Output, real WVAL(NVAL), the value of the Lagrange factor at XVAL.
%
  wval = zeros ( nval, 1 );

  for ival = 1 : nval

    wval(ival) = prod ( xval(ival) - xpol(1:npol) );

  end

  return
end


function r=ksrlin(x,y,h,N)
% KSRLIN   Local linear kernel smoothing regression
%
% r=ksrlin(x,y) returns the local linear Gaussian kernel regression in
% structure r such that r.f(r.x) = y(x) + e. The bandwidth and number of
% samples are also stored in r.h and r.n respectively.
%
% r=ksrlin(x,y,h) performs the regression using the specified bandwidth, h.
%
% r=ksrlin(x,y,h,n) calculates the regression in n points (default n=100).
%
% Without output, ksrlin(x,y) or ksrlin(x,y,h) will display the regression
% plot.
%
% Algorithm
% The kernel regression is a non-parametric approach to
% estimate the conditional expectation of a random variable:
%
% E(Y|X) = f(X)
%
% where f is a non-parametric function. The normal kernel regression is a
% local constant estimator. The extension of local linear estimator is
% obtained by solving the least squares problem:
%
% min sum (y-alpha-beta(x-X))^2 kerf((x-X)/h)
%
% The local linear estimator can be given an explicit formula
%
% f(x) = 1/n sum((s2-s1*(x-X)).*kerf((x-X)/h).*Y)/(s2*s0-s1^2)
%
%where si = sum((x-X)^i*kerf((x-X)/h))/n. Compare with local constant
%estimator, the local linear estimator can improve the estimation near the
%edge of the region over which the data have been collected.
%
% See also gkde, ksdensity, ksr

% Reference:
% Bowman, A.W. and Azzalini, A., Applied Smoothing Techniques for Data
% Analysis, Clarendon Press, 1997 (p.50)

% Example 1: smooth curve with noise
%{
x = 1:100;
y = sin(x/10)+(x/50).^2;
yn = y + 0.2*randn(1,100);
r=ksrlin(x,yn);
r1=ksr(x,yn); % downloadable from FEX ID 19195
plot(x,y,'b-',x,yn,'co',r.x,r.f,'r--',r1.x,r1.f,'m-.','linewidth',2)
legend('true','data','local linear','local constant','location','northwest');
title('Gaussian kernel regression')
%}
% Example 2: with missing data
%{
x = sort(rand(1,100)*99)+1;
y = sin(x/10)+(x/50).^2;
y(round(rand(1,20)*100)) = NaN;
yn = y + 0.2*randn(1,100);
r=ksrlin(x,yn);
r1=ksr(x,yn); % downloadable from FEX ID 19195
plot(x,y,'b-',x,yn,'co',r.x,r.f,'r--',r1.x,r1.f,'m-.','linewidth',2)
legend('true','data','local linear','local constant','location','northwest');
title('Gaussian kernel regression with 20% missing data')
%}

% By Yi Cao at Cranfield University on 12 April 2008.
%

% Check input and output
error(nargchk(2,4,nargin));
error(nargoutchk(0,1,nargout));
if numel(x)~=numel(y)
    error('x and y are in different sizes.');
end

x=x(:);
y=y(:);
% clean missing or invalid data points
inv=(x~=x)|(y~=y)|(abs(x)==Inf)|(abs(y)==Inf);
x(inv)=[];
y(inv)=[];

% Default parameters
if nargin<4
    N=100;
elseif ~isscalar(N)
    error('N must be a scalar.')
end
r.n=length(x);
if nargin<3
    % optimal bandwidth suggested by Bowman and Azzalini (1997) p.31
    hx=median(abs(x-median(x)))/0.6745*(4/3/r.n)^0.2;
    hy=median(abs(y-median(y)))/0.6745*(4/3/r.n)^0.2;
    h=sqrt(hy*hx);
elseif ~isscalar(h)
    error('h must be a scalar.')
end
r.h=h;

% Gaussian kernel function
kerf=@(z)exp(-z.*z/2)/sqrt(2*pi);

r.x=linspace(min(x),max(x),N);
r.f=zeros(1,N);
for k=1:N
    d=r.x(k)-x;
    z=kerf(d/h);
    s1=d.*z;
    s2=sum(d.*s1);
    s1=sum(s1);
    r.f(k)=sum((s2-s1*d).*z.*y)/(s2*sum(z)-s1*s1);
end

% Plot
if ~nargout
    plot(r.x,r.f,'r',x,y,'bo')
    ylabel('f(x)')
    xlabel('x')
    title('Kernel Smoothing Regression');
end


function cdf = half_normal_cdf ( x, a, b )

%*****************************************************************************80
%
%% HALF_NORMAL_CDF evaluates the Half Normal CDF.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    12 September 2004
%
%  Author:
%
%    John Burkardt
%
%  Parameters:
%
%    Input, real X, the argument of the CDF.
%
%    Input, real A, B, the parameters of the PDF.
%    0.0 < B.
%
%    Output, real CDF, the value of the CDF.
%
  if ( x <= a )
    cdf = 0.0;
  else
    cdf2 = normal_cdf ( x, a, b );
    cdf = 2.0 * cdf2 - 1.0;
  end

  return
end


function varargout = ackley(x)
%ACKLEY Ackley's function

if nargin == 0
    varargout{1} = [-8,-8];     % LB
    varargout{2} = [2,2];       % UB
    varargout{3} = [0,20];      % Zlim
    varargout{4} = [-26.5,60];  % view
    varargout{5} = [0,0];       % xmin
    varargout{6} = 'Ackley function';    % name
    varargout{7} = [-6,-5];     % default x0
    return;
end
    varargout{1} = -20 * exp(-0.2 * sqrt( sum(x .* x, 2)/ 2)) ...
        - exp(sum(cos(2*pi*x),2) / 2) ...
        + 20 + 2.7182818284590452353602874713526625;
    varargout{2} = [];
end


function [J, grad] = lrCostFunction(theta, X, y, lambda)
%LRCOSTFUNCTION Compute cost and gradient for logistic regression with
%regularization
%   J = LRCOSTFUNCTION(theta, X, y, lambda) computes the cost of using
%   theta as the parameter for regularized logistic regression and the
%   gradient of the cost w.r.t. to the parameters.

% Initialize some useful values
m = length(y); % number of training examples

% You need to return the following variables correctly
J = 0;
grad = zeros(size(theta));

% ====================== YOUR CODE HERE ======================
% Instructions: Compute the cost of a particular choice of theta.
%               You should set J to the cost.
%               Compute the partial derivatives and set grad to the partial
%               derivatives of the cost w.r.t. each parameter in theta
%

hx = sigmoid(X * theta); %hypothesis, m * 1
% Hint: The computation of the cost function and gradients can be
%       efficiently vectorized. For example, consider the computation
%
%           sigmoid(X * theta)
%
%       Each row of the resulting matrix will contain the value of the
%       prediction for that example. You can make use of this to vectorize
%       the cost function and gradient computations.
%
J = 1 / m * sum(-y' * log(hx) - (1 - y)' * log(1 - hx)) + lambda / (2 * m) * theta(2:end)' * theta(2:end);

% Hint: When computing the gradient of the regularized cost function,
%       there're many possible vectorized solutions, but one solution
%       looks like:
%           grad = (unregularized gradient for logistic regression)
%           temp = theta;
%           temp(1) = 0;   % because we don't add anything for j = 0
%           grad = grad + YOUR_CODE_HERE (using the temp variable)
%
gradf = (1 / m) * (X(:, 1)' * (hx - y));
gradb = (1 / m) * (X(:, 2:end)' * (hx - y)) + lambda * theta(2:end) / m;

grad = [gradf;gradb];
size(grad)

% =============================================================

grad = grad(:);

end


function area = sphere_cap_area_nd ( ndim, r, h )

%*****************************************************************************80
%
%% SPHERE_CAP_AREA_ND computes the area of a spherical cap in ND.
%
%  Discussion:
%
%    The spherical cap is a portion of the surface of the sphere:
%
%      sum ( X(1:N)^2 ) = R^2
%
%    which is no more than H units from the uppermost point on the sphere.
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    18 July 2005
%
%  Author:
%
%    John Burkardt
%
%  Reference:
%
%    Thomas Ericson and Victor Zinoviev,
%    Codes on Euclidean Spheres,
%    Elsevier, 2001, pages 439-441.
%    QA166.7 E75
%
%  Parameters:
%
%    Input, integer NDIM, the dimension of the space.
%
%    Input, real R, the radius of the sphere.
%
%    Input, real H, the "thickness" of the spherical cap,
%    which is normally between 0 and 2 * R.
%
%    Output, real AREA, the area of the spherical cap.
%
  if ( h <= 0.0 )
    area = 0.0;
    return
  end

  if ( 2.0 * r <= h )
    area = sphere_area_nd ( ndim, r );
    return
  end
%
%  For cases where R < H < 2 * R, work with the complementary region.
%
  haver_sine = sqrt ( ( 2.0 * r - h ) * h );

  theta = r8_asin ( haver_sine / r );

  if ( ndim < 1 )

    area = -1.0;

  elseif ( ndim == 1 )

    area = 0.0;

  elseif ( ndim == 2 )

    area = 2.0 * theta * r;

  else

    ti = theta;

    tj = ti;
    ti = 1.0 - cos ( theta );

    for i = 2 : ndim-2
      tk = tj;
      tj = ti;
      ti = ( ( i - 1 ) * tk - cos ( theta ) * sin ( theta )^( i - 1 ) ) / i;
    end

    area = sphere_k ( ndim-1 ) * ti * r^( ndim - 1 );

  end
%
%  Adjust for cases where R < H < 2R.
%
  if ( r < h )
    area2 = sphere_area_nd ( ndim, r );
    area = area2 - area;
  end

  return
end


% book : Signals and Systems Laboratory with MATLAB
% authors : Alex Palamides & Anastasia Veloni
%
%
% Problem 4
% Inverse z-Transform computation of X(z)=(2z+3)/(z^2+5z+6)

% a)in rational form
syms n z
X=(2*z+3)/(z^2+5*z+6);
iztrans(X,n)

% b) in partial fraction form
num=[ 2 3]
den= [ 1 5 6];
[R,P,K]=residue(num,den)

X=3/(z+3)-1/(z+2)
iztrans(X,n)


%
% lellipf(phi, k, errtol)
%
% Inputs:
%
%   phi     Input angle vector size 1 or 1xN.
%   k       Input parameter vector size 1 or 1xN.
%   errtol  Error tolerance for Carlson's algorithms.
%
% Matlab function to compute Legendre's (incomplete) elliptic integral
% F(phi, k).  Uses a vectorized implementation of Carlson's Duplication Algorithms
% for symmetric elliptic integrals as found in "Computing Elliptic
% Integrals by Duplication," by B. C. Carlson, Numer. Math. 33, 1-16 (1979)
% and also found in ACM TOMS Algorithm 577.  Section 4 in the paper cited
% here describes how to convert between the symmetric elliptic integrals
% and Legendre's elliptic integrals.
%
% Returns NaN's for any argument values outside input range.
%

function f = lellipf(phi, k, errtol)

% Argument checking for vectorization:
lphi = length(phi);
lk = length(k);
errflag = logical(0);
if (lphi ~= lk)
    if (lphi==1)
        phivec = phi * ones(1,lk);
        kvec = k;
    elseif (lk==1)
        kvec = k * ones(1,lphi);
        phivec = phi;
    else
        disp('Incompatible input vector dimensions in lellipf!');
        errflag = logical(1);
    end
else
    phivec = phi;
    kvec = k;
end

if ~errflag
    snphi = sin(phivec);
    csphi = cos(phivec);
    csphi2 = csphi .* csphi;
    onesvec = ones(1,length(phivec));
    y = onesvec - kvec.*kvec .* snphi.*snphi;
    f = snphi .* rf(csphi2,  y, onesvec, errtol);
else
    f = NaN;
end


% Euclidean distance between polyhedra
% Section 8.2.1, Boyd & Vandenberghe "Convex Optimization"
% Joelle Skaf - 10/09/05
%
% Given two polyhedra C = {x | A1*x <= b1} and D = {x | A2*x <= b2}, the
% distance between them is the optimal value of the problem:
%           minimize    || x - y ||_2
%               s.t.    A1*x <= b1
%                       A2*y <= b2

% Input data
randn('state',0);
rand('state',0);

n  = 5;
m1 = 2*n;
m2 = 3*n;
A1 = randn(m1,n);
A2 = randn(m2,n);
b1 = rand(m1,1);
b2 = rand(m2,1) + A2*randn(n,1);

% Solution via CVX
cvx_begin
    variables x(n) y(n)
    minimize (norm(x - y))
    A1*x <= b1;
    A2*y <= b2;
cvx_end

% Displaying results
disp('------------------------------------------------------------------');
disp('The distance between the 2 polyhedra C and D is: ' );
disp(['dist(C,D) = ' num2str(cvx_optval)]);


% Euclidean distance between polyhedra in 2D
% Section 8.2.1, Boyd & Vandenberghe "Convex Optimization"
% Joelle Skaf - 10/09/05
% (a figure is generated)
%
% Given two polyhedra C = {x | A1*x <= b1} and D = {x | A2*x <= b2}, the
% distance between them is the optimal value of the problem:
%           minimize    || x - y ||_2
%               s.t.    A1*x <= b1
%                       A2*y <= b2
% Note: here x is in R^2

% Input data
randn('seed',0);
n = 2;
m = 2*n;
A1 = randn(m,n);
b1 = randn(m,1);
A2 = randn(m,n);
b2 = randn(m,1);

fprintf(1,'Computing the distance between the 2 polyhedra...');
% Solution via CVX
cvx_begin
    variables x(n) y(n)
    minimize (norm(x - y))
    norm(x,1) <= 2; %#ok
    norm(y-[4;3],inf) <=1; %#ok
cvx_end

fprintf(1,'Done! \n');

% Displaying results
disp('------------------------------------------------------------------');
disp('The distance between the 2 polyhedra C and D is: ' );
disp(['dist(C,D) = ' num2str(cvx_optval)]);
disp('The optimal points are: ')
disp('x = '); disp(x);
disp('y = '); disp(y);

%Plotting
figure;
fill([-2; 0; 2; 0],[0;2;0;-2],'b', [3;5;5;3],[2;2;4;4],'r')
axis([-3 6 -3 6])
axis square
hold on;
plot(x(1),x(2),'k.')
plot(y(1),y(2),'k.')
plot([x(1) y(1)],[x(2) y(2)])
title('Euclidean distance between 2 polyhedron in R^2');
xlabel('x_1');
ylabel('x_2');


function [out] = GLCM_Features1(glcmin,pairs)
%
% GLCM_Features1 helps to calculate the features from the different GLCMs
% that are input to the function. The GLCMs are stored in a i x j x n
% matrix, where n is the number of GLCMs calculated usually due to the
% different orientation and displacements used in the algorithm. Usually
% the values i and j are equal to 'NumLevels' parameter of the GLCM
% computing function graycomatrix(). Note that matlab quantization values
% belong to the set {1,..., NumLevels} and not from {0,...,(NumLevels-1)}
% as provided in some references
% http://www.mathworks.com/access/helpdesk/help/toolbox/images/graycomatrix
% .html
%
% Although there is a function graycoprops() in Matlab Image Processing
% Toolbox that computes four parameters Contrast, Correlation, Energy,
% and Homogeneity. The paper by Haralick suggests a few more parameters
% that are also computed here. The code is not fully vectorized and hence
% is not an efficient implementation but it is easy to add new features
% based on the GLCM using this code. Takes care of 3 dimensional glcms
% (multiple glcms in a single 3D array)
%
% If you find that the values obtained are different from what you expect
% or if you think there is a different formula that needs to be used
% from the ones used in this code please let me know.
% A few questions which I have are listed in the link
% http://www.mathworks.com/matlabcentral/newsreader/view_thread/239608
%
% I plan to submit a vectorized version of the code later and provide
% updates based on replies to the above link and this initial code.
%
% Features computed
% Autocorrelation: [2]                      (out.autoc)
% Contrast: matlab/[1,2]                    (out.contr)
% Correlation: matlab                       (out.corrm)
% Correlation: [1,2]                        (out.corrp)
% Cluster Prominence: [2]                   (out.cprom)
% Cluster Shade: [2]                        (out.cshad)
% Dissimilarity: [2]                        (out.dissi)
% Energy: matlab / [1,2]                    (out.energ)
% Entropy: [2]                              (out.entro)
% Homogeneity: matlab                       (out.homom)
% Homogeneity: [2]                          (out.homop)
% Maximum probability: [2]                  (out.maxpr)
% Sum of sqaures: Variance [1]              (out.sosvh)
% Sum average [1]                           (out.savgh)
% Sum variance [1]                          (out.svarh)
% Sum entropy [1]                           (out.senth)
% Difference variance [1]                   (out.dvarh)
% Difference entropy [1]                    (out.denth)
% Information measure of correlation1 [1]   (out.inf1h)
% Informaiton measure of correlation2 [1]   (out.inf2h)
% Inverse difference (INV) is homom [3]     (out.homom)
% Inverse difference normalized (INN) [3]   (out.indnc)
% Inverse difference moment normalized [3]  (out.idmnc)
%
% The maximal correlation coefficient was not calculated due to
% computational instability
% http://murphylab.web.cmu.edu/publications/boland/boland_node26.html
%
% Formulae from MATLAB site (some look different from
% the paper by Haralick but are equivalent and give same results)
% Example formulae:
% Contrast = sum_i(sum_j(  (i-j)^2 * p(i,j) ) ) (same in matlab/paper)
% Correlation = sum_i( sum_j( (i - u_i)(j - u_j)p(i,j)/(s_i.s_j) ) ) (m)
% Correlation = sum_i( sum_j( ((ij)p(i,j) - u_x.u_y) / (s_x.s_y) ) ) (p[2])
% Energy = sum_i( sum_j( p(i,j)^2 ) )           (same in matlab/paper)
% Homogeneity = sum_i( sum_j( p(i,j) / (1 + |i-j|) ) ) (as in matlab)
% Homogeneity = sum_i( sum_j( p(i,j) / (1 + (i-j)^2) ) ) (as in paper)
%
% Where:
% u_i = u_x = sum_i( sum_j( i.p(i,j) ) ) (in paper [2])
% u_j = u_y = sum_i( sum_j( j.p(i,j) ) ) (in paper [2])
% s_i = s_x = sum_i( sum_j( (i - u_x)^2.p(i,j) ) ) (in paper [2])
% s_j = s_y = sum_i( sum_j( (j - u_y)^2.p(i,j) ) ) (in paper [2])
%
%
% Normalize the glcm:
% Compute the sum of all the values in each glcm in the array and divide
% each element by it sum
%
% Haralick uses 'Symmetric' = true in computing the glcm
% There is no Symmetric flag in the Matlab version I use hence
% I add the diagonally opposite pairs to obtain the Haralick glcm
% Here it is assumed that the diagonally opposite orientations are paired
% one after the other in the matrix
% If the above assumption is true with respect to the input glcm then
% setting the flag 'pairs' to 1 will compute the final glcms that would result
% by setting 'Symmetric' to true. If your glcm is computed using the
% Matlab version with 'Symmetric' flag you can set the flag 'pairs' to 0
%
% References:
% 1. R. M. Haralick, K. Shanmugam, and I. Dinstein, Textural Features of
% Image Classification, IEEE Transactions on Systems, Man and Cybernetics,
% vol. SMC-3, no. 6, Nov. 1973
% 2. L. Soh and C. Tsatsoulis, Texture Analysis of SAR Sea Ice Imagery
% Using Gray Level Co-Occurrence Matrices, IEEE Transactions on Geoscience
% and Remote Sensing, vol. 37, no. 2, March 1999.
% 3. D A. Clausi, An analysis of co-occurrence texture statistics as a
% function of grey level quantization, Can. J. Remote Sensing, vol. 28, no.
% 1, pp. 45-62, 2002
% 4. http://murphylab.web.cmu.edu/publications/boland/boland_node26.html
%
%
% Example:
%
% Usage is similar to graycoprops() but needs extra parameter 'pairs' apart
% from the GLCM as input
% I = imread('circuit.tif');
% GLCM2 = graycomatrix(I,'Offset',[2 0;0 2]);
% stats = GLCM_features1(GLCM2,0)
% The output is a structure containing all the parameters for the different
% GLCMs
%
% [Avinash Uppuluri: avinash_uv@yahoo.com: Last modified: 11/20/08]

% If 'pairs' not entered: set pairs to 0
if ((nargin > 2) || (nargin == 0))
   error('Too many or too few input arguments. Enter GLCM and pairs.');
elseif ( (nargin == 2) )
    if ((size(glcmin,1) <= 1) || (size(glcmin,2) <= 1))
       error('The GLCM should be a 2-D or 3-D matrix.');
    elseif ( size(glcmin,1) ~= size(glcmin,2) )
        error('Each GLCM should be square with NumLevels rows and NumLevels cols');
    end
elseif (nargin == 1) % only GLCM is entered
    pairs = 0; % default is numbers and input 1 for percentage
    if ((size(glcmin,1) <= 1) || (size(glcmin,2) <= 1))
       error('The GLCM should be a 2-D or 3-D matrix.');
    elseif ( size(glcmin,1) ~= size(glcmin,2) )
       error('Each GLCM should be square with NumLevels rows and NumLevels cols');
    end
end

format long e
if (pairs == 1)
    newn = 1;
    for nglcm = 1:2:size(glcmin,3)
        glcm(:,:,newn)  = glcmin(:,:,nglcm) + glcmin(:,:,nglcm+1);
        newn = newn + 1;
    end
elseif (pairs == 0)
    glcm = glcmin;
end

size_glcm_1 = size(glcm,1);
size_glcm_2 = size(glcm,2);
size_glcm_3 = size(glcm,3);

% checked
out.autoc = zeros(1,size_glcm_3); % Autocorrelation: [2]
out.contr = zeros(1,size_glcm_3); % Contrast: matlab/[1,2]
out.corrm = zeros(1,size_glcm_3); % Correlation: matlab
out.corrp = zeros(1,size_glcm_3); % Correlation: [1,2]
out.cprom = zeros(1,size_glcm_3); % Cluster Prominence: [2]
out.cshad = zeros(1,size_glcm_3); % Cluster Shade: [2]
out.dissi = zeros(1,size_glcm_3); % Dissimilarity: [2]
out.energ = zeros(1,size_glcm_3); % Energy: matlab / [1,2]
out.entro = zeros(1,size_glcm_3); % Entropy: [2]
out.homom = zeros(1,size_glcm_3); % Homogeneity: matlab
out.homop = zeros(1,size_glcm_3); % Homogeneity: [2]
out.maxpr = zeros(1,size_glcm_3); % Maximum probability: [2]

out.sosvh = zeros(1,size_glcm_3); % Sum of sqaures: Variance [1]
out.savgh = zeros(1,size_glcm_3); % Sum average [1]
out.svarh = zeros(1,size_glcm_3); % Sum variance [1]
out.senth = zeros(1,size_glcm_3); % Sum entropy [1]
out.dvarh = zeros(1,size_glcm_3); % Difference variance [4]
%out.dvarh2 = zeros(1,size_glcm_3); % Difference variance [1]
out.denth = zeros(1,size_glcm_3); % Difference entropy [1]
out.inf1h = zeros(1,size_glcm_3); % Information measure of correlation1 [1]
out.inf2h = zeros(1,size_glcm_3); % Informaiton measure of correlation2 [1]
%out.mxcch = zeros(1,size_glcm_3);% maximal correlation coefficient [1]
%out.invdc = zeros(1,size_glcm_3);% Inverse difference (INV) is homom [3]
out.indnc = zeros(1,size_glcm_3); % Inverse difference normalized (INN) [3]
out.idmnc = zeros(1,size_glcm_3); % Inverse difference moment normalized [3]

% correlation with alternate definition of u and s
%out.corrm2 = zeros(1,size_glcm_3); % Correlation: matlab
%out.corrp2 = zeros(1,size_glcm_3); % Correlation: [1,2]

glcm_sum  = zeros(size_glcm_3,1);
glcm_mean = zeros(size_glcm_3,1);
glcm_var  = zeros(size_glcm_3,1);

% http://www.fp.ucalgary.ca/mhallbey/glcm_mean.htm confuses the range of
% i and j used in calculating the means and standard deviations.
% As of now I am not sure if the range of i and j should be [1:Ng] or
% [0:Ng-1]. I am working on obtaining the values of mean and std that get
% the values of correlation that are provided by matlab.
u_x = zeros(size_glcm_3,1);
u_y = zeros(size_glcm_3,1);
s_x = zeros(size_glcm_3,1);
s_y = zeros(size_glcm_3,1);

% % alternate values of u and s
% u_x2 = zeros(size_glcm_3,1);
% u_y2 = zeros(size_glcm_3,1);
% s_x2 = zeros(size_glcm_3,1);
% s_y2 = zeros(size_glcm_3,1);

% checked p_x p_y p_xplusy p_xminusy
p_x = zeros(size_glcm_1,size_glcm_3); % Ng x #glcms[1]
p_y = zeros(size_glcm_2,size_glcm_3); % Ng x #glcms[1]
p_xplusy = zeros((size_glcm_1*2 - 1),size_glcm_3); %[1]
p_xminusy = zeros((size_glcm_1),size_glcm_3); %[1]
% checked hxy hxy1 hxy2 hx hy
hxy  = zeros(size_glcm_3,1);
hxy1 = zeros(size_glcm_3,1);
hx   = zeros(size_glcm_3,1);
hy   = zeros(size_glcm_3,1);
hxy2 = zeros(size_glcm_3,1);

%Q    = zeros(size(glcm));

for k = 1:size_glcm_3 % number glcms

    glcm_sum(k) = sum(sum(glcm(:,:,k)));
    glcm(:,:,k) = glcm(:,:,k)./glcm_sum(k); % Normalize each glcm
    glcm_mean(k) = mean2(glcm(:,:,k)); % compute mean after norm
    glcm_var(k)  = (std2(glcm(:,:,k)))^2;

    for i = 1:size_glcm_1

        for j = 1:size_glcm_2

            out.contr(k) = out.contr(k) + (abs(i - j))^2.*glcm(i,j,k);
            out.dissi(k) = out.dissi(k) + (abs(i - j)*glcm(i,j,k));
            out.energ(k) = out.energ(k) + (glcm(i,j,k).^2);
            out.entro(k) = out.entro(k) - (glcm(i,j,k)*log(glcm(i,j,k) + eps));
            out.homom(k) = out.homom(k) + (glcm(i,j,k)/( 1 + abs(i-j) ));
            out.homop(k) = out.homop(k) + (glcm(i,j,k)/( 1 + (i - j)^2));
            % [1] explains sum of squares variance with a mean value;
            % the exact definition for mean has not been provided in
            % the reference: I use the mean of the entire normalized glcm
            out.sosvh(k) = out.sosvh(k) + glcm(i,j,k)*((i - glcm_mean(k))^2);

            %out.invdc(k) = out.homom(k);
            out.indnc(k) = out.indnc(k) + (glcm(i,j,k)/( 1 + (abs(i-j)/size_glcm_1) ));
            out.idmnc(k) = out.idmnc(k) + (glcm(i,j,k)/( 1 + ((i - j)/size_glcm_1)^2));
            u_x(k)          = u_x(k) + (i)*glcm(i,j,k); % changed 10/26/08
            u_y(k)          = u_y(k) + (j)*glcm(i,j,k); % changed 10/26/08
            % code requires that Nx = Ny
            % the values of the grey levels range from 1 to (Ng)
        end

    end
    out.maxpr(k) = max(max(glcm(:,:,k)));
end
% glcms have been normalized:
% The contrast has been computed for each glcm in the 3D matrix
% (tested) gives similar results to the matlab function

for k = 1:size_glcm_3

    for i = 1:size_glcm_1

        for j = 1:size_glcm_2
            p_x(i,k) = p_x(i,k) + glcm(i,j,k);
            p_y(i,k) = p_y(i,k) + glcm(j,i,k); % taking i for j and j for i
            if (ismember((i + j),[2:2*size_glcm_1]))
                p_xplusy((i+j)-1,k) = p_xplusy((i+j)-1,k) + glcm(i,j,k);
            end
            if (ismember(abs(i-j),[0:(size_glcm_1-1)]))
                p_xminusy((abs(i-j))+1,k) = p_xminusy((abs(i-j))+1,k) +...
                    glcm(i,j,k);
            end
        end
    end

%     % consider u_x and u_y and s_x and s_y as means and standard deviations
%     % of p_x and p_y
%     u_x2(k) = mean(p_x(:,k));
%     u_y2(k) = mean(p_y(:,k));
%     s_x2(k) = std(p_x(:,k));
%     s_y2(k) = std(p_y(:,k));

end

% marginal probabilities are now available [1]
% p_xminusy has +1 in index for matlab (no 0 index)
% computing sum average, sum variance and sum entropy:
for k = 1:(size_glcm_3)

    for i = 1:(2*(size_glcm_1)-1)
        out.savgh(k) = out.savgh(k) + (i+1)*p_xplusy(i,k);
        % the summation for savgh is for i from 2 to 2*Ng hence (i+1)
        out.senth(k) = out.senth(k) - (p_xplusy(i,k)*log(p_xplusy(i,k) + eps));
    end

end
% compute sum variance with the help of sum entropy
for k = 1:(size_glcm_3)

    for i = 1:(2*(size_glcm_1)-1)
        out.svarh(k) = out.svarh(k) + (((i+1) - out.senth(k))^2)*p_xplusy(i,k);
        % the summation for savgh is for i from 2 to 2*Ng hence (i+1)
    end

end
% compute difference variance, difference entropy,
for k = 1:size_glcm_3
% out.dvarh2(k) = var(p_xminusy(:,k));
% but using the formula in
% http://murphylab.web.cmu.edu/publications/boland/boland_node26.html
% we have for dvarh
    for i = 0:(size_glcm_1-1)
        out.denth(k) = out.denth(k) - (p_xminusy(i+1,k)*log(p_xminusy(i+1,k) + eps));
        out.dvarh(k) = out.dvarh(k) + (i^2)*p_xminusy(i+1,k);
    end
end

% compute information measure of correlation(1,2) [1]
for k = 1:size_glcm_3
    hxy(k) = out.entro(k);
    for i = 1:size_glcm_1

        for j = 1:size_glcm_2
            hxy1(k) = hxy1(k) - (glcm(i,j,k)*log(p_x(i,k)*p_y(j,k) + eps));
            hxy2(k) = hxy2(k) - (p_x(i,k)*p_y(j,k)*log(p_x(i,k)*p_y(j,k) + eps));
%             for Qind = 1:(size_glcm_1)
%                 Q(i,j,k) = Q(i,j,k) +...
%                     ( glcm(i,Qind,k)*glcm(j,Qind,k) / (p_x(i,k)*p_y(Qind,k)) );
%             end
        end
        hx(k) = hx(k) - (p_x(i,k)*log(p_x(i,k) + eps));
        hy(k) = hy(k) - (p_y(i,k)*log(p_y(i,k) + eps));
    end
    out.inf1h(k) = ( hxy(k) - hxy1(k) ) / ( max([hx(k),hy(k)]) );
    out.inf2h(k) = ( 1 - exp( -2*( hxy2(k) - hxy(k) ) ) )^0.5;
%     eig_Q(k,:)   = eig(Q(:,:,k));
%     sort_eig(k,:)= sort(eig_Q(k,:),'descend');
%     out.mxcch(k) = sort_eig(k,2)^0.5;
% The maximal correlation coefficient was not calculated due to
% computational instability
% http://murphylab.web.cmu.edu/publications/boland/boland_node26.html
end

corm = zeros(size_glcm_3,1);
corp = zeros(size_glcm_3,1);
% using http://www.fp.ucalgary.ca/mhallbey/glcm_variance.htm for s_x s_y
for k = 1:size_glcm_3
    for i = 1:size_glcm_1
        for j = 1:size_glcm_2
            s_x(k)  = s_x(k)  + (((i) - u_x(k))^2)*glcm(i,j,k);
            s_y(k)  = s_y(k)  + (((j) - u_y(k))^2)*glcm(i,j,k);
            corp(k) = corp(k) + ((i)*(j)*glcm(i,j,k));
            corm(k) = corm(k) + (((i) - u_x(k))*((j) - u_y(k))*glcm(i,j,k));
            out.cprom(k) = out.cprom(k) + (((i + j - u_x(k) - u_y(k))^4)*...
                glcm(i,j,k));
            out.cshad(k) = out.cshad(k) + (((i + j - u_x(k) - u_y(k))^3)*...
                glcm(i,j,k));
        end
    end
    % using http://www.fp.ucalgary.ca/mhallbey/glcm_variance.htm for s_x
    % s_y : This solves the difference in value of correlation and might be
    % the right value of standard deviations required
    % According to this website there is a typo in [2] which provides
    % values of variance instead of the standard deviation hence a square
    % root is required as done below:
    s_x(k) = s_x(k) ^ 0.5;
    s_y(k) = s_y(k) ^ 0.5;
    out.autoc(k) = corp(k);
    out.corrp(k) = (corp(k) - u_x(k)*u_y(k))/(s_x(k)*s_y(k));
    out.corrm(k) = corm(k) / (s_x(k)*s_y(k));
%     % alternate values of u and s
%     out.corrp2(k) = (corp(k) - u_x2(k)*u_y2(k))/(s_x2(k)*s_y2(k));
%     out.corrm2(k) = corm(k) / (s_x2(k)*s_y2(k));
end
% Here the formula in the paper out.corrp and the formula in matlab
% out.corrm are equivalent as confirmed by the similar results obtained

% % The papers have a slightly different formular for Contrast
% % I have tested here to find this formula in the papers provides the
% % same results as the formula provided by the matlab function for
% % Contrast (Hence this part has been commented)
% out.contrp = zeros(size_glcm_3,1);
% contp = 0;
% Ng = size_glcm_1;
% for k = 1:size_glcm_3
%     for n = 0:(Ng-1)
%         for i = 1:Ng
%             for j = 1:Ng
%                 if (abs(i-j) == n)
%                     contp = contp + glcm(i,j,k);
%                 end
%             end
%         end
%         out.contrp(k) = out.contrp(k) + n^2*contp;
%         contp = 0;
%     end
%
% end

%       GLCM Features (Soh, 1999; Haralick, 1973; Clausi 2002)
%           f1. Uniformity / Energy / Angular Second Moment (done)
%           f2. Entropy (done)
%           f3. Dissimilarity (done)
%           f4. Contrast / Inertia (done)
%           f5. Inverse difference
%           f6. correlation
%           f7. Homogeneity / Inverse difference moment
%           f8. Autocorrelation
%           f9. Cluster Shade
%          f10. Cluster Prominence
%          f11. Maximum probability
%          f12. Sum of Squares
%          f13. Sum Average
%          f14. Sum Variance
%          f15. Sum Entropy
%          f16. Difference variance
%          f17. Difference entropy
%          f18. Information measures of correlation (1)
%          f19. Information measures of correlation (2)
%          f20. Maximal correlation coefficient
%          f21. Inverse difference normalized (INN)
%          f22. Inverse difference moment normalized (IDN)


function fx = p54_fun ( n, x )

%*****************************************************************************80
%
%% P54_FUN evaluates the integrand for problem 54.
%
%  Discussion:
%
%    The reference claims that this integrand is more closely approximated
%    by the trapezoid rule than by Gauss-Legendre quadrature.
%
%    Points  Trapezoid  Gauss-Legendre
%     4      1.91667    2.53883
%    12      2.1594     2.25809
%
%    However, the stated results hardly give one confindence in
%    the convergence of the trapezoid results, and I am unable to
%    confirm them, because my results for 4 points give good results
%    (about 1.14) for BOTH Trapezoid and Gauss-Legendre%
%
%  Interval:
%
%    0 <= x <= 1
%
%  Integrand:
%
%    2 / ( 2 + sin ( 10 * PI * x ) )
%
%  Exact Integral:
%
%    2 / sqrt ( 3 ) = 1.1547...
%
%  Licensing:
%
%    This code is distributed under the GNU LGPL license.
%
%  Modified:
%
%    04 November 2009
%
%  Author:
%
%    John Burkardt
%
%  Reference:
%
%    Prem Kythe, Pratap Puri,
%    Computational Methods for Linear Integral Equations,
%    Birkhaeuser, 2002.
%
%  Parameters:
%
%    Input, integer N, the number of evaluation points.
%
%    Input, real X(N), the evaluation points.
%
%    Output, real FX(N), the integrand values.
%
  fx(1:n) = 2.0 ./ ( 2.0 + sin ( 10.0 * pi * x(1:n) ) );

  return
end


% .............................................................
    function [shape,naturalDerivatives]=shapeFunctionL2(xi)

    % shape function and derivatives for L2 elements
    % shape : Shape functions
    % naturalDerivatives: derivatives w.r.t. xi
    % xi: natural coordinates (-1 ... +1)

    shape=([1-xi,1+xi]/2);

    naturalDerivatives=[-1,1]/2;

    end % end function shapeFunctionL2


function [zProj,AProj]=projEllipsOntoSpace(z,A,gammaVal,d,T)
%%PROJELLIPSONTOSPACE Determine the orthogonal projection of an ellipsoid
%                    onto an affine space. The projection itself will be an
%                    ellipsoid. For example, in 3D, the projection onto a
%                    plane will be an ellipse.
%
%INPUTS: z  The numDimX1 center of the ellipsoid.
%        A  A numDimXnumDim symmetric, positive definite matrix that
%           specifies the size and shape of the ellipse or ellipsoid, where
%           a point zp is on the ellipse/ellipsoid if
%           (zp-z)'*A*(zp-z)=gammaVal.
%  gammaVal The threshold for declaring a point to be in the ellipsoid. If
%           an empty matrix is passed, the default value of 1 is used.
%      d,T  A numDimX1 point d and a numDImXn matrix such that points in
%           the affine subspace are defined by x=d+T*t where t is an nX1
%           real vector, n<numDim.
%
%OUTPUTS: zProj, AProj Parameters defining the projected ellipsoid. zProj
%           is numDImX1 and AProj is numDimXnumDim. However, AProj will be
%           a singular matrix as everything resides on a subspace. A point
%           is on the projected ellipsoid if
%           (zp-zProj)'*AProj*(zp-zProj)=1
%           The example below shows gives an example of using such a
%           singular matrix to display the projection of a 3D ellipsoid
%           onto a 2D plane.
%
%The algorithm is taken from Section 13 of [1].
%
%EXAMPLE:
%We will project an ellipsoid in 3D onto a plane.
%Define the ellipsoid:
% A=[10,0,0;
%     0,20,0;
%     0,0,1];
% %A rotation matrix
% M=[-0.039622269933159, -0.378261925022885, -0.924850253718582;
%     0.765018603761816, -0.606901366847761,  0.215446668149313;
%    -0.642788154545232, -0.698991163746939,  0.313424219517309];
% A=M*A*M';
% c=[0;0;0];%The center of the ellipsoid
% %Define a plane.
% d=[1;0;3];
% T=[1,1;
%    0,1;
%    0,1];
% T=orth(T);%Must be othonormal.
% %Determine the projection of the ellipsoid onto the plane.
% [cProj,AProj]=projEllipsOntoSpace(c,A,1,d,T);
%
% figure(1)
% clf
% hold on
% axis([-4,4,-4,4,-4,4])
% %Draw the ellipsoid
% drawEllipse(c,A,1,'EdgeColor','none')
% view(104,-29)
% %Compute points on the plane to display.
% numPoints=50;
% tSpan=linspace(-10,10,numPoints);
% [tx,ty]=meshgrid(tSpan,tSpan);
% t=[tx(:)';ty(:)'];
% placeSurfVals=bsxfun(@plus,d,T*t);
% X=reshape(placeSurfVals(1,:),numPoints,numPoints);
% Y=reshape(placeSurfVals(2,:),numPoints,numPoints);
% Z=reshape(placeSurfVals(3,:),numPoints,numPoints);
% %Draw the plane
% surf(X,Y,Z)
% %Draw the projection of the ellipsoid onto the plane
% drawEllipse(cProj,AProj,1)
% %This makes the ellipsoid shiny
% light('Position',[0;1;-1])
%
%REFERENCES:
%[1] S. B. Pope, "Algorithms for ellipsoids," Cornell University, Tech.
%    Rep. FDA-08-01, Feb. 2008. [Online].
%    Available: https://tcg.mae.cornell.edu/pubs/Pope_FDA_08.pdf
%
%May 2016 David F. Crouse, Naval Research Laboratory, Washington D.C.
%(UNCLASSIFIED) DISTRIBUTION STATEMENT A. Approved for public release.

if(isempty(gammaVal))
   gammaVal=1;
end

A=A/gammaVal;

L=chol(A,'lower');

%Make T into an orthonormal matrix. This does not rotate or shift the
%affine space at all. It just changes the parameterization as in x=d+T*t
%still holds, but if T is orthonormalized, then a different value of t
%is needed to get the same x.
T=orth(T);%Must be othonormal.

%Extracted from Equation 126
zProj=d+T*T'*(z-d);
[U,Sigma,~]=svd(T'/(L'),'econ');

L=T*U/Sigma;
AProj=L*L';

end


%DEMGLM1 Demonstrate simple classification using a generalized linear model.
%
%	Description
%	 The problem consists of a two dimensional input matrix DATA and a
%	vector of classifications T.  The data is  generated from two
%	Gaussian clusters, and a generalized linear model with logistic
%	output is trained using iterative reweighted least squares. A plot of
%	the data together with the 0.1, 0.5 and 0.9 contour lines of the
%	conditional probability is generated.
%
%	See also
%	DEMGLM2, GLM, GLMTRAIN
%

%	Copyright (c) Ian T Nabney (1996-2001)

% Generate data from two classes in 2d
input_dim = 2;

% Fix seeds for reproducible results
randn('state', 42);
rand('state', 42);

ndata = 100;
% Generate mixture of two Gaussians in two dimensional space
mix = gmm(2, 2, 'spherical');
mix.priors = [0.4 0.6];              % Cluster priors
mix.centres = [2.0, 2.0; 0.0, 0.0];  % Cluster centres
mix.covars = [0.5, 1.0];

[data, label] = gmmsamp(mix, ndata);
targets = label - ones(ndata, 1);

% Plot the result

clc
disp('This demonstration illustrates the use of a generalized linear model')
disp('to classify data from two classes in a two-dimensional space. We')
disp('begin by generating and plotting the data.')
disp(' ')
disp('Press any key to continue.')
pause

fh1 = figure;
plot(data(label==1,1), data(label==1,2), 'bo');
hold on
axis([-4 5 -4 5])
set(gca, 'box', 'on')
plot(data(label==2,1), data(label==2,2), 'rx')
title('Data')

clc
disp('Now we fit a model consisting of a logistic sigmoid function of')
disp('a linear combination of the input variables.')
disp(' ')
disp('The model is trained using the IRLS algorithm for 5 iterations')
disp(' ')
disp('Press any key to continue.')
pause

net = glm(input_dim, 1, 'logistic');
options = foptions;
options(1) = 1;
options(14) = 5;
net = glmtrain(net, options, data, targets);

disp(' ')
disp('We now plot some density contours given by this model.')
disp('The contour labelled 0.5 is the decision boundary.')
disp(' ')
disp('Press any key to continue.')
pause
x = -4.0:0.2:5.0;
y = -4.0:0.2:5.0;
[X, Y] = meshgrid(x,y);
X = X(:);
Y = Y(:);
grid = [X Y];
Z = glmfwd(net, grid);
Z = reshape(Z, length(x), length(y));
v = [0.1 0.5 0.9];
[c, h] = contour(x, y, Z, v);
title('Generalized Linear Model')
set(h, 'linewidth', 3)
clabel(c, h);

clc
disp('Note that the contours of constant density are straight lines.')
disp(' ')
disp('Press any key to end.')
pause
close(fh1);
clear all;