from __future__ import division  # take the division operator from future versions
import numpy as np
import math
from sklearn import linear_model
from regularizedtau import toolboxutilities_latest as util
import matplotlib.pyplot as plt

# import matlab.engine

def least_squares(matrix_a, vector_y):
    '''
    This function computes the estimate :math:`\\hat x` given by the least squares method
    :math:`\\mathbf{\\hat x} = {\\rm arg}\\min_x\\,\\lVert \\mathbf{y - Ax} \\rVert_2^2`.
    This is the simplest algorithm to solve a linear inverse problem of the form :math:`\\mathbf{y = Ax + n}`, where
    :math:`\\mathbf{y}` (vector) and :math:`\\mathbf{A}` (matrix) are known and :math:`\\mathbf{x}`  (vector)
     and :math:`\\mathbf{n}`  (vector) are unknown.

    :param matrix_a: (np.matrix) matrix :math:`\\mathbf{A}`
    :param vector_y: (array) vector :math:`\\mathbf{y}`

    :return vector_x: (array) estimate :math:`\\mathbf{\\hat x}` given by least squares

    Example : compute the least squares solution of a system :math:`\\mathbf{y = Ax}`

    .. code-block:: python

        import numpy as np
        import linvpy as lp

        A = np.matrix([[1,3],[3,4],[4,5]])
        y = [-6,1,-2]

        # Returns x_hat, the least squares solution of y = Ax
        lp.least_squares(A,y)

        # [ 3.86666667 -3.18666667]

    '''

    # Ensures np.matrix type
    matrix_a = np.matrix(matrix_a)

    # x = (A' A)^-1 A' y
    vector_x = np.dot(
        np.dot(
            np.linalg.inv(
                np.dot(
                    matrix_a.T,  # A.T returns the transpose of A
                    matrix_a
                )
            ),
            matrix_a.T
        ),
        vector_y
    )

    # Flattens result into an array
    vector_x = np.squeeze(np.asarray(vector_x))

    return vector_x

def tikhonov_regularization(matrix_a, vector_y, lambda_parameter=0):
    '''
    The standard approach to solve the problem :math:`\\mathbf{y = Ax + n}` explained above is to use the  ordinary
    least squares method. However if your matrix :math:`\\mathbf{A}` is a fat matrix (it has more columns than rows)
    or it has a large condition number, then you should use a regularization to your problem in order to get a
    meaningful estimation of :math:`\\mathbf{x}`.

    The Tikhonov regularization is a tradeoff between the least squares
    solution and the minimization of the L2-norm of the output :math:`x` (L2-norm =
    sum of squared values of the vector :math:`x`),
    :math:`\\hat{\\mathbf{x}} = {\\rm arg}\\min_x\\,\\lVert \\mathbf{y - Ax} \\rVert_2^2 + \\lambda\\lVert \\mathbf{x} \\rVert_2^2`

    The parameter lambda tells how close to the least squares solution the
    output :math:`\\mathbf{x}` will be; a large lambda will make :math:`\\mathbf{x}` close to
    :math:`\\lVert\\mathbf{x}\\rVert_2^2 = 0`, while
    a small lambda will approach the least squares solution (Running
    the function with lambda=0 will behave like the ordinary least_squares()
    method).

    The Tikhonov solution has an analytic solution and it is given
    by :math:`\\hat{\\mathbf{x}} = (\\mathbf{A^{T}A}+ \\lambda^{2} \\mathbf{I})^{-1}\\mathbf{A}^{T}\\mathbf{y}`,
    where :math:`\\mathbf{I}` is the identity matrix.

    Raises a ValueError if lambda < 0.

    :param matrix_a: (np.matrix) matrix A in :math:`\\mathbf{y = Ax + n}`
    :param vector_y: (array) vector y in :math:`\\mathbf{y = Ax + n}`
    :param lambda: (int) lambda non-negative parameter to regulate the trade off.

    :return vector_x: (array) Tikhonov estimate :math:`\\hat{\\mathbf{x}}`

    :raises ValueError: raises an exception if lambda_parameter < 0

    Example : compute the solution of a system :math:`\\mathbf{y = Ax}` (knowing :math:`\\mathbf{y, A}`) which is a
    trade off between the least squares solution and the minimization of x's
    L2-norm. The greater lambda, the smaller the norm of the given solution.
    We take a matrix :math:`\\mathbf{A}` which is ill-conditionned.

    .. code-block:: python

        import numpy as np
        import linvpy as lp

        A = np.matrix([[7142.80730214, 6050.32000196],
                       [6734.4239248, 5703.48709251],
                       [4663.22591408, 3949.23319264]])

        y = [0.83175086, 0.60012918, 0.89405644]

        # Returns x_hat, the tradeoff solution of y = Ax
        print lp.tikhonov_regularization(A, y, 50)

        # [8.25871731e-05   4.39467106e-05]

    '''

    if lambda_parameter < 0:
        raise ValueError('lambda_parameter must be zero or positive.')
    if lambda_parameter == 0:
        return np.linalg.lstsq(matrix_a, vector_y)[0].reshape(-1)

    # Ensures np.matrix type
    matrix_a = np.matrix(matrix_a)

    # Generates an identity matrix of the same shape as A'A.
    # matrix_a.shape() returns a tuple (#row,#columns) so with [1] with take the
    # number of columns to build the identity because A'A yields a square
    # matrix of the same size as the number of columns of A and rows of A'.
    identity_matrix = np.identity(matrix_a.shape[1])

    try:
        # x = (A' A + lambda I)^-1 A' y
        vector_x = np.dot(
            np.dot(
                np.linalg.inv(
                    np.add(
                        np.dot(matrix_a.T, matrix_a),  # A.T transpose of A
                        np.dot(lambda_parameter, identity_matrix)
                    ),
                ),
                matrix_a.T
            ),
            vector_y
        )

        # Flattens result into an array
        vector_x = np.squeeze(np.asarray(vector_x))
        return vector_x

    # catches Singular matrix error (or any other error), prints the trace
    except:
        print("Lambda parameter may be too small.")
        raise

def lasso_regularization(matrix_a, vector_y, lambda_parameter=0):
    """
    Lasso algorithm that solves min ||y - Ax||_2^2 + lambda ||x||_1
    :param matrix_a:
    :param vector_y:
    :param lambda_parameter:
    :return: estimated x
    """

    # convert regularization parameter (sklearn considers (1/2m factor))
    reg_parameter = lambda_parameter / (2 * len(vector_y))

    # initialize model
    clf = linear_model.Lasso(reg_parameter, fit_intercept=False, normalize=False)

    # fit it
    clf.fit(matrix_a, vector_y)

    # return estimate
    x = clf.coef_

    return x

def rho_huber(input, clipping=1.345):
    '''
    The regular huber loss function; the "rho" version.

    :math:`\\rho(x)=\\begin{cases}
    \\frac{1}{2}{x^2}& \\text{if |x| <=} clipping, \\\\
    clipping (|x| - \\dfrac{1}{2} clipping)& \\text{otherwise}.
    \\end{cases}`

    This function is quadratic for small inputs, and linear for large
    inputs.

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: :math:`\\rho(x)`

    Example : run huber loss on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        loss = [lp.rho_huber(e, 4) for e in x]

        # [0.5, 2.0, 4.5, 8.0, 12, 16, 20, 24, 28]
    '''
    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    if (np.absolute(input) <= clipping):
        return math.pow(input, 2) / 2.0
    else:
        return clipping * (np.subtract(np.absolute(input), clipping / 2.0))

def psi_huber(input, clipping=1.345):
    '''
    Derivative of the Huber loss function; the "psi" version. Used in the weight
    function of the M-estimator.

    :math:`\\psi(x)=\\begin{cases}
    x& \\text{if |x| <=} clipping, \\\\
    clipping \\cdot sign(x) & \\text{otherwise}.
    \\end{cases}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: :math:`\\psi(x)`

    Example : run psi_huber derivative on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        derivative = [lp.psi_huber(e, 4) for e in x]

        # [1, 2, 3, 4, 4, 4, 4, 4, 4]

    '''
    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    if (np.absolute(input) >= clipping):
        return clipping * np.sign(input)
    else:
        return input

def rho_bisquare(input, clipping=4.685):
    '''
    The regular bisquare loss (or Tukey's loss), "rho" version.

    :math:`\\rho(x)=\\begin{cases}
    (c^2 / 6)(1-(1-(x/c)^2)^3)& \\text{if |x|} \\leq 0, \\\\
    c^2 / 6& \\text{if |x| > 0}.
    \\end{cases}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return: (float) result :math:`\\rho(x)` of bisquare function

    Example : run bisquare loss on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        result = [lp.rho_bisquare(e, 4) for e in x]

        # [0.46940104166666663, 1.5416666666666665, 2.443359375, 2.6666666666666665, 2.6666666666666665, 2.6666666666666665, 2.6666666666666665, 2.6666666666666665, 2.6666666666666665]
    '''
    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    if (np.absolute(input) <= clipping):
        return (
                   (clipping ** 2.0) / 6.0) * (
                   1 - (
                       (1 - (
                           input / clipping) ** 2)
                       ** 3)
               )
    else:
        return (clipping ** 2) / 6.0

def psi_bisquare(input, clipping=4.685):
    '''
    The derivative of bisquare loss (or Tukey's loss), "psi" version.

    :math:`\\psi(x)=\\begin{cases}
    x((1-(x/c)^2)^2)& \\text{if |x|} \\leq 0, \\\\
    0& \\text{if |x| > 0}.
    \\end{cases}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return: (float) :math:`\\psi(x)`

    Example : run psi_bisquare on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        result = [lp.psi_bisquare(e, 4) for e in x]

        # [0.87890625, 1.125, 0.57421875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
    '''
    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    if (np.absolute(input) <= clipping):
        return input * ((1 - (input / clipping) ** 2) ** 2)
    else:
        return 0.0

def rho_cauchy(input, clipping=2.3849):
    '''
    Cauchy loss function; the "rho" version.

    :math:`\\rho(x)=(c^2/2)log(1+(x/c)^2)`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: :math:`\\rho(x)`

    Example : run Cauchy loss on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        result = [lp.rho_cauchy(e, 4) for e in x]

        # [0.4849969745314787, 1.7851484105136781, 3.5702968210273562, 5.545177444479562, 7.527866755716213, 9.42923997073317, 11.214388381246847, 12.875503299472802, 14.416978050108813]
    '''
    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    return (
               (clipping ** 2) / 2
           ) * math.log(
        1 + (
            input / clipping) ** 2)

def psi_cauchy(input, clipping=2.3849):
    '''
    Derivative of Cauchy loss function; the "psi" version.

    :math:`\\psi(x)=\\frac{x}{1+(x/c)^2}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: result of the Cauchy's derivative function

    Example : run psi_cauchy on a vector

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5,6,7,8,9]

        result = [lp.psi_cauchy(e, 4) for e in x]

        # [0.9411764705882353, 1.6, 1.92, 2.0, 1.951219512195122, 1.8461538461538463, 1.7230769230769232, 1.6, 1.4845360824742269]
    '''

    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    return input / (
        1 + (
            input / clipping
        ) ** 2)

def rho_optimal(input, clipping=3.270):
    '''
    The so-called optimal loss function is given by
    :math:`\\rho(x)=\\begin{cases}
    1.38(x/c)^2 & \\text{if |x/c|} \\leq 2/3, \\\\
    0.55 - 2.69(x/c)^2 + 10.76(x/c)^4 - 11.66(x/c)^6 + 4.04(x/c)^8 & \\text{if 2/3 }<|x/c| \\leq 1, \\\\
    1 &  \\text{if |x/c| > 1}.
    \\end{cases}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: :math:`\\rho(x)`
    '''

    # Casting input to float to avoid divisions rounding
    input = float(input)

    if clipping <= 0:
        raise ValueError('clipping must be positive.')
    if abs(input / clipping) <= 2.0 / 3.0:
        return 1.38 * (input / clipping) ** 2
    elif abs(input / clipping) <= 1.0:
        return 0.55 - (2.69 * (input / clipping) ** 2) + (
            10.76 * (input / clipping) ** 4) - (
                   11.66 * (input / clipping) ** 6) + (
                   4.04 * (input / clipping) ** 8)
    elif abs(input / clipping) > 1:
        return 1.0

def psi_optimal(input, clipping=3.270):
    '''
    The derivative of the optimal 'rho' function is given by
    :math:`\\psi(x)=\\begin{cases}
    2\\cdot1.38 x / c^2 & \\text{if |x/c|} \\leq 2/3, \\\\
    2\\cdot2.69x / c^2 + 4\\cdot10.76x^3 / c^4 - 6\\cdot11.66x^5/ c^6 + 8\\cdot4.04x^7 /c^8 & \\text{if 2/3 }<|x/c| \\leq 1, \\\\
    0 &  \\text{if |x/c| > 1}.
    \\end{cases}`

    :param input: (float) :math:`x`
    :param clipping: (optional)(float) clipping parameter. Default value is optimal for normalized distributions.

    :return float: :math:`\\psi(x)`
    '''

    if clipping <= 0:
        raise ValueError('clipping must be positive.')

    if abs(input / clipping) <= 2.0 / 3.0:
        return 2 * 1.38 * (input / clipping ** 2)
    elif abs(input / clipping) <= 1.0:
        return (- 2 * 2.69 * (input / clipping ** 2)) + (
            4 * 10.76 * (input ** 3 / clipping ** 4)) - (
                   6 * 11.66 * (input ** 5 / clipping ** 6)) + (
                   8 * 4.04 * (input ** 7 / clipping ** 8))
    elif abs(input / clipping) > 1:
        return 0

def weights(input, loss_function, clipping=None, nmeasurements=None):
    '''
    Returns an array of :

    :math:`\\begin{cases}
    \\frac{loss\\_function(x_i)}{x_i}& \\text{if } r_i \\neq 0, \\\\
    1& \\text{otherwise}.
    \\end{cases}`

    Weights function designed to be used with loss functions like rho_huber,
    psi_huber, rho_cauchy... Note that the loss_function passed in argument must
    support two inputs.

    :param input: (array or float) vector or float to be processed, r_i's
    :param loss_function: (loss_function) f(x) in f(x)/x.
    :param clipping: (optional) clipping parameter of the huber loss function.

    :return array or float: element-wise result of f(x)/x if x!=0, 0 otherwise

    Example : run the weight function with the psi_huber with default
    clipping or with another function like rho_cauchy and another clipping.

    .. code-block:: python

        import linvpy as lp

        x = [1,2,3,4,5]

        # psi_huber, default clipping
        lp.weights(x, lp.psi_huber)

        # [1.0, 0.67249999999999999, 0.44833333333333331, 0.33624999999999999, 0.26900000000000002]

        # rho_cauchy, clipping=2.5
        lp.weights(x, lp.rho_cauchy, 2.5)

        # [0.46381251599460444, 0.7729628778689174, 0.9291646242761568, 0.9920004256749526, 1.0058986952713127]

    '''

    # kwargs = keyword arguments : if clipping is not specified, kwargs=None
    # and we use the default loss function's clipping, otherwise we use the one
    # passed in weights() with **kwargs
    kwargs = {}
    if clipping != None:
        kwargs['clipping'] = clipping

    if isinstance(input, (int, float)):
        if (input == 0):
            return 1.0
        # print 'ENTER 1'
        # return loss_function(input, **kwargs)/float(input)
        return util.scorefunction(input, 'huber', **kwargs) / float(input)

    # only used for the tau estimator
    elif nmeasurements != None:

        # print 'Marta ENTER 2, rhat = ', input

        z = util.scorefunction(input, 'tau', **kwargs)

        # print 'Marta z = ', z

        # for zero inputs, set weight to one
        w = np.ones(input.shape)

        # only for the non zero u elements
        i = np.nonzero(input)
        w[i] = z[i] / (2 * input[i] * nmeasurements)

        # print 'Marta w = ', w

        return w

    else:
        # Ensures the input is an array and not a matrix.
        # Turns [[a b c]] into [a b c].

        # print 'ENTER 3!'

        input = np.squeeze(
            np.asarray(
                input
            )
        ).flatten()

        # output = [0 if (i == 0) else 0.5*loss_function(i, **kwargs)/float(i) for i in input]
        output = [1 if (i == 0) else 0.5 * util.scorefunction(i, 'huber', **kwargs) / float(i) for i in input]
        return np.array(output)

# scale = sigma that divides; if sigma if given in parameter => preliminary scale
# lmb = lambda for tikhonov => if lambda is given : regularized m-estimator
# if lamb and scale are given : regularized m-estimator with preliminary scale
def irls(
        matrix_a,
        vector_y,
        loss_function,
        clipping=None,
        scale=None,
        lamb=0,
        initial_x=None,
        regularization=tikhonov_regularization,
        kind=None,
        b=0.5,
        tolerance=1e-10,
        max_iterations=100
):
    '''
    The method of iteratively reweighted least squares (IRLS) minimizes iteratively the function:

    :math:`\\boldsymbol x^{(t+1)} =`
    :math:`{\\rm arg}\\min_x\\,`
    :math:`\\big|\\big| \\boldsymbol W (\\boldsymbol x^{(t)})(\\boldsymbol y - \\boldsymbol A \\boldsymbol x )\\big | \\big |_2^2.`

    The IRLS is used, among other things, to compute the M-estimate and the tau-estimate.

    :param matrix_a: (np.matrix) matrix A in y - Ax
    :param vector_y: (ndarray) vector y in y - Ax
    :param loss_function: the loss function to be used in the M estimator
    :param clipping: clipping parameter for the loss function

    :return array: vector x solution of IRLS

    '''

    # print 'Marta 0 A,y = ', matrix_a,vector_y

    # If a scale parameter is given, m-estimator runs with preliminary scale.
    # This checks that scale is int or float and nonzero.
    # If no scale is given, scale = 1.0
    if (scale == None) or (scale == 0):
        scale = 1.0

    # kwargs = keyword arguments : if clipping is not specified, kwargs=None
    # and we use the default loss function's clipping, otherwise we use the one
    # passed in weights() with **kwargs
    # If the tau option is used, the function needs a clipping as tuple,
    # otherwise only one clipping is given.
    kwargs = {}
    if clipping != None:
        kwargs['clipping'] = clipping

    # if an initial value for x is specified, use it, otherwise generate a
    # vector of ones
    if initial_x is not None:
        vector_x = initial_x
    else:
        # Generates a ones vector_x with length = matrix_a.columns
        vector_x = np.ones(matrix_a.shape[1])
        initial_x = np.ones(matrix_a.shape[1])

    # number of measurements and unknowns; by default None, if tau is used it
    # takes value m,n = matrix_a.shape
    m = None

    # Ensures numpy types
    matrix_a = np.matrix(matrix_a)
    vector_y = vector_y.reshape(-1, 1)

    # Residuals = y - Ax, difference between measured values and model
    residuals = vector_y - np.dot(matrix_a, initial_x).reshape(-1, 1)

    # print 'Marta residuals 1 = ', residuals

    for i in range(1, max_iterations):

        # if we are computing the tau estimator, we need to upgrade the estimation of the scale in each iteration
        if kind == 'tau':

            # print 'Marta TAU'

            m, n = matrix_a.shape

            residuals = np.asarray(residuals.reshape(-1)).flatten()

            # print 'Marta residuals 2 = ', residuals

            # scale = scale * (mean(loss_function(residuals/scale))/b)^1/2
            if (scale != 0):
                scale *= np.sqrt(
                    np.mean(
                        # array_loss(residuals / scale, loss_function, clipping[0])
                        util.rhofunction(residuals / scale, loss_function, clipping[0])
                    ) / b
                )

        # normalize residuals ((y - Ax)/ scale)
        if (scale != 0):
            rhat = np.array(residuals / scale).flatten()
        else:
            rhat = np.array(residuals).flatten()

        # print "Marta rhat = ", rhat
        #
        # print "Marta irls rhat = ", rhat
        # print 'Marta irls scale = ', scale
        # print 'Marta irls residuals = ', residuals

        # weights(y-Ax, loss_function, clipping)
        weights_vector = weights(rhat, loss_function, nmeasurements=m, **kwargs)

        # print "Marta weights vector 1 = ", weights_vector

        # Makes a diagonal matrix with values of w(y-Ax)
        # np.squeeze(np.asarray()) is there to flatten the matrix into a vector
        weights_matrix = np.diag(
            np.squeeze(
                np.asarray(weights_vector)
            )
        )

        # Square root of the weights matrix, sqwm = W^1/2
        # sqwm = np.sqrt(weights_vector.reshape(-1,1))
        sqwm = np.sqrt(weights_matrix)

        # print 'sqwm, A = ', sqwm, matrix_a

        # A_weighted = W^1/2 A
        a_weighted = np.dot(sqwm, matrix_a)

        # y_weighted = diagonal of W^1/2 y
        # sqwm = sqwm.reshape(-1)

        y_weighted = np.dot(sqwm, vector_y)

        # vector_x_new is there to keep the previous value to compare
        vector_x_new = regularization(a_weighted, y_weighted, lamb)

        # ensures flattened array type
        vector_x = np.asarray(vector_x).flatten()

        # print 'Marta vector x, vector x new = ', vector_x, vector_x_new

        # Normalized distance between previous and current iteration
        xdis = np.linalg.norm(vector_x - vector_x_new)

        # print 'Marta xdis =', xdis
        # print 'x irls ', vector_x_new

        # TODO : changed by Guillaume
        vector_x_new = np.array(vector_x_new).reshape(-1)

        # New residuals
        residuals = vector_y.reshape(-1) - np.dot(matrix_a, vector_x_new).reshape(-1)

        # Divided by the specified optional scale, otherwise scale = 1
        # residuals = np.array(residuals / scale).flatten()

        vector_x = vector_x_new

        # if the difference between iteration n and iteration n+1 is smaller
        # than tolerance, return vector_x
        if (xdis < tolerance):
            return vector_x

    return vector_x

def m_estimator(a, y, lossfunction, clipping, preliminaryscale, regularization=tikhonov_regularization, lmbd=0):
    """
    Function that coputes the M estimator
    :param y: array that contains the measurements
    :param a: model matrix
    :param lossfunction: rho function
    :param clipping: clipping parameter for the rho function
    :param preliminaryscale: value of the scale of the residuals
    :param regularization: type of regularization used (default: tikhonov)
    :param lmbd: regularization parameter (default=0)
    :return: M estimate
    """
    m, n = a.shape
    initialx = np.ones((n, 1))  # initial solution
    xhat = irls(
        a,
        y,
        lossfunction,
        clipping,
        scale=preliminaryscale,
        initial_x=initialx,
        kind='M',
        regularization=regularization,
        lamb=lmbd
    )

    return xhat

def tau_derivative(a, y, x, clipping_1, clipping_2):
    '''
    Returns the derivative of the tau function f(x) for the given parameters y = ax + n
    Derovative defined in page 9 of
    'The regularized tau estimator: a robust and efficient solution to ill-posed problems'

    :param a: model matrix
    :param y: measurement vector
    :param x: unknown value
    :param clipping_1: clipping parameter for the rho_1 function (robust one)
    :param clipping_2: clipping paramter for the rho-2 function (non-robust one)
    :return: f'(x)

    '''
    # Get number of measurements
    m = a.shape[0]
    # get the normalized residuals
    residuals = y.reshape(-1, 1) - np.dot(a, x).reshape(-1, 1)
    scale = util.mscaleestimator(residuals, tolerance=1e-5, b=0.5, clipping=clipping_1, kind='optimal')
    rhat = np.array(residuals / scale).flatten()

    # weights for the tau rho function - independent of i
    wm = util.tauweights(rhat, lossfunction='optimal', clipping=[clipping_1, clipping_2])

    # derivative (for the moment no regularization)
    der = 0
    for element in range(m):
        # contribution to the sum of column element
        contribution = (- 1 / m) * (wm *
                                    util.scorefunction(rhat[element], kind='optimal', clipping=clipping_1
                                                       ) + util.scorefunction(rhat[element], kind='optimal',
                                                                              clipping=clipping_2)) * scale * a[element,
                                                                                :].T

        # add the contribution to the total sum
        der += contribution

    return der

def tau_gradient_descent(a, y, clipping_1, clipping_2, initial_x):
    '''
    Gradient descent algorithm to find the minima of the tau function
    :param a: model matrix
    :param y: measurement vector
    :param x: unknown value
    :param clipping_1: clipping parameter for the rho_1 function (robust one)
    :param clipping_2: clipping paramter for the rho-2 function (non-robust one)
    :param initial_x: initial starting point for the algorithm
    :return: local minimum
    '''

    # initialization point
    # x_init = np.random.rand(a.shape[1], 1)
    x_init = initial_x

    # number of iterations
    iter = 1000

    # step in each iteration
    step = 2

    # initialized gradient descent
    x = x_init

    for iteration in range(iter):
        # gradient descent step
        dr = tau_derivative(a, y, x, clipping_1, clipping_2)

        x -= step * tau_derivative(a, y, x, clipping_1, clipping_2).reshape(-1, 1)
        plt.scatter(x[0], x[1])

    plt.xlabel('coordinate 0')
    plt.ylabel('coordinate 1')

    return x

def soft_thresholding(v, threshold):
    """
    soft thresholding function
    :param v: scalar
    :param threshold: positive scalar
    :return: st(v)

    """

    if v >= threshold:
        return (v - threshold)
    elif v <= - threshold:
        return (v + threshold)
    else:
        return 0

def proximal_operator_l2(v, step_size):
    """
    Function to compute the proximal operator with g(x) = lambda||x||_2^2
    :param v: argument of the proximal operator
    :param step_size: parameter for the proxima operator
    :return value of the proximal operator
    """

    x = v / (2 * step_size + 1)
    # if np.linalg.norm(v) >= step_size:
    #     x = (1 - step_size/np.linalg.norm(v)) * v
    # else:
    #     x = np.zeros(v.shape)

    return x

def proximal_operator_l1(v, step_size):
    """
    Computes the proximal operator of the l1 norm, g(x) = lambda||x||_1
    :param v: argument for the proximal operator (vector)
    :param step_size: parameter in the prox. operator (lambda in the paper)
    :param lmbd: regularization parameter
    :return: value of the proximal operator
    """

    return np.sign(v) * np.maximum(np.abs(v) - step_size, 0.)

def line_search(a, y, x_k, init_step, lmbd, clipping_1, clipping_2, proximal_operator):
    """
    Given x_k, we find the step size for each iteration
    :param a: model matrix
    :param y: measurement vector
    :param x_k: x at iteration k
    :param init_step: initialization of the step
    :param lmbd: regularization parameter
    :param clipping_1: clipping parameter for the robust rho function
    :param clipping_2: clipping parameter for the efficient rho function
    :param proximal_operator: function to compute proximal operator (options: proximal_operator_l2 and
    proximal_operator_l2)
    :return: suitable step size
    """
    # Find suitable step size
    step_size = init_step  # initial guess

    # f'(x_k)
    grad_fk = tau_derivative(a, y, x_k, clipping_1, clipping_2)

    # r(x_k)
    r_k = y.reshape(-1, 1) - np.dot(a, x_k).reshape(-1, 1)

    # f(x_k)
    fk = util.tauscale(r_k, 'optimal', clipping=[clipping_1, clipping_2], b=0.5)
    while True:  # adjust step size
        xk_grad = x_k - step_size * grad_fk

        # proximal operator with x_k
        z = proximal_operator(xk_grad, step_size * lmbd)

        # residual for z
        r_values = y.reshape(-1, 1) - np.dot(a, z).reshape(-1, 1)

        # left part of the inequality, f(z)
        lhand = util.tauscale(r_values, 'optimal', clipping=[clipping_1, clipping_2], b=0.5)

        # right part of the inequality, majorization function
        rhand = fk + grad_fk.dot(z - x_k) + (0.5 * (1 / step_size)) * (z - x_k).dot(z - x_k)

        if lhand <= rhand:
            # if f(z) is smaller than the majorization function, the step size is good
            break
        else:
            # we update the step size, and check again
            step_size *= .5

    return step_size

def tau_apg(a, y, lmbd, clipping_1, clipping_2, initial_x, proximal_operator, rtol=0.0000000001):
    """
    Gradient descent algorithm to find the minima of the tau function
    :param a: model matrix
    :param y: measurement vector
    :param lmbd: regularization parameter
    :param clipping_1: clipping parameter for the rho_1 function (robust one)
    :param clipping_2: clipping paramter for the rho-2 function (non-robust one)
    :param initial_x: initial starting point for the algorithm
    :param proximal_operator: function for the proximal operator (proximal_operator_l2 or proximal_operator_l1)
    :param rtol: tol to stop the algorithm
    :return: local minimum
    """

    # initialization point
    # x_init = np.random.rand(a.shape[1], 1)
    x_init = initial_x

    # number of iterations
    iter = 100

    # initialized z and x to the same value
    x = [x_init, x_init]  # x_(k-1) and x_k
    z = x_init

    # initialize parameters for the acceleration extension
    t = [0, 1]  # t_(k-1) and t_k

    # initialize the step for prox. gradient
    default_step_size = 100

    for iteration in range(1, iter):

        # algo step without acceleration (Eq. 12)

        # parameter for the proximal operator
        alpha_x = line_search(a, y, x[1], default_step_size, lmbd, clipping_1, clipping_2, proximal_operator)

        # argument for the proximal operator
        arg_v = x[1] - alpha_x * tau_derivative(a, y, x[1], clipping_1, clipping_2)

        # update v
        v = proximal_operator(arg_v, alpha_x * lmbd)

        # extension for the acceleration (Eq. 10)

        l = x[1] + (t[0] / t[1]) * (np.array(z) - x[1]) + ((t[0] - 1) / t[1]) * (x[1] - np.array(x[0]))

        # algo step with acceleration (Eq. 11)

        # parameter for the proximal operator
        alpha_l = line_search(a, y, l, default_step_size, lmbd, clipping_1, clipping_2, proximal_operator)

        # argument for the proximal operator
        arg_l = l - alpha_l * tau_derivative(a, y, l, clipping_1, clipping_2)

        # update z
        z = proximal_operator(arg_l, alpha_l * lmbd)

        # update t
        t[0] = t[1]
        t[1] = 0.5 * (np.sqrt(4 * t[0] ** 2 + 1) + 1)

        # update x (Eq. 14)
        x[0] = x[1]

        # compare objective functions
        res_z = y.reshape(-1, 1) - np.dot(a, z).reshape(-1, 1)
        tscale_z = util.tauscale(res_z, 'optimal', [clipping_1, clipping_2], b=0.5)
        F_z = tscale_z + lmbd * np.dot(z, z)

        res_v = y.reshape(-1, 1) - np.dot(a, v).reshape(-1, 1)
        tscale_v = util.tauscale(res_v, 'optimal', [clipping_1, clipping_2], b=0.5)
        F_v = tscale_v + lmbd * np.dot(v, v)

        # take the best one
        if F_z <= F_v:
            x[1] = z
        else:
            x[1] = v

        # sct = x[1]
        # plt.scatter(sct[0], sct[1])

        # Check if the algorithm already converged
        res_old = y.reshape(-1, 1) - np.dot(a, x[0]).reshape(-1, 1)
        res_new = y.reshape(-1, 1) - np.dot(a, x[1]).reshape(-1, 1)

        F_old = util.tauscale(res_old, 'optimal', [clipping_1, clipping_2], b=0.5) + lmbd * np.dot(x[0], x[0])
        F_new = util.tauscale(res_new, 'optimal', [clipping_1, clipping_2], b=0.5) + lmbd * np.dot(x[1], x[1])
        # print 'x_k = ', x[1]

        # print 'Shrink obj func ', np.abs(F_old - F_new)
        # print 'x ', x[1]

        if (np.abs(F_old - F_new) / F_new) < rtol:
            print("Achieved relative tolerance at iteration %s" % iteration)
            break
        xdis = np.linalg.norm(x[0] - x[1])
        if xdis < rtol:
            print("Achieved relative tolerance at iteration %s" % iteration)
            break

    # plt.show()
    obj_distance = np.abs(F_old - F_new)
    # print 's_z = ', tscale_z
    # print 's_v = ', tscale_v
    # print 'z = ', z
    # print 'v = ', v

    return x[1], obj_distance, alpha_l, alpha_x

def basictau(
        a,
        y,
        loss_function,
        clipping,
        ninitialx,
        maxiter=100,
        nbest=1,
        initialx=None,
        b=0.5,
        regularization=tikhonov_regularization,
        lamb=0
):
    '''
    This routine minimizes the objective function associated with the tau-estimator.
    For more information on the tau estimator see http://arxiv.org/abs/1606.00812

    This function is hard to minimize because it is non-convex. This means that it has several local minima. Depending on
    the initial x that we use for our minimization, we will end up in a different local minimum.

    In this algorithm we take the 'brute force' approach: let's try many different initial solutions, and let's pick the
    minimum with smallest value. The output of basictau are the best nbest minima (we will need them later)

    :param a: matrix A in y - Ax
    :param y: vector y in y - Ax
    :param loss_function: type of the rho function we are using
    :param clipping: clipping parameters. In this case we need two, because the rho function for the tau is composed two rho functions.
    :param ninitialx: how many different solutions do we want to use to find the global minimum (this function is not convex!)
                      if ninitialx=0, means the user introduced a predefined initial solution
    :param maxiter: maximum number of iteration for the irls algorithm
    :param nbest: we return the best nbest solutions. This will be necessary for the fast algorithm
    :param initialx: the user can define here the initial x he wants
    :param b: this is a parameter to estimate the scale
    :param regularization: function for the regularization we want to use. Default: tikhonov_regularization
    :param lamb: parameter for the regularization. It has to be non-negative.

    :return xhat: contains the best nmin estimations of x
    :return mintauscale: value of the objective function when x = xhat
    '''

    # TODO : remove this, for testing only
    np.set_printoptions(precision=20)

    # to store the minimum values of the objective function (in this case is
    # the scale)
    mintauscale = np.empty((nbest, 1))

    # initializing objective function with infinite. When we have a x that gives a smaller value for the obj. function,
    # we store the value of the objective function here
    mintauscale[:] = float("inf")

    # count how many initial solutions are we trying
    k = 0

    # store here the best xhat (nbest of them)
    xhat = np.zeros((a.shape[1], nbest))  # to store the best nmin minima

    # auxiliary variable to check if the user introduced a predefined initial solution.
    # = 0 if we do not have initial x. =1 if we have a given initial x
    givenx = 0

    if initialx is None:
        initialx = np.ones(a.shape[1])

    if ninitialx == 0:
        # we have a predefined initial x
        ninitialx = initialx.shape[1]

        # set givenx to 1
        givenx = 1

    while k < ninitialx:
        # if still we did not reach the number of initial solutions that we want to try,
        if givenx == 1:
            # if we have a given initial solution initx, we take it
            initx = np.expand_dims(initialx[:, k], axis=1)
        else:
            # otherwise get randomly  a new initial solution initx
            # TODO : remove this line and put back initx = util.getinitialsolution(y.reshape(-1, 1), a)
            initx = np.ones(a.shape[1])
            # initx = util.getinitialsolution(y.reshape(-1, 1), a)

        # compute the residual y - Ainitx
        initialres = y.reshape(-1, 1) - np.dot(a, initx)

        # TODO : added by Guillaume
        initialres = np.array(initialres)

        # estimate the scale using initialres
        initials = np.median(np.abs(initialres)) / 0.6745

        # print 'MARTA initial_residuals = ', initialres
        # print 'MARTA scale = ', initials
        # print 'Marta initial x = ', initx

        # solve irls using y, a, the tau weights, initx and initals. We get an
        # estimation of x, xhattmp
        xhattmp = irls(
            a,
            y,
            loss_function,
            clipping,
            scale=initials,
            initial_x=initx,
            kind='tau',
            b=0.5,
            max_iterations=maxiter,
            regularization=regularization,
            lamb=lamb)

        # print 'MARTA IRLS RESULT = ', xhattmp
        # last working version
        # xhattmp, scaletmp, ni, w, steps = inv.irls(y, a, 'tau', 'optimal', 'none', 0, initx, initials, clipping, maxiter)

        # print 'Marta y = ', y
        #
        # print 'Marta a * xhattmp = ', np.dot(a, xhattmp).reshape(-1, 1)

        # compute the value of the objective function using xhattmp
        # we compute the res first
        res = y.reshape(-1, 1) - np.dot(a, xhattmp).reshape(-1, 1)
        # res = np.subtract(y.reshape(-1, 1), np.dot(a, xhattmp).reshape(-1, 1))

        # Value of the objective function using xhattmp
        # tscalesquare = util.tauscale(res, lossfunction, clipping, b)

        # print 'Marta residuals = ', res

        tscalesquare = util.tauscale(res, 'optimal', clipping, b)

        # print 'Marta tscalesquare = ', tscalesquare

        # update counter
        k += 1

        # we checks if the objective function has a smaller value that then
        # ones we found before
        if tscalesquare < np.amax(mintauscale):
            # it is smaller, so we keep it!
            # store value for the objective function
            mintauscale[np.argmax(mintauscale)] = tscalesquare

            # store value of xhat
            xhat[:, np.argmax(mintauscale)] = np.squeeze(xhattmp)

    # we return the best solutions we found, with the value of the objective
    # function associated with the xhats
    return xhat, mintauscale

def fasttau(y,
            a,
            loss_function,
            clipping,
            ninitialx,
            regularization=tikhonov_regularization,
            nmin=5,
            initialiter=5,
            lamb=0
            ):
    '''
    Fast version of the basic tau algorithm.
    To save some computational cost, this algorithm exploits the speed of convergence of the basic algorithm.
    It has two steps: in the first one, for every initial solution, it only performs initialiter iterations. It keeps
    value of the objective function.
    In the second step, it compares the value of all the objective functions, and it select the nmin smaller ones.
    It iterates them until convergence. Finally, the algorithm select the x that produces the smallest objective
    function.
    For more details see http://arxiv.org/abs/1606.00812

    :param a: matrix A in y - Ax
    :param y: vector y in y - Ax
    :param loss_function: type of the rho function we are using
    :param clipping: clipping parameters. In this case we need two, because the rho function for the tau is composed two rho functions.
    :param ninitialx: how many different solutions do we want to use to find the global minimum (this function is not convex!)
                    if ninitialx=0, means the user introduced a predefined initial solution
    :param regularization: function for the regularization we want to use. Default: tikhonov_regularization
    :param nmin: how many solutions do we keep for the second round.
    :param initialiter: number of iterations during the first round
    :param lamb: parameter for the regularization. It has to be non-negative.

    :return xhat: contains the best nmin estimations of x
    :return mintauscale: value of the objective function when x = xhat
    '''
    xhat, mintauscale = basictau(
        a=a,
        y=y,
        loss_function=loss_function,
        clipping=clipping,
        ninitialx=ninitialx,
        maxiter=initialiter,
        nbest=nmin,
        regularization=regularization,
        lamb=lamb
    )  # first round: only initialiter iterations. We keep the nmin best solutions

    xfinal, tauscalefinal = basictau(
        a=a,
        y=y,
        loss_function=loss_function,
        clipping=clipping,
        regularization=regularization,
        ninitialx=0,
        maxiter=100,
        nbest=1,
        initialx=xhat,
        lamb=lamb
    )  # iterate the best solutions until convergence

    return xfinal, tauscalefinal

# function to apply a loss function to any data structure; scalar, array or matrix
# returns the exact same data structure with the loss function applied element-wise
def array_loss(values, loss_function, clipping=None):
    # if we input incorrect values (due to division by too tiny numbers in irls)
    # inside array_loss, it returns zeroes

    # checks if there is a 'nan' value in the input
    if np.any(np.isnan(values)):
        print "incorrect input in array loss !"
        return np.zeros(values.shape)

    kwargs = {}
    if clipping != None:
        kwargs['clipping'] = clipping

    vfunc = np.vectorize(loss_function)
    return vfunc(values, **kwargs)

def tau_weights_new(input, clipping, loss_function=rho_optimal):
    '''
    This is the beginning of the refactoring of the whole tau estimator. We did not have to finish it, so we used the
    architecture...
    :return array or float: element-wise result of f(x)/x if x!=0, 0 otherwise
    :param input: (array or float) vector or float to be processed, x_i's
    :param clipping: clipping parameter for loss function -- they should be two!
    :param loss_function (optional): family of loss functions we use
    :return: ndarray with tau weights
    '''

    weights = 2.0 * array_loss(input, loss_function, clipping[1])
    weights -= (util.scoreoptimal(input, clipping[1]) * input)
    scaling = np.sum(util.scoreoptimal(input, clipping[0]) * input)

    if (scaling == 0):
        scaling = 1.0

    return weights / scaling

def sridge_matlab(A, y):
    '''
    This function runs the sridge function written in Matlab code.
    :param A: model matrix
    :param y: measurement vector
    :return: S-estimat
    '''
    # import Sridge.matlab as sr
    # build the path for the matlab code
    # matlab_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Sridge/matlab')
    # python_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Sridge/python')
    # # append it to the current path
    # sys.path.append(python_folder)

    # start matlab engine
    eng = matlab.engine.start_matlab()

    # call matlab function
    tf = eng.sridge(A, y, 3, 3)

# ===============================================================================
# ===================================TESTING AREA================================
# ===============================================================================

def main():
    import array
    # defining linear problem, create data
    m = 20  # number of measurements
    n = 3  # dimensions of x
    a = np.random.rand(m, n)  # model matrix
    x_grountruth = [3, 5, 1]  # ground truth
    y = np.dot(a, x_grountruth).reshape(-1, 1) + 0.5 * np.random.rand(m, 1)

    # define any x
    x = [7, 9, 3]

    # clipping parameters
    clipping_1 = 1.21
    clipping_2 = 3.27
    reg_parameter = 0.001

    # testing apg algo

    # x_apg = tau_apg(a, y, reg_parameter, clipping_1, clipping_2, x, proximal_operator_l1, rtol=1e-5)

    # print 'x_apg =', x_apg
    initx = np.array(x)
    initx = initx.reshape(-1, 1)

    # x_irls = basictau(
    #     a,
    #     y,
    #     'optimal',
    #     [clipping_1, clipping_2],
    #     ninitialx=0,
    #     maxiter=100,
    #     nbest=1,
    #     initialx=initx,
    #     b=0.5,
    #     regularization=lasso_regularization,
    #     lamb=reg_parameter
    #     )

    # print 'x_irls = ', x_irls[0]

    x_m = m_estimator(a, y, 'optimal', clipping_1, 1, regularization=lasso_regularization, lmbd=reg_parameter)

    print 'estimate x_m =', x_m

    # cast to matlab types
    # amat = matlab.double(a.tolist())

    # ymat = matlab.double(y.tolist())

    # sridge_matlab(amat, ymat)

if __name__ == '__main__':
    # To launch manually in console:
    # python plotFunctions.py
    main()


import numpy as np
import random
import pandas as pd
from matplotlib import pyplot as plt

'''
Perceptron algorithm that finds the idea weights given a data set and labels

Parameters
----------
coords: data points in 2 dimensions ([x1, x2])
labels: binary target output for each data point (-1 or 1)
epochs: number of iterations that needs to be run
misclassifiedList

Returns
-------
w: resulting weight vector after learning the data points

'''
def perceptron_algorithm(coords, labels, epochs, misclassifiedList):
	w = np.zeros((coords.shape[1]) + 1)

	for i in range(epochs):
		misclassified = 0
		for x, y in zip(coords, labels):
			dotProd = np.dot(x, w[1:]) + w[0]

			target = 1.0 if (dotProd > 0.0) else -1.0

			if(target != y):
				print(y, target, "classified incorrectly")
				misclassified += 1
				w[1:] += y * x
				w[0] += x[0]
			else:
				print(y, target, "classified correctly")

			print("PRINTING W", w)

		misclassifiedList.append(misclassified)

	return w

if __name__ == "__main__":
	# Set up the random generator seed
	np.random.seed(121232141)

	# Total number of random points
	total = 1000

	# Create 20 random points that are linearly separable
	A = 2 * np.random.random_sample((total//2, 2)) + 0
	B = 2 * np.random.random_sample((total//2, 2)) - 2

	# Create the lists that hold the respective binary labels for the 20 points
	A1 = np.ones((total//2, 1))
	B1 = np.negative(np.ones((total//2, 1)))

	x = np.linspace(-3, 3)
	y = -1 * x

	# Plot the scatter points and the target function, y = -x
	plt.plot(x, y, color="blue", label="target function f")
	plt.scatter(A[:,0], A[:,1], marker="o")
	plt.scatter(B[:,0], B[:,1], marker="x")
	plt.title("PLA Testing")
	plt.xlabel("x1")
	plt.ylabel("x2")

	coords = np.concatenate((A, B))
	labels = np.concatenate((A1, B1))

	# Set up and run perceptron algorithm
	misclassifiedList = []
	epochs = 10
	w = perceptron_algorithm(coords, labels, epochs, misclassifiedList)

	# Create the coordinates to plot the separator line
	x = np.linspace(-3, 3)
	slope = -(w[0]/w[2]) / (w[0]/w[1])
	intercept = -(w[0]/w[2])
	y = (slope * x) + intercept

	# # Plot the classified points along with separator line to show that the dataset is linearly separable
	plt.plot(x, y, color="red", label="final hypothesis g")
	plt.legend(loc="upper right")

	# # Plot the number of iterations needed before termination
	# iterations = np.arange(1, epochs+1)
	# plt.plot(iterations, misclassifiedList, color="red")
	# plt.xlabel("Iterations")
	# plt.ylabel("Misclassified")

	plt.show()


#!/usr/bin/python

import sys
import time
import numpy as np
from scipy import linalg, stats

def rsvd(X, dstar, power_iters=2, delta=10):
	""" Perform rsvd algorithm on input matrix.
		Method must be supplied dstar.
		Returns truncated svd (U,S,V).
	Parameters
	----------
	X : int matrix
    	Matrix of n x m integers, where m <= n. If n < m,
    	matrix will be transposed to enforce m <= n.
   	dstar : int
   		The latent (underlying) matrix rank that will be
   		used to truncate the larger dimension (m).
   	power_iters : int
   		default: 2
   		Number of power iterations used (random matrix multiplications)
   	delta : int
   		default: 10
   		oversampling parameter (to improve numerical stability)
    Returns
	-------
	int matrix
    	Matrix of left singular vectors.
    int matrix
    	Matrix of singular values.
    int matrix
    	Matrix of right singular vectors.
    """
        transpose = False
	if(X.shape[0] < X.shape[1]):
		X = X.T
                transpose = True
	if(power_iters < 1):
		power_iters = 1
	# follows manuscript notation as closely as possible
	P = np.random.randn(X.shape[0],dstar+delta)
	for i in range(power_iters):
		P = np.dot(X.T,P)
		P = np.dot(X,P)
	Q,R = np.linalg.qr(P)
	B = np.dot(Q.T,X)
	U,S,V = linalg.svd(B)
	U = np.dot(Q,U)

        # remove extra dimensionality incurred by delta
        U = U[:,0:dstar]
        S = S[0:dstar]

        if(transpose):
                return V.T,S,U.T
        # else
	return U,S,V

def stabilityMeasure(X, d_max, B=5, power_iters=2):
	""" Calculate stability of
	Parameters
	----------
	X : int matrix
		input matrix to determine rank of
	d_max : int
		upper bound rank to estimate
	B : int
		default: 5
		number of projections to correlate
	power_iters : int
		default: 2
   		Number of power iterations used (random matrix multiplications)
	Returns
	-------
	int
		Latent (lower-dimensional) matrix rank
	"""
	singular_basis = np.zeros((B,X.shape[0],d_max))
	# calculate singular basis under multiple projections
	for i in range(B):
		U = rsvd(X,d_max)[0]
		singular_basis[i,:,:] = U[:,0:d_max]

	# calculate score for each singular vector
	stability_vec = np.zeros((d_max))
	for k in range(d_max):
		stability = 0
		for i in range(0,B-1):
			for j in range(i+1,B):
				corr = stats.spearmanr(singular_basis[i,:,k],singular_basis[j,:,k])[0]
				stability = stability + abs(corr)
		N = B*(B-1)/2
		stability = stability/N
		stability_vec[k] = stability

	# wilcoxon rank-sum test p-values
	p_vals = np.zeros(d_max-2)
	for k in range(2,d_max):
		p_vals[k-2] = stats.ranksums(stability_vec[0:k-1],stability_vec[k-1:d_max])[1]

	dstar = np.argmin(p_vals)

	return dstar


'''
## Python Example

Consider
'''
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size': 22, 'figure.autolayout': True, 'figure.figsize': (15,5)})
# set seed for reproductibility
np.random.seed(999)

#%% Multinoulli example

n = 12                      # number of trials (games in one tourment)
pvals = [0.4, 0.35, 0.25]   # probabilities on a single trial

sizes =[] # number of tournments played
p = []    # a list to hold ratios (converge to prob) that player 1 wins 7 times, player 2 wins 2 times and 3 ties

for size in np.logspace(2,4):
    # the line below is where we actually generate discrete random variables according the multinomial distribution
    outcomes = np.random.multinomial(n, pvals, size=int(size))

    # let's count the ratio of the expected outcome over all the outcomes - this will lastly converge to the probability
    prob = sum((outcomes[:,0]==7)&(outcomes[:,1]==2)&(outcomes[:,2]==3))/len(outcomes)

    p.append(prob)
    sizes.append(int(size))

# Plotting
fig1 = plt.figure()
plt.plot(sizes,p,'o-')
plt.plot(sizes,[0.0248]*len(sizes),'--r')
plt.grid()
plt.xlim(xmin=0)
plt.xlabel('Number of Drawings')
plt.ylabel('p(X=K)')
plt.title('Theoretical p(X=K) = 0.0248')

#%% save figures

fig1.savefig('multinomial.png')


def newEq(theEquation, ans, theTerm, theVar):
    """
      Content under Creative Commons Attribution license CC-BY 4.0,
      code under MIT license (c)2018 Sergio Rojas (srojas@usb.ve)

      http://en.wikipedia.org/wiki/MIT_License
      http://creativecommons.org/licenses/by/4.0/

      Created on april, 2018
      Last Modified on: may 15, 2018

      This is a helper function to find step by step the solution
      of any linear equation via operations on both
      sides of the equation. In case of doubt, option 5 can be
      used to check the answer. This also can also be used
      to find solutions to any one variable non-linear equation.
    """
    from sympy import symbols, Eq, solveset, S, sympify
    from sympy import simplify, factor, expand, collect, nsimplify

    LHS, RHS = theEquation.split('=')
    LHS = sympify(LHS)
    RHS = sympify(RHS)
    if ans == 1:
         term = sympify(theTerm)
         RHS = nsimplify(RHS + term)
         LHS = nsimplify(LHS + term)
    elif ans == 2:
         term = sympify(theTerm)
         RHS = nsimplify(RHS - term)
         LHS = nsimplify(LHS - term)
    elif ans == 3:
         term = sympify(theTerm)
         #RHS = nsimplify( simplify(factor(RHS * term)) )
         #LHS = nsimplify( simplify(factor(LHS * term)) )
         RHS = nsimplify( simplify(RHS * term) )
         LHS = nsimplify( simplify(LHS * term) )
    elif ans == 4:
         term = sympify(theTerm)
         if term != 0:
            #RHS = nsimplify( simplify(expand(RHS / term)) )
            #LHS = nsimplify( simplify(expand(LHS / term)) )
            RHS = nsimplify( simplify(RHS / term) )
            LHS = nsimplify( simplify(LHS / term) )
    elif ans == 5:
         theVar = symbols(theVar)
         RHS = nsimplify( simplify(RHS) )
         LHS = nsimplify( simplify(LHS) )
        #RHS = solveset( Eq(LHS, RHS), theVar) ).args[0]
         RHS = list( solveset( Eq(LHS, RHS), theVar) )
         if len(RHS) == 1:
            RHS = RHS[0]
         LHS = theVar
    else:
         RHS = RHS
         LHS = LHS

    theneweq = '{0} = {1}'.format(LHS,RHS)
    return theneweq


"""
# !/usr/bin/env python
# -*- coding: utf-8 -*-
@Time        : 2022/3/26 13:47
@File        : gardent_descent.py
"""

import numpy as np

def sigmoid(x):
    """
    Calculate sigmoid
    """
    return 1 / (1 + np.exp(-x))

# Derivative of the sigmoid function
def sigmoid_prime(x):
    return sigmoid(x) * (1 - sigmoid(x))

learnrate = 0.5
x = np.array([1, 2])
y = np.array(0.5)

# Initial weights
w = np.array([0.5, -0.5])

# Calculate one gradient descent step for each weight
# TODO: Calculate output of neural network
nn_output = sigmoid(x[0] * w[0] + x[1] * w[1])
# or nn_output = sigmoid(np.dot(x, w))

# TODO: Calculate error of neural network
error = y - nn_output

# TODO: Calculate change in weights
# error term (lowercase delta)
# 第一种方法
# error_term = error * sigmoid_prime(np.dot(x,w))
# del_w = learnrate * error_term * x

# 第二种方法
del_w = learnrate * error * nn_output * (1 - nn_output) * x

print('Neural Network output:')
print(nn_output)

print('Amount of Error:')
print(error)

print('Change in Weights:')
print(del_w)

udacity-program_self_driving_car_engineer_v1.0/part01-computer vision and deep learning/module03-deep learning/lesson01-introduction to neural networks/exercise05-gardient descent/gardent_descent.py


# import necessary libraries
import matplotlib.pyplot as plt
import numpy as np

# import the Distribution class
from .GenericDistribution import Distribution

class Gaussian(Distribution):
    """ Gaussian distribution class for calculating and
    visualizing a Gaussian distribution.

    Attributes:
        mean (float) representing the mean value of the distribution
        stdev (float) representing the standard deviation of the distribution
        data_list (list of floats) a list of floats extracted from the data file

    """

    def __init__(self, mu=0, sigma=1):

        Distribution.__init__(self, mu, sigma)

    def calculate_mean(self):

        """Method to calculate the mean of the data set.

        Args:
            None

        Returns:
            float: mean of the data set

        """

        # Calculate the mean of the data set. Remember that the data set is stored in self.data
        # Change the value of the mean attribute to be the mean of the data set
        # Return the mean of the data set
        self.mean = np.mean(self.data)

        return self.mean

    def calculate_stdev(self, sample=True):

        """Method to calculate the standard deviation of the data set.

        Args:
            sample (bool): whether the data represents a sample or population

        Returns:
            float: standard deviation of the data set

        """

        #   Calculate the standard deviation of the data set
        #
        #   The sample variable determines if the data set contains a sample or a population
        #   If sample = True, this means the data is a sample.
        #   Keep the value of sample in mind for calculating the standard deviation
        #
        #   Make sure to update self.stdev and return the standard deviation as well
        # """

        if sample:
            number_of_data = len(self.data) - 1
        else:
            number_of_data = len(self.data)
        # convert the data to a numpy array
        data_np = np.array(self.data)
        sigma = np.sqrt(np.sum(np.square(data_np - self.mean)) / number_of_data)
        self.stdev = sigma

        return self.stdev

    def plot_histogram(self):
        """Method to output a histogram of the instance variable data using
        matplotlib pyplot library.

        Args:
            None

        Returns:
            None
        """

        # Plot a histogram of the data_list using the matplotlib package.

        plt.hist(self.data)
        plt.xlabel('Data')
        plt.ylabel('Count')
        plt.title('Histogram of the data')

    def pdf(self, x):
        """Probability density function calculator for the gaussian distribution.

        Args:
            x (float): point for calculating the probability density function

        Returns:
            float: probability density function output
        """

        # Calculate the probability density function of the Gaussian distribution
        #       at the value x. You'll need to use self.stdev and self.mean to do the calculation
        pdf = 1 / (self.stdev * np.sqrt(2 * np.pi)) * np.exp(-0.5 * (np.square(x - self.mean)
                                                                     * (1 / np.square(self.stdev))))

        return pdf

    def plot_histogram_pdf(self, n_spaces=50):

        """Method to plot the normalized histogram of the data and a plot of the
        probability density function along the same range

        Args:
            n_spaces (int): number of data points

        Returns:
            list: x values for the pdf plot
            list: y values for the pdf plot

        """

        mu = self.mean
        sigma = self.stdev

        min_range = min(self.data)
        max_range = max(self.data)

        # calculates the interval between x values
        interval = 1.0 * (max_range - min_range) / n_spaces

        x = []
        y = []

        # calculate the x values to visualize
        for i in range(n_spaces):
            tmp = min_range + interval * i
            x.append(tmp)
            y.append(self.pdf(tmp))

        # make the plots
        fig, axes = plt.subplots(2, sharex=True)
        fig.subplots_adjust(hspace=.5)
        axes[0].hist(self.data, density=True)
        axes[0].set_title('Normed Histogram of Data')
        axes[0].set_ylabel('Density')

        axes[1].plot(x, y)
        axes[1].set_title('Normal Distribution for \n Sample Mean and Sample Standard Deviation')
        axes[0].set_ylabel('Density')
        plt.show()

        return x, y

    def __add__(self, other):

        """Magic method to add together two Gaussian distributions

        Args:
            other (Gaussian): Gaussian instance

        Returns:
            Gaussian: Gaussian distribution

        """

        # Calculate the results of summing two Gaussian distributions
        #   When summing two Gaussian distributions, the mean value is the sum
        #       of the means of each Gaussian.
        #
        #   When summing two Gaussian distributions, the standard deviation is the
        #       square root of the sum of square ie sqrt(stdev_one ^ 2 + stdev_two ^ 2)

        # create a new Gaussian object
        result = Gaussian()

        # calculate the mean and standard deviation of the sum of two Gaussians
        # mean of the result distribution
        result.mean = self.mean + other.mean
        # standard deviation of the result distribution
        result.stdev = np.sqrt(np.square(self.stdev) + np.square(other.stdev))

        return result

    def __repr__(self):

        """Magic method to output the characteristics of the Gaussian instance

        Args:
            None

        Returns:
            string: characteristics of the Gaussian

        """

        # TODO: Return a string in the following format -
        # "mean mean_value, standard deviation standard_deviation_value"
        # where mean_value is the mean of the Gaussian distribution
        # and standard_deviation_value is the standard deviation of
        # the Gaussian.
        # For example "mean 3.5, standard deviation 1.3"
        # "mean {}, standard deviation {}".format(self.mean, self.stdev)
        # represent = print("mean {}, standard deviation {}".format(self.mean, self.stdev))

        return "mean {}, standard deviation {}".format(self.mean, self.stdev)


'''
Linear regression with multiple variables. Feature normalisation.

Bence Mélykúti
22/02/2018
'''

import numpy as np
import csv
import itertools
import matplotlib.pyplot as plt
from sklearn import linear_model, preprocessing

csvfile = open('../machine-learning-ex1/ex1/ex1data2.txt', 'r', newline='')
csvdata = csv.reader(csvfile, delimiter=',')
dataiter, dataiter2 = itertools.tee(csvdata) # https://docs.python.org/3/library/itertools.html
length=len(next(dataiter))
data=np.empty([0,length])
for row in dataiter2:
    data=np.vstack((data, [float(entry) for entry in row]))

X = data[:,:2]
X_orig = X
y = data[:,2]
m = len(y)

# https://matplotlib.org/2.0.2/api/pyplot_api.html
# fig = plt.figure()
plt.scatter(X[:,0], y, marker="x", c='r')
# fig.suptitle('Size-price plot', fontsize=20)
plt.ylabel('Price')
plt.show()

# Own implementation of feature normalisation; just use sklearn.preprocessing.StandardScaler instead
def featureNormalize(X):
    mu = np.mean(X, axis=0)
    sigma = np.std(X, axis=0, ddof=0)
    mu_matrix = np.concatenate([mu for i in range(X.shape[0])], axis=0)
    sigma_matrix = np.concatenate([sigma for i in range(X.shape[0])], axis=0)
    X = (X - mu)/sigma
    return X, mu, sigma

Xsc_own, mu_own, sigma_own = featureNormalize(X)

# feature normalisation with sklearn.preprocessing.StandardScaler:
# http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html
scaler = preprocessing.StandardScaler().fit(X)
Xsc = scaler.transform(X) # scaled X
mu = scaler.mean_
sigma2 = scaler.var_ # variance, i.e. sigma^2
sigma = scaler.scale_ # standard deviation, i.e. sigma

# Linear regression, classic fitting

# Linear regression example
# http://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html

reg = linear_model.LinearRegression()
reg.fit(Xsc,y) # for X already scaled from featureNormalize or StandardScaler
# reg.fit(scaler.transform(X),y)
print('\nTheta found by linear regression:')
print(reg.intercept_, reg.coef_)

data_prediction = reg.predict(Xsc) # for X already scaled from featureNormalize or StandardScaler
#data_prediction = reg.predict(scaler.transform(X))

# These four are equivalent:
#predict1 = reg.predict((((1650,3)-mu_own)/sigma_own).reshape(1,-1)) # using featureNormalize
#predict1 = reg.predict((((1650,3)-mu)/sigma).reshape(1,-1)) # using output of sklearn.preprocessing.StandardScaler
#predict1 = reg.predict((((1650,3)-mu)/sigma2**0.5).reshape(1,-1)) # using output of sklearn.preprocessing.StandardScaler
predict1 = reg.predict(scaler.transform(np.asarray((1650,3)).astype(float).reshape(1,-1))) # recommended way of using sklearn.preprocessing.StandardScaler
print('Predicted price of a 1650 sq-ft, 3 br house: ',predict1[0])

# Linear regression, normal equation

Xe = np.concatenate((np.ones((X.shape[0],1)), Xsc), axis=1)
theta=np.dot(np.linalg.inv(np.dot(np.transpose(Xe),Xe)), np.dot(np.transpose(Xe), y))
print('\nTheta found by normal equation:')
print(theta)

'''
# In comparison to ex1.py, there is no point in plotting it in 2 dim, as there are two features and one value, together 3 dim:
# https://matplotlib.org/2.0.2/api/pyplot_api.html
# fig = plt.figure()
plt.scatter(X[:,1],y, marker="x", c='r')
# fig.suptitle('Population-profit plot', fontsize=20)
plt.plot(X[:,1], data_prediction, color='blue', linewidth=3)
plt.ylabel('Price')
plt.show()
'''


import numpy as np
from sklearn.tree import DecisionTreeRegressor

# paste your get_features_targets function here
def get_features_targets(data):
  features = np.zeros((data.shape[0], 4))

  features[:, 0] = data['u'] - data['g']
  features[:, 1] = data['g'] - data['r']
  features[:, 2] = data['r'] - data['i']
  features[:, 3] = data['i'] - data['z']

  targets = data['redshift']

  return (features, targets)

# paste your median_diff function here
def median_diff(predicted, actual):
  return (np.median(abs(predicted-actual)))

# write a function that splits the data into training and testing subsets
# trains the model and returns the prediction accuracy with median_diff
def validate_model(model, features, targets):
  # split the data into training and testing features and predictions
  split = features.shape[0]//2
  train_features = features[:split]
  test_features = features[split:]
  train_targets = targets[:split]
  test_targets = targets[split:]
  # train the model
  model.fit(train_features, train_targets)
  # get the predicted_redshifts
  predicted_redshifts = model.predict(test_features)
  # use median_diff function to calculate the accuracy
  return median_diff(test_targets, predicted_redshifts)

if __name__ == "__main__":
  data = np.load('sdss_galaxy_colors.npy')
  features, targets = get_features_targets(data)

  # initialize model
  dtr = DecisionTreeRegressor()

  # validate the model and print the med_diff
  diff = validate_model(dtr, features, targets)
  print('Median difference: {:f}'.format(diff))

Week5/Assignment 1 - Building a regression classifier/Model_Validation.py


"""E-Divisive Numpy Calculator."""
import numpy as np

from scipy.special import comb

def _calculate_q(cross_term: float, x_term: float, y_term: float, x_len: int, y_len: int) -> float:
    """
    Calculate the q value from the terms and coefficients.

    :param cross_term: The sum of differences between sample distributions X, Y.
    :param x_term: The sum of differences within sample distribution X.
    :param y_term: The sum of differences within sample distribution Y.
    :param x_len: The length of sample distribution X.
    :param y_len: The length of sample distribution Y.

    :return: The q value generated from the terms.
    """
    term1_reg = 0 if x_len < 1 or y_len < 1 else cross_term * (2.0 / (x_len * y_len))
    term2_reg = 0 if x_len < 2 else x_term * (comb(x_len, 2) ** -1)
    term3_reg = 0 if y_len < 2 else y_term * (comb(y_len, 2) ** -1)
    newq = (x_len * y_len / (x_len + y_len)) * (term1_reg - term2_reg - term3_reg)
    return newq

def calculate_diffs(series: np.ndarray) -> np.ndarray:
    """
    Given an array N calculate an NxN difference matrix.

    :param series: The array to calculate the matrix for.
    :return: The difference matrix.
    """
    row, col = np.meshgrid(series, series)
    diffs = abs(row - col)
    return diffs

def calculate_qhat_values(diffs: np.ndarray) -> np.ndarray:
    """
    Find Q-Hat values given a difference matrix.

    :param diffs: The difference matrix.
    :return: The qhat values.
    """
    qhat_values = np.zeros(len(diffs), dtype=np.float)

    # We will partition our signal into:
    # X = {Xi; 0 <= i < tau}
    # Y = {Yj; tau <= j < len(signal) }
    # and look for argmax(tau)Q(tau)

    # sum |Xi - Yj| for i < tau <= j
    cross_term = 0
    # sum |Xi - Xj| for i < j < tau
    x_term = 0
    # sum |Yi - Yj| for tau <= i < j
    y_term = 0

    for row in range(0, len(diffs)):
        y_term += np.sum(diffs[row, row:])

    for tau in range(0, len(diffs)):
        qhat_values[tau] = _calculate_q(cross_term, x_term, y_term, tau, len(diffs) - tau)

        column_delta = np.sum(diffs[:tau, tau])
        row_delta = np.sum(diffs[tau, tau:])

        cross_term = cross_term - column_delta + row_delta
        x_term = x_term + column_delta
        y_term = y_term - row_delta

    return qhat_values


from pprint import pprint
import numpy as np
from utils.lineq import Solver
import warnings
warnings.filterwarnings('ignore')

A = [[2, 1, 2],
     [3, -3, -1],
     [1, -2,  3]]

b = [-1,5,6]

lineq = Solver(A,b)
lineq.find_solution(method='jacobi', x0=[1,0,1])

print('A is diagonally dominant:', lineq.is_diag(lineq.A_orig))
print('transformed A is diagonally dominant:', lineq.is_diag(lineq.A))
print('')
print('Exact solution:                   ', lineq.exact_sol().T) # Not computationally feasible for very large matrices
print('Solution using Gauss elimination: ', lineq.gauss_elim().T) # Not computationally feasible for very large matrices
print('Approximate solution:             ', lineq.sol.T)


import numpy as np
from deepnumpy.activation._activation import Activation
import logging

logger = logging.getLogger(__name__)

class Sigmoid(Activation):

    def __init__(self, output_dim):
        super().__init__()
        self.units = output_dim
        self.type = 'Sigmoid'

    def forward(self, input_x):
        '''
        Sigmoid =  1/(1+exp(-x))
        '''
        logger.debug('This layer input', input_x.shape)
        self.prev_activation_output = 1 / (1 + np.exp(-1 * input_x))
        logger.debug('This layer Output', self.prev_activation_output.shape)
        return self.prev_activation_output

    def backward(self, dJ):
        '''
        Sigmoid =  1/(1+exp(-x))
        derivative of Sigmoid = exp(-x)/(1+exp(-1))^2 = Sig(x) * (1-Sig(x))
        '''
        logger.debug('dJ shape ', dJ.shape)
        sig = self.prev_activation_output
        dSigmoid = dJ * sig * (1 - sig)
        logger.debug('dSigmoid shape ', dSigmoid.mean(axis=1).shape)
        return dSigmoid.mean(axis=1)


import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
    return x * (1 - x)

class NeuronLayer:
    def __init__(self, n_neurons, n_inputs):
        self.synaptic_weights = 2 * np.random.random((n_inputs, n_neurons)) - 1

class ShallowNetwork:
    def __init__(self, data, seed=None):
        if seed != None:
            np.random.seed(seed)
        self.set_training_data(data)
        self.layer = NeuronLayer(1, len(self.training_inputs[0]))

    def get_training_data(self):
        return [[i, o] for i, o in zip(self.training_inputs, self.training_outputs)]

    def set_training_data(self, data):
        self.training_inputs = np.array([e[0] for e in data])
        self.training_outputs = np.array([[e[1] for e in data]]).T

    def train(self, iterations):
        for _ in range(iterations):
            output = self.think(self.training_inputs)
            error = self.training_outputs - output
            adjustment = np.dot(
                self.training_inputs.T, error * sigmoid_derivative(output)
            )
            self.layer.synaptic_weights += adjustment

    def think(self, inputs):
        return sigmoid(np.dot(np.array(inputs), self.layer.synaptic_weights))

class DeepNetwork:
    def __init__(self, data, depth, neurons, seed=None):
        if seed != None:
            np.random.seed(seed)
        self.set_training_data(data)
        self.layer = []

        assert depth > 1

        self.layer.append(NeuronLayer(neurons, len(self.training_inputs[0])))
        for _ in range(depth - 2):
            self.layer.append(NeuronLayer(neurons, neurons))
        self.layer.append(NeuronLayer(1, neurons))

    def get_training_data(self):
        return [[i, o] for i, o in zip(self.training_inputs, self.training_outputs)]

    def set_training_data(self, data):
        self.training_inputs = np.array([e[0] for e in data])
        self.training_outputs = np.array([[e[1] for e in data]]).T

    def train(self, iterations):
        for _ in range(iterations):
            thoughts = self.study(self.training_inputs)

            e = len(self.layer) - 1
            layer_error = {e: self.training_outputs - thoughts[e]}
            layer_delta = {e: layer_error[e] * sigmoid_derivative(thoughts[e])}
            for t in reversed(range(e)):
                layer_error[t] = layer_delta[t + 1].dot(
                    self.layer[t + 1].synaptic_weights.T
                )
                layer_delta[t] = layer_error[t] * sigmoid_derivative(thoughts[t])

            layer_adjustment = {0: self.training_inputs.T.dot(layer_delta[0])}
            for t in range(1, len(thoughts)):
                layer_adjustment[t] = thoughts[t - 1].T.dot(layer_delta[t])

            for t in range(len(self.layer)):
                self.layer[t].synaptic_weights += layer_adjustment[t]

    def study(self, inputs):
        thoughts = [np.array(inputs)]
        for e in self.layer:
            thoughts.append(sigmoid(np.dot(thoughts[-1], e.synaptic_weights)))
        return thoughts[1:]

    def think(self, inputs):
        iterative = np.array(inputs)
        for e in self.layer:
            iterative = self.__sigmoid(np.dot(iterative, e.synaptic_weights))
        return iterative


# Exercise #2: Generalize the code which was used to determine a limiting
# value at infinity.  Apply to Lial Section 11.1 Example 11.

import numpy
from numpy import sin, arange
import matplotlib.pyplot
from matplotlib.pyplot import *

# f = ( 12*x**2 - 15*x + 12 ) / ( x**2 + 1 )

# Lial Section 11.1 Example 11
def f(x):
    f = ( 12*x**2 - 15*x + 12 ) / ( x**2 + 1 )
    return f

# The next section shows convergence to a limit at infinity.
# The coding shows list manipulations resulting in a plot.
# For simplicity, equal intervals between calculated points will be used.

number = 210  # This is the number of points calculated (minus the increment).
increment = 10  # This is the increment between the points.

y = []
x = []

# The for loop traverses between 10 and 200 in increments of 10.
# A range statement is inclusive of the first number and exclusive of the last.

for k in range( increment, number, increment ):
    w = float( k )
    x = x + [ k ]
    y = y + [ f(w) ]

print( 'Final value equals' , y[-1] )  #Floating point with 4 decimals.

figure()
xlim( 0, number + increment )
ylim( min( y ) - 0.1 , max( y ) + 0.1 )

# The black points were computed.  The yellow point indicates the limit.

plot( x,y, color='r' )
scatter( x,y, color='k', s=30 )
scatter( number, y[-1], c='y', s=40 )

title( 'Example of Convergence to a Limit at Infinity' )
xlabel( 'x-axis' )
ylabel( 'y-axis' )
show()


# demo of antithetic sampling
# https://en.wikipedia.org/wiki/Antithetic_variates

import numpy as np
np.random.seed(0)

N = 750 #1500
u1 = np.random.uniform(size=N)
u2 = np.random.uniform(size=N)
u = np.concatenate((u1,u2))
f = 1 / (1+u)
mu_naive = np.mean(f)
se_naive  = np.sqrt(np.var(f)/(2*N))
print('naive {:0.4f}, se {:0.4f}'.format(mu_naive, se_naive))

# antithetic version
uprime = 1-u1
f1 = 1 / (1+u1)
fprime = 1 / (1+uprime)
f = (f1 + fprime) / 2.0 # paired samples!
mu_anti = np.mean(f)
se_anti  = np.sqrt(np.var(f)/(2*N))
print('anti {:0.4f}, se {:0.4f}'.format(mu_anti, se_anti))


"""Defines the most commonly used kernels."""
import math
from scipy import stats
import numpy as np

__author__ = "Miguel Carbajo Berrocal"
__email__ = "miguel.carbajo@estudiante.uam.es"

def normal(u):
    r"""Evaluate a normal kernel.

    .. math::
        K(x) = \frac{1}{\sqrt{2\pi}}e^{-\frac{x^2}{2}}

    """
    return stats.norm.pdf(u)

def cosine(u):
    r"""Cosine kernel.

    .. math::
        K(x) =
        \begin{cases}
        \frac{\pi}{4} cos\left( \frac{\pi x}{2} \right) & \mbox{if } |x| \le
        1 \\
        0 & \mbox{elsewhere}
        \end{cases}

    """
    if isinstance(u, np.ndarray):
        res = np.zeros(u.shape)
        res[abs(u) <= 1] = math.pi / 4 * (math.cos(math.pi * u[abs(u) <= 1]
                                                   / 2))
        return res
    if abs(u) <= 1:
        return math.pi / 4 * (math.cos(math.pi * u / 2))
    return 0

def epanechnikov(u):
    r"""Epanechnikov kernel.

    .. math::
        K(x) =
        \begin{cases}
        0.75(1-x^2) & \mbox{if } |x| \le 1 \\
        0 & \mbox{elsewhere}
        \end{cases}

    """
    if isinstance(u, np.ndarray):
        res = np.zeros(u.shape)
        res[abs(u) <= 1] = 0.75 * (1 - u[abs(u) <= 1] ** 2)
        return res
    if abs(u) <= 1:
        return 0.75 * (1 - u ** 2)
    return 0

def tri_weight(u):
    r"""Tri-weight kernel.

    .. math::
        K(x) =
        \begin{cases}
        \frac{35}{32} \left(1 - u^2 \right) ^3 & \mbox{if } |x| \le 1 \\
        0 & \mbox{elsewhere}
        \end{cases}

    """
    if isinstance(u, np.ndarray):
        res = np.zeros(u.shape)
        res[abs(u) <= 1] = 35 / 32 * (1 - u[abs(u) <= 1] ** 2) ** 3
        return res
    if abs(u) <= 1:
        return 35 / 32 * (1 - u ** 2) ** 3
    return 0

def quartic(u):
    r"""Quartic kernel.

    .. math::
        K(x) =
        \begin{cases}
        \frac{15}{16} \left( 1- u^2 \right) ^2 & \mbox{if } |x| \le 1 \\
        0 & \mbox{elsewhere}
        \end{cases}

    """
    if isinstance(u, np.ndarray):
        res = np.zeros(u.shape)
        res[abs(u) <= 1] = 15 / 16 * (1 - u[abs(u) <= 1] ** 2) ** 2
        return res
    if abs(u) <= 1:
        return 15 / 16 * (1 - u ** 2) ** 2
    return 0

def uniform(u):
    r"""Uniform kernel.

    .. math::
        K(x) =
        \begin{cases}
        0.5 & \mbox{if } |x| \le
        1 \\
        0 & \mbox{elsewhere}
        \end{cases}

    """
    if isinstance(u, np.ndarray):
        res = np.zeros(u.shape)
        res[abs(u) <= 1] = 0.5
        return res
    if abs(u) <= 1:
        return 0.5
    return 0


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Circle, Rectangle
import logging
import math
# import pywt

def read_image(img_name):
    img = plt.imread(img_name).astype(float)
    logging.debug(img.shape)
    return img

def show_img(img, file_name = None):
    plt.imshow(img, cmap = 'gray')
    if not file_name:
        plt.show()
    else:
        plt.savefig('output/' + file_name)

h = np.array([1 + math.sqrt(3), 3 + math.sqrt(3), 3 - math.sqrt(3), \
                1 - math.sqrt(3)]) / (4 * math.sqrt(2))
g = np.array([h[3], -h[2], h[1], -h[0]])

def D4_1D(input_list, compress = False):

    odd_padded = False
    if len(input_list) % 2 == 1:
        odd_padded = True

    # repeat the last pixel at the right boundary
    # note we make a copy of the input list
    # because we do NOT want to change the length of it
    # len(input_padded) == n + 4

    input_padded = [input_list[1], input_list[0]]
    input_padded.extend(input_list)
    input_padded.extend([input_list[-1], input_list[-2]])

    if odd_padded:
        input_padded.append(input_list[-3])

    n = len(input_padded)
    half = n / 2 - 1
    # print(half)

    high_pass = [0] * half
    low_pass = [0] * half

    for i in range(0, half):
        for j in range(4):
            high_pass[i] += h[j] * input_padded[2 * i + j]
            # if we are doing a compression, the low_pass will be left zeros
            if not compress:
                low_pass[i] += g[j] * input_padded[2 * i + j]

    return high_pass, low_pass, odd_padded

def D4_1D_inv(high_pass, low_pass, odd_padded = False):

    h_inv = [h[2], g[2], h[0], g[0]]
    g_inv = [h[3], g[3], h[1], g[1]]

    n = len(high_pass)

    # len(input_padded) == 2 * n

    input_padded = []
    for i in range(n):
        input_padded.append(high_pass[i])
        input_padded.append(low_pass[i])

    # print(input_padded)

    output_list = [0] * (2 * n - 2)

    for i in range(0, n - 1):
        for j in range(4):
            output_list[2 * i] += h_inv[j] * input_padded[2 * i + j]
            output_list[2 * i + 1] += g_inv[j] * input_padded[2 * i + j]

    if odd_padded:
        output_list = output_list[ : -1]

    return output_list

def D4_2D(img, levels = 3):

    new_input = img.copy()
    results = []

    for level in range(levels):

        res_img, row_odd_padded, col_odd_padded = D4_2D_one_level(new_input, level + 1)
        row, col = res_img.shape
        new_input = res_img[0 : row / 2, 0: col / 2]

        results.append([res_img, row_odd_padded, col_odd_padded])

    return results

def D4_2D_one_level(img, level):

    show_img(img, 'input-level-%d' % level)

    # for level in range(levels):

    row_wave_img = []

    for row in img:

        high_pass, low_pass, row_odd_padded = D4_1D(row)
        row_wave_img.append(high_pass + low_pass)

    row_wave_img = np.array(row_wave_img)

    col_row_wave_img = []

    # transpose to iterate over the columns
    for col in row_wave_img.transpose():

        high_pass, low_pass, col_odd_padded = D4_1D(col)
        col_row_wave_img.append(high_pass + low_pass)

    # transpose back to its original arrangement
    col_row_wave_img = np.array(col_row_wave_img).transpose()

    show_img(col_row_wave_img, 'output-level-%d' % level)

    return col_row_wave_img, row_odd_padded, col_odd_padded

def D4_2D_inv_one_level(img, row_odd_padded, col_odd_padded, \
                                 level, decompress = False):

    col_inv_img = []
    for col in img.transpose():
        half = len(col) / 2
        high_pass = col[0 : half]
        low_pass = col[half :]
        col_inv = D4_1D_inv(high_pass, low_pass, col_odd_padded)
        col_inv_img.append(col_inv)

    col_inv_img = np.array(col_inv_img).transpose()

    row_col_inv_img = []

    for row in col_inv_img:
        half = len(row) / 2
        high_pass = row[0 : half]
        low_pass = row[half :]
        row_inv = D4_1D_inv(high_pass, low_pass, row_odd_padded)
        row_col_inv_img.append(row_inv)

    row_col_inv_img = np.array(row_col_inv_img)

    if not decompress:
        file_name = 'invesre-level-%d' % level
    else:
        file_name = 'decompress-level-%d' % level
    show_img(row_col_inv_img, file_name)

    return row_col_inv_img

def D4_2D_inv(wavelet_results, levels = 3):

    curr_level = levels - 1
    next_image = wavelet_results[curr_level][0]
    row_odd_padded = wavelet_results[curr_level][1]
    col_odd_padded = wavelet_results[curr_level][2]

    while curr_level >= 0:

        level_inv = D4_2D_inv_one_level(next_image, row_odd_padded, \
                            col_odd_padded, level = curr_level + 1)

        if curr_level == 0:
            break

        rows, cols = level_inv.shape

        curr_level -= 1
        next_image = wavelet_results[curr_level][0]
        row_odd_padded = wavelet_results[curr_level][1]
        col_odd_padded = wavelet_results[curr_level][2]

        next_image[ : rows, : cols] = level_inv

def compress_image(img, levels = 2):

    results = D4_2D(img, levels = levels)
    res_img, _, _ = results[-1]
    rows, cols = res_img.shape

    img_wavelet = res_img[ : rows / 2, : cols / 2]
    return img_wavelet

def decompress_image(img_wavelet, levels = 2):

    next_image = img_wavelet.copy()
    while levels > 0:

        rows, cols = next_image.shape
        img_wavelet_expanded = np.zeros([2 * rows, 2 * cols])
        img_wavelet_expanded[: rows, : cols] = next_image

        next_image = D4_2D_inv_one_level(img_wavelet_expanded, \
                                False, False, levels, decompress = True)

        levels -= 1

    return next_image

def test_pywt(l = None, compress = False):

    print('\n\n')
    if l == None:
        l = [3, 7, 1, 1, -2, 5, 4, 6]

    print(l)
    cA, cD = pywt.dwt(l, 'db2')
    print(cA)
    print(cD)

    if compress:
        cD = [0] * len(cD)

    l_inv = pywt.idwt(cA, cD, 'db2')
    print(l_inv)

    w = pywt.Wavelet('db2')
    print(w.dec_hi)
    print(w.dec_lo)

    print(w.rec_hi)
    print(w.rec_lo)

def test_1D(l = None, compress = False):
    if l == None:
        l = [3, 7, 1, 1, -2, 5, 4, 6]

    print(l)

    high_pass, low_pass, odd_padded = D4_1D(l, compress)
    print(high_pass)
    print(low_pass)

    l_inv = D4_1D_inv(high_pass, low_pass, odd_padded)
    print(l_inv)

def main():

    img = read_image('input.png')

    l = [3, 7, 1, 1, -2, 5, 4, 6]

    # test_1D(l)

    # test_pywt(l)

    # only handle the first channel
    img = img[ : , : , 0]
    print(img.shape)

    d4_3_level = D4_2D(img, levels = 3)
    D4_2D_inv(d4_3_level, levels = 3)

    img_wavelet = compress_image(img, levels = 2)
    # print(img_wavelet.shape)
    decompressed_img = decompress_image(img_wavelet, levels = 2)

if __name__ == '__main__':
    main()


r"""
Solve Helmholtz equation in 1D with Dirichlet bcs

.. math::

    \alpha u - \nabla^2 u = f, \quad u(\pm 1) = 0

The equation to solve is

.. math::

    \alpha (u, v)_w - (\nabla^2 u, v)_w = (f, v)_w

The weighted inner product over the domain :math:`\Omega` is defined as

.. math::

    (u, v)_w = \int_{\Omega} u v w dx

where :math:`w(x)` is a weight function.

For either Chebyshev, Legendre or Jacobi we choose a basis that satsifies
the boundary conditions.

"""
import sys
import importlib
from sympy import symbols, sin
import numpy as np
from shenfun import inner, div, grad, TestFunction, TrialFunction, \
    Array, Function, FunctionSpace, dx

assert len(sys.argv) == 3
assert sys.argv[-1].lower() in ('legendre', 'chebyshev', 'jacobi')
assert isinstance(int(sys.argv[-2]), int)

# Collect basis and solver from either Chebyshev or Legendre submodules
family = sys.argv[-1].lower()
base = importlib.import_module('.'.join(('shenfun', family)))
Solver = base.la.Helmholtz

# Use sympy to compute a rhs, given an analytical solution
alfa = 2.
x = symbols("x", real=True)
ue = sin(4*np.pi*x)*(1-x**2)
fe = alfa*ue - ue.diff(x, 2)

# Size of discretization
N = int(sys.argv[-2])

SD = FunctionSpace(N, family=family, bc=(0, 0))
u = TrialFunction(SD)
v = TestFunction(SD)

# Get f on quad points
fj = Array(SD, buffer=fe)

# Compute right hand side of Poisson equation
f_hat = Array(SD)
f_hat = inner(v, fj, output_array=f_hat)

# Get left hand side of Poisson equation
A = inner(v, -div(grad(u)))
B = inner(v, alfa*u)

H = Solver(A, B)
u_hat = Function(SD)           # Solution spectral space
u_hat = H(u_hat, f_hat)
uj = SD.backward(u_hat)

# Compare with analytical solution
ua = Array(SD, buffer=ue)

error = dx((uj-ua)**2)
print('Error=%2.6e'%(np.sqrt(error)))
assert np.allclose(uj, ua)


import numpy as np

class LinearRegression():
    def __init__(self):
        self.w = None

    def __featureNormaliza(self, X):
        """

        Normalized feature dataset.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape(n_samples, n_features)
            Training data

        Returns
        -------
        X_norm : norm data

        mu : mean of every row

        sigma : standard deviation of every row
        """

        X_norm = np.array(X)

        mu = np.mean(X_norm, 0) # average
        sigma = np.std(X_norm, 0) # standard deviation

        for i in range(X.shape[1]):
            X_norm[:, i] = (X_norm[:,i] - mu[i])/sigma[i]

        return X_norm

    def __gradientDescent(self, X, y, alpha=0.001, max_iteration_count=10000):
        """

        batch gradient descent

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape(n_samples, n_features)
            Training data

        y : array-like of shape(n_samples,) or (n_samples, n_targets)
            Target values.

        alpha : learning rate

        Returns
        -------
        w : the final value list formed after gradient descent

        J_history : w change log
        """
        y = np.array(y)
        m,n = X.shape
        w = np.ones((n))

        iteration_count = 0

        J_history = []
        w_change_log = []

        while iteration_count < max_iteration_count:

            X_w_ip = np.dot(X, w) # w^T * X

            temp_d = X_w_ip.flatten() - y # h_theta(X) - y
            temp_dot = np.dot(np.transpose(X), temp_d) # (h_theta(X) - y)*X
            temp = w - ((alpha/m) * temp_dot)
            w = temp

            J = np.dot((np.dot(X,w) - y) , (np.transpose(np.dot(X,w) - y)))/(2 * m) # cost J
            J_history.append(J)
            iteration_count += 1

        self.w = w
        return w, J_history

    def fit(self, X, y):
        """

        Fit linear model.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape(n_samples, n_features)
            Training data

        y : array-like of shape(n_samples,) or (n_samples, n_targets)
            Target values.

        Returns
        -------
        self : returns an instance of self.
        """
        X = self.__featureNormaliza(X)
        X = np.insert(X, 0, 1, axis=1)

        # X_inv = np.linalg.inv(X.T.dot(X)) # X^T.dot(X)
        # self.w = X_inv.dot(X.T).dot(y)

        w, J_history = self.__gradientDescent(X, y)
        # print(J_history)

    def predict(self, X):
        """

        Predict using the linear model.

        Parameters
        ----------
        X : array-like or sparse matrix, shape(n_samples, n_features)
            Samples.

        Returns
        -------
        C : array, shape [n_samples]
            Predicted class label per sample.
        """
        # h(theta) = theta.T.dot(X)
        X = self.__featureNormaliza(X)
        X = np.insert(X, 0, 1, axis=1)
        C = X.dot(self.w)
        return C


import math
import numpy as np

b0 = 1.129552
b1 = 1.160436
b2 = 0.2642821
b3 = 0.01131791
b4 = 0.5900113
b5 = 0.04589742
b6 = 1.00
b7 = 0.5002034
b8 = 1.500
b9 = 1.979139

c0 = 4.39890
c1 = 0.43693
c2 = -4.16078
c3 = 0.090

# aquifer constant function
def b_function(poro,h,cw,cf,ro,f):
    b = (1.119*poro*h*(cw+cf)*ro*ro*f)
    return b

# fraction of perimeter function
def f_function(thitha):
    f = (thitha /360)
    return f

# dimensionless radius function
def rd_function(re,ro):
    rd = re/ro
    return rd

# dimensionless time function in years
def td_function(array,k,poro,miu,cw,cf,ro):
    td_result_array = []
    for t in array:
        td = (2.309*k*t)/(poro*miu*(cw+cf)*ro*ro)
        td_result_array.append(td)
    return td_result_array

# Here the Polynomial approach by Klins et al., 1988 was used
# to iterate for qd values at respective dimensionless radius
def qd_function(rd,td_array):
    try:
        qd_array = []
        for td in td_array:
            if td <= 0.01:
                qd = (2/math.sqrt(math.pi))*(math.sqrt(td))
            if (td > 0.01)  and (td < 200):
                qd = ((b0*(td**b7))+(b1*(td))+(b2*(td**b8))+(b3*(td**b9)))/((b4*(td**b7))+(b5*td)+(b6))
            if (td > 200)  and (td < 2000000000000):
                qd = 10**(c0 + (c1*(math.log(td)))+(c2*(math.log(td))**c3))
            qd_array.append(qd)  # add each value of qd to the qd_array
            reversed_qd=list(reversed(qd_array))  #reversed qd_array
        return reversed_qd, qd_array

    except:
        return "sorry! something went wrong"

# Water influx function
def water_influx_function(b,dp_array, reversed_qd_array):
    try:
        z = np.multiply(dp_array, reversed_qd_array)
        z_sum = np.sum(z)
        we = (b*z_sum)/1000000

    except Exception:
        we  = "sorry! something went wrong"
    return we

#Wd summation function
#  NB: this function is faulty, I am yet to figure out the algorithm for yearly  Wd summation
# majorly due to time constraint and also i would be feasible for me to get round it in the limited time
# Hence i brought in the actual values from the text book to get the real plot. Thanks
def wd_function(b,dp_array,qd_array):
    new_array = []
    correct_array = [0, 3.829, 13.46, 26.462, 41.935, 59.207, 77.628, 96.805, 116.284, 135.601, 154.4] #actual wd values from textbook
    for qd in qd_array:
        z = np.multiply(dp_array, qd)
        z_sum = np.sum(z)
        we = (b*z_sum)/1000000
        new_array.append(we)
    # return new_array    #return the wd values obtained from the function itself.
    return correct_array  #return the textbook values


import numpy as np
from cvxopt import solvers, matrix
import matplotlib.pyplot as plt

# Given data
P = 2*matrix([[0.2, -0.2, -0.12, 0.02], [-0.2, 1.4, 0.02, 0.0], [-0.12, 0.02, 1.0, -0.4], [0.02, 0.0, -0.4, 0.2]])
q = matrix([0.0, 0.0, 0.0, 0.0])
G = matrix([[-1.1, -1.0, 0.0, 0.0, 0.0], [-1.35, 0.0, -1.0, 0.0, 0.0], [-1.25, 0.0, 0.0, -1.0, 0.0], [-1.05, 0.0, 0.0, 0.0, -1.0]])
A = matrix([1.0, 1.0, 1.0, 1.0], (1, 4))
b = matrix(1.0)

# Vary the minimum expected return between 1.05 and 1.35
r_min_range = np.arange(1.05, 1.35, 0.01)

# For each minimum expected return find the QP solution and variance
variance_range = []
IBM_allocation = []
google_allocation = []
apple_allocation = []
intel_allocation = []

for r_min in r_min_range:
    h = matrix([-1 * r_min, 0.0, 0.0, 0.0, 0.0])
    sol = solvers.qp(P, q, G, h, A, b)
    IBM_allocation.append(sol['x'][0])
    google_allocation.append(sol['x'][1])
    apple_allocation.append(sol['x'][2])
    intel_allocation.append(sol['x'][3])
    variance_range.append((sol['x'].T * matrix([[0.2, -0.2, -0.12, 0.02], [-0.2, 1.4, 0.02, 0.0], [-0.12, 0.02, 1.0, -0.4], [0.02, 0.0, -0.4, 0.2]]) * sol['x'])[0])

# Plot variance for each minimum expected return
fig1 = plt.figure()
plt.plot(r_min_range, variance_range)
fig1.suptitle('Expected Return to Variance')
plt.xlabel('Expected Return (r_min)')
plt.ylabel('Variance')
plt.grid()

# Plot portfolio composition
fig2 = plt.figure()
plt.plot(r_min_range, IBM_allocation, label="IBM")
plt.plot(r_min_range, google_allocation, label="Google")
plt.plot(r_min_range, apple_allocation, label="Apple")
plt.plot(r_min_range, intel_allocation, label="Intel")
fig2.suptitle('Portfolio Composition over Varying Risk/Returns')
plt.xlabel('Expected Return (r_min)')
plt.ylabel('Allocation (out of total of 1)')
plt.legend()
plt.grid()

plt.show()


from math import asin, cos, sqrt

import numpy as np

def distance(lat1, lon1, lat2, lon2):
    p = 0.017453292519943295
    a = 0.5 - cos((lat2 - lat1) * p) / 2 + cos(lat1 * p) * cos(lat2 * p) * (1 - cos((lon2 - lon1) * p)) / 2
    return 12742 * asin(sqrt(a)) * 1000

def distances_vectorized(lats, lons, lat1, lon1):
    p = 0.017453292519943295
    lats_arr = np.array(lats)
    lons_arr = np.array(lons)
    a = 0.5 - np.cos((lats_arr - lat1) * p) / 2 + np.cos(lat1 * p) * \
        np.cos(lats_arr * p) * (1 - np.cos((lons_arr - lon1) * p)) / 2
    return list(12742 * np.arcsin(np.sqrt(a)) * 1000)

def coords_with_data(bdy_lon_vec, bdy_lat_vec, bdy_data):
    coords = []

    if bdy_data.dimensions[-1] == 'time':
        i_range = range(bdy_data.shape[0])
        j_range = range(bdy_data.shape[1])
    else:
        i_range = range(bdy_data.shape[1])
        j_range = range(bdy_data.shape[2])

    for i in i_range:
        for j in j_range:
            if bdy_data.dimensions[-1] == 'time':
                data_cell = bdy_data[i, j, 0]
            else:
                data_cell = bdy_data[0, i, j]
            if data_cell is not np.ma.masked:
                if len(bdy_lat_vec.shape) > 1:
                    coords.append({'lat': bdy_lat_vec[i][j],
                                   'lon': bdy_lon_vec[i][j],
                                   'i': i,
                                   'j': j})
                else:
                    coords.append({'lat': bdy_lat_vec[i],
                                   'lon': bdy_lon_vec[j],
                                   'i': i,
                                   'j': j})
    return coords

def get_nearest_index(coords, pt_lon, pt_lat):
    # if len(pt_lon.shape)==1:
    #    res = min(coords, key=lambda p: distance(pt_lat, pt_lon, p['lat'], p['lon']))
    # else:
    dists = distances_vectorized([p['lat'] for p in coords],
                                 [p['lon'] for p in coords],
                                 pt_lat, pt_lon)

    try:
        res_ind = dists.index(min(dists))
    except:
        md = [min(_) for _ in dists]
        res_ind = md.index(min(md))
    min_ind_i, min_ind_j = coords[res_ind]['i'], coords[res_ind]['j']

    return min_ind_i, min_ind_j