"""
SARIMAX specification class.

Author: Chad Fulton
License: BSD-3
"""
import numpy as np
import pandas as pd

from statsmodels.tools.data import _is_using_pandas
from statsmodels.tsa.base.tsa_model import TimeSeriesModel
from statsmodels.tsa.statespace.tools import (
    is_invertible, constrain_stationary_univariate as constrain,
    unconstrain_stationary_univariate as unconstrain,
    prepare_exog, prepare_trend_spec, prepare_trend_data)

from statsmodels.tsa.arima.tools import standardize_lag_order, validate_basic

class SARIMAXSpecification:
    """
    SARIMAX specification.

    Parameters
    ----------
    endog : array_like, optional
        The observed time-series process :math:`y`.
    exog : array_like, optional
        Array of exogenous regressors.
    order : tuple, optional
        The (p,d,q) order of the model for the autoregressive, differences, and
        moving average components. d is always an integer, while p and q may
        either be integers or lists of integers. May not be used in combination
        with the arguments `ar_order`, `diff`, or `ma_order`.
    seasonal_order : tuple, optional
        The (P,D,Q,s) order of the seasonal component of the model for the
        AR parameters, differences, MA parameters, and periodicity. Default
        is (0, 0, 0, 0). D and s are always integers, while P and Q
        may either be integers or lists of positive integers. May not be used
        in combination with the arguments `seasonal_ar_order`, `seasonal_diff`,
        or `seasonal_ma_order`.
    ar_order : int or list of int
        The autoregressive order of the model. May be an integer, in which case
        all autoregressive lags up to and including it will be included.
        Alternatively, may be a list of integers specifying which lag orders
        are included. May not be used in combination with `order`.
    diff : int
        The order of integration of the model. May not be used in combination
        with `order`.
    ma_order : int or list of int
        The moving average order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` for details.
        May not be used in combination with `order`.
    seasonal_ar_order : int or list of int
        The seasonal autoregressive order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` for examples.
        Note that if `seasonal_periods = 4` and `seasonal_ar_order = 2`, then
        this implies that the overall model will include lags 4 and 8.
        May not be used in combination with `seasonal_order`.
    seasonal_diff : int
        The order of seasonal integration of the model. May not be used in
        combination with `seasonal_order`.
    seasonal_ma_order : int or list of int
        The moving average order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` and
        `seasonal_ar_order` for additional details. May not be used in
        combination with `seasonal_order`.
    seasonal_periods : int
        Number of periods in a season. May not be used in combination with
        `seasonal_order`.
    enforce_stationarity : bool, optional
        Whether or not to require the autoregressive parameters to correspond
        to a stationarity process. This is only possible in estimation by
        numerical maximum likelihood.
    enforce_invertibility : bool, optional
        Whether or not to require the moving average parameters to correspond
        to an invertible process. This is only possible in estimation by
        numerical maximum likelihood.
    concentrate_scale : bool, optional
        Whether or not to concentrate the scale (variance of the error term)
        out of the likelihood. This reduces the number of parameters by one.
        This is only applicable when considering estimation by numerical
        maximum likelihood.
    dates : array_like of datetime, optional
        If no index is given by `endog` or `exog`, an array-like object of
        datetime objects can be provided.
    freq : str, optional
        If no index is given by `endog` or `exog`, the frequency of the
        time-series may be specified here as a Pandas offset or offset string.
    missing : str
        Available options are 'none', 'drop', and 'raise'. If 'none', no nan
        checking is done. If 'drop', any observations with nans are dropped.
        If 'raise', an error is raised. Default is 'none'.

    Attributes
    ----------
    order : tuple, optional
        The (p,d,q) order of the model for the autoregressive, differences, and
        moving average components. d is always an integer, while p and q may
        either be integers or lists of integers.
    seasonal_order : tuple, optional
        The (P,D,Q,s) order of the seasonal component of the model for the
        AR parameters, differences, MA parameters, and periodicity. Default
        is (0, 0, 0, 0). D and s are always integers, while P and Q
        may either be integers or lists of positive integers.
    ar_order : int or list of int
        The autoregressive order of the model. May be an integer, in which case
        all autoregressive lags up to and including it will be included. For
        example, if `ar_order = 3`, then the model will include lags 1, 2,
        and 3. Alternatively, may be a list of integers specifying exactly
        which lag orders are included. For example, if `ar_order = [1, 3]`,
        then the model will include lags 1 and 3 but will exclude lag 2.
    diff : int
        The order of integration of the model.
    ma_order : int or list of int
        The moving average order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` for examples.
    seasonal_ar_order : int or list of int
        The seasonal autoregressive order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` for examples.
        Note that if `seasonal_periods = 4` and `seasonal_ar_order = 2`, then
        this implies that the overall model will include lags 4 and 8.
    seasonal_diff : int
        The order of seasonal integration of the model.
    seasonal_ma_order : int or list of int
        The moving average order of the model. May be an integer or
        list of integers. See the documentation for `ar_order` and
        `seasonal_ar_order` for additional details.
    seasonal_periods : int
        Number of periods in a season.
    trend : str{'n','c','t','ct'} or iterable, optional
        Parameter controlling the deterministic trend polynomial :math:`A(t)`.
        Can be specified as a string where 'c' indicates a constant (i.e. a
        degree zero component of the trend polynomial), 't' indicates a
        linear trend with time, and 'ct' is both. Can also be specified as an
        iterable defining the polynomial as in `numpy.poly1d`, where
        `[1,1,0,1]` would denote :math:`a + bt + ct^3`. Default is to not
        include a trend component.
    ar_lags : list of int
        List of included autoregressive lags. If `ar_order` is a list, then
        `ar_lags == ar_order`. If `ar_lags = [1, 2]`, then the overall model
        will include the 1st and 2nd autoregressive lags.
    ma_lags : list of int
        List of included moving average lags. If `ma_order` is a list, then
        `ma_lags == ma_order`. If `ma_lags = [1, 2]`, then the overall model
        will include the 1st and 2nd moving average lags.
    seasonal_ar_lags : list of int
        List of included seasonal autoregressive lags. If `seasonal_ar_order`
        is a list, then `seasonal_ar_lags == seasonal_ar_order`. If
        `seasonal_periods = 4` and `seasonal_ar_lags = [1, 2]`, then the
        overall model will include the 4th and 8th autoregressive lags.
    seasonal_ma_lags : list of int
        List of included seasonal moving average lags. If `seasonal_ma_order`
        is a list, then `seasonal_ma_lags == seasonal_ma_order`. See the
        documentation to `seasonal_ar_lags` for examples.
    max_ar_order : int
        Largest included autoregressive lag.
    max_ma_order : int
        Largest included moving average lag.
    max_seasonal_ar_order : int
        Largest included seasonal autoregressive lag.
    max_seasonal_ma_order : int
        Largest included seasonal moving average lag.
    max_reduced_ar_order : int
        Largest lag in the reduced autoregressive polynomial. Equal to
        `max_ar_order + max_seasonal_ar_order * seasonal_periods`.
    max_reduced_ma_order : int
        Largest lag in the reduced moving average polynomial. Equal to
        `max_ma_order + max_seasonal_ma_order * seasonal_periods`.
    enforce_stationarity : bool
        Whether or not to transform the AR parameters to enforce stationarity
        in the autoregressive component of the model. This is only possible
        in estimation by numerical maximum likelihood.
    enforce_invertibility : bool
        Whether or not to transform the MA parameters to enforce invertibility
        in the moving average component of the model. This is only possible
        in estimation by numerical maximum likelihood.
    concentrate_scale : bool
        Whether or not to concentrate the variance (scale term) out of the
        log-likelihood function. This is only applicable when considering
        estimation by numerical maximum likelihood.
    is_ar_consecutive
    is_ma_consecutive
    is_integrated
    is_seasonal
    k_exog_params
    k_ar_params
    k_ma_params
    k_seasonal_ar_params
    k_seasonal_ma_params
    k_params
    exog_names
    ar_names
    ma_names
    seasonal_ar_names
    seasonal_ma_names
    param_names

    Examples
    --------
    >>> SARIMAXSpecification(order=(1, 0, 2))
    SARIMAXSpecification(endog=y, order=(1, 0, 2))

    >>> spec = SARIMAXSpecification(ar_order=1, ma_order=2)
    SARIMAXSpecification(endog=y, order=(1, 0, 2))

    >>> spec = SARIMAXSpecification(ar_order=1, seasonal_order=(1, 0, 0, 4))
    SARIMAXSpecification(endog=y, order=(1, 0, 0), seasonal_order=(1, 0, 0, 4))
    """

    def __init__(self, endog=None, exog=None, order=None,
                 seasonal_order=None, ar_order=None, diff=None, ma_order=None,
                 seasonal_ar_order=None, seasonal_diff=None,
                 seasonal_ma_order=None, seasonal_periods=None, trend=None,
                 enforce_stationarity=None, enforce_invertibility=None,
                 concentrate_scale=None, trend_offset=1, dates=None, freq=None,
                 missing='none', validate_specification=True):

        # Basic parameters
        self.enforce_stationarity = enforce_stationarity
        self.enforce_invertibility = enforce_invertibility
        self.concentrate_scale = concentrate_scale
        self.trend_offset = trend_offset

        # Validate that we were not given conflicting specifications
        has_order = order is not None
        has_specific_order = (ar_order is not None or diff is not None or
                              ma_order is not None)
        has_seasonal_order = seasonal_order is not None
        has_specific_seasonal_order = (seasonal_ar_order is not None or
                                       seasonal_diff is not None or
                                       seasonal_ma_order is not None or
                                       seasonal_periods is not None)
        if has_order and has_specific_order:
            raise ValueError('Cannot specify both `order` and either of'
                             ' `ar_order` or `ma_order`.')
        if has_seasonal_order and has_specific_seasonal_order:
            raise ValueError('Cannot specify both `seasonal_order` and any of'
                             ' `seasonal_ar_order`, `seasonal_ma_order`,'
                             ' or `seasonal_periods`.')

        # Compute `order`
        if has_specific_order:
            ar_order = 0 if ar_order is None else ar_order
            diff = 0 if diff is None else diff
            ma_order = 0 if ma_order is None else ma_order
            order = (ar_order, diff, ma_order)
        elif not has_order:
            order = (0, 0, 0)

        # Compute `seasonal_order`
        if has_specific_seasonal_order:
            seasonal_ar_order = (
                0 if seasonal_ar_order is None else seasonal_ar_order)
            seasonal_diff = 0 if seasonal_diff is None else seasonal_diff
            seasonal_ma_order = (
                0 if seasonal_ma_order is None else seasonal_ma_order)
            seasonal_periods = (
                0 if seasonal_periods is None else seasonal_periods)
            seasonal_order = (seasonal_ar_order, seasonal_diff,
                              seasonal_ma_order, seasonal_periods)
        elif not has_seasonal_order:
            seasonal_order = (0, 0, 0, 0)

        # Validate shapes of `order`, `seasonal_order`
        if len(order) != 3:
            raise ValueError('`order` argument must be an iterable with three'
                             ' elements.')
        if len(seasonal_order) != 4:
            raise ValueError('`seasonal_order` argument must be an iterable'
                             ' with four elements.')

        # Validate differencing parameters
        if validate_specification:
            if order[1] < 0:
                raise ValueError('Cannot specify negative differencing.')
            if order[1] != int(order[1]):
                raise ValueError('Cannot specify fractional differencing.')
            if seasonal_order[1] < 0:
                raise ValueError('Cannot specify negative seasonal'
                                 ' differencing.')
            if seasonal_order[1] != int(seasonal_order[1]):
                raise ValueError('Cannot specify fractional seasonal'
                                 ' differencing.')
            if seasonal_order[3] < 0:
                raise ValueError('Cannot specify negative seasonal'
                                 ' periodicity.')

        # Standardize to integers or lists of integers
        order = (
            standardize_lag_order(order[0], 'AR'),
            int(order[1]),
            standardize_lag_order(order[2], 'MA'))
        seasonal_order = (
            standardize_lag_order(seasonal_order[0], 'seasonal AR'),
            int(seasonal_order[1]),
            standardize_lag_order(seasonal_order[2], 'seasonal MA'),
            int(seasonal_order[3]))

        # Validate seasonals
        if validate_specification:
            if seasonal_order[3] == 1:
                raise ValueError('Seasonal periodicity must be greater'
                                 ' than 1.')
            if ((seasonal_order[0] != 0 or seasonal_order[1] != 0 or
                    seasonal_order[2] != 0) and seasonal_order[3] == 0):
                raise ValueError('Must include nonzero seasonal periodicity if'
                                 ' including seasonal AR, MA, or'
                                 ' differencing.')

        # Basic order
        self.order = order
        self.ar_order, self.diff, self.ma_order = order

        self.seasonal_order = seasonal_order
        (self.seasonal_ar_order, self.seasonal_diff, self.seasonal_ma_order,
         self.seasonal_periods) = seasonal_order

        # Lists of included lags
        if isinstance(self.ar_order, list):
            self.ar_lags = self.ar_order
        else:
            self.ar_lags = np.arange(1, self.ar_order + 1).tolist()
        if isinstance(self.ma_order, list):
            self.ma_lags = self.ma_order
        else:
            self.ma_lags = np.arange(1, self.ma_order + 1).tolist()

        if isinstance(self.seasonal_ar_order, list):
            self.seasonal_ar_lags = self.seasonal_ar_order
        else:
            self.seasonal_ar_lags = (
                np.arange(1, self.seasonal_ar_order + 1).tolist())
        if isinstance(self.seasonal_ma_order, list):
            self.seasonal_ma_lags = self.seasonal_ma_order
        else:
            self.seasonal_ma_lags = (
                np.arange(1, self.seasonal_ma_order + 1).tolist())

        # Maximum lag orders
        self.max_ar_order = self.ar_lags[-1] if self.ar_lags else 0
        self.max_ma_order = self.ma_lags[-1] if self.ma_lags else 0

        self.max_seasonal_ar_order = (
            self.seasonal_ar_lags[-1] if self.seasonal_ar_lags else 0)
        self.max_seasonal_ma_order = (
            self.seasonal_ma_lags[-1] if self.seasonal_ma_lags else 0)

        self.max_reduced_ar_order = (
            self.max_ar_order +
            self.max_seasonal_ar_order * self.seasonal_periods)
        self.max_reduced_ma_order = (
            self.max_ma_order +
            self.max_seasonal_ma_order * self.seasonal_periods)

        # Check that we don't have duplicate AR or MA lags from the seasonal
        # component
        ar_lags = set(self.ar_lags)
        seasonal_ar_lags = set(np.array(self.seasonal_ar_lags)
                               * self.seasonal_periods)
        duplicate_ar_lags = ar_lags.intersection(seasonal_ar_lags)
        if validate_specification and len(duplicate_ar_lags) > 0:
            raise ValueError('Invalid model: autoregressive lag(s) %s are'
                             ' in both the seasonal and non-seasonal'
                             ' autoregressive components.'
                             % duplicate_ar_lags)

        ma_lags = set(self.ma_lags)
        seasonal_ma_lags = set(np.array(self.seasonal_ma_lags)
                               * self.seasonal_periods)
        duplicate_ma_lags = ma_lags.intersection(seasonal_ma_lags)
        if validate_specification and len(duplicate_ma_lags) > 0:
            raise ValueError('Invalid model: moving average lag(s) %s are'
                             ' in both the seasonal and non-seasonal'
                             ' moving average components.'
                             % duplicate_ma_lags)

        # Handle trend
        self.trend = trend
        self.trend_poly, _ = prepare_trend_spec(trend)

        # Check for a constant column in the provided exog
        exog_is_pandas = _is_using_pandas(exog, None)
        if (validate_specification and exog is not None and
                len(self.trend_poly) > 0 and self.trend_poly[0] == 1):
            # Figure out if we have any constant columns
            x = np.asanyarray(exog)
            ptp0 = np.ptp(x, axis=0)
            col_is_const = ptp0 == 0
            nz_const = col_is_const & (x[0] != 0)
            col_const = nz_const

            # If we already have a constant column, raise an error
            if np.any(col_const):
                raise ValueError('A constant trend was included in the model'
                                 ' specification, but the `exog` data already'
                                 ' contains a column of constants.')

        # This contains the included exponents of the trend polynomial,
        # where e.g. the constant term has exponent 0, a linear trend has
        # exponent 1, etc.
        self.trend_terms = np.where(self.trend_poly == 1)[0]
        # Trend order is either the degree of the trend polynomial, if all
        # exponents are included, or a list of included exponents. Here we need
        # to make a distinction between a degree zero polynomial (i.e. a
        # constant) and the zero polynomial (i.e. not even a constant). The
        # former has `trend_order = 0`, while the latter has
        # `trend_order = None`.
        self.k_trend = len(self.trend_terms)
        if len(self.trend_terms) == 0:
            self.trend_order = None
            self.trend_degree = None
        elif np.all(self.trend_terms == np.arange(len(self.trend_terms))):
            self.trend_order = self.trend_terms[-1]
            self.trend_degree = self.trend_terms[-1]
        else:
            self.trend_order = self.trend_terms
            self.trend_degree = self.trend_terms[-1]

        # Handle endog / exog
        # Standardize exog
        self.k_exog, exog = prepare_exog(exog)

        # Standardize endog (including creating a faux endog if necessary)
        faux_endog = endog is None
        if endog is None:
            endog = [] if exog is None else np.zeros(len(exog)) * np.nan

        # Add trend data into exog
        nobs = len(endog) if exog is None else len(exog)
        if self.trend_order is not None:
            # Add in the data
            trend_data = self.construct_trend_data(nobs, trend_offset)
            if exog is None:
                exog = trend_data
            elif exog_is_pandas:
                trend_data = pd.DataFrame(trend_data, index=exog.index,
                                          columns=self.construct_trend_names())
                exog = pd.concat([trend_data, exog], axis=1)
            else:
                exog = np.c_[trend_data, exog]

        # Create an underlying time series model, to handle endog / exog,
        # especially validating shapes, retrieving names, and potentially
        # providing us with a time series index
        self._model = TimeSeriesModel(endog, exog=exog, dates=dates, freq=freq,
                                      missing=missing)
        self.endog = None if faux_endog else self._model.endog
        self.exog = self._model.exog

        # Validate endog shape
        if (validate_specification and not faux_endog and
                self.endog.ndim > 1 and self.endog.shape[1] > 1):
            raise ValueError('SARIMAX models require univariate `endog`. Got'
                             ' shape %s.' % str(self.endog.shape))

        self._has_missing = (
            None if faux_endog else np.any(np.isnan(self.endog)))

    @property
    def is_ar_consecutive(self):
        """
        (bool) Is autoregressive lag polynomial consecutive.

        I.e. does it include all lags up to and including the maximum lag.
        """
        return (self.max_seasonal_ar_order == 0 and
                not isinstance(self.ar_order, list))

    @property
    def is_ma_consecutive(self):
        """
        (bool) Is moving average lag polynomial consecutive.

        I.e. does it include all lags up to and including the maximum lag.
        """
        return (self.max_seasonal_ma_order == 0 and
                not isinstance(self.ma_order, list))

    @property
    def is_integrated(self):
        """
        (bool) Is the model integrated.

        I.e. does it have a nonzero `diff` or `seasonal_diff`.
        """
        return self.diff > 0 or self.seasonal_diff > 0

    @property
    def is_seasonal(self):
        """(bool) Does the model include a seasonal component."""
        return self.seasonal_periods != 0

    @property
    def k_exog_params(self):
        """(int) Number of parameters associated with exogenous variables."""
        return len(self.exog_names)

    @property
    def k_ar_params(self):
        """(int) Number of autoregressive (non-seasonal) parameters."""
        return len(self.ar_lags)

    @property
    def k_ma_params(self):
        """(int) Number of moving average (non-seasonal) parameters."""
        return len(self.ma_lags)

    @property
    def k_seasonal_ar_params(self):
        """(int) Number of seasonal autoregressive parameters."""
        return len(self.seasonal_ar_lags)

    @property
    def k_seasonal_ma_params(self):
        """(int) Number of seasonal moving average parameters."""
        return len(self.seasonal_ma_lags)

    @property
    def k_params(self):
        """(int) Total number of model parameters."""
        k_params = (self.k_exog_params + self.k_ar_params + self.k_ma_params +
                    self.k_seasonal_ar_params + self.k_seasonal_ma_params)
        if not self.concentrate_scale:
            k_params += 1
        return k_params

    @property
    def exog_names(self):
        """(list of str) Names associated with exogenous parameters."""
        exog_names = self._model.exog_names
        return [] if exog_names is None else exog_names

    @property
    def ar_names(self):
        """(list of str) Names of (non-seasonal) autoregressive parameters."""
        return ['ar.L%d' % i for i in self.ar_lags]

    @property
    def ma_names(self):
        """(list of str) Names of (non-seasonal) moving average parameters."""
        return ['ma.L%d' % i for i in self.ma_lags]

    @property
    def seasonal_ar_names(self):
        """(list of str) Names of seasonal autoregressive parameters."""
        s = self.seasonal_periods
        return ['ar.S.L%d' % (i * s) for i in self.seasonal_ar_lags]

    @property
    def seasonal_ma_names(self):
        """(list of str) Names of seasonal moving average parameters."""
        s = self.seasonal_periods
        return ['ma.S.L%d' % (i * s) for i in self.seasonal_ma_lags]

    @property
    def param_names(self):
        """(list of str) Names of all model parameters."""
        names = (self.exog_names + self.ar_names + self.ma_names +
                 self.seasonal_ar_names + self.seasonal_ma_names)
        if not self.concentrate_scale:
            names.append('sigma2')
        return names

    @property
    def valid_estimators(self):
        """
        (list of str) Estimators that could be used with specification.

        Note: does not consider the presense of `exog` in determining valid
        estimators. If there are exogenous variables, then feasible Generalized
        Least Squares should be used through the `gls` estimator, and the
        `valid_estimators` are the estimators that could be passed as the
        `arma_estimator` argument to `gls`.
        """
        estimators = {'yule_walker', 'burg', 'innovations',
                      'hannan_rissanen', 'innovations_mle', 'statespace'}

        # Properties
        has_ar = self.max_ar_order != 0
        has_ma = self.max_ma_order != 0
        has_seasonal = self.seasonal_periods != 0

        # Only state space can handle missing data or concentrated scale
        if self._has_missing:
            estimators.intersection_update(['statespace'])

        # Only numerical MLE estimators can enforce restrictions
        if ((self.enforce_stationarity and self.max_ar_order > 0) or
                (self.enforce_invertibility and self.max_ma_order > 0)):
            estimators.intersection_update(['innovations_mle', 'statespace'])

        # Innovations: no AR, non-consecutive MA, seasonal
        if has_ar or not self.is_ma_consecutive or has_seasonal:
            estimators.discard('innovations')
        # Yule-Walker/Burg: no MA, non-consecutive AR, seasonal
        if has_ma or not self.is_ar_consecutive or has_seasonal:
            estimators.discard('yule_walker')
            estimators.discard('burg')
        # Hannan-Rissanen: no seasonal
        if has_seasonal:
            estimators.discard('hannan_rissanen')
        # Innovations MLE: cannot have enforce_stationary=False or
        # concentratre_scale=True
        if self.enforce_stationarity is False or self.concentrate_scale:
            estimators.discard('innovations_mle')

        return estimators

    def validate_estimator(self, estimator):
        """
        Validate an SARIMA estimator.

        Parameters
        ----------
        estimator : str
            Name of the estimator to validate against the current state of
            the specification. Possible values are: 'yule_walker', 'burg',
            'innovations', 'hannan_rissanen', 'innovoations_mle', 'statespace'.

        Notes
        -----
        This method will raise a `ValueError` if an invalid method is passed,
        and otherwise will return None.

        This method does not consider the presense of `exog` in determining
        valid estimators. If there are exogenous variables, then feasible
        Generalized Least Squares should be used through the `gls` estimator,
        and a "valid" estimator is one that could be passed as the
        `arma_estimator` argument to `gls`.

        This method only uses the attributes `enforce_stationarity` and
        `concentrate_scale` to determine the validity of numerical maximum
        likelihood estimators. These only include 'innovations_mle' (which
        does not support `enforce_stationarity=False` or
        `concentrate_scale=True`) and 'statespace' (which supports all
        combinations of each).

        Examples
        --------
        >>> spec = SARIMAXSpecification(order=(1, 0, 2))

        >>> spec.validate_estimator('yule_walker')
        ValueError: Yule-Walker estimator does not support moving average
                    components.

        >>> spec.validate_estimator('burg')
        ValueError: Burg estimator does not support moving average components.

        >>> spec.validate_estimator('innovations')
        ValueError: Burg estimator does not support autoregressive components.

        >>> spec.validate_estimator('hannan_rissanen')  # returns None
        >>> spec.validate_estimator('innovations_mle')  # returns None
        >>> spec.validate_estimator('statespace')       # returns None

        >>> spec.validate_estimator('not_an_estimator')
        ValueError: "not_an_estimator" is not a valid estimator.
        """
        has_ar = self.max_ar_order != 0
        has_ma = self.max_ma_order != 0
        has_seasonal = self.seasonal_periods != 0
        has_missing = self._has_missing

        titles = {
            'yule_walker': 'Yule-Walker',
            'burg': 'Burg',
            'innovations': 'Innovations',
            'hannan_rissanen': 'Hannan-Rissanen',
            'innovations_mle': 'Innovations MLE',
            'statespace': 'State space'
        }

        # Only state space form can support missing data
        if estimator != 'statespace':
            if has_missing:
                raise ValueError('%s estimator does not support missing'
                                 ' values in `endog`.' % titles[estimator])

        # Only state space and innovations MLE can enforce parameter
        # restrictions
        if estimator not in ['innovations_mle', 'statespace']:
            if self.max_ar_order > 0 and self.enforce_stationarity:
                raise ValueError('%s estimator cannot enforce a stationary'
                                 ' autoregressive lag polynomial.'
                                 % titles[estimator])
            if self.max_ma_order > 0 and self.enforce_invertibility:
                raise ValueError('%s estimator cannot enforce an invertible'
                                 ' moving average lag polynomial.'
                                 % titles[estimator])

        # Now go through specific disqualifications for each estimator
        if estimator in ['yule_walker', 'burg']:
            if has_seasonal:
                raise ValueError('%s estimator does not support seasonal'
                                 ' components.' % titles[estimator])
            if not self.is_ar_consecutive:
                raise ValueError('%s estimator does not support'
                                 ' non-consecutive autoregressive lags.'
                                 % titles[estimator])
            if has_ma:
                raise ValueError('%s estimator does not support moving average'
                                 ' components.' % titles[estimator])
        elif estimator == 'innovations':
            if has_seasonal:
                raise ValueError('Innovations estimator does not support'
                                 ' seasonal components.')
            if not self.is_ma_consecutive:
                raise ValueError('Innovations estimator does not support'
                                 ' non-consecutive moving average lags.')
            if has_ar:
                raise ValueError('Innovations estimator does not support'
                                 ' autoregressive components.')
        elif estimator == 'hannan_rissanen':
            if has_seasonal:
                raise ValueError('Hannan-Rissanen estimator does not support'
                                 ' seasonal components.')
        elif estimator == 'innovations_mle':
            if self.enforce_stationarity is False:
                raise ValueError('Innovations MLE estimator does not support'
                                 ' non-stationary autoregressive components,'
                                 ' but `enforce_stationarity` is set to False')
            if self.concentrate_scale:
                raise ValueError('Innovations MLE estimator does not support'
                                 ' concentrating the scale out of the'
                                 ' log-likelihood function')
        elif estimator == 'statespace':
            # State space form supports all variations of SARIMAX.
            pass
        else:
            raise ValueError('"%s" is not a valid estimator.' % estimator)

    def split_params(self, params, allow_infnan=False):
        """
        Split parameter array by type into dictionary.

        Parameters
        ----------
        params : array_like
            Array of model parameters.
        allow_infnan : bool, optional
            Whether or not to allow `params` to contain -np.Inf, np.Inf, and
            np.nan. Default is False.

        Returns
        -------
        split_params : dict
            Dictionary with keys 'exog_params', 'ar_params', 'ma_params',
            'seasonal_ar_params', 'seasonal_ma_params', and (unless
            `concentrate_scale=True`) 'sigma2'. Values are the parameters
            associated with the key, based on the `params` argument.

        Examples
        --------
        >>> spec = SARIMAXSpecification(ar_order=1)
        >>> spec.split_params([0.5, 4])
        {'exog_params': array([], dtype=float64),
         'ar_params': array([0.5]),
         'ma_params': array([], dtype=float64),
         'seasonal_ar_params': array([], dtype=float64),
         'seasonal_ma_params': array([], dtype=float64),
         'sigma2': 4.0}
        """
        params = validate_basic(params, self.k_params,
                                allow_infnan=allow_infnan,
                                title='joint parameters')

        ix = [self.k_exog_params, self.k_ar_params, self.k_ma_params,
              self.k_seasonal_ar_params, self.k_seasonal_ma_params]
        names = ['exog_params', 'ar_params', 'ma_params',
                 'seasonal_ar_params', 'seasonal_ma_params']
        if not self.concentrate_scale:
            ix.append(1)
            names.append('sigma2')
        ix = np.cumsum(ix)

        out = dict(zip(names, np.split(params, ix)))
        if 'sigma2' in out:
            out['sigma2'] = out['sigma2'].item()

        return out

    def join_params(self, exog_params=None, ar_params=None, ma_params=None,
                    seasonal_ar_params=None, seasonal_ma_params=None,
                    sigma2=None):
        """
        Join parameters into a single vector.

        Parameters
        ----------
        exog_params : array_like, optional
            Parameters associated with exogenous regressors. Required if
            `exog` is part of specification.
        ar_params : array_like, optional
            Parameters associated with (non-seasonal) autoregressive component.
            Required if this component is part of the specification.
        ma_params : array_like, optional
            Parameters associated with (non-seasonal) moving average component.
            Required if this component is part of the specification.
        seasonal_ar_params : array_like, optional
            Parameters associated with seasonal autoregressive component.
            Required if this component is part of the specification.
        seasonal_ma_params : array_like, optional
            Parameters associated with seasonal moving average component.
            Required if this component is part of the specification.
        sigma2 : array_like, optional
            Innovation variance parameter. Required unless
            `concentrated_scale=True`.

        Returns
        -------
        params : ndarray
            Array of parameters.

        Examples
        --------
        >>> spec = SARIMAXSpecification(ar_order=1)
        >>> spec.join_params(ar_params=0.5, sigma2=4)
        array([0.5, 4. ])
        """
        definitions = [
            ('exogenous variables', self.k_exog_params, exog_params),
            ('AR terms', self.k_ar_params, ar_params),
            ('MA terms', self.k_ma_params, ma_params),
            ('seasonal AR terms', self.k_seasonal_ar_params,
                seasonal_ar_params),
            ('seasonal MA terms', self.k_seasonal_ma_params,
                seasonal_ma_params),
            ('variance', int(not self.concentrate_scale), sigma2)]

        params_list = []
        for title, k, params in definitions:
            if k > 0:
                # Validate
                if params is None:
                    raise ValueError('Specification includes %s, but no'
                                     ' parameters were provided.' % title)
                params = np.atleast_1d(np.squeeze(params))
                if not params.shape == (k,):
                    raise ValueError('Specification included %d %s, but'
                                     ' parameters with shape %s were provided.'
                                     % (k, title, params.shape))

                # Otherwise add to the list
                params_list.append(params)

        return np.concatenate(params_list)

    def validate_params(self, params):
        """
        Validate parameter vector by raising ValueError on invalid values.

        Parameters
        ----------
        params : array_like
            Array of model parameters.

        Notes
        -----
        Primarily checks that the parameters have the right shape and are not
        NaN or infinite. Also checks if parameters are consistent with a
        stationary process if `enforce_stationarity=True` and that they are
        consistent with an invertible process if `enforce_invertibility=True`.
        Finally, checks that the variance term is positive, unless
        `concentrate_scale=True`.

        Examples
        --------
        >>> spec = SARIMAXSpecification(ar_order=1)
        >>> spec.validate_params([-0.5, 4.])  # returns None
        >>> spec.validate_params([-0.5, -2])
        ValueError: Non-positive variance term.
        >>> spec.validate_params([-1.5, 4.])
        ValueError: Non-stationary autoregressive polynomial.
        """
        # Note: split_params includes basic validation
        params = self.split_params(params)

        # Specific checks
        if self.enforce_stationarity:
            if self.k_ar_params:
                ar_poly = np.r_[1, -params['ar_params']]
                if not is_invertible(ar_poly):
                    raise ValueError('Non-stationary autoregressive'
                                     ' polynomial.')
            if self.k_seasonal_ar_params:
                seasonal_ar_poly = np.r_[1, -params['seasonal_ar_params']]
                if not is_invertible(seasonal_ar_poly):
                    raise ValueError('Non-stationary seasonal autoregressive'
                                     ' polynomial.')

        if self.enforce_invertibility:
            if self.k_ma_params:
                ma_poly = np.r_[1, params['ma_params']]
                if not is_invertible(ma_poly):
                    raise ValueError('Non-invertible moving average'
                                     ' polynomial.')
            if self.k_seasonal_ma_params:
                seasonal_ma_poly = np.r_[1, params['seasonal_ma_params']]
                if not is_invertible(seasonal_ma_poly):
                    raise ValueError('Non-invertible seasonal moving average'
                                     ' polynomial.')

        if not self.concentrate_scale:
            if params['sigma2'] <= 0:
                raise ValueError('Non-positive variance term.')

    def constrain_params(self, unconstrained):
        """
        Constrain parameter values to be valid through transformations.

        Parameters
        ----------
        unconstrained : array_like
            Array of model unconstrained parameters.

        Returns
        -------
        constrained : ndarray
            Array of model parameters transformed to produce a valid model.

        Notes
        -----
        This is usually only used when performing numerical minimization
        of the log-likelihood function. This function is necessary because
        the minimizers consider values over the entire real space, while
        SARIMAX models require parameters in subspaces (for example positive
        variances).

        Examples
        --------
        >>> spec = SARIMAXSpecification(ar_order=1)
        >>> spec.constrain_params([10, -2])
        array([-0.99504,  4.     ])
        """
        unconstrained = self.split_params(unconstrained)
        params = {}

        if self.k_exog_params:
            params['exog_params'] = unconstrained['exog_params']
        if self.k_ar_params:
            if self.enforce_stationarity:
                params['ar_params'] = constrain(unconstrained['ar_params'])
            else:
                params['ar_params'] = unconstrained['ar_params']
        if self.k_ma_params:
            if self.enforce_invertibility:
                params['ma_params'] = -constrain(unconstrained['ma_params'])
            else:
                params['ma_params'] = unconstrained['ma_params']
        if self.k_seasonal_ar_params:
            if self.enforce_stationarity:
                params['seasonal_ar_params'] = (
                    constrain(unconstrained['seasonal_ar_params']))
            else:
                params['seasonal_ar_params'] = (
                    unconstrained['seasonal_ar_params'])
        if self.k_seasonal_ma_params:
            if self.enforce_invertibility:
                params['seasonal_ma_params'] = (
                    -constrain(unconstrained['seasonal_ma_params']))
            else:
                params['seasonal_ma_params'] = (
                    unconstrained['seasonal_ma_params'])
        if not self.concentrate_scale:
            params['sigma2'] = unconstrained['sigma2']**2

        return self.join_params(**params)

    def unconstrain_params(self, constrained):
        """
        Reverse transformations used to constrain parameter values to be valid.

        Parameters
        ----------
        constrained : array_like
            Array of model parameters.

        Returns
        -------
        unconstrained : ndarray
            Array of parameters with constraining transformions reversed.

        Notes
        -----
        This is usually only used when performing numerical minimization
        of the log-likelihood function. This function is the (approximate)
        inverse of `constrain_params`.

        Examples
        --------
        >>> spec = SARIMAXSpecification(ar_order=1)
        >>> spec.unconstrain_params([-0.5, 4.])
        array([0.57735, 2.     ])
        """
        constrained = self.split_params(constrained)
        params = {}

        if self.k_exog_params:
            params['exog_params'] = constrained['exog_params']
        if self.k_ar_params:
            if self.enforce_stationarity:
                params['ar_params'] = unconstrain(constrained['ar_params'])
            else:
                params['ar_params'] = constrained['ar_params']
        if self.k_ma_params:
            if self.enforce_invertibility:
                params['ma_params'] = unconstrain(-constrained['ma_params'])
            else:
                params['ma_params'] = constrained['ma_params']
        if self.k_seasonal_ar_params:
            if self.enforce_stationarity:
                params['seasonal_ar_params'] = (
                    unconstrain(constrained['seasonal_ar_params']))
            else:
                params['seasonal_ar_params'] = (
                    constrained['seasonal_ar_params'])
        if self.k_seasonal_ma_params:
            if self.enforce_invertibility:
                params['seasonal_ma_params'] = (
                    unconstrain(-constrained['seasonal_ma_params']))
            else:
                params['seasonal_ma_params'] = (
                    constrained['seasonal_ma_params'])
        if not self.concentrate_scale:
            params['sigma2'] = constrained['sigma2']**0.5

        return self.join_params(**params)

    def construct_trend_data(self, nobs, offset=1):
        if self.trend_order is None:
            trend_data = None
        else:
            trend_data = prepare_trend_data(
                self.trend_poly, int(np.sum(self.trend_poly)), nobs, offset)

        return trend_data

    def construct_trend_names(self):
        names = []
        for i in self.trend_terms:
            if i == 0:
                names.append('const')
            elif i == 1:
                names.append('drift')
            else:
                names.append('trend.%d' % i)
        return names

    def __repr__(self):
        """Represent SARIMAXSpecification object as a string."""
        components = []
        if self.endog is not None:
            components.append('endog=%s' % self._model.endog_names)
        if self.k_exog_params:
            components.append('exog=%s' % self.exog_names)
        components.append('order=%s' % str(self.order))
        if self.seasonal_periods > 0:
            components.append('seasonal_order=%s' % str(self.seasonal_order))
        if self.enforce_stationarity is not None:
            components.append('enforce_stationarity=%s'
                              % self.enforce_stationarity)
        if self.enforce_invertibility is not None:
            components.append('enforce_invertibility=%s'
                              % self.enforce_invertibility)
        if self.concentrate_scale is not None:
            components.append('concentrate_scale=%s' % self.concentrate_scale)
        return 'SARIMAXSpecification(%s)' % ', '.join(components)


import torchtext
from macros import *
from torchtext.data import Dataset
import torch
from torch import nn
from torch.nn.utils import clip_grad_norm_
from torch.optim.lr_scheduler import ReduceLROnPlateau
import numpy as np
import utils
import random
import os
from torch.nn import functional as F
import json
import nets
import tqdm
import logging
from .m10ae_utils import gen_batch

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)-8s %(message)s')
import copy
from sklearn.metrics import accuracy_score, \
    precision_score, recall_score, f1_score

def gen_batch_train(opt):
    return gen_batch(opt.min_len_train, opt.max_len_train, opt.min_lopr_train, opt.max_lopr_train, opt.modd, opt.bsz)

def gen_batch_valid(opt):
    return gen_batch(opt.min_len_valid, opt.max_len_valid, opt.min_lopr_valid, opt.max_lopr_valid, opt.modd, opt.bsz)

def log_init(opt):
    basename = "{}-{}-{}-{}".format(opt.task,
                                    opt.sub_task,
                                    opt.enc_type,
                                    utils.time_int())
    log_fname = basename + ".json"
    log_path = os.path.join(LOGS, log_fname)
    with open(log_path, 'w') as f:
        f.write(str(utils.param_str(opt)) + '\n')
    logging.info(f'Logging file path: {log_path}')
    return log_path, basename

def log_print(log_path, log_str, optim):
    log_str = json.dumps(log_str)
    logging.info(f'{log_str}')
    with open(log_path, 'a+') as f:
        f.write(log_str + '\n')

    # for param_group in optim.param_groups:
    #     print('learning rate:', param_group['lr'])

def valid_along(**kwargs):
    model = kwargs['model']
    diter_valid = kwargs['diter_valid_along']

    nc = 0
    nt = 0

    with torch.no_grad():
        model.eval()
        for inp, tar in diter_valid:
            tlen, bsz, _ = tar.shape
            # out: (seq_len, bsz, odim)
            out = model(inp, tlen)
            pred = out.max(dim=-1)[1]
            nc += torch.abs(out_binarized - tar).sum(dim=-1).eq(0).sum(dim=1)
            nt += bsz

    return (nc.float() / nt).cpu().numpy().tolist()

def valid(model, valid_iter, args):
    pred_lst = []
    tar_lst = []

    with torch.no_grad():
        model.eval()
        for batch in valid_iter:
            inp, tar = batch
            out = run_iter(model, batch, None, None, args, is_training=False)
            pred = out.max(dim=-1)[1]
            pred_lst.extend(pred.cpu().numpy())
            tar_lst.extend(tar.cpu().numpy())

    return accuracy_score(tar_lst, pred_lst)

def run_iter(model, batch, criterion, optimizer, args, is_training):
    model.train(is_training)
    (inp, tar) = batch
    if is_training:
        out = model(inp)
        loss = criterion(out, tar)
        optimizer.zero_grad()
        loss.backward()
        gnorm = clip_grad_norm_(parameters=model.parameters(), max_norm=args.gclip)
        optimizer.step()
        return loss, gnorm
    else:
        out = model(inp)
        pred = out.data.gt(0.5).float()
        return pred

def train(args):
    encoder = nets.select_enc(args)
    device = utils.build_device(args)
    model = Model(encoder, args).to(device)
    utils.init_model(model)
    if args.fload is not None and args.continue_training:
        utils.model_loading(args, model)
    train_iter = utils.DataIter(args, args.nbatch_train, gen_batch_train)
    valid_iter = utils.DataIter(args, args.nbatch_valid, gen_batch_valid)
    optim = utils.select_optim(args, model)
    scheduler = ReduceLROnPlateau(optim,
                                  mode='min',
                                  factor=0.1,
                                  patience=args.patience,
                                  min_lr=args.lr / 10)

    criterion = nn.CrossEntropyLoss()
    log_path, basename = log_init(args)

    best_perform = -1
    losses = []
    gnorms = []
    for epoch in range(args.nepoch):
        train_iter_tqdm = tqdm.tqdm(train_iter)
        for i, batch in enumerate(train_iter_tqdm):
            loss, gnorm = run_iter(model, batch, criterion, optim, args, is_training=True)
            loss = loss.item()
            losses.append(loss)
            gnorms.append(gnorm)
            train_iter_tqdm.set_description(f'Epoch {epoch} loss {loss:.4f}')

            if (i + 1) % (len(train_iter_tqdm) // args.valid_times) == 0:
                loss_ave = np.mean(losses)
                gnorm_ave = np.mean(gnorms)
                losses = []
                gnorms = []
                acc = valid(model, valid_iter, args)
                log_str = {'Epoch': epoch,
                           'acc': round(acc, 4),
                           'loss': round(float(loss_ave), 4),
                           'gnorm': round(float(gnorm_ave), 4)}

                log_print(log_path, log_str, optim)
                scheduler.step(loss_ave)

                if acc >= best_perform:
                    best_perform = acc
                    utils.mdl_save(model, basename, epoch, loss_ave, acc)

class Model(nn.Module):

    def __init__(self, encoder, args):
        super(Model, self).__init__()
        odim = args.odim
        self.encoder = encoder
        self.hdim = self.encoder.odim
        self.odim = odim
        self.clf = nn.Sequential(nn.Linear(self.hdim, self.odim))

    def forward(self, inp):
        out = self.encoder(embs=inp)
        encoded = out[-1]
        logtis = self.clf(encoded)

        return logtis


# Correlations sellingprice/time and housing info - Felicia
# This needs to be added in the wrapper.py file!

import pandas.io.sql as sqlio
import pandas as pd
import psycopg2
import mysql.connector

def correlation_housing_data_sellingprice_sellingtime():
    #start connection with database
    with open ('db_login.txt', 'r') as myfile:
        data = myfile.read()
    conn = psycopg2.connect(data)
    cur = conn.cursor()

    #Create dataframe to select columns of housing_data
    housinginfo_sellingpricetime_table = "SELECT sellingPrice, fullDescription, houseType, categoryObject, yearOfBuilding, garden, parcelSurface, numberRooms, numberBathrooms, energylabelClass, surface, sellingtime FROM funda;"
    housinginfo_sellingpricetime = sqlio.read_sql_query(housinginfo_sellingpricetime_table, conn)

    #Look for correlations between columns housing_data and sellingprice and sellingtime
    print(housinginfo_sellingpricetime.corr(method ='pearson'))

    ''''
    Conclusions with regard to sellingprice: 1)garden+sellingprice=-0,258484 2)parcelSurface+sellingprice=0.076516 3)numberrooms+sellingprice=0.100043
    4)numberbathrooms+sellingprice=0.069725 5)surface+sellingprice=0.580748 6)sellingtime+sellingprice=0.145279

    Conclusion with reagrd to sellingtime: 1)garden+sellingtime=0.145279 2)garden+sellingtime=-0.085790 3)parcelsurface+sellingtime=0.002927
    4)numberrooms+sellingtime= 0.136939 5)numberbathrooms+sellingtime=-0.073602 6)surface+sellingtime=0.153849
    '''

    return print('Analysis succesfully done')
correlation_housing_data_sellingprice_sellingtime()


import numpy as np
import scipy.constants as const
import math
from uncertainties import ufloat
import uncertainties.unumpy as unp
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt

#Expeiment

U = np.array([0.15, 0.48, 0.7, 0.85, 0.95, 1.1, 1.2, 1.25, 1.3, 1.31, 1.4])
U1 = unp.uarray(U, 0.02*U)

I1 = np.array([88,69,56,47,40,36,32,29,27,24,23])
I = 0.001*I1

Ra = U/I
N_ex = Ra*I**2

plt.plot(Ra, N_ex, 'rx')
print(Ra)
print(N_ex)
#Theorie

U0 = 1.6
RI = 19
x = np.linspace(1, 61, 1000)

def f (U0,x,RI):
    return U0**2 * x /(RI+x)**2

plt.plot(x, f(U0,x,RI), 'b-')

plt.xlabel('Belastungswiderstand $R_{a}$ [Ω]')
plt.ylabel('Leistung $P$ [W]')
plt.legend(loc='best')
plt.show()


from django import forms
from django.forms import widgets

from items.models import Item

sizes = (
    ('S', 'S'),
    ('M', 'M'),
    ('L', 'L'),
    ('XL', 'XL')
)
PAYMENT_CHOICES = (
    ('IB', 'Internet Banking'),
    ('C', 'COD')
)
class OrderItemForm(forms.Form):
    item_size = forms.ChoiceField(choices=sizes, widget=forms.RadioSelect(attrs={'require':True}))
    # quantity = forms.DecimalField(min_value=1, max_value=100, max_digits=4,widget=forms.NumberInput(attrs={'class': 'text-black',
    #                                                                 }),)


import torch
import torchvision
import os
import numpy as np
import matplotlib.pyplot as plt
from tensorboardX import SummaryWriter
import torchvision.transforms as transforms
from main import ResNet18
from main import ResBlock

model = ResNet18(ResBlock)
model.load_state_dict(torch.load("./resnet18_10000_rate_0_001.pth"))

def imshow(img):
    img = img / 2 + 0.5
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1,2,0)),cmap='Greys')
    plt.show()

if __name__=='__main__':
    tensor = model.conv1[0].weight.cpu().data.clone()
    transform_test = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.4914, 0.4822, 0.4465),(0.2023, 0.1994, 0.2010))])
    testset = torchvision.datasets.CIFAR10(root='./data',train=False,download=False,transform=transform_test)
    testloader = torch.utils.data.DataLoader(testset,batch_size=1,shuffle=False,num_workers=2)
    for data in testloader:
        model.eval()
        images, labels = data
        feature_conv5 = model.get_conv5(images).cpu()
        feature_conv1 = model.get_conv1(images).cpu()
        feature_conv1_bn = model.get_conv1_bn(images).cpu()
        feature_conv1 = feature_conv1.permute(1,0,2,3)
        feature_conv1_bn = feature_conv1_bn.permute(1,0,2,3)
        feature_conv5 = feature_conv5.permute(1,0,2,3)

        Map1 = torchvision.utils.make_grid(feature_conv1,nrow=8,padding=2,normalize=False)
        Map2 = torchvision.utils.make_grid(feature_conv1_bn,nrow=8,padding=2,normalize=False)
        img1 = Map1.data.clone()
        img2 = Map2.data.clone()

        nrow = 8
        padding = 1
        rows = np.min((feature_conv5.shape[0] // nrow + 1, 64))
        grid = torchvision.utils.make_grid(feature_conv5.data, nrow=nrow, normalize=True, padding=padding)
        plt.figure( figsize=(nrow,rows) )
        plt.imshow(grid.numpy().transpose((1, 2, 0)), cmap = plt.get_cmap('gray_r'))
        plt.show()
        # Map3 = torchvision.utils.make_grid(feature_conv5,nrow=8,padding=2,normalize=False)
        # img3 = Map3.data.clone()
        imshow(img1)
        imshow(img2)
        # imshow(img3)
        imshow(torchvision.utils.make_grid(images))
        break # just test one image


import re
import glob
import hashlib
import inspect
import logging
import collections
import os.path as osp
import schedula as sh
from docutils.parsers.rst import directives
from sphinx.ext.autodoc import DataDocumenter, bool_option
from doctest import DocTestParser, DocTestRunner, NORMALIZE_WHITESPACE, ELLIPSIS
from sphinx.ext.autodoc.directive import AutodocDirective

logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------
# Doctest handling
# ------------------------------------------------------------------------------

_re_doctest = re.compile(r'^\s*>>>', re.M)

def contains_doctest(text):
    try:
        # check if it's valid Python as-is
        compile(text, '<string>', 'exec')
        return False
    except SyntaxError:
        pass
    return bool(_re_doctest.search(text))

# ------------------------------------------------------------------------------
# Auto dispatcher content
# ------------------------------------------------------------------------------

def get_grandfather_content(content, level=2, offset=None):
    if offset is None:
        offset = len(content)
    if content.parent and level:
        return get_grandfather_content(
            content.parent, level - 1, offset=offset + content.parent_offset
        )
    return content[:offset]

def _import_docstring_code_content(documenter):
    content = getattr(documenter, 'content', None)
    if content:
        import textwrap

        def get_code(source, c=''):
            s = "\n%s" % c
            return textwrap.dedent(s.join(map(str, source)))

        is_doctest = contains_doctest(get_code(content))
        if is_doctest:
            code = get_code(get_grandfather_content(content))
        else:
            code = get_code(content, '>>> ')
        return code, content

def _import_docstring(documenter):
    code_content = _import_docstring_code_content(documenter)
    if code_content:
        # noinspection PyBroadException
        try:
            code, content = code_content
            parser = DocTestParser()
            runner = DocTestRunner(
                verbose=False, optionflags=NORMALIZE_WHITESPACE | ELLIPSIS
            )

            glob = {}
            if documenter.modname:
                exec('from %s import *\n' % documenter.modname, glob)

            tests = parser.get_doctest(code, glob, '', '', 0)
            runner.run(tests, clear_globs=False)

            documenter.object = tests.globs[documenter.name]
            documenter.code = content
            documenter.is_doctest = True
            return True
        except Exception:
            pass

def _description(lines, dsp, documenter):
    docstring = dsp.__doc__

    if documenter.objpath and documenter.analyzer:
        attr_docs = documenter.analyzer.find_attr_docs()
        key = ('.'.join(documenter.objpath[:-1]), documenter.objpath[-1])
        if key in attr_docs:
            docstring = attr_docs[key]

    if isinstance(docstring, str):
        docstring = docstring.split('\n') + ['']

    lines.extend(docstring)

def _code(lines, documenter):
    if documenter.code:
        if documenter.is_doctest:
            lines += [row.rstrip() for row in documenter.code]
        else:
            lines.extend(['.. code-block:: python', ''])
            lines.extend(['    %s' % r.rstrip() for r in documenter.code])

        lines.append('')

def _plot(lines, dsp, dot_view_opt, documenter, dsp_opt):
    code_content = _import_docstring_code_content(documenter)
    hashkey = str(sorted(dot_view_opt.items())) + '\n'
    if code_content:
        hashkey += code_content[0]
    else:
        modname, objpath = documenter.modname, documenter.objpath
        if modname:
            hashkey += 'import %s\n' % modname
        if objpath:
            hashkey += 'from %s import %s\n' % (modname, '.'.join(objpath))

    fname = 'dispatcher-%s' % hashlib.sha1(hashkey.encode('utf-8')).hexdigest()
    env = documenter.env
    if osp.isabs(env.config.dispatchers_out_dir):
        dspdir = env.config.dispatchers_out_dir
    else:
        dspdir = osp.join(env.srcdir, env.config.dispatchers_out_dir)
    dspdir = osp.join(dspdir, fname)
    index = dot_view_opt.get('index', False)
    viz_js = dot_view_opt.get('viz', False)
    executor = dot_view_opt.get('executor', 'async')
    if osp.isdir(dspdir):
        fpath = glob.glob(osp.join(dspdir, '*.gv'))[0]
    elif hasattr(dsp, 'plot'):
        smap = dsp.plot(**dot_view_opt)
        folder = next(iter(smap))
        fpath = osp.join(dspdir, '%s.gv' % folder.sitemap.foldername)
        dot = folder.dot(smap.rules(index=index, viz_js=viz_js))
        smap.render(
            directory=dspdir, index=index, viz_js=viz_js, executor=executor
        )
        dot.save(fpath, directory='')
    else:
        return

    dsource = osp.dirname(osp.join(env.srcdir, env.docname))
    p = osp.relpath(fpath, dsource).replace('\\', '/')
    dsp_opt = dsp_opt.copy()
    dsp_opt['graphviz_dot'] = dot_view_opt.get('engine', 'dot')
    if index:
        dsp_opt['index'] = ''

    lines.append('.. dsp:: %s' % p)
    lines.extend('   :{}: {}'.format(k, v) for k, v in dsp_opt.items())
    lines.append('')

def _table_heather(lines, title, dsp_name):
    q = 's' if dsp_name and dsp_name[-1] != 's' else ''
    lines.extend(['.. csv-table:: **%s\'%s %s**' % (dsp_name, q, title), ''])

def _data(lines, dsp):
    if isinstance(dsp, sh.SubDispatch):
        dsp = dsp.dsp

    data = sorted(dsp.data_nodes.items())
    if data:
        _table_heather(lines, 'data', dsp.name)
        from schedula.utils.des import get_summary
        from schedula.utils.alg import _search_node_description
        for k, v in data:
            des, link = _search_node_description(dsp, k)

            link = ':obj:`%s <%s>`' % (_node_name(str(k)), link)
            str_format = u'   "%s", "%s"'

            lines.append(str_format % (
                link.replace('"', "'"),
                get_summary(des.split('\n')).replace('"', '""')
            ))

        lines.append('')

dsp_classes = sh.Dispatcher, sh.SubDispatch, sh.Blueprint

def _functions(lines, dsp, node_type='function'):
    if isinstance(dsp, sh.SubDispatch):
        dsp = dsp.dsp

    def check_fun(node_attr):
        if node_attr['type'] not in ('function', 'dispatcher'):
            return False

        if 'function' in node_attr:
            func = sh.parent_func(node_attr['function'])
            c = isinstance(func, dsp_classes)
            return c if node_type == 'dispatcher' else not c
        return node_attr['type'] == node_type

    fun = [v for v in sorted(dsp.nodes.items()) if check_fun(v[1])]

    if fun:
        from schedula.utils.alg import _search_node_description
        _table_heather(lines, '%ss' % node_type, dsp.name)

        for k, v in fun:
            des, full_name = _search_node_description(dsp, k)
            lines.append(u'   ":func:`%s <%s>`", "%s"' % (
                k.replace('"', '""'), full_name.replace('"', '""'),
                des.replace('"', '""')
            ))
        lines.append('')

def _node_name(name):
    return name.replace('<', '\\<').replace('>', '\\>')

PLOT = object()

def _dsp2dot_option(arg):
    """Used to convert the :dmap: option to auto directives."""

    # noinspection PyUnusedLocal
    def map_args(*args, **kwargs):
        from schedula.utils.base import Base
        a = collections.OrderedDict(inspect.signature(
            Base.plot
        ).bind(None, *args, **kwargs).arguments)
        a.popitem(last=False)
        return a

    kw = eval('map_args(%s)' % arg)

    return kw if kw else PLOT

# ------------------------------------------------------------------------------
# Registration hook
# ------------------------------------------------------------------------------

class DispatcherDocumenter(DataDocumenter):
    """
    Specialized Documenter subclass for dispatchers.
    """
    content_indent = ''
    objtype = 'dispatcher'
    directivetype = 'data'
    option_spec = dict(DataDocumenter.option_spec)
    option_spec.update({
        'description': bool_option,
        'opt': _dsp2dot_option,
        'code': bool_option,
        'data': bool_option,
        'func': bool_option,
        'dsp': bool_option,
        'height': directives.length_or_unitless,
        'width': directives.length_or_percentage_or_unitless,
    })
    default_opt = {
        'depth': 0,
        'view': False
    }
    code = None
    is_doctest = False
    blue_cache = {}

    def get_real_modname(self):
        return self.modname

    @classmethod
    def can_document_member(cls, member, *args, **kwargs):
        return isinstance(member, dsp_classes)

    def add_directive_header(self, sig):
        if not self.code:
            if not self.options.annotation:
                self.options.annotation = ' = %s' % self.object.name
            super(DispatcherDocumenter, self).add_directive_header(sig)

    def parse_name(self):
        return super(DispatcherDocumenter, self).parse_name() or True

    def generate(self, more_content=None, **kw):
        # noinspection PyAttributeOutsideInit
        self.content = kw['more_content'] = more_content
        return super(DispatcherDocumenter, self).generate(**kw)

    def import_object(self, *args, **kwargs):
        if _import_docstring(self):
            return True
        self.is_doctest = False
        self.code = None
        res = super(DispatcherDocumenter, self).import_object(*args, **kwargs)
        if res and isinstance(self.object, sh.Blueprint):
            self.object = self.object.register(memo=self.blue_cache)
        return res

    def format_signature(self):
        return ''

    def add_content(self, more_content, no_docstring=False):
        # noinspection PyUnresolvedReferences
        sourcename = self.get_sourcename()
        dsp = self.object
        opt = self.options

        dot_view_opt = self.default_opt.copy()
        if opt.opt and opt.opt is not PLOT:
            dot_view_opt.update(opt.opt)

        lines = []

        if opt.code:
            _code(lines, self)

        if not opt or opt.des:
            _description(lines, dsp, self)

        _plot(lines, dsp, dot_view_opt, self, {
            k: getattr(opt, k) for k in ('height', 'width')
            if getattr(opt, k) is not None
        })

        if not opt or opt.data:
            _data(lines, dsp)

        if not opt or opt.func:
            _functions(lines, dsp)

        if not opt or opt.dsp:
            _functions(lines, dsp, 'dispatcher')

        for line in lines:
            self.add_line(line, sourcename)

class DispatcherDirective(AutodocDirective):
    _default_flags = set(DispatcherDocumenter.option_spec)

    def __init__(self, name, arguments, options, content, lineno,
                 content_offset, *args, **kwargs):
        content._offset = content_offset
        super(DispatcherDirective, self).__init__(
            name, arguments, options, content, lineno, content_offset, *args,
            **kwargs
        )

        if name == 'dispatcher':
            self.name = 'autodispatcher'

def add_autodocumenter(app, cls):
    logger.debug('[app] adding autodocumenter: %r', cls)
    app.registry.add_documenter(cls.objtype, cls)
    app.add_directive('auto' + cls.objtype, DispatcherDirective)

def setup(app):
    """Setup `dispatcher` Sphinx extension module. """
    app.setup_extension('sphinx.ext.autodoc')
    add_autodocumenter(app, DispatcherDocumenter)
    app.add_directive('dispatcher', DispatcherDirective)
    app.add_config_value('dispatchers_out_dir', '_build/_dispatchers', 'html')


from .db_wrapper import db
from ..utils.consts import credentials
from ..utils.utils import (
    write_file,
    extract_value_from_a_query,
    extract_list_of_values_from_a_query,
    hash_password)
from ..utils.log import logger

#------------------------------------------------------------
#  QUERY - USERS  (customers)
#------------------------------------------------------------
def user_exist(user_id):
    print('user_exist()')
    command = "SELECT * FROM customers WHERE id = %s"
    user_exist = bool(db.execute_a_query(command, (user_id,)))
    return user_exist

#------------------------------------------------------------
def user_in_credentials_file(username):
    print('user_in_credentials_file()')
    admins = credentials["admins_username"]
    return username in admins

#------------------------------------------------------------
def is_admin(user_id):
    print('is_admin()')
    command = "SELECT is_admin FROM customers WHERE id = %s"
    user_is_admin = db.execute_a_query(command, (user_id,))
    user_exist = bool(user_is_admin)
    if user_exist:
        return extract_value_from_a_query(user_is_admin)
    else: return False

#------------------------------------------------------------
def get_all_available_by_user_id(user_id):
    print('get_all_available_by_user_id()')
    command = """ SELECT * FROM customers
        WHERE id = %s """
    user_with_id = db.execute_a_query(
        command, (user_id,))
    return user_with_id

#------------------------------------------------------------
def username_from_user_id(user_id):
    print('username_from_user_id()')
    command = "SELECT username FROM customers WHERE id = %s"
    user_name = db.execute_a_query(command, (user_id,))
    return extract_value_from_a_query(user_name)

#------------------------------------------------------------
#  QUERY - PHOTO
#------------------------------------------------------------
def extract_blob(photo_id):
    print('extract_blob()')
    command = "SELECT image_blob FROM photo WHERE id = %s"
    blob = bytes(
        extract_value_from_a_query(
        db.execute_a_query(command, (photo_id,)))
        )
    return blob

#------------------------------------------------------------
def save_photo_in_file(photo_id, file_path):
    print('save_photo_in_file()')
    blob = extract_blob(photo_id)
    write_file(blob, file_path)

#------------------------------------------------------------
#  QUERY - EFILE
#------------------------------------------------------------
def extract_efile_blob(efile_id):
    print('extract_efile_blob()')
    command = "SELECT efile_blob FROM efile WHERE id = %s"
    blob = bytes(
        extract_value_from_a_query(
        db.execute_a_query(command, (efile_id,)))
        )
    return blob

#------------------------------------------------------------
def save_efile_in_file(efile_id, file_path):
    print('save_efile_in_file()')
    blob = extract_blob(efile_id)
    write_file(blob, file_path)

#------------------------------------------------------------
#  QUERY - CATEGORY    (CREATORS)
#------------------------------------------------------------
def get_name_of_all_categories():
    print('get_name_of_all_categories()')
    command = "SELECT name FROM category"
    all_names_query = db.execute_a_query(command)
    names = extract_list_of_values_from_a_query(all_names_query)
    return names

#------------------------------------------------------------
def get_category_id_from_name(name):
    print('get_category_id_from_name()')
    command = "SELECT id FROM category WHERE name = %s"
    category_id = db.execute_a_query(command, (name,))
    return extract_value_from_a_query(category_id)

#------------------------------------------------------------
def get_category_id_from_creator_id(creator_id):
    print('get_category_id_from_creator_id()')
    command = "SELECT id FROM category WHERE creator_id = %s"
    category_id = db.execute_a_query(command, (creator_id,))
    return extract_value_from_a_query(category_id)

#------------------------------------------------------------
def get_all_descriptions_from_all_creators():
    print('get_all_descriptions_from_all_creators()')
    command = """ SELECT description FROM category
        WHERE id > %s """
    descriptions_from_creators = db.execute_a_query(
        command, (0,))
    return descriptions_from_creators

#------------------------------------------------------------
def get_description_from_creator_id(creator_id):
    print('get_description_from_creator_id()')
    command = """ SELECT description FROM category
        WHERE creator_id = %s """
    creator_description = db.execute_a_query(
        command, (creator_id,))
    return extract_value_from_a_query(creator_description)

#------------------------------------------------------------
def get_all_available_by_category_name(name):
    print('get_all_available_by_category_name()')
    category_id = get_category_id_from_name(name)
    return get_all_available_by_category_id(category_id)

#------------------------------------------------------------
#  QUERY - PRODUCTS
#------------------------------------------------------------
def get_quantity_purchased(product_id):
    print('get_quantity_purchased()')
    command = "SELECT total_sold FROM products WHERE id = %s"
    quantity_purchased = db.execute_a_query(command, (product_id,))
    return extract_value_from_a_query(quantity_purchased)

#------------------------------------------------------------
def get_quantity_in_stock(product_id):
    print('get_quantity_in_stock()')
    command = "SELECT in_stock FROM products WHERE id = %s"
    quantity_in_stock = db.execute_a_query(command, (product_id,))
    return extract_value_from_a_query(quantity_in_stock)

#------------------------------------------------------------
def get_efile_id_from_product_id(product_id):
    print('get_efile_id_from_product_id()')
    command = """ SELECT efile_id FROM products WHERE id = %s """
    efile_id_from_db = db.execute_a_query(
        command, (product_id,))
    return extract_value_from_a_query(efile_id_from_db)

#------------------------------------------------------------
def get_all_available_by_category_id(category_id):
    print('get_all_available_by_category_id()')
    command = """ SELECT * FROM products
        WHERE category_id = %s AND in_stock > 0"""
    products_with_category_id = db.execute_a_query(
        command, (category_id,))
    return products_with_category_id

#------------------------------------------------------------
#  QUERY - RECEIPTS
#------------------------------------------------------------

def customers_names_from_receipts_by_product_id(product_id):

    command = ("""
        SELECT buyer_id FROM receipts WHERE product_id = %s""")
    command_args = (product_id,)
    ids_query = db.execute_a_query(command,  command_args)
    return extract_list_of_values_from_a_query(ids_query)

##
# REMOVE RATINGS
#------------------------------------------------------------
def get_ratings_of_a_product(product_id):
    print('get_ratings_of_a_product()')
    command = """ SELECT rating FROM orders
        WHERE product_id = %s AND rating IS NOT NULL"""
    ratings_query = db.execute_a_query(command, (product_id,))
    return extract_list_of_values_from_a_query(ratings_query)

#------------------------------------------------------------
def count_occurrence_of_specified_rating(product_id, rating):
    print('count_occurrence_of_specified_rating()')
    all_ratings = get_ratings_of_a_product(product_id)
    return all_ratings.count(rating)

#------------------------------------------------------------
def search_products(string_to_search):
    print('search_products()')
    command = """SELECT * FROM products
        WHERE MATCH(name, description) AGAINST(%s)"""
    products_that_match = (
        db.execute_a_query(command, (string_to_search,)))
    return products_that_match


import json
import requests
import sys
import os

WEBHOOK_URI = os.environ['uri']
GIPHY_API = os.environ['api_key']

def post_message(event, content):
    response = None
    msg = format_chime_msg(get_random_dog_image())
    print (msg)
    try:
        response = requests.post(
            url=WEBHOOK_URI,
            json={"Content": msg})
        return json.loads(response.text)
    except:
        return response.text

def get_random_dog_image():
    giphy_url = "http://api.giphy.com/v1/gifs/random"
    response = None
    PARAMS = {
            "api_key":GIPHY_API,
            "tag" : "dog"
        }
    response = requests.get(
        url = giphy_url,
        params = PARAMS
    )

    dog_gif_url = json.loads(response.text)["data"]["image_url"]
    return dog_gif_url

def format_chime_msg(url):
    return "/md **Good Morning Ariel!**   ![Alt Text](" + url + ")   Here is your daily dog dose: " + url

# def main():
#     post_message("test", "test")

# main()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import yfinance as yf

tickers = [ 'PETZ3.SA', 'VIVA3.SA','VIIA3.SA','AZUL4.SA']
carteira = pd.DataFrame()
for i in tickers:
  carteira[i] = yf.download(i, period='1y')['Adj Close']

retorno = carteira.pct_change()

num_acoes = len(tickers)
num_carteiras = 10
Rf = 0

lista_retornos = []
lista_volatilidade = []
lista_pesos = []
lista_sharpe_ratio = []

for carteira in range(num_carteiras):
  #gerando pesos aleartorios para as ações
  peso = np.random.random(num_acoes)
  peso = np.round((peso / np.sum(peso)), 3)
  lista_pesos.append(peso)
  #calculo de retorno esperado
  retorno_esperado = np.sum(retorno.mean() * peso) * 264
  lista_retornos.append(retorno_esperado)
  #calculo risco
  Matrix_cov = retorno.cov() * 264
  variancia = np.dot(peso.T,
                     np.dot(Matrix_cov, peso))
  desvio_padrao = np.sqrt(variancia)
  lista_volatilidade.append(desvio_padrao)

  #calculo Sharpe ratio
  sharpe_ratio = (retorno_esperado - Rf) / desvio_padrao
  lista_sharpe_ratio.append(sharpe_ratio)

lista_retornos = np.array(lista_retornos)
lista_volatilidade = np.array(lista_volatilidade)
lista_sharpe_ratio = np.array(lista_sharpe_ratio)

metricas = [lista_retornos, lista_volatilidade, lista_sharpe_ratio]
portifolios_df = pd.DataFrame(metricas).T
for contar,acao in enumerate(tickers):
  portifolios_df[acao+' Peso'] = [Peso[contar] for Peso in lista_pesos]
portifolios_df.columns = ['Retorno', 'Risco', 'Sharpe'] + [acao+' Peso' for acao in tickers]

min_Risco = portifolios_df.iloc[portifolios_df['Risco'].astype(float).idxmin()]
max_Retorno = portifolios_df.iloc[portifolios_df['Retorno'].astype(float).idxmax()]
max_Sharpe = portifolios_df.iloc[portifolios_df['Sharpe'].astype(float).idxmax()]

portifolios_df.to_csv("10portifolios_4smallcaps.csv")

print(portifolios_df)
print('')
print('Minimo risco')
print(min_Risco)
print('')
print('Maximo Retorno')
print(max_Retorno)
print('')
print('Max Sharpe')
print(max_Sharpe)
print('')

#grafico

plt.figure(figsize = (10,5))
plt.scatter(lista_volatilidade, lista_retornos,
            c = lista_retornos / lista_volatilidade)
plt.title('Otimização de Portifólio', fontsize = 26)
plt.xlabel('Volatilidade', fontsize = 20)
plt.ylabel('Retorno', fontsize = 20)
plt.xticks(fontsize = 15)
plt.yticks(fontsize = 15)
plt.colorbar(label = 'Sharpe Ratio')
plt.show()


from django.urls import path
from . import views

app_name = "ruta_apps"

urlpatterns = [

    path(
        'add-cargue-recepcion/',
        views.RecepcioCreateView.as_view(),
        name = 'cargue-recepcion'
    ),

    path(
        'listar-planillas/<full_name>/',
        views.ListEmpleadosPdf.as_view(),
        name = 'impresion',
    ),

    path(
        'lista/planillas/asignar/',
        views.AsignarCreateView.as_view(),
        name='asignar',
    ),

     path(
        'lista/planillas/generar/',
        views.AsignarListview.as_view(),
        name='planillas',
    ),

    path(
        'generar/destino/',
        views.DestinoCreate.as_view(),
        name='destino'
    ),

    path(
        'generar/descargue/',
        views.DescargueCreateView.as_view(),
        name='destino-descargue'
    ),

    # path(
    #     'generar/historial/',
    #     views.HistorialListview.as_view(),
    #     name='historial'
    # ),
    path('<int:rr>/results/', views.detail, name='results'),

    path(
        'inf/destino/ciudad/',
        views.InformeRutaCiudadListView.as_view(),
        name='informe-destino',
    ),

    path(
        'update/recepcion/view/<int:pk>/',
        views.CorreccionRecepcion.as_view(),
        name='update-recepcion',
    ),

    path(
        'lista/recepcion/',
        views.RecepcionListView.as_view(),
        name='lista-recepcion',
    ),
    # path('about/', TemplateView.as_view(template_name="ruta/    prueba.html")),

    ]


#!/usr/bin/env python3

import os,sys
import pandas as pd
import numpy as np
import json

def cut_nodes(data,num_nodes):

    # identify inner n nodes based on num_nodes input
    total_nodes = len(data['nodeID'].unique())
    cut_nodes = int((total_nodes - num_nodes) / 2)

    # remove cut_nodes from dataframe
    data = data[data['nodeID'].between((cut_nodes)+1,(num_nodes+cut_nodes))]

    # replace empty spaces with nans
    data = data.replace(r'^\s+$', np.nan, regex=True)

    return data

def main():

    # load config.json
    with open('config.json','r') as config_f:
        config = json.load(config_f)

    # set input variables
    tractmeasures = pd.read_csv(config['tractmeasures'])
    num_nodes = config['num_nodes']

    # cut nodes
    tractmeasures_cut = cut_nodes(tractmeasures,num_nodes)

    # make output directory
    if not os.path.exists('output'):
        os.mkdir('output')

    # save cut nodes csv
    tractmeasures_cut.to_csv('./output/tractmeasures.csv',index=False)

if __name__ == '__main__':
    main()


import os
import ast
import torch
from model_dispatcher import MODEL_DISPATCHER
from dataset import TestDataset
from torch import nn
import pandas as pd
from tqdm import tqdm

DEVICE = 'cuda'

IMAGE_HEIGHT = int(os.environ.get("IMAGE_HEIGHT"))
IMAGE_WIDTH = int(os.environ.get("IMAGE_WIDTH"))

TEST_BATCH_SIZE = int(os.environ.get("TEST_BATCH_SIZE"))

MODEL_MEAN = ast.literal_eval(os.environ.get("MODEL_MEAN"))
MODEL_STD = ast.literal_eval(os.environ.get("MODEL_STD"))

BASE_MODEL = os.environ.get("BASE_MODEL")

def predict_to_numpy(predict):
    return torch.nn.functional.softmax(predict, dim=1).data.cpu().numpy().argmax(axis=1)

def get_prediction( data_loader, models):
    predictions = []
    total_model = float(len(models))

    with torch.no_grad():
        for bi, d in tqdm(enumerate(data_loader), total=int(len(data_loader))):
            image = d["image"]
            img_id = d["image_id"]
            image = image.to(DEVICE, dtype = torch.float)

            preds = torch.zeros(image.shape[0], 4, dtype = torch.float).to(DEVICE)

            for i in range(len(models)):
                p = models[i](image)
                preds += p/float(len(models))

            preds = predict_to_numpy(preds)

            for ii, imid in enumerate(img_id):
                predictions.append((f"{imid}", preds[ii]))

    return predictions

def get_models():
    models =[]
    i =0
    for j in [0,1,2,3,4,5,6]:
        models.append(MODEL_DISPATCHER[BASE_MODEL](pretrained=False))
        models[i].load_state_dict(torch.load(f'../save_model/{BASE_MODEL}_folds({j},).bin'))
        models[i].to(DEVICE)
        models[i].eval()
        i+=1
    return models

def save_submission(predictions):
    inverse_map={0:'Food',1:'Attire',2:'Decorationandsignage',3:'misc'}

    sub = pd.DataFrame(predictions, columns = ["Image", "label"])
    sub['Class']=sub['label'].map(inverse_map)
    sub = sub[['Image', 'Class']]
    sub.to_csv("../save_submission/submission.csv", index = False)
    print(sub.head(5))

def main():

    models = get_models()

    test_dataset  = TestDataset(
        img_height = IMAGE_HEIGHT,
        img_width = IMAGE_WIDTH,
        mean = MODEL_MEAN,
        std = MODEL_STD
    )

    test_loader = torch.utils.data.DataLoader(
        dataset = test_dataset,
        batch_size = TEST_BATCH_SIZE,
        shuffle = False,
        num_workers = 4
    )

    predictions = get_prediction(test_loader, models)

    save_submission(predictions)

if __name__ == "__main__":
    main()


from typing import Tuple

import tensorflow as tf
from tensorflow.keras.utils import image_dataset_from_directory

DATASET_PATH = './dataset/'
IMAGE_SIZE = 48

label_to_text = {
    0: 'angry',
    1: 'disgust',
    2: 'fear',
    3: 'happy',
    4: 'neutral',
    5: 'sad',
    6: 'surprise'
}

def load() -> Tuple[tf.data.Dataset, tf.data.Dataset]:
    train_set = image_dataset_from_directory(
        directory=DATASET_PATH + 'train',
        image_size=(IMAGE_SIZE, IMAGE_SIZE),
        labels='inferred',
        label_mode='categorical',
        batch_size=64,
        seed=9000
    )

    test_set = image_dataset_from_directory(
        directory=DATASET_PATH + 'test',
        image_size=(IMAGE_SIZE, IMAGE_SIZE),
        labels='inferred',
        label_mode='categorical',
        batch_size=64,
        seed=9000
    )

    return train_set, test_set


from gensim.models import word2vec
import numpy as np
import argparse
import networkx as nx
from node2vec import Node2Vec

from util.helper import ensure_dir
from util.setting import log

class Node2vec:
    def __init__(self, args:argparse.Namespace, inter_data) -> None:
        self.nx_G = self.read_graph(inter_data)
        self.walk_length = args.walk_length
        self.num_walks = args.num_walks

        self.type_dim = args.type_dim
        self.window = args.word2vec_window
        self.min_count = args.word2vec_count
        self.worker = args.word2vec_worker
        self.model = None
        self.embedding = None

    def read_graph(self, inter_data):
        '''
        Reads the input network in networkx.
        '''
        G = nx.Graph()
        G.add_edges_from(inter_data)

        for edge in G.edges():
            G[edge[0]][edge[1]]['weight'] = 1

        return G

    def init_embedding(self, e2t_list:list, typetoken_seq:list):
        """"""
        log.info('Initing type/token embeddings with node2vec')
        node2vec = Node2Vec(self.nx_G,
            dimensions=self.type_dim, walk_length=self.walk_length,
            num_walks=self.num_walks, workers=self.worker, seed=2022)

        model = node2vec.fit(window=self.window, min_count=self.min_count, batch_words=4)

        init_embedding = np.zeros(shape=(len(e2t_list), 2*self.type_dim), dtype=np.float32)

        # Embedding: entity = type || token
        for idx, typetoken in enumerate(e2t_list):
            if typetoken[1] != -1:
                init_embedding[idx] = np.append(model.wv[typetoken[0]], model.wv[typetoken[1]])
            else:
                init_embedding[idx] = np.append(model.wv[typetoken[0]], np.zeros(self.type_dim))

        self.embedding = init_embedding
        self.model = model

    def store_embedding(self, pretrain_path:str):
        """"""
        embedding_save_dir = '%s/node2vec/%s_%s_%s/' % \
            (pretrain_path, self.type_dim, self.window, self.min_count)
        ensure_dir(embedding_save_dir)

        embedding_path = embedding_save_dir + 'node2vec.embedding'

        with open (embedding_path, 'wb') as f:
            np.save(f, self.embedding)

        log.info("save node2vec embeddings in %s" % embedding_path)

    def load_embedding(self, pretrain_path:str):
        """"""
        embedding_path = '%s/node2vec/%s_%s_%s/node2vec.embedding' % \
            (pretrain_path, self.type_dim, self.window, self.min_count)

        with open(embedding_path, 'rb') as f:
            self.embedding = np.load(embedding_path)

        log.info("load node2vec embeddings in %s" % embedding_path)


"""This file produces results for when the Global Model uses all data in the train set for
fitting the Global Model"""
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.linear_model import (
    LinearRegression,
    HuberRegressor,
)
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from main import run_LESST, performance_LESST
from data_prep import to_array
from preprocessing import read_m4test_series, read_m4_series
import numpy as np
import pickle

np.seed = 1

# models to consider
models = {
    "xgb": XGBRegressor(tree_method="gpu_hist"),
    "lgbm": LGBMRegressor(),
    "rf": RandomForestRegressor(
        bootstrap=True, max_depth=80, max_features=6, n_estimators=100
    ),
    "ols": LinearRegression(),
    "huber": HuberRegressor(),
    "gradient": GradientBoostingRegressor(),
}

datasets = [
    "Yearly",
    "Quarterly",
    "Monthly",
    "Weekly",
    "Daily",
    "Hourly",
]
frequencys = [1, 4, 12, 52, 7, 24]
n_clusters = [100, 3, 3, 30, 30, 100]
deseasons = [False, True]

localmodels = [
    models["huber"],
    models["huber"],
    models["huber"],
    models["huber"],
    models["huber"],
    models["huber"],
]
globalmodels = [
    models["huber"],
    models["huber"],
    models["huber"],
    models["rf"],
    models["huber"],
    models["huber"],
]
lesst_owas = {}
lesst_smapes = {}
lesst_rmses = {}
lesst_mases = {}
for dataset, frequency, n_cluster, deseason, localmodel, globalmodel in zip(
    datasets, frequencys, n_clusters, deseasons, localmodels, globalmodels
):
    test = read_m4test_series(dataset)
    train = read_m4_series(dataset)
    res_train = to_array(train)
    res_test = to_array(test)
    horizon = test.shape[1]
    predictions, less = run_LESST(
        dataset,
        train,
        n_cluster,
        horizon,
        frequency,
        localmodel,
        globalmodel,
        deseason,
        rolling=True,
    )
    del less
    lesst_owa, lesst_smape, lesst_mase, lesst_rmse = performance_LESST(
        predictions,
        dataset,
        res_train,
        res_test,
        horizon,
        frequency,
    )
    lesst_owas.update({f"{dataset}": lesst_owa})
    lesst_smapes.update({f"{dataset}": lesst_smape})
    lesst_mases.update({f"{dataset}": lesst_mase})
    lesst_rmses.update({f"{dataset}": lesst_rmse})
    with open(
        f"E:/documents/work/thesis/rolling_{dataset}.pkl",
        "wb",
    ) as handle:
        pickle.dump(
            {
                "owa": lesst_owas,
                "smape": lesst_smapes,
                "mase": lesst_mases,
                "rmse": lesst_rmses,
            },
            handle,
            protocol=pickle.HIGHEST_PROTOCOL,
        )
    print("=====================================================")
    print(f"{dataset} is DONE")
    print("=====================================================")


# -*- coding: utf-8 -*-
import os
from sklearn.metrics import confusion_matrix, precision_score
import numpy as np

class result_summary():
    def __init__(self, pred, true, score = None, score_th = None, labels = None):
        self.pred = pred
        self.true = true
        self.labels = labels
        if labels is None:
            self.Get_labels()

        self.score = score
        self.score_th = score_th
        if not (self.score is None or self.score_th is None):
            self.Pred_score_filter()
        self.confusion_mat = self.Get_Confusion_matrix()

    def Get_labels(self):
        true_label = np.ndarray.tolist(self.true)
        self.labels = list(set(true_label))
        return self.labels

    def Pred_score_filter(self):
        filter_index = self.score < self.score_th
        self.pred[filter_index] = 0
        self.score[filter_index] = 0.0

    def Gwt_diag(self):
        return np.diag(self.Get_Confusion_matrix())

    # def Get_FP(self):
    def Get_Confusion_matrix(self):
        return confusion_matrix(self.true, self.pred, self.labels)

    def Get_TP(self):
        tp_list = np.diag(self.confusion_mat)
        tp_dict = {}
        for index_i, label_i in enumerate(self.labels):
            tp_dict[label_i] = tp_list[index_i]
        return tp_dict

    def Get_FP(self):
        fp_list = np.sum(self.confusion_mat, 0) - np.diag(self.confusion_mat)
        fp_dict = {}
        for index_i, label_i in enumerate(self.labels):
            fp_dict[label_i] = fp_list[index_i]
        return fp_dict

    def Get_FN(self):
        fn_list = np.sum(self.confusion_mat, 1) - np.diag(self.confusion_mat)
        fn_dict = {}
        for index_i, label_i in enumerate(self.labels):
            fn_dict[label_i] = fn_list[index_i]
        return fn_dict

    def Get_Accuracy(self):
        All_num = np.sum(self.confusion_mat)
        Ture_num = np.sum(np.diag(self.confusion_mat))
        return 1.0*Ture_num/np.maximum(All_num, np.finfo(np.float64).eps)

    def Get_Recall(self):
        tp_list = np.diag(self.confusion_mat)
        recall_list = 1.0*tp_list/np.maximum(np.sum(self.confusion_mat, 1), np.finfo(np.float64).eps)
        recall_dict = {}
        for index_i, label_i in enumerate(self.labels):
            recall_dict[label_i] = recall_list[index_i]
        return recall_dict

    def Get_Precision(self):
        # precision_list = precision_score(self.true, self.pred, labels=self.labels, average=None)

        tp_list = np.diag(self.confusion_mat)
        precision_list = 1.0*tp_list/np.maximum(np.sum(self.confusion_mat, 0), np.finfo(np.float64).eps)
        precision_dict = {}
        for index_i, label_i in enumerate(self.labels):
            precision_dict[label_i] = precision_list[index_i]
        return precision_dict

    def Get_AP(self):
        if self.score is None:
            return None
        pred_dict = {}
        sorted_ind = np.argsort(-self.score)
        sort_pred = self.pred[sorted_ind]
        sort_true = self.true[sorted_ind]
        for label_i in self.labels:
            pred_dict[label_i] = []
        for index_i, pred_i in enumerate(sort_pred):
            if pred_i == sort_true[index_i]:
                pred_dict[pred_i].append(1.0)
            else:
                pred_dict[pred_i].append(0.0)

        pred_total_num = np.sum(self.confusion_mat, 0)
        true_total_num = np.sum(self.confusion_mat, 1)

        AP_dict = {}
        for i, label_i in enumerate(self.labels):
            pred_num_i = pred_total_num[i]
            true_num_i = true_total_num[i]

            tp_i = np.array(pred_dict[label_i])
            tp_i = np.cumsum(tp_i)

            recall_i = tp_i/np.maximum(true_num_i, np.finfo(np.float64).eps)
            precision_i = tp_i/np.maximum(pred_num_i, np.finfo(np.float64).eps)
            AP_dict[label_i] = self.voc_ap(recall_i, precision_i)
        return AP_dict

    def voc_ap(self, rec, prec):
        # correct AP calculation
        # first append sentinel values at the end
        mrec = np.concatenate(([0.], rec, [1.]))
        mpre = np.concatenate(([0.], prec, [0.]))

        # compute the precision envelope
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])

        # to calculate area under PR curve, look for points
        # where X axis (recall) changes value
        i = np.where(mrec[1:] != mrec[:-1])[0]

        # and sum (\Delta recall) * prec
        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
        return ap


import re, datetime
import numpy as np

# Default date dictionary : maps month in string format to int number
date_dictionary = {'january':1,'february':2,'march':3,'april':4,'may':5,'june':6,'july':7,'august':8,'september':9,
                   'october':10,'november':11,'december':12}

def update_date_dictionary(dic):
    # Update the date dictionary by taking first three letters of the month in the string format to map to int.
    new = {}
    for key in dic:
        new[key] = dic[key]
        new[key[:3]] = dic[key]
    return new

date_dictionary = update_date_dictionary(date_dictionary)

def get_month(text):
    # extract month value
    pattern_month = re.compile(r"[A-Za-z]+")
    try:
        return date_dictionary[pattern_month.findall(text)[0].lower()]
    except:
        return -1

def process_year(year):
    # transform year value to have same format everywhere
    if len(year)==2:
        return int('20'+year)
    elif len(year)==4:
        return int(year)
    else:
        return -1

def get_day(text):
    # extract the day for the date
    pattern = re.compile(r"(\d+)([^\d]+)(\d+)")
    try:
        return int(pattern.search(text).group(1))
    except:
        return -1

def get_year(text):
    # extract the year
    pattern = re.compile(r"(\d+)([^\d]+)(\d+)")
    try:
        return process_year(pattern.search(text).group(3))
    except:
        return -1

def extract_date(text,string = True):
    # wrapper of previous function to extract the full date in datetime format
    day = get_day(text)
    month = get_month(text)
    year = get_year(text)
    try:
        if not string:
            return datetime.date(year, month, day)
        else:
            return str(datetime.date(year, month, day))
    except:
        return np.nan