"""
3D convolutions using GPU accelereration for Theano (using conv2d)
https://github.com/jaberg/TheanoConv3d2d

"""

import theano
from theano.gradient import DisconnectedType
from theano.gof import Op, Apply
from theano import tensor
import theano.sandbox.cuda as cuda

def get_diagonal_subtensor_view(x, i0, i1):
    """Helper function for DiagonalSubtensor and
    IncDiagonalSubtensor

    :note: it return a partial view of x, not a partial copy.
    """

    if x.shape[i0] < x.shape[i1]:
        raise NotImplementedError('is this allowed?')
    idx = [slice(None)] * x.ndim
    idx[i0] = slice(x.shape[i1] - 1, None, None)
    xview = x.__getitem__(tuple(idx))
    strides = list(xview.strides)
    strides[i1] -= strides[i0]
    xview.strides = strides
    return xview

class DiagonalSubtensor(Op):
    """Return a form a nd diagonal subtensor.

    :param x: n-d tensor
    :param i0: axis index in x
    :param i1: axis index in x
    :note: Work on the GPU.

    ``x`` is some n-dimensional tensor, but this Op only deals with a
    matrix-shaped slice, using axes i0 and i1. Without loss of
    generality, suppose that ``i0`` picks out our ``row`` dimension,
    and i1 the ``column`` dimension.

    So the relevant part of ``x`` is some matrix ``u``. Suppose it has 7 rows
    and 4 columns::

        [ 0 0 0 0 ]
        [ 0 0 0 0 ]
        [ 0 0 0 0 ]
        [ 0 0 0 0 ]
        [ 0 0 0 0 ]
        [ 0 0 0 0 ]

    The view returned by this function is also a matrix. It's a thick,
    diagonal ``stripe`` across u that discards the lower left triangle
    and the upper right triangle:

        [ x 0 0 0 ]
        [ x x 0 0 ]
        [ x x x 0 ]
        [ 0 x x x ]
        [ 0 0 x x ]
        [ 0 0 0 x ]

    In this case the return value would be this view of shape 3x4. The
    returned view has the same number of dimensions as the input
    ``x``, and the only difference is that the shape along dimension
    ``i0`` has been reduced by ``shape[i1] - 1`` because of the
    triangles that got chopped out.

    The NotImplementedError is meant to catch the case where shape[i0]
    is too small for the stripe to reach across the matrix, in which
    case it's not clear what this function should do. Maybe always
    raise an error. I'd look back to the call site in the Conv3D to
    see what's necessary at that point.

    """
    def __str__(self):
        if self.inplace:
            return "%s{inplace}" % self.__class__.__name__
        return "%s" % self.__class__.__name__

    def __init__(self, inplace=False):
        self.inplace = inplace
        if inplace:
            self.view_map = {0: [0]}

    def __eq__(self, other):
        return type(self) == type(other) and self.inplace == other.inplace

    def __hash__(self):
        return hash((type(self), self.inplace))

    def make_node(self, x, i0, i1):
        _i0 = tensor.as_tensor_variable(i0)
        _i1 = tensor.as_tensor_variable(i1)
        return Apply(self, [x, _i0, _i1], [x.type()])

    def perform(self, node, inputs, output_storage):
        xview = get_diagonal_subtensor_view(*inputs)
        if self.inplace:
            output_storage[0][0] = xview
        else:
            output_storage[0][0] = xview.copy()

    def grad(self, inputs, g_outputs):
        z = tensor.zeros_like(inputs[0])
        gx = inc_diagonal_subtensor(z, inputs[1], inputs[2], g_outputs[0])
        return [gx, DisconnectedType()(), DisconnectedType()()]

    def connection_pattern(self, node):
        rval = [[True], [False], [False]]
        return rval

diagonal_subtensor = DiagonalSubtensor(False)

class IncDiagonalSubtensor(Op):
    """
    The gradient of DiagonalSubtensor
    """
    def __str__(self):
        if self.inplace:
            return "%s{inplace}" % self.__class__.__name__
        return "%s" % self.__class__.__name__

    def __init__(self, inplace=False):
        self.inplace = inplace
        if inplace:
            self.destroy_map = {0: [0]}

    def __eq__(self, other):
        return type(self) == type(other) and self.inplace == other.inplace

    def __hash__(self):
        return hash((type(self), self.inplace))

    def make_node(self, x, i0, i1, amt):
        _i0 = tensor.as_tensor_variable(i0)
        _i1 = tensor.as_tensor_variable(i1)
        return Apply(self, [x, _i0, _i1, amt], [x.type()])

    def perform(self, node, inputs, output_storage):
        x, i0, i1, amt = inputs
        if not self.inplace:
            x = x.copy()
        xview = get_diagonal_subtensor_view(x, i0, i1)
        xview += amt
        output_storage[0][0] = x

    def grad(self, inputs, g_outputs):
        x, i0, i1, amt = inputs
        gy = g_outputs[0]
        return [gy, DisconnectedType()(), DisconnectedType()(),
                diagonal_subtensor(gy, i0, i1)]

    def connection_pattern(self, node):
        rval = [[True], [False], [False], [True]]
        return rval
inc_diagonal_subtensor = IncDiagonalSubtensor(False)

def conv3d(signals, filters,
           signals_shape=None, filters_shape=None,
           border_mode='valid'):
    """Convolve spatio-temporal filters with a movie.

    :param signals: timeseries of images whose pixels have color channels.
            shape: [Ns, Ts, C, Hs, Ws]
    :param filters: spatio-temporal filters
            shape: [Nf, Tf, C, Hf, Wf]
    :param signals_shape: None or a tuple/list with the shape of signals
    :param filters_shape: None or a tuple/list with the shape of filters
    :param border_mode: The only one tested is 'valid'.

    :note: Work on the GPU.
    """

    if isinstance(border_mode, str):
        border_mode = (border_mode, border_mode, border_mode)

    _signals_shape_5d = signals.shape if signals_shape is None else signals_shape
    _filters_shape_5d = filters.shape if filters_shape is None else filters_shape

    _signals_shape_4d = (
        _signals_shape_5d[0] * _signals_shape_5d[1],
        _signals_shape_5d[2],
        _signals_shape_5d[3],
        _signals_shape_5d[4],
        )
    _filters_shape_4d = (
        _filters_shape_5d[0] * _filters_shape_5d[1],
        _filters_shape_5d[2],
        _filters_shape_5d[3],
        _filters_shape_5d[4],
        )

    if border_mode[1] != border_mode[2]:
        raise NotImplementedError('height and width bordermodes must match')
    conv2d_signal_shape = _signals_shape_4d
    conv2d_filter_shape = _filters_shape_4d
    if signals_shape is None:
        conv2d_signal_shape = None
    if filters_shape is None:
        conv2d_filter_shape = None

    out_4d = tensor.nnet.conv2d(
        signals.reshape(_signals_shape_4d),
        filters.reshape(_filters_shape_4d),
        image_shape=conv2d_signal_shape,
        filter_shape=conv2d_filter_shape,
        border_mode = border_mode[1])  # ignoring border_mode[2]

    # reshape the output to restore its original size
    # shape = Ns, Ts, Nf, Tf, W-Wf+1, H-Hf+1
    if border_mode[1] == 'valid':
        out_tmp = out_4d.reshape((
            _signals_shape_5d[0],  # Ns
            _signals_shape_5d[1],  # Ts
            _filters_shape_5d[0],  # Nf
            _filters_shape_5d[1],  # Tf
            _signals_shape_5d[3] - _filters_shape_5d[3] + 1,
            _signals_shape_5d[4] - _filters_shape_5d[4] + 1,
            ))
    elif border_mode[1] == 'full':
        out_tmp = out_4d.reshape((
            _signals_shape_5d[0],  # Ns
            _signals_shape_5d[1],  # Ts
            _filters_shape_5d[0],  # Nf
            _filters_shape_5d[1],  # Tf
            _signals_shape_5d[3] + _filters_shape_5d[3] - 1,
            _signals_shape_5d[4] + _filters_shape_5d[4] - 1,
            ))
    elif border_mode[1] == 'same':
        raise NotImplementedError()
    else:
        raise ValueError('invalid border mode', border_mode[1])

    # now sum out along the Tf to get the output
    # but we have to sum on a diagonal through the Tf and Ts submatrix.
    if border_mode[0] == 'valid':
        out_5d = diagonal_subtensor(out_tmp, 1, 3).sum(axis=3)
    elif border_mode[0] in ('full', 'same'):
        out_5d = out_4d.reshape((_signals_shape_5d))
        # raise NotImplementedError('sequence border mode', border_mode[0])
    else:
        raise ValueError('invalid border mode', border_mode[1])
    return out_5d

def make_gpu_optimizer(op, to_gpu):
    """This function create optimizer that move some inputs to the GPU
    for op that work on both CPU and GPU.

    The op object is created by calling op(), so good default value
    are needed.

    We suppose the same op work with CPU and GPU inputs.

    :param op: the op that support GPU inputs
    :param to_gpu: a list of op inputs that are moved to the GPU.

    """
    @theano.gof.local_optimizer([])
    def local_to_gpu(node):
        """
        op(host_from_gpu()) -> host_from_gpu(op)
        gpu_from_host(op) -> op(gpu_from_host)
        """
        if isinstance(node.op, op):
            #op(host_from_gpu()) -> host_from_gpu(op)
            #If any of the input that go on the GPU are on the GPU,
            #move the op to the gpu.
            if any(node.inputs[idx].owner and
                   isinstance(node.inputs[idx].owner.op, cuda.HostFromGpu)
                   for idx in to_gpu):
                new_inp = list(node.inputs)
                for idx in to_gpu:
                    new_inp[idx] = cuda.gpu_from_host(new_inp[idx])
                return [cuda.host_from_gpu(op()(*new_inp))]
        if node.op == cuda.gpu_from_host:
            #gpu_from_host(op) -> op(gpu_from_host)
            host_input = node.inputs[0]
            if host_input.owner and isinstance(host_input.owner.op,
                                               op):
                op_node = host_input.owner
                new_inp = list(op_node.inputs)
                for idx in to_gpu:
                    new_inp[idx] = cuda.gpu_from_host(new_inp[idx])
                return [op()(*new_inp)]
        return False
    local_to_gpu.__name__ = "local_to_gpu_" + op.__name__
    cuda.opt.register_opt()(local_to_gpu)

if cuda.cuda_available:
    make_gpu_optimizer(DiagonalSubtensor, [0])
    make_gpu_optimizer(IncDiagonalSubtensor, [0, 3])


import pandas as pd
import pathlib as pl
import numpy as np
import RootPath
from abc import abstractmethod
from Utils.Data.Features.RawFeatures import *
from Utils.Data.Dictionary.MappingDictionary import *

def map_column_single_value(series, dictionary):
    mapped_series = series.map(dictionary).astype(np.int32)
    return pd.DataFrame(mapped_series)

def map_column_array(series, dictionary):
    mapped_series = series.map(
        lambda x: np.array([dictionary[y] for y in x.split('\t')], dtype=np.int32) if x is not pd.NA else None)
    return pd.DataFrame(mapped_series)

class MappedFeaturePickle(Feature):
    """
    Abstract class representing a dictionary that works with pickle file.
    """

    def __init__(self, feature_name: str, dataset_id: str):
        super().__init__(feature_name, dataset_id)
        self.pck_path = pl.Path(f"{Feature.ROOT_PATH}/{self.dataset_id}/mapped/{self.feature_name}.pck.gz")
        self.csv_path = pl.Path(f"{Feature.ROOT_PATH}/{self.dataset_id}/mapped/{self.feature_name}.csv.gz")

    def has_feature(self):
        return self.pck_path.is_file()

    def load_feature(self):
        assert self.has_feature(), f"The feature {self.feature_name} does not exists. Create it first."
        df = pd.read_pickle(self.pck_path, compression="gzip")
        # Renaming the column for consistency purpose
        df.columns = [self.feature_name]
        return df

    @abstractmethod
    def create_feature(self):
        pass

    def save_feature(self, dataframe: pd.DataFrame):
        # Changing column name
        dataframe.columns = [self.feature_name]
        self.pck_path.parent.mkdir(parents=True, exist_ok=True)
        dataframe.to_pickle(self.pck_path, compression='gzip')
        # For backup reason
        # self.csv_path.parent.mkdir(parents=True, exist_ok=True)
        # dataframe.to_csv(self.csv_path, compression='gzip', index=True)

class MappedFeatureTweetLanguage(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_language", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetLanguage(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingLanguageDictionary().load_or_create()
        mapped_dataframe = map_column_single_value(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureGroupedTweetLanguage(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_grouped_tweet_language", dataset_id)

        self.group_id_dict = {}
        self.current_mapping = 0

    def get_grouped_id(self, language_id):
        # ??? inglese misto altre cose
        if language_id == 16 or language_id == 18 or language_id == 20:
            return 16
        # [UNK]
        elif language_id == 26 or language_id == 56 or language_id == 57 or language_id == 58 or language_id == 59 or language_id == 61:
            return 26
        # ???
        elif language_id == 28 or language_id == 36 or language_id == 37 or language_id == 43 or language_id == 45 or language_id == 46:
            return 28
        # persian / pashto
        elif language_id == 25 or language_id == 44 or language_id == 41:
            return 25
        # lingue indiane
        elif language_id == 8 or language_id == 32 or language_id == 34 or language_id == 35 or language_id == 47 or language_id == 48 or language_id == 49 or language_id == 50 or language_id == 52 or language_id == 53 or language_id == 54 or language_id == 60 or language_id == 62:
            return 8
        # lingue est europa
        elif language_id == 14 or language_id == 23 or language_id == 24 or language_id == 55:
            return 14
        # lingue nord europa
        elif language_id == 21 or language_id == 31 or language_id == 38 or language_id == 39:
            return 21
        # lingue centro europa / balcani
        elif language_id == 29 or language_id == 40 or language_id == 42:
            return 29
        # others (vietnamita, birmano, armeno, georgiano, uiguro)
        elif language_id == 30 or language_id == 51 or language_id == 63 or language_id == 64 or language_id == 65:
            return 30
        else:
            return language_id

    def remap_language_id(self, group_id):
        if group_id not in self.group_id_dict:
            self.group_id_dict[group_id] = self.current_mapping
            self.current_mapping += 1
        return self.group_id_dict[group_id]

    def create_feature(self):
        feature = MappedFeatureTweetLanguage(self.dataset_id)
        dataframe = feature.load_or_create()

        #dataframe = dataframe.head()
        grouped_dataframe = pd.DataFrame(dataframe["mapped_feature_tweet_language"].map(lambda x: self.get_grouped_id(x)))
        #print(grouped_dataframe)
        mapped_dataframe = pd.DataFrame(dataframe["mapped_feature_tweet_language"].map(lambda x: self.remap_language_id(x)))
        #print(mapped_dataframe)

        self.save_feature(mapped_dataframe)

class MappedFeatureTweetId(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_id", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetId(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingTweetIdDictionary().load_or_create()
        mapped_dataframe = map_column_single_value(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureCreatorId(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_creator_id", dataset_id)

    def create_feature(self):
        feature = RawFeatureCreatorId(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingUserIdDictionary().load_or_create()
        mapped_dataframe = map_column_single_value(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureEngagerId(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_engager_id", dataset_id)

    def create_feature(self):
        feature = RawFeatureEngagerId(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingUserIdDictionary().load_or_create()
        mapped_dataframe = map_column_single_value(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureTweetHashtags(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_hashtags", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetHashtags(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingHashtagDictionary().load_or_create()
        mapped_dataframe = map_column_array(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureTweetLinks(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_links", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetLinks(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingLinkDictionary().load_or_create()
        mapped_dataframe = map_column_array(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureTweetDomains(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_domains", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetDomains(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingDomainDictionary().load_or_create()
        mapped_dataframe = map_column_array(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)

class MappedFeatureTweetMedia(MappedFeaturePickle):

    def __init__(self, dataset_id: str):
        super().__init__("mapped_feature_tweet_media", dataset_id)

    def create_feature(self):
        feature = RawFeatureTweetMedia(self.dataset_id)
        dataframe = feature.load_or_create()
        dictionary = MappingMediaDictionary().load_or_create()
        mapped_dataframe = map_column_array(dataframe[feature.feature_name], dictionary)

        self.save_feature(mapped_dataframe)


from datetime import datetime

import pandas as pd
import pydash as _
from bs4 import BeautifulSoup

from Base import NSEBase

class NSE(NSEBase):
    """
        A class to interact with NSE (National Stock Exchange) API.

        Attributes:
            valid_pcr_fields : list of valid fields for put-call ratio calculation

        Methods:
            __init__ : Initialize the NSE class
            get_option_chain : Get the option chain for a given ticker
            get_raw_option_chain : Get the raw option chain data for a given ticker
            get_options_expiry : Get the next expiry date for a given ticker
            get_all_derivatives_enabled_stocks : Get the list of equities available for derivatives trading
            get_equity_future_trade_info : Get the trade information of active future contracts for a given ticker
            get_equity_options_trade_info : Get the trade information of equity options for a given ticker
            _mapped_index_ticker_for_futures : Get the mapped index ticker for index futures
            get_index_futures_data : Get the data for index futures of a given index or ticker
            get_currency_futures : Get the data for currency futures
            get_commodity_futures : Get the data for commodity futures
            get_pcr : Get the put-call ratio for a given ticker and expiry date
    """

    def __init__(self) -> None:
        """
            The __init__ function is called when the class is instantiated.
            It sets up the session and headers for all subsequent requests.

            :param self: Represent the instance of the class

            :return: Nothing
        """

        super().__init__()
        self.valid_pcr_fields = ['oi', 'volume']

    # ----------------------------------------------------------------------------------------------------------------
    # Utility Functions

    def get_option_chain(self, ticker: str, is_index: bool = True, expiry: datetime = None) -> pd.DataFrame:
        """
            The get_option_chain function takes a ticker as input and returns the option chain for that ticker. The
            function uses the try_n_times_get_response function to get a response from NSE's API, which is then converted
            into a DataFrame using pd.json_normalize.

            :param self: Represent the instance of the class
            :param ticker: Specify the stock ticker for which we want to get the option chain its also called symbol in
            NSE
            :param is_index: (optional) Boolean value Specifies the given ticker is an index or not
            :param expiry: (optional) It takes the `expiry date` in the datetime format of the options contracts,
            default is very next expiry day

            :return: A dataframe with option chain
        """

        params = {'symbol': ticker}

        if is_index:
            url = f'{self._base_url}/api/option-chain-indices'
        else:
            url = f'{self._base_url}/api/option-chain-equities'

        response = self.hit_and_get_data(url, params=params)
        if expiry is None:
            df = pd.DataFrame(pd.json_normalize(_.get(response, 'filtered.data', {}), sep='_')).set_index('strikePrice')
        else:
            df = pd.DataFrame(pd.json_normalize(_.get(response, 'records.data', {}), sep='_')).set_index('strikePrice')
            df = df[df['expiryDate'] == expiry.strftime('%d-%b-%Y')]
        return df

    def get_raw_option_chain(self, ticker: str, is_index: bool = True) -> dict:
        """
            The get_option_chain function takes a ticker as input and returns the option chain for that ticker.
            The function uses the try_n_times_get_response function to get a response from NSE's API, which is
            then converted into a DataFrame using pd.json_normalize.

            :param is_index: Boolean value Specifies the given ticker is an index or not
            :param self: Represent the instance of the class
            :param ticker: Specify the stock ticker for which we want to get the option chain

            :return: A dataframe with option chain data
        """

        params = {'symbol': ticker}

        if is_index:
            url = f'{self._base_url}/api/option-chain-indices'
        else:
            url = f'{self._base_url}/api/option-chain-equities'

        response = self.hit_and_get_data(url, params=params)
        return response

    def get_options_expiry(self, ticker: str, is_index: bool = False) -> datetime:
        """
            The get_expiry function takes in a ticker and returns the next expiry date for that ticker.
            The function uses the NSE API to get all expiry dates for a given ticker, sorts them in ascending order,
            and then returns the nth element of this sorted list.

            :param self: Represent the instance of the class
            :param ticker: Specify the ticker / symbol for which we want to get the expiry date
            :param is_index: Boolean value Specifies the given ticker is an index or not

            :return: The very next expiry date
        """

        params = {'symbol': ticker}
        if is_index:
            url = f'{self._base_url}/api/option-chain-indices'
        else:
            url = f'{self._base_url}/api/option-chain-equities'
        response = self.hit_and_get_data(url, params=params)
        dates = sorted([datetime.strptime(date_str, "%d-%b-%Y") for date_str in
                        response.get('records', {}).get('expiryDates', [])])
        return dates

    # ----------------------------------------------------------------------------------------------------------------_
    # Equity Futures
    def get_all_derivatives_enabled_stocks(self) -> list:
        """
            The get_all_derivatives_enabled_stocks provides the list of Equities available for derivative trading

            :param self: Represent the instance of the class

            :return: List of all Equities tickers / symbols for which derivative trading is allowed
        """

        response = self.hit_and_get_data(f'{self._base_url}/api/master-quote')
        return response

    def get_equity_future_trade_info(self, ticker: str) -> pd.DataFrame:
        """
            The get_equity_future_trade_info provides all active future contracts trade information including its price
            details

            :param self: Represent the instance of the class
            :param ticker: Specify the ticker / symbol for which we want to get the expiry date

            :return: A DataFrame of trade info data of Equity Future contracts
        """

        params = {'symbol': ticker}
        response = self.hit_and_get_data(f'{self._base_url}/api/quote-derivative', params=params)
        future_data = []
        for fno_data in response.get('stocks', []):
            if fno_data.get('metadata', {}).get('instrumentType') == 'Stock Futures':
                future_data.append(fno_data)

        df = pd.DataFrame(pd.json_normalize(future_data, sep='_'))
        df['ticker'] = response.get('info', {}).get('symbol', '')
        df['companyName'] = response.get('info', {}).get('companyName', '')
        df['industry'] = response.get('info', {}).get('industry', '')
        df['fut_timestamp'] = response.get('fut_timestamp', '')
        return df

    # ----------------------------------------------------------------------------------------------------------------
    # Equity Options

    def get_equity_options_trade_info(self, ticker: str) -> pd.DataFrame:
        """
            Gets equity options trade information for a given ticker.

            :param ticker: Ticker symbol of the equity options trade.

            :return: DataFrame containing the trade information.
        """

        params = {'symbol': ticker}
        response = self.hit_and_get_data(f'{self._base_url}/api/quote-derivative', params=params)
        future_data = []
        for fno_data in response.get('stocks', []):
            if fno_data.get('metadata', {}).get('instrumentType') == 'Stock Options':
                future_data.append(fno_data)

        df = pd.DataFrame(pd.json_normalize(future_data, sep='_'))
        df['ticker'] = response.get('info', {}).get('symbol', '')
        df['companyName'] = response.get('info', {}).get('companyName', '')
        df['industry'] = response.get('info', {}).get('industry', '')
        df['opt_timestamp'] = response.get('opt_timestamp', '')
        return df

    # ----------------------------------------------------------------------------------------------------------------
    # Index Futures
    def _mapped_index_ticker_for_futures(self) -> dict:
        """
            Mapped index ticker will give dict of available options with its corresponding ticker value

            :param self: Represent the instance of the class

            :return: A dict obj with all FUTURES mappings
        """

        response = self.session.get(f'{self._base_url}//market-data/equity-derivatives-watch',
                                    headers=self.headers)
        soup = BeautifulSoup(response.text, features="html5lib")
        all_derivative_options = soup.find_all('option', attrs={"rel": "derivative"})
        mapped_index_ticker = {}
        for i in all_derivative_options:
            mapped_index_ticker[i.get_text().lower()] = i['value']

        return mapped_index_ticker

    def get_index_futures_data(self, index_or_ticker: str) -> pd.DataFrame:
        """
            Fetches index futures data.

            :param self: Represent the instance of the class
            :param index_or_ticker: Name or ticker symbol of the index.

            :return: DataFrame containing the FUTURES data
        """

        index_or_ticker = index_or_ticker.lower()
        mapped_tickers = {}

        try:
            mapped_tickers = self._mapped_index_ticker_for_futures()
        except Exception as err:
            print(
                f'Exception in fetching mapped ticker for this index try to pass actual ticker in the next call,  '
                f'Exact error : {err}')

        if index_or_ticker in mapped_tickers.keys():
            ticker_to_used = mapped_tickers[index_or_ticker]
        else:
            ticker_to_used = index_or_ticker
        params = {'index': ticker_to_used}
        response = self.hit_and_get_data(f'{self._base_url}/api/liveEquity-derivatives', params=params)
        df = pd.DataFrame(response.get('data', []))
        return df

    # ----------------------------------------------------------------------------------------------------------------
    # Currency

    def get_currency_futures(self) -> pd.DataFrame:
        """
            Fetches currency futures data.

            :param self: Represent the instance of the class

            :return: DataFrame containing the currency futures data
        """

        params = {'index': 'live_market_currency', 'key': 'INR'}
        response = self.hit_and_get_data(
            f'{self._base_url}/api/liveCurrency-derivatives', params=params)
        df = pd.DataFrame(response.get('data', []))
        return df

    # ----------------------------------------------------------------------------------------------------------------
    # Commodity
    def get_commodity_futures(self) -> pd.DataFrame:
        """
            Fetches commodity futures data.

            :param self: Represent the instance of the class

            :return: Pd.DataFrame: DataFrame containing the currency futures data
        """
        response = self.hit_and_get_data(f'{self._base_url}/api/liveCommodity-derivatives')
        df = pd.DataFrame(response.get('data', []))
        return df

    def get_pcr(self, ticker: str, is_index: bool = True, on_field: str = 'OI', expiry: datetime = None) -> float:
        """
            Calculate the put-call ratio (PCR) for a given ticker.

            :param self: Represent the instance of the class
            :param ticker: The ticker symbol.
            :param is_index: Boolean value Specifies the given ticker is an index or not
            :param expiry: The expiry date of the option contract. Defaults to None.
            :param on_field: The field to calculate PCR on. `Volume` or `oi` (open-interest) Default to 'OI'.

            :return: The calculated PCR value
        """

        on_field = on_field.lower()
        if on_field not in self.valid_pcr_fields:
            print(f'Un-supported filed is passed only these are the fields available : {self.valid_pcr_fields}')
            return 0

        if expiry is None:
            df = self.get_option_chain(ticker, is_index=is_index)
        else:
            df = self.get_option_chain(ticker, is_index=is_index, expiry=expiry)

        if df.shape[0] == 0:
            print('Your filters lead to empty DataSet check all params, expiry, etc; returning 0 as default')
            return 0
        if on_field == 'oi':
            put_oi = df['PE_openInterest'].sum()
            call_oi = df['CE_openInterest'].sum()
            return put_oi / call_oi
        else:
            put_vol = df['PE_totalTradedVolume'].sum()
            call_vol = df['CE_totalTradedVolume'].sum()
            return put_vol / call_vol


import asyncio
from flask import Blueprint, abort, flash, redirect, render_template, request, jsonify, url_for, Response
from werkzeug.utils import secure_filename
import socket
from flask_socketio import SocketIO, emit
from app import app, db, socketio
import os
import time

HOST = "127.0.1.1"
WEBSOCKET_PORT = 9999
CHUNK_SIZE = 4096  # Define o tamanho do pacote. Pode ser ajustado conforme necessário.

# Lista de endereços IP dos servidores para armazenamento de réplicas
REPLICA_SERVERS = [HOST, HOST, HOST] #ips locais mockados
#REPLICA_SERVERS = ["192.168.1.2", "192.168.1.3", "192.168.1.4"]  # IPs das máquinas das réplicas

main = Blueprint('main', __name__)

MIME_TYPES = {
    "mp4": "video/mp4",
    "avi": "video/x-msvideo",
    "mkv": "video/x-matroska",
    "flv": "video/x-flv"
}

class StreamingError(Exception):
    """Exceção personalizada para erros de streaming."""
    pass

class Video(db.Model):
    __tablename__ = 'video'
    __table_args__ = {'extend_existing': True}

    id = db.Column(db.Integer, primary_key=True)
    filename = db.Column(db.String(150), unique=True, nullable=False)
    description = db.Column(db.String(500), nullable=True)

with app.app_context():
    db.create_all()

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS']

def upload_to_replica(filename, file_content):
    for server_ip in REPLICA_SERVERS:
        try:
            client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            client.connect((server_ip, WEBSOCKET_PORT))
            # Enviar comando UPLOAD
            header = f"UPLOAD"
            client.send(header.encode())
            # Enviar tamanho do arquivo como uma string de tamanho 10
            client.send(str(len(file_content)).encode().zfill(10))
            # Enviar tamanho do nome do arquivo
            client.send(str(len(filename)).encode().zfill(10))
            # Enviar nome do arquivo
            client.send(filename.encode())
            # Enviar os dados do arquivo
            client.sendall(file_content)
            client.close()
        except Exception as e:
            print(f"Erro ao enviar para servidor {server_ip}: {e}")

@main.route('/upload', methods=['POST'])
def upload_file():
    if 'file' not in request.files:
        return jsonify({"error": "No file provided"}), 400
    file = request.files['file']
    if file.filename == '':
        return jsonify({"error": "No file selected"}), 400
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
        file.save(file_path)
        new_video = Video(filename=filename)
        db.session.add(new_video)
        db.session.commit()

        with open(file_path, 'rb') as f:
            file_content = f.read()
        upload_to_replica(filename, file_content)
        return "File uploaded successfully! You can now upload another file."
    return jsonify({"error": "Invalid file type"}), 400

@main.route('/', methods=['GET'])
def show_upload():
    return render_template('upload.html')

@main.route('/videos', methods=['GET'])
def list_videos():
    videos = Video.query.all()
    return render_template('video_list.html', videos=videos)

from websockets import connect as ws_connect

@main.route('/play/<int:video_id>', methods=['GET'])
def play_video(video_id):
    video = Video.query.get(video_id)
    video_name = video.filename

    # Adicionando failover para o streaming de vídeo
    for _ in range(3):  # Tenta até 3 vezes, uma para cada réplica
        try:
            return stream_video(video_name)
        except StreamingError:
            continue  # Se ocorrer um erro, tenta a próxima réplica

    return "Não foi possível reproduzir o vídeo."

def stream_video(video_name):
    try:
        client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        client.connect((HOST, 9999))

        header = f"STREAM"
        client.send(header.encode())
        client.send(str(len(video_name)).zfill(10).encode())
        client.send(video_name.encode())

        def generate():
            while True:
                chunk = client.recv(CHUNK_SIZE)
                if not chunk:
                    break
                yield chunk

        ext = video_name.split('.')[-1]
        mime_type = MIME_TYPES.get(ext, "video/mp4")
        return Response(generate(), content_type=mime_type)

    except ConnectionError:
        # Esta exceção pode ser lançada se houver um problema de conexão de rede
        raise StreamingError("Erro de conexão durante o streaming do vídeo")

@main.route('/delete_video/<int:video_id>', methods=['POST'])
def delete_video(video_id):
    video = Video.query.get(video_id)
    if video:
        db.session.delete(video)
        db.session.commit()
        return redirect(url_for('main.list_videos'))
    else:
        # Caso o vídeo não seja encontrado no banco de dados
        flash('Vídeo não encontrado', 'error')
        return redirect(url_for('main.list_videos'))

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-

"""The Simulator takes in a :obj:`seagull.Board`, and runs a simulation given a
set number of iterations and a rule. For each iteration, the rule is applied to
the Board in order to evolve the lifeforms. After the simulation, run
statistics are returned.

.. code-block:: python

    import seagull as sg

    board = sg.Board()
    board.add(Blinker(), loc=(0,0))

    # Initialize a simulator
    sim = sg.Simulator(board)
    stats = sim.run(sg.rules.conway_classic, iters=1000)

You can always get the history of the whole simulation by calling the
`get_history()` method. The length of the history will always be equal to
:code:`iters + 1` since we include the initial state

.. note::

    Running a simulation does not change the :code:`state` attribute of the
    board. Internally, the simulator makes a copy of that layout and updates
    that instead. This is to avoid unintended behaviour when running
    simulations again and again.

Various statistics such as entropy, peak cell coverage, and the like are
returned as a dictionary. This gives us an idea on the characteristics of the
simulation experiment.

.. note::

    Some statistics are highly-dependent on the size of the board and the
    number of iterations. For example, peak cell coverage (pertaining to the
    max. amount of active cells during the whole run) depends on board size. If
    you have better ideas for computing these statistics, please open-up an
    Issue!

The :code:`run()` method only computes the progress of the board for the whole
simulation, but it does not animate it yet. To create an animation, call the
:code:`animate()` method:

.. code-block:: python

    sim.animate()

This returns a :obj:`matplotlib.animation.FuncAnimation` that you can turn into
an interactive animation in your notebook or exported as a GIF.

.. note::

    When exporting to GIF, it is required to have the ffmpeg backend installed.

"""

# Import standard library
from typing import Callable, Union

# Import modules
import matplotlib.pyplot as plt
import numpy as np
from loguru import logger
from matplotlib import animation

from .board import Board
from .utils import statistics as stats

class Simulator:
    def __init__(self, board: Board):
        """Initialize the class

        Parameters
        ----------
        board : seagull.Board
            The board to run the simulation on
        """
        self.board = board
        self.history = []  # type: list
        self.stats = {}  # type: dict

    def run(self, rule: Callable, iters: int, **kwargs) -> dict:
        """Run the simulation for a given number of iterations

        Parameters
        ----------
        rule : callable
            Callable that takes in an array and returns an array of the same
            shape.
        iters : int
            Number of iterations to run the simulation.

        Returns
        -------
        dict
           Computed statistics for the simulation run
        """
        layout = self.board.state.copy()

        # Append the initial state
        self.history.append(layout)

        # Run simulation
        for i in range(iters):
            layout = rule(layout, **kwargs)
            self.history.append(layout)

        self.stats = self.compute_statistics(self.get_history())
        return self.stats

    def compute_statistics(self, history: Union[list, np.ndarray]) -> dict:
        """Compute various statistics for the board

        Parameters
        ----------
        history : list or numpy.ndarray
            The simulation history

        Returns
        -------
        dict
            Compute statistics
        """
        logger.info("Computing simulation statistics...")
        sim_stats = {
            "peak_cell_coverage": np.max(
                [stats.cell_coverage(h) for h in history]
            ),
            "avg_cell_coverage": np.mean(
                [stats.cell_coverage(h) for h in history]
            ),
            "avg_shannon_entropy": np.mean(
                [stats.shannon_entropy(h) for h in history]
            ),
            "peak_shannon_entropy": np.max(
                [stats.shannon_entropy(h) for h in history]
            ),
        }

        return sim_stats

    def get_history(self, exclude_init=False) -> np.ndarray:
        """Get the simulation history

        Parameters
        ----------
        exclude_init: bool
            If True, then excludes the initial state in the history

        Returns
        -------
        numpy.ndarray
            Simulation history of shape :code:`(iters+1, board.size[0],
            board.size[1])`
        """
        history = self.history[1:] if exclude_init else self.history
        return np.asarray(history)

    def animate(self, figsize=(5, 5), interval=100) -> animation.FuncAnimation:
        """Animate the resulting simulation

        Parameters
        ----------
        figsize : tuple
            Size of the output figure
        interval : int
            Interval for transitioning between frames

        Returns
        -------
        matplotlib.animation.FuncAnimation
            Animation generated from the run
        """
        if not self.history:
            msg = "The run() argument must be executed first"
            logger.error(msg)
            raise ValueError(msg)

        logger.info("Rendering animation...")
        fig = plt.figure(figsize=figsize)
        ax = fig.add_axes([0, 0, 1, 1], xticks=[], yticks=[], frameon=False)
        X_blank = np.zeros(self.board.size, dtype=bool)
        im = ax.imshow(X_blank, cmap=plt.cm.binary, interpolation="nearest")
        im.set_clim(-0.05, 1)

        def _animate(i, history):
            current_pos = history[i]
            im.set_data(current_pos)
            return (im,)

        def _init():
            im.set_data(X_blank)
            return (im,)

        history = self.get_history()
        anim = animation.FuncAnimation(
            fig,
            func=_animate,
            frames=range(history.shape[0]),
            init_func=_init,
            interval=interval,
            fargs=(history,),
            blit=True,
        )
        return anim


import torch
import torch.nn as nn
from torch.optim import Adam
import torch.nn.functional as F
from random import randint
import numpy as np
# import subprocess
# import multiprocessing
# import concurrent.futures
from time import time
from math import sqrt

CHANNEL = 256
BLOCKNUM = 40
BOARDSIZE = 8
BATCH = 50
EPOCHS = 20
DATASIZE = 7200
DATAUSE = 2000
ROUNDLIMIT = 500
PROCESS = 3
OUTPUT_INFO = 1

class resBlock(nn.Module):
    def __init__(self, x):
        super(resBlock, self).__init__()
        self.resBlock = nn.Sequential(
            nn.Conv2d(x, x, kernel_size=3, stride=1, padding=1, bias=False),
            nn.BatchNorm2d(x),
            nn.ReLU(inplace=True),
            nn.Conv2d(x, x, kernel_size=3, stride=1, padding=1, bias=False),
            nn.BatchNorm2d(x)
        )
        self.relu = nn.ReLU(inplace=True)

    def forward(self, x):
        shortCut = x
        out = self.resBlock(x)
        out += shortCut
        out = self.relu(out)
        return out

class resCNN(nn.Module):
    def __init__(self):
        super(resCNN, self).__init__()
        self.input = nn.Sequential(
            nn.Conv2d(3, CHANNEL, kernel_size=3, stride=1, padding=1, bias=False),
            nn.BatchNorm2d(CHANNEL),
            nn.ReLU(inplace=True)
        )
        self.resnet = nn.Sequential()
        for i in range(BLOCKNUM):
            self.resnet.add_module(str(i),resBlock(CHANNEL))

        self.ph = nn.Sequential(
            nn.Conv2d(CHANNEL, 2, kernel_size=1, stride=1, bias=False),
            nn.BatchNorm2d(2),
            nn.ReLU(inplace=True),
            nn.Flatten(),
            nn.Linear(BOARDSIZE*BOARDSIZE*2, BOARDSIZE*BOARDSIZE),
            # nn.Softmax(dim=1)
        )
        self.vh = nn.Sequential(
            nn.Conv2d(CHANNEL, 1, kernel_size=1, stride=1, bias=False),
            nn.BatchNorm2d(1),
            nn.ReLU(inplace=True),
            nn.Flatten(),
            nn.Linear(BOARDSIZE*BOARDSIZE, CHANNEL),
            nn.ReLU(inplace=True),
            nn.Linear(CHANNEL, 1),
            nn.Tanh()
        )

    def forward(self, x):
        model = self.input(x)
        model = self.resnet(model)
        p = self.ph(model)
        v = self.vh(model)
        return p, v

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

cnn = resCNN()
cnn.load_state_dict(torch.load(r'./rescnn.pth'))
cnn.to(device)
optimizer = Adam(cnn.parameters(), weight_decay=1e-4)

stateData = torch.zeros(DATASIZE, 3, 8, 8, dtype=float)
policyData = torch.zeros(DATASIZE, 64, dtype=float)
valueData = torch.zeros(DATASIZE, 1, dtype=float)
policyLossFunc = nn.CrossEntropyLoss()
valueLossFunc = nn.MSELoss()

def calc(cood):
    return cood[0] * BOARDSIZE + cood[1]

def lossFunction(policyOutput, valueOutput, policyTarget, valueTarget):
    policyLoss = policyLossFunc(policyOutput, policyTarget)
    valueLoss = valueLossFunc(valueOutput, valueTarget)
    return policyLoss + valueLoss

def train():
    cnn.train()
    use = torch.zeros(DATASIZE)
    inputData = torch.zeros(DATAUSE,3,8,8)
    policyTargetData = torch.zeros(DATAUSE,64)
    valueTargetData = torch.zeros(DATAUSE,1)

    i = 0
    while i < DATAUSE:
        x = randint(0, DATASIZE - 1)
        if use[x] == 1:
            continue
        inputData[i] = stateData[x]
        policyTargetData[i] = policyData[x]
        valueTargetData[i] = valueData[x]
        use[x] = 1
        i += 1

    optimizer.zero_grad()
    for i in range(EPOCHS):
        policyLossAvg = 0.0
        valueLossAvg = 0.0

        if OUTPUT_INFO:
            print(f'epoch {i+1}:')

        for j in range(0, DATAUSE, BATCH):

            input = inputData[j:j+BATCH]
            policyTarget = policyTargetData[j:j+BATCH]
            valueTarget = valueTargetData[j:j+BATCH]

            policyOutput, valueOutput = cnn(input.to(device))

            policyLoss = policyLossFunc(policyOutput, policyTarget.to(device))
            valueLoss = valueLossFunc(valueOutput, valueTarget.to(device))
            loss = policyLoss + valueLoss
            policyLossAvg += float(policyLoss)
            valueLossAvg += float(valueLoss)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        if OUTPUT_INFO:
            print(f'    policy loss: {policyLossAvg / (DATAUSE / BATCH)}')
            print(f'    value loss: {valueLossAvg / (DATAUSE / BATCH)}')
            print(f'    total loss: {(policyLossAvg + valueLossAvg) / (DATAUSE / BATCH)}')

    torch.save(cnn.state_dict(), r'./rescnn.pth')

class GameState:
    def __init__(self):
        self.board = np.zeros((8, 8), dtype=np.int8) # 0 ~ 7
        self.board[3, 3] = self.board[4, 4] = -1
        self.board[3, 4] = self.board[4, 3] = 1 #Black 1 White -1
        self.history = []

    def copy(self):
        state = GameState()
        state.board = np.copy(self.board)
        state.history = self.history[:]
        return state

    def makeMove(self, move, player):
        self.history.append(move)
        self.board[move] = player
        for d in (-1, 0, 1):
            for e in (-1, 0, 1):
                if d == 0 and e == 0:
                    continue
                x, y = move
                x += d
                y += e
                to_flip = []
                while x >= 0 and y >= 0 and x < 8 and y < 8 and self.board[x, y] == -player:
                    to_flip.append((x, y))
                    x += d
                    y += e
                if x >= 0 and y >= 0 and x < 8 and y < 8 and self.board[x, y] == player:
                    for f in to_flip:
                        self.board[f] = player

    def isValid(self, move, player):
        if self.board[move] != 0:
            return False
        for d in (-1, 0, 1):
            for e in (-1, 0, 1):
                if d == 0 and e == 0:
                    continue
                x, y = move
                x += d
                y += e
                num = 0
                while x >= 0 and y >= 0 and x < 8 and y < 8 and self.board[x, y] == -player:
                    x += d
                    y += e
                    num += 1
                if num > 0 and x >= 0 and y >= 0 and x < 8 and y < 8 and self.board[x, y] == player:
                    return True
        return False

    def getValidMoves(self, player):
        moves = []
        for i in range(8):
            for j in range(8):
                if self.isValid((i, j), player):
                    moves.append((i, j))
        return moves

    def isTerminal(self):
        if len(self.getValidMoves(1)) > 0:
            return False
        if len(self.getValidMoves(-1)) > 0:
            return False
        return True

    def getWinner(self):
        count = np.sum(self.board)
        if count > 0:
            return 1
        elif count < 0:
            return -1
        else:
            return 0

    def getScore(self, player):
        cnt = 0
        for i in range(8):
            for j in range(8):
                if self.board[i,j] == player:
                    cnt += 1
        return cnt

    def print(self):
        print('  ',end='')
        for i in range(8):
            print(i,end=' ')
        print('')
        for i in range(8):
            print(i,end=' ')
            for j in range(8):
                if self.board[i,j] == 1:
                    print('#',end=' ')
                elif self.board[i,j] == -1:
                    print('O',end=' ')
                else:
                    print('.',end=' ')
            print('')

PUCT_CONSTANT = 1
class MCTSNode:
    def __init__(self, state:GameState, player):
        self.state:GameState = state.copy()
        self.parent:MCTSNode = None
        self.children = []
        self.unexploredMoves = state.getValidMoves(player)
        self.player = player
        self.n = 0
        self.v = 0.0
        self.p = 0.0
        self.policyPredict = torch.zeros(64)
        self.valuePredict = 0.0
        if type == 2:
            input = torch.zeros(3,8,8)
            for i in range(8):
                for j in range(8):
                    if state.board[i,j] == 1:
                        input[0,i,j] = 1
            for i in range(8):
                for j in range(8):
                    if state.board[i,j] == -1:
                        input[1,i,j] = 1
            for i in range(8):
                for j in range(8):
                    input[2,i,j] = player
            input.unsqueeze_(0)
            output = cnn(input.to(device))
            self.policyPredict = F.softmax(output[0][0], dim=-1)
            self.valuePredict = float(output[1][0])

    def expand(self):
        if len(self.unexploredMoves) <= 0:
            return None

        move = self.unexploredMoves.pop()
        newState = self.state.copy()
        newState.makeMove(move, self.player)

        child = None
        if len(newState.getValidMoves(-self.player)) > 0:
            child = MCTSNode(newState, -self.player)
        else:
            child = MCTSNode(newState, self.player)
        child.parent = self
        child.p = float(self.policyPredict[calc(move)])
        self.children.append(child)
        return child

    def puct(self, player):
        Q = self.v / self.n
        U = PUCT_CONSTANT * self.p * sqrt(self.parent.n + 1) / (self.n + 1)
        if player == -1:
            Q = -Q
        return Q + U

    def select(self, player):
        return max(self.children, key=lambda c: c.puct(player))

    def backpropagate(self, v):
        self.n += 1
        self.v += v
        if self.parent:
            self.parent.backpropagate(v)

class CNNMCTS:
    def __init__(self):
        return

    def CNNMCTSBestMove(self, state, player, timeIterations):
        rootNode = MCTSNode(state, player)

        for i in range(timeIterations):
            node = rootNode

            while len(node.unexploredMoves) == 0 and node.state.isTerminal() == False:
                if len(node.children) > 0:
                    node = node.select(player)
                else:
                    break

            if len(node.unexploredMoves) > 0 and node.state.isTerminal() == False:
                node = node.expand()

            if node.state.isTerminal() == False:
                node.backpropagate(node.valuePredict)
            else:
                node.backpropagate(node.state.getWinner())

        bestChild = rootNode.children[0]
        for child in rootNode.children:
            if child.n > bestChild.n:
                bestChild = child
        return bestChild.state.history[-1]

def gen_py():
    MCTS = CNNMCTS()
    cnt = 0
    cnn.eval()
    while cnt < DATASIZE:
        c_state = GameState()
        currentPlayer = 1
        cur = 0
        lst = cnt

        while c_state.isTerminal() == 0:
            if len(c_state.getValidMoves(currentPlayer)) <= 0:
                currentPlayer = -currentPlayer
                continue
            bestMove = MCTS.CNNMCTSBestMove(c_state, currentPlayer, ROUNDLIMIT)
            cur += 1
            if 5 <= cur and cur <= 54 and cnt < DATASIZE:
                for i in range(8):
                    for j in range(8):
                        if c_state.board[i,j] == 1:
                            stateData[cnt,0,i,j] = 1
                for i in range(8):
                    for j in range(8):
                        if c_state.board[i,j] == -1:
                            stateData[cnt,1,i,j] = 1
                for i in range(8):
                    for j in range(8):
                        stateData[cnt,2,i,j] = currentPlayer
                policyData[cnt] = calc(bestMove)
                cnt += 1

            c_state.makeMove(bestMove, currentPlayer)
            currentPlayer = -currentPlayer

        valueData[lst:cnt] = c_state.getWinner()
        if OUTPUT_INFO:
            print(f'{cnt} / {DATASIZE}\r', end='')

    if OUTPUT_INFO:
        print('')

if __name__ == '__main__':
    np.set_printoptions(suppress=True, precision=7)
    # multiprocessing.freeze_support()
    times = 0
    while 1 :
        if OUTPUT_INFO:
            print(f'iteration {times}:')
            print('self-matching:')

        gen_py()
        # gen_cpp()
        # gen_mainProcess() # in train.py

        if OUTPUT_INFO:
            print('train start:')

        train()
        # archivePath = 'D:/Desktop/yanxue/rescnn_archive/rescnn-iteration' + str(times) +'.pth'
        # torch.save(cnn.state_dict(), archivePath)


import cv2
import pandas as pd
import numpy as np
from PIL import Image
from torch.utils.data import Dataset

class COVIDChestXRayDataset(Dataset):
    def __init__(self, path, size=128, augment=None):
        super(COVIDChestXRayDataset, self).__init__()
        print('{} initialized with size={}, augment={}'.format(self.__class__.__name__, size, augment))
        print('Dataset is located in {}'.format(path))
        self.size = size
        self.augment = augment

        image_dir = path / 'images'
        metadata_path = path / 'metadata.csv'

        df_metadata = pd.read_csv(metadata_path, header=0)
        # Drop CT scans
        df_metadata = df_metadata[df_metadata['modality'] == 'X-ray']
        # Keep only PA/AP/AP Supine, drop Axial, L (lateral)
        allowed_views = ['PA', 'AP', 'AP Supine']
        df_metadata = df_metadata[df_metadata['view'].isin(allowed_views)]

        # COVID-19 = 1, SARS/ARDS/Pneumocystis/Streptococcus/No finding = 0
        self.labels = (df_metadata.finding == 'COVID-19').values.reshape(-1, 1)
        images = df_metadata.filename
        images = images.apply(lambda x: image_dir / x).values.reshape(-1, 1)

        self.df = pd.DataFrame(np.concatenate((images, self.labels), axis=1), columns=['image', 'label'])

        del images

        print("Dataset: {}".format(self.df))

    @staticmethod
    def _load_image(path, size):
        img = Image.open(path)
        img = cv2.resize(np.array(img), (size, size), interpolation=cv2.INTER_AREA)
        if len(img.shape) == 2:
            img = np.expand_dims(img, axis=2)
            img = np.dstack([img, img, img])
        else:
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        # size, size, chan -> chan, size, size
        img = np.transpose(img, axes=[2, 0, 1])

        return img

    def __getitem__(self, index):
        row = self.df.iloc[index]
        img = self._load_image(row['image'], self.size)
        label = row['label']

        if self.augment is not None:
            img = self.augment(img)

        return img, label

    def __len__(self):
        return self.df.shape[0]


# Import packages.
import glob
import numpy as np
import os

# import cvxpy as cp

##################################################################################
#           Data import
##################################################################################

folder = 'runD'
n_robots = '30'
min_votes = '3'
seed = '1'
storage = '10'
routing = '10'
hashing_bucket = '5'
name = '/vcsbppfile_' + min_votes + '_' + n_robots + '_' + seed + '_' + storage + '_' + routing + '_' + hashing_bucket + '.dat'

path = os.path.abspath(os.path.join(os.getcwd(), '..', 'argos-application', 'data', folder))
file_names = glob.glob(path + name)
file_names = sorted(file_names)
print(file_names)

def get_data(file_name):

    results = []
    items = []

    ################ Reading file ##################################
    with open(file_name, 'r') as file:
        while True:
            line = file.readline()
            if not line:
                break
            timestep, num_robots = map(int, line.split(' '))
            total_tuples = 0
            for _ in range(num_robots):
                line = file.readline().strip('\n').split(' ')
                rid = int(line[0][1:])
                node_id = int(line[1])
                num_tuples = int(line[2])
                neighbors = int(line[3])
                total_tuples += num_tuples
                results.append((timestep, rid, node_id, num_tuples, neighbors))
            items.append(total_tuples)
    return results, items

results_dict = {}
load_dict = {}

for name in file_names:
    n = name.split('/')
    n = n[-1].split('.')
    n = n[0].split('_')
    min_votes = n[1]
    num_robots = int(n[2])

    results, load = get_data(name)
    results_dict[min_votes] = results
    load_dict[min_votes] = load

##################################################################################
#           Optimization
##################################################################################

# https://stackoverflow.com/questions/10035752/elegant-python-code-for-integer-partitioning
# See vscbpp_testing for more detail
def accel_asc(n):
    a = [0 for i in range(n + 1)]
    k = 1
    y = n - 1
    while k != 0:
        x = a[k - 1] + 1
        k -= 1
        while 2 * x <= y:
            a[k] = x
            y -= x
            k += 1
        l = k + 1
        while x <= y:
            a[k] = x
            a[l] = y
            yield a[:k + 2]
            x += 1
            y -= 1
        a[k] = x + y
        y = x + y - 1
        yield a[:k + 1]

def solve_vscbpp_accel(total_tuples, num_robots, neighbors, memory_capacity):
    min_cost = num_robots
    optimal_partition = []
    # Sort in ascending order
    a_neighbors = sorted(neighbors)
    idx_neighbors = np.argsort(neighbors)
    assignment_partitions = accel_asc(total_tuples)
    for partition in assignment_partitions:
        num_parts = len(partition)
        # Ignore partitions with too many parts
        if(num_parts >  num_robots):
            continue
        # Sort in ascending order
        a_partition = sorted(partition)
        # Impose volume constraint
        if(a_partition[-1] > memory_capacity):
            continue
        # Match largest num neighbors with largest part size
        prod = np.multiply(a_neighbors[-num_parts:], memory_capacity - np.array(a_partition))
        cur_cost = sum(np.divide(1, prod))
        if (cur_cost < min_cost):
            # print (min_cost, cur_cost)
            min_cost = cur_cost
            optimal_partition = list([0] * (num_robots - len(partition)) + list(a_partition))
    # Unsort back to initial neighbor order
    idx_unsort = idx_neighbors.argsort()
    opt_partition = np.array(optimal_partition)[idx_unsort]
    return min_cost, opt_partition

##################################################################################
#           Bin Packing
##################################################################################

########### Saving bin packing cost over time ###########################

#### In simulation ####

M = int(storage) + int(routing)
for key in results_dict.keys():
    results = results_dict[key]
    tuples = load_dict[key]
    x = []
    y = []
    # assignments = []
    cost = 0
    # assignment = np.zeros(num_robots)
    for i, result in enumerate(results):
        # assignment[result[1] - 1] = result[3]
        free_memory = float(M - result[3])
        if (result[3] != 0):
            cost += 1 / max(result[4] * free_memory, 1)
        if((i+1)%num_robots == 0):
            print(result[0], tuples[result[0]-1])
            x.append(result[0] / 10)
            y.append(cost)
            cost = 0
            # assignments.append(assignment)
            # assignment = np.zeros(num_robots)
    # Write to file (made to match optimal, want to have a partial file if takes too long)
    with open("heuristic_" + folder + '_' + key + '_' + n_robots + ".txt", "w") as f:
        for i,j in zip(x,y):
            f.write(str(i) +"\n")
            f.write(str(j) +"\n")

#### Optimal solution ####

# M = int(storage) + int(routing)
# for key in results_dict.keys():
#     results = results_dict[key]
#     tuples = load_dict[key]
#     x_opt = []
#     y_opt = []
#     neighbors = np.zeros(num_robots)
#     for i, result in enumerate(results):
#         neighbors[result[1] - 1] = result[4]
#         if((i+1)%num_robots == 0):
#             print("t", result[0])
#             # Skip time steps
#             if (result[0]%10 != 0):
#                 continue
#             opt_cost, pa = solve_vscbpp_accel(tuples[result[0]-1], num_robots, neighbors, M)
#             x_opt = result[0] / 10
#             y_opt = opt_cost
#             neighbors = np.zeros(num_robots)
#             with open("optimal_" + folder + '_' + key + '_' + n_robots + ".txt", "a") as f:
#                 f.write(str(x_opt) +"\n")
#                 f.write(str(y_opt) +"\n")

#### Worst cost ####

M = int(storage) + int(routing)
for key in results_dict.keys():
    results = results_dict[key]
    tuples = load_dict[key]
    x_worst = []
    y_worst = []
    worst_cost = 0
    neighbors = np.zeros(num_robots)
    for i, result in enumerate(results):
        neighbors[result[1] - 1] = result[4]
        if((i+1)%num_robots == 0):
            items = tuples[result[0]-1]
            # Put one item in all bins with 0 neighbors (assuming low enough load factor)
            zero_neighbors = len(neighbors) - np.count_nonzero(neighbors)
            if(items > zero_neighbors):
                worst_cost += zero_neighbors
                items -= zero_neighbors
            if items > 0 and items < M:
                # Put all in one bin
                worst_cost += 1
            else:
                # Sort number of neighbors in ascending order
                a_neighbors = sorted(neighbors)
                # Fill out memory of bins with lowest degree
                num_bins_to_fill = items // M
                worst_cost += num_bins_to_fill
                items -= num_bins_to_fill * M # same as modulo
                # Put remaining items in next lowest
                if(items > 0):
                    n_low = a_neighbors[zero_neighbors + num_bins_to_fill - 1]
                    free_memory = M - items
                    worst_cost += 1 / (max(n_low * free_memory, 1))
            x_worst = result[0] / 10
            y_worst = worst_cost
            neighbors = np.zeros(num_robots)
            worst_cost = 0
            with open("worst_" + folder + '_' + key + '_' + n_robots + ".txt", "a") as f:
                f.write(str(x_worst) +"\n")
                f.write(str(y_worst) +"\n")

# # Generate data.
# bins = 10
# items = 40
# np.random.seed(1)
# neighbors = np.random.randint(1, bins, bins)
# M = 20

# # Define and solve the CVXPY problem.
# assignment = cp.Variable((items, bins), boolean=True)
# selection = cp.Variable(bins, boolean=True)

# # cost = 1/cp.multiply(neighbors, M - cp.sum(assignment, axis=0))
# cost = 1/neighbors
# # objective = cp.sum(cp.multiply(cost, selection) + cp.multiply(cost/M, cp.max(cp.sum(assignment, axis=0))) )
# objective = cp.sum(cp.multiply(cost, selection) + cp.multiply(1/M, cp.max(cp.sum(assignment, axis=0))) )
# constraints = [
#     cp.sum(assignment, axis=1) == 1,
#     cp.sum(assignment, axis=0) <= M * selection
# ]
# prob = cp.Problem(cp.Minimize(objective), constraints)
# prob.solve()

# # Print result.
# print("Neigbors", neighbors)
# print("\nThe optimal value is", prob.value)
# # print("The optimal assignment is")
# # print(assignment.value)
# # print("The optimal selection is")
# # print(selection.value)

# # for tau in range(len(assignment.value))
# print("The optimal assignment per bin is")
# print(np.sum(assignment.value, axis=0))


""" Problem 3: Largest Prime Factor

https://projecteuler.net/problem=3

Goal: Find the largest prime factor of N.

Constraints: 10 <= N <= 1e12

Fundamental Theorem of Arithmetic: There will only ever be a unique set of prime
factors for any number.

e.g.: N = 10
      prime factors = {2, 5}
      largest = 5
"""
from math import isqrt
from util.maths.reusable import prime_factors

def largest_prime_factor(n: int) -> int:
    """
    Uses prime decomposition via the Sieve of Eratosthenes algorithm to return
    the largest prime factor.

    SPEED (WORSE for N with small factors)
        53.54ms for N = 1e12
    SPEED (WORST for N with large factors)
        39.56ms for N = 600_851_475_143
    """

    factors = prime_factors(n)
    return max(factors.keys())

def largest_prime_factor_simple(n: int) -> int:
    """
    Uses prime decomposition via trial division without any optimisation.

    SPEED (BEST for N with small factors)
        3743ns for N = 1e12
    SPEED (BEST for N with large factors)
        2.7e+05ns for N = 600_851_475_143
    """

    factor = 2
    while factor * factor <= n:
        while n % factor == 0 and n != factor:
            n //= factor
        factor += 1
    return n

def largest_prime_factor_recursive(n: int, f: int = 2) -> int:
    """
    Original solution used a floored square root to get an integer value. This
    was replaced with math.isqrt(), introduced in Py 3.8.

    SPEED (WORSE for N with small factors)
        52.41ms for N = 1e12
    SPEED (BETTER for N with large factors)
        12.85ms for N = 600_851_475_143
    """

    factors = [2]
    factors.extend(range(3, isqrt(n) + 1, 2))
    for factor in factors:
        if n % factor == 0:
            return largest_prime_factor_recursive(n // factor, factor)
    if n > 2:
        f = max(f, n)
    return f


import os
import ast
import subprocess
import uuid
import json
import hashlib
import socket
import psutil
from ipykernel.ipkernel import IPythonKernel

def make_except_safe(code):
    code = code.replace('\n', '\n ')
    code = 'try:\n ' + code
    code = code + '\nexcept: pass\n'
    try:
        ast.parse(code)
        return code
    except:
        return ''

SCIUNIT_HOME = os.path.expanduser('~/sciunit/')
SCIUNIT_PROJECT_FILE = os.path.join(SCIUNIT_HOME, '.activated')
SCIUNIT_SOCKET_FILE = os.path.join(SCIUNIT_HOME, 'listener.socket')

class SciunitKernel(IPythonKernel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        implementation = super().implementation + ' sciunit'

        if (os.path.exists(SCIUNIT_PROJECT_FILE)):
            self.project = open(SCIUNIT_PROJECT_FILE).read().strip()
            self.project_name = os.path.basename(os.path.normpath(self.project))
            if (os.path.exists(os.path.join(self.project, 'kernel'))):
                self.recording = False
            else:
                self.recording = True
                open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps([]))

        else:
            self.project_name = 'Project_' + str(uuid.uuid4())
            self.project = os.path.join(SCIUNIT_HOME, self.project_name)
            subprocess.run(['sciunit', 'create', self.project_name])
            self.recording = True
            open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps([]))

        self.eid = 1
        self.file = os.path.join(self.project, 'run.py')
        self.valid = True

        files = psutil.Process().open_files()
        for file in files:
            os.close(file.fd)

        criu_path = os.path.join(self.project, 'criu0')
        data = ['Dump', os.getpid(), os.getppid(), criu_path, 0]
        client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        client.connect(SCIUNIT_SOCKET_FILE)
        client.sendall(json.dumps(data).encode())
        client.close()

    def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
        criu_path = os.path.join(self.project, f'criu{self.eid}')
        if (os.path.exists(criu_path)): self.recording = False
        hashes = json.loads(open(os.path.join(self.project, 'kernel')).read())
        if not self.recording and (len(hashes) == self.eid - 1): self.valid = False

        data = []
        if self.valid:
            with open(self.file[1], 'a') as file:
                safe_code = make_except_safe(code)
                if safe_code:
                    if self.recording:
                        print('Recording e{}'.format(self.eid))
                        open(self.file, 'a').write(safe_code)
                        subprocess.Popen(['sciunit', 'exec', 'python3', self.file], stdout=subprocess.PIPE).communicate()

                        hashes.append(hashlib.sha256(safe_code.encode()).hexdigest())
                        open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps(hashes))

                        data = ['Dump', os.getpid(), os.getppid(), criu_path, self.eid]
                    else:
                        if (hashlib.sha256(safe_code.encode()).hexdigest() != hashes[self.eid - 1]):
                            print('Invalid, stopped repeating')
                            self.valid = False
                        else:
                            print('Valid, repeating e{}'.format(self.eid))
                            subprocess.Popen(['sciunit', 'repeat', 'e{}'.format(self.eid)], stdout=subprocess.PIPE).communicate()
                            data = ['Restore', os.getpid(), os.getppid(), criu_path, self.eid]

                    self.eid += 1

        output = super().do_execute(code, silent, False, user_expressions, allow_stdin)

        if data:
            client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            client.connect(SCIUNIT_SOCKET_FILE)
            client.sendall(json.dumps(data).encode())
            client.close()
            # TODO: Wait without Socket

        return output

if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=SciunitKernel)


# Crea una función llamada promedio que tome una lista de números como parámetro y devuelva el promedio de esos números.

usuario = input(
    'Ingresa un una lista de numeros separados por coma "," : ').split(",")

def promedio(args):
    acumulador = 0
    cantidad = len(args)
    for i in args:
        acumulador += int(i)
    resultado = acumulador / cantidad
    print(resultado)

promedio(usuario)


import os
import torch
import datetime
import numpy as np
import pandas as pd

from src.attn_analysis import gradcam
from src.attn_analysis import iou_analysis
from src.attn_analysis import blue_heatmap
from src.attn_analysis import extract_disease_reps
from src.attn_analysis import make_2d_plot_and_3d_gif

import warnings
warnings.filterwarnings('ignore')

class AttentionAnalysis(object):
    def __init__(self, results_dir_force,
                 base_results_dir, task,
                 attention_type, attention_type_args,
                 setname, valid_results_dir,
                 custom_net, custom_net_args, params_path,
                 stop_epoch, which_scans, dataset_class, dataset_args):
        """
        Variables:
        <results_dir_force>: path to a results directory. If this is a valid
            path, then all results will be stored in here. If this is NOT a
            valid path, a new directory for the new results will be created
            based on <base_results_dir>.
        <base_results_dir>: path to the base results directory. A new directory
            will be created within this directory to store the results of
            this experiment.
        <task>: a list of strings. The strings may include 'iou_analysis',
            'blue_heatmaps', and/or 'attn_plots'.

            If <task> contains 'iou_analysis' then calculate approximate IOU
            statistics for the final epoch of a model.
            Specifically, the 'IOU' is calculated as the ratio of raw scores
            within the allowed area to raw scores outside of the allowed
            area.
            Produces iou_wide_df, a dataframe with the following 5 columns:
            'Epoch': int, the epoch in which the IOU was calculated.
            'IOU': float, the 'IOU' value for this label's attention map vs. the
                segmentation ground truth (which in this case is the approximate
                attention ground truth.)
            'Label': string for the label for which IOU was calculated e.g. 'airplane'
            'VolumeAccession': volume accession number
            'LabelsPerImage': total number of labels present in this image
            Also produces dfs that summarize the IOU across different ways
            of grouping the data.

            If <task> contains 'blue_heatmaps' then make a blue heatmap showing
            the disease scores for each slice.

            If <task> contains 'attn_plots' then make visualizations of the
            attention superimposed on the CT scan (as a 3D gif, and as a 2D plot
            for the slice with the highest score for that disease). Also if
            doing Grad-CAM, make a 2d debugging plot.

        <attention_type>: str; either
            'gradcam-vanilla' for vanilla Grad-CAM, or
            'hirescam' for HiResCAM, in which feature maps and gradients are
                element-wise multiplied and then we take the avg over the
                feature dimension, or
            'hirescam-check' for alternative implementation of HiResCAM
                attention calculation, which can be used in a model that
                has convolutional layers followed by a single FC layer.
                In this implementation, the HiResCAM attention is calculated
                during the forward pass of the model by element-wise multiplying
                the final FC layer weights (the gradients) against the final
                representation. This option is called 'hirescam-check'
                because for models that meet the architecture requirements this
                implementation is a 'check' on the 'hirescam' option which
                actually accesses the gradients.
                'hirescam-check' and 'hirescam' on the output of the last conv
                layer produce identical results on AxialNet as expected, since
                AxialNet is a CNN with one FC layer at the end.
        <attention_type_args>: dict; additional arguments needed to calculate
            the specified kind of attention. If the attention_type is one of the
            GradCAMs then in this dict we need to specify
            'model_name' and 'target_layer_name' (see gradcam.py for
            more documentation)
        <setname>: str; which split to use e.g. 'train' or 'val' or 'test'; will
            be passed to the <dataset_class>
        <valid_results_dir>: path to a directory that contains the validation
            set IOU analysis results. Only needed if setname=='test' because we
            need to use validation set per-label thresholds to calculate
            results.
        <custom_net>: a PyTorch model
        <custom_net_args>: dict; arguments to pass to the PyTorch model
        <params_path>: str; path to the model parameters that will be loaded in
        <stop_epoch>: int; epoch at which the model saved at <params_path> was
            saved
        <which_scans>: a pandas DataFrame specifying what scans and/or
            abnormalities to use.
            It can be an empty pandas DataFrame, in which case all available
            scans in the set will be used and named with whatever volume
            accession they were saved with (real or fake).
            Or, it can be a filled in pandas DataFrame, with columns
            ['VolumeAcc','VolumeAcc_ForOutput','Abnormality'] where
            VolumeAcc is the volume accession the scan was saved with,
            VolumeAcc_ForOutput is the volume accession that should be used in
            the file name of any output files of this module (e.g. a DEID acc),
            and Abnormality is either 'all' to save all abnormalities for that
            scan, or it's comma-separated names of specific abnormalities to
            save for that scan.
        <dataset_class>: a PyTorch dataset class
        <dataset_args>: dict; arguments to pass to the <dataset_class>"""
        self.base_results_dir = base_results_dir
        self.task = task
        for specific_task in self.task:
            assert ((specific_task == 'iou_analysis')
                or (specific_task == 'blue_heatmaps')
                or (specific_task == 'attn_plots'))
        assert len(self.task) <= 2
        if 'blue_heatmaps' in self.task:
            #only allow calculation of the blue_heatmaps if we are using
            #attention_type hirescam-check. Why? Because for both the blue
            #heatmaps and the hirescam-check visualizations, we need to run
            #the model to get out. And in gradcam we need to run the model again
            #later so we get a memory error if we try to do this after getting
            #out.
            assert attention_type == 'hirescam-check'
        self.attention_type = attention_type
        assert self.attention_type in ['gradcam-vanilla','hirescam','hirescam-check']
        self.attention_type_args = attention_type_args
        if self.attention_type in ['gradcam-vanilla','hirescam']:
            assert 'model_name' in self.attention_type_args.keys()
            assert 'target_layer_name' in self.attention_type_args.keys()
        self.setname = setname
        self.valid_results_dir = valid_results_dir
        self.custom_net = custom_net
        self.custom_net_args = custom_net_args #dict of args
        self.params_path = params_path
        self.stop_epoch = stop_epoch
        self.which_scans = which_scans
        self.CTDatasetClass = dataset_class
        self.dataset_args = dataset_args #dict of args
        self.device = torch.device('cuda:0')
        self.verbose = self.dataset_args['verbose'] #True or False

        #Run
        self.set_up_results_dirs(results_dir_force)
        self.run()

    def set_up_results_dirs(self, results_dir_force):
        if os.path.isdir(results_dir_force):
            results_dir = results_dir_force
        else:
            #If you're not forcing a particular results_dir, then make a new
            #results dir:
            #Example params_path = '/home/rlb61/data/img-hiermodel2/results/2020-09/2020-09-27_AxialNet_Mask_CORRECT_dilateFalse_nearest/params/AxialNet_Mask_CORRECT_dilateFalse_nearest_epoch23'
            old_results_dir = os.path.split(os.path.split(os.path.split(self.params_path)[0])[0])[1] #e.g. '2020-09-27_AxialNet_Mask_CORRECT_dilateFalse_nearest'
            date = datetime.datetime.today().strftime('%Y-%m-%d')
            results_dir = os.path.join(self.base_results_dir,date+'_'+self.setname.capitalize()+'AttnAnalysis_of_'+old_results_dir)
            if not os.path.isdir(results_dir):
                os.mkdir(results_dir)

        #Subdirs for particular analyses:
        if 'iou_analysis' in self.task:
            self.iou_analysis_dir = os.path.join(results_dir,'iou_analysis_'+self.attention_type)
            if not os.path.exists(self.iou_analysis_dir): os.mkdir(self.iou_analysis_dir)
        if 'blue_heatmaps' in self.task:
            #Note that the blue heatmaps depend only on the model, and not on the
            #attention type
            self.blue_heatmaps_dir = os.path.join(results_dir,'blue_heatmaps')
            if not os.path.exists(self.blue_heatmaps_dir): os.mkdir(self.blue_heatmaps_dir)
        if 'attn_plots' in self.task:
            self.attn_2dplot_dir = os.path.join(results_dir,'attn_2dplot_'+self.attention_type)
            self.attn_3dgif_dir = os.path.join(results_dir,'attn_3dgif_dir_'+self.attention_type)
            for directory in [self.attn_2dplot_dir,self.attn_3dgif_dir]:
                if not os.path.exists(directory): os.mkdir(directory)
            for key in ['g1p1', 'g1p0', 'g0p1', 'g0p0']:
                if not os.path.exists(os.path.join(self.attn_2dplot_dir,key)):
                    os.mkdir(os.path.join(self.attn_2dplot_dir,key))
                if not os.path.exists(os.path.join(self.attn_3dgif_dir,key)):
                    os.mkdir(os.path.join(self.attn_3dgif_dir,key))

            if self.attention_type in ['gradcam-vanilla','hirescam']:
                self.gradcam_debug_dir = os.path.join(results_dir,self.attention_type+'_debug_dir')
                if not os.path.exists(self.gradcam_debug_dir): os.mkdir(self.gradcam_debug_dir)
        else: #even if attn_plots is not in task, we need to have a placeholder for
            #this directory to avoid an error later:
            self.gradcam_debug_dir = None

    def run(self):
        self.load_model()
        self.load_dataset()
        self.load_chosen_indices()
        if 'blue_heatmaps' in self.task:
            self.blue_heatmap_baseline = blue_heatmap.get_baseline(self.chosen_dataset, self.model, self.blue_heatmaps_dir)
        if 'iou_analysis' in self.task:
            thresh_perf_df_filename = 'Determine_Best_Threshold_For_Each_Label_Epoch'+str(self.stop_epoch)+'.csv'
            valid_thresh_perf_df_path = os.path.join(os.path.join(self.valid_results_dir,'iou_analysis_'+self.attention_type), thresh_perf_df_filename)
            self.iou_analysis_object = iou_analysis.DoIOUAnalysis(self.setname, self.stop_epoch,
                                    self.label_meanings, self.iou_analysis_dir, valid_thresh_perf_df_path)
        self.loop_over_dataset_and_labels()
        if 'iou_analysis' in self.task:
             self.iou_analysis_object.do_all_final_steps()

    ######################################################
    # Methods to Load Model, Dataset, and Chosen Indices #----------------------
    ######################################################
    def load_model(self):
        print('Loading model')
        self.model = self.custom_net(**self.custom_net_args).to(self.device)
        check_point = torch.load(self.params_path, map_location='cpu') #map to CPU to avoid memory issue #TODO check if you need this
        self.model.load_state_dict(check_point['params'])
        self.model.eval()
        #If everything loads correctly you will see the following message:
        #IncompatibleKeys(missing_keys=[], unexpected_keys=[])

    def load_dataset(self):
        print('Loading dataset')
        self.chosen_dataset = self.CTDatasetClass(setname = self.setname, **self.dataset_args)
        self.label_meanings = self.chosen_dataset.return_label_meanings()

    def load_chosen_indices(self):
        print('Loading chosen indices')
        if len([x for x in self.which_scans.columns.values.tolist() if x in ['VolumeAcc','VolumeAcc_ForOutput','Abnormality']])==3:
            #you did specify which scans to use, so figure out what indices
            #you need to query in the dataset to get those chosen scans:
            for df_idx in range(self.which_scans.shape[0]):
                volume_acc = self.which_scans.at[df_idx,'VolumeAcc']
                self.which_scans.at[df_idx,'ChosenIndex'] = np.where(self.chosen_dataset.volume_accessions == volume_acc)[0][0]
        else:
            assert (self.which_scans == pd.DataFrame()).all().all()
            #you didn't specify which scans to use, so use all the scans in the dataset
            self.which_scans['ChosenIndex'] = [x for x in range(len(self.chosen_dataset))]
        self.which_scans['ChosenIndex'] = self.which_scans['ChosenIndex'].astype('int')

    ###########
    # Looping #-----------------------------------------------------------------
    ###########
    def loop_over_dataset_and_labels(self):
        if (self.task == ['iou_analysis'] and self.iou_analysis_object.loaded_from_existing_file):
            return #don't need to loop again if iou_wide_df already created
        print('Looping over dataset and labels')
        five_percent = max(1,int(0.05*self.which_scans.shape[0]))
        #Iterate through the examples in the dataset. df_idx is an integer
        for df_idx in range(self.which_scans.shape[0]):
            if self.verbose: print('Starting df_idx',df_idx)
            idx = self.which_scans.at[df_idx,'ChosenIndex'] #int, e.g. 5
            example = self.chosen_dataset[idx]
            ctvol = example['data'].unsqueeze(0).to(self.device) #unsqueeze to create a batch dimension. out shape [1, 135, 3, 420, 420]
            gr_truth = example['gr_truth'].cpu().data.numpy() #out shape [80]
            volume_acc = example['volume_acc'] #this is a string, e.g. 'RHAA12345_5.npz'
            attn_gr_truth = example['attn_gr_truth'].data.cpu().numpy() #out shape [80, 135, 6, 6]

            #Get out and x_perslice_scores when using attention_type hirescam-check
            out = self.get_out_and_blue_heatmaps(ctvol, gr_truth, volume_acc)

            if self.verbose: print('Analyzing',volume_acc)
            #volume_acc sanity check and conversion to FAKE volume acc if indicated
            if 'VolumeAcc' in self.which_scans.columns.values.tolist():
                intended_volume_acc = self.which_scans.at[df_idx,'VolumeAcc']
                assert volume_acc == intended_volume_acc
                #Now, because which_scans is not empty, you can switch volume_acc
                #from the actual volume acc e.g. RHAA12345_6 to the fake ID,
                #because from here onwards, the volume acc is only used in file
                #names:
                volume_acc = self.which_scans.at[df_idx,'VolumeAcc_ForOutput'].replace('.npz','').replace('.npy','') #e.g. fake ID 'val12345'

            #Now organize the labels for this particular image that you want to
            #make heatmap visualizations for into g1p1, g1p0, g0p1, and g0p0
            #g1p1=true positive, g1p0=false negative, g0p1=false positive, g0p0=true negative
            #we pass in volume_acc twice because the variable volume_acc could
            #be fake OR real, depending on the preceding logic, but
            #example['volume_acc'] is guaranteed to always be real.
            label_indices_dict = make_label_indices_dict(volume_acc, example['volume_acc'], gr_truth, self.params_path, self.label_meanings)

            for key in ['g1p1', 'g1p0', 'g0p1', 'g0p0']:
                chosen_label_indices = label_indices_dict[key] #e.g. [32, 37, 43, 46, 49, 56, 60, 62, 64, 67, 68, 71]

                if (('Abnormality' not in self.which_scans.columns.values.tolist()) or (self.which_scans.at[df_idx,'Abnormality'] == 'all')): #plot ALL abnormalities
                    pass
                else: #plot only chosen abnormalities
                    chosen_abnormalities = self.which_scans.at[df_idx,'Abnormality'].split(',')
                    chosen_label_indices = [x for x in chosen_label_indices if self.label_meanings[x] in chosen_abnormalities]

                #Calculate label-specific attn and make label-specific attn figs
                for chosen_label_index in chosen_label_indices:
                    #Get label_name and seg_gr_truth:
                    label_name = self.label_meanings[chosen_label_index] #e.g. 'lung_atelectasis'
                    seg_gr_truth = attn_gr_truth[chosen_label_index,:,:,:] #out shape [135, 6, 6]
                    #segprediction is the raw attention. slice_idx is the index of
                    #the slice with the highest raw score for this label
                    segprediction, x_perslice_scores_this_disease = self.return_segprediction(out, ctvol, gr_truth, volume_acc, chosen_label_index) #out shape [135, 6, 6]
                    segprediction_clipped_and_normed = clip_and_norm_volume(segprediction)

                    if 'iou_analysis' in self.task:
                        if key in ['g1p1','g1p0']: #TODO: implement IOU analysis for other options! also make this more efficient so no excessive calculations are done
                            if self.verbose: print('Adding example to IOU analysis')
                            self.iou_analysis_object.add_this_example_to_iou_wide_df(segprediction_clipped_and_normed,
                                                    seg_gr_truth, volume_acc, label_name, num_labels_this_ct=int(gr_truth.sum()))
                    if 'attn_plots' in self.task:
                        if self.verbose: print('Making 2D and 3D attn figures')
                        make_2d_plot_and_3d_gif.plot_attn_over_ct_scan(ctvol,
                            segprediction_clipped_and_normed, x_perslice_scores_this_disease, volume_acc,
                            label_name, os.path.join(self.attn_2dplot_dir,key), os.path.join(self.attn_3dgif_dir,key))

            #Report progress
            if df_idx % five_percent == 0:
                print('Done with',df_idx,'=',round(100*df_idx/self.which_scans.shape[0],2),'%')
            del example, ctvol, gr_truth, volume_acc, attn_gr_truth, out

    def get_out_and_blue_heatmaps(self, ctvol, gr_truth, volume_acc):
        """Calculate 'out' which will be used for:
            1. the blue heatmap figure (the 'x_perslice_scores') which is
               specific to a particular scan, NOT a particular label;
            2. the 'hirescam-check' attention (the 'disease_reps')
        Note that we don't do this within the label for loop below
        because it's computationally wasteful to run a fixed model again
        and again on the same input CT scan.

        To avoid memory issues of running the model twice,
        for determining true positives/false positives/true negatives/false
        negatives, we use the pre-calculated predicted probabilities that were
        saved when the model was first run.

        out['out'] contains the prediction scores and has shape [1,80]
        out['disease_reps'] contains the 'hirescam-check' attention for
            all diseases and has shape [80, 135, 16, 6, 6]
        out['x_perslice_scores'] contains the abnormality scores for each
            slice and has shape [1, 80, 135]"""
        if self.attention_type == 'hirescam-check':
            out = self.model(ctvol)
            if 'blue_heatmaps' in self.task:
                if self.verbose: print('Making blue heatmap')
                blue_heatmap.visualize_slicediseases(out['out'], gr_truth,
                                    out['x_perslice_scores'].cpu().data.numpy(),
                                    volume_acc, self.blue_heatmaps_dir, self.label_meanings,
                                    self.blue_heatmap_baseline)
            return out
        else:
            return None

    def return_segprediction(self, out, ctvol, gr_truth, volume_acc, chosen_label_index):
        """Return the <segprediction> which is a volume of scores for a particular
        label"""
        if self.attention_type == 'hirescam-check':
            return extract_disease_reps.return_segprediction_from_disease_rep(out, chosen_label_index)
        elif self.attention_type in ['gradcam-vanilla','hirescam']:
            #note that if 'make_figure' is in self.task, then a 2d debugging
            #figure for Grad-CAM will also be saved in this step
            return gradcam.RunGradCAM(self.attention_type, self.model, self.device,
                      self.label_meanings, self.gradcam_debug_dir, self.task,
                      **self.attention_type_args).return_segprediction_from_grad_cam(ctvol, gr_truth, volume_acc, chosen_label_index)

def make_label_indices_dict(possibly_fake_volume_acc, real_volume_acc, gr_truth, params_path, label_meanings):
    """Based on the <gr_truth> and the predicted probability that was
    pre-calculated, figure out which abnormalities are true positives (g1p1),
    false negatives (g1p0), false positives (g0p1), and true negatives (g0p0).
    g stands for ground truth and p stands for predicted probability.

    The predicted probabilities are read in from the predicted probabilities
    that were saved from the final model when it was done training.
    The path for these is inferred from params_path based on known
    directory structure. We also need to use this pre-calculated file because
    we need to get the median predicted probability for each abnormality.

    The predicted probabilities are binarized as 0 or 1 according to being
    above or below the median (50th percentile) for that abnormality.

    Returns a dictionary with keys g1p1, g1p0, g0p1, and g0p0
    and values that are numpy arrays of numeric indices of the corresponding
    abnormalities e.g. array([32, 37, 64, 67, 68, 71])"""
    #Infer paths to the precomputed pred probs based on known directory organization:
    #e.g. precomputed_path = '/home/rlb61/data/img-hiermodel2/results/results_2019-2020/2020-10/2020-10-09_WHOLEDATA_BodyAvg_Baseline_FreshStart/pred_probs'
    precomputed_path = os.path.join(os.path.split(os.path.split(params_path)[0])[0],'pred_probs')
    files = os.listdir(precomputed_path) #e.g. ['valid_grtruth_ep4.csv', 'valid_predprob_ep4.csv']
    pred_probs_file = [x for x in files if 'predprob' in x][0] #e.g. 'valid_predprob_ep4.csv'
    gr_truth_file = [x for x in files if 'grtruth' in x][0] #e.g. 'valid_grtruth_ep4.csv'

    #Open the pred probs and gr truth for this data subset
    #Each of them has volume accesions as the index, and abnormalities as
    #the columns. Example shape: [2085,80]
    pred_probs_all = pd.read_csv(os.path.join(precomputed_path, pred_probs_file),header=0,index_col=0)
    gr_truth_all = pd.read_csv(os.path.join(precomputed_path, gr_truth_file),header=0,index_col=0)

    #Sanity checks:
    for df in [pred_probs_all, gr_truth_all]:
        assert df.columns.values.tolist()==label_meanings
    assert (gr_truth_all.loc[real_volume_acc,:]==gr_truth).all()

    #Calculate the medians of the different abnormalities across the whole
    #data subset.
    medians = np.median(pred_probs_all,axis=0) #np array, e.g. shape [80]

    #Select out the predicted probabilities for just this scan
    pred_probs = pred_probs_all.loc[real_volume_acc,:] #pd Series w abn labels and float values, e.g. shape [80]

    #Get binary vector that's equal to 1 if the corresponding abnormality
    #has a pred prob greater than the median
    pred_probs_geq = (pred_probs >= medians).astype('int') #pd Series w abn labels and binary int values, e.g. shape [80]

    #Now divide up the abnormalities for this particular CT scan based on whether
    #they are above or below the median pred prob, and whether the gr truth
    #is 1 or 0
    g0p0 = np.intersect1d(np.where(gr_truth==0)[0], np.where(pred_probs_geq==0)[0])
    g0p1 = np.intersect1d(np.where(gr_truth==0)[0], np.where(pred_probs_geq==1)[0])
    g1p0 = np.intersect1d(np.where(gr_truth==1)[0], np.where(pred_probs_geq==0)[0])
    g1p1 = np.intersect1d(np.where(gr_truth==1)[0], np.where(pred_probs_geq==1)[0])

    #Checks
    assert len(g1p0)+len(g1p1)==int(gr_truth.sum())
    assert len(g0p0)+len(g0p1)+len(g1p0)+len(g1p1)==len(gr_truth)

    label_indices_dict = {'g0p0':g0p0.tolist(),
                          'g0p1':g0p1.tolist(),
                          'g1p0':g1p0.tolist(),
                          'g1p1':g1p1.tolist()}
    #uncomment the next line to print detailed info to the terminal:
    #print_for_future_reference(params_path, label_indices_dict, possibly_fake_volume_acc, pred_probs, medians, label_meanings)
    return label_indices_dict

def print_for_future_reference(params_path, label_indices_dict, possibly_fake_volume_acc, pred_probs, medians, label_meanings):
    model_description = os.path.split(params_path)[1]
    for key in list(label_indices_dict.keys()): #the keys are ['g0p0','g0p1','g1p0','g1p1']
        for idx in label_indices_dict[key]:
            print('\t'.join([model_description, possibly_fake_volume_acc, key, label_meanings[idx], str(round(pred_probs[idx],4)),'median:',str(round(medians[idx],4))]))

#############
# Functions #-------------------------------------------------------------------
#############
def clip_and_norm_volume(volume):
    volume = np.maximum(volume, 0) #ReLU operation
    volume = volume - np.min(volume)
    if np.max(volume)!=0:
        volume = volume / np.max(volume)
    return volume