import logging
from typing import List

from allennlp.common import Tqdm
from allennlp.common.checks import ConfigurationError
from allennlp.data.vocabulary import DEFAULT_OOV_TOKEN
from allennlp.modules.token_embedders.embedding import EmbeddingsTextFile
import numpy
import torch

from contexteval.contextualizers import Contextualizer

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

@Contextualizer.register("glove_contextualizer")
class GloveContextualizer(Contextualizer):
    """
    This "contextualizer" simply assigns each word to its pre-trained GloVe vector.

    Parameters
    ----------
    glove_path: str
        Path to the GloVe embeddings.
    embedding_dim: int
        The dimensionality of the GloVe embeddings.
    trainable: bool
        Whether to update the GloVe embeddings.
    """
    def __init__(self,
                 glove_path: str,
                 embedding_dim: int,
                 trainable: bool = False) -> None:
        super(GloveContextualizer, self).__init__()
        self.embedding_dim = embedding_dim
        self.trainable = trainable
        # Read the GloVe file, and produce a dictionary of tokens to indices, a dictionary
        # of indices to tokens, and a PyTorch Embedding object.
        self.token_to_idx = {DEFAULT_OOV_TOKEN: 0}
        self.idx_to_token = {0: DEFAULT_OOV_TOKEN}

        # First we read the embeddings from the file, only keeping vectors for the words we need.
        logger.info("Reading pretrained embeddings from file")
        embeddings = {}
        with EmbeddingsTextFile(glove_path) as embeddings_file:
            for line in Tqdm.tqdm(embeddings_file):
                token = line.split(' ', 1)[0]
                fields = line.rstrip().split(' ')
                if len(fields) - 1 != self.embedding_dim:
                    # Sometimes there are funny unicode parsing problems that lead to different
                    # fields lengths (e.g., a word with a unicode space character that splits
                    # into more than one column).  We skip those lines.  Note that if you have
                    # some kind of long header, this could result in all of your lines getting
                    # skipped.  It's hard to check for that here; you just have to look in the
                    # embedding_misses_file and at the model summary to make sure things look
                    # like they are supposed to.
                    logger.warning("Found line with wrong number of dimensions (expected: %d; actual: %d): %s",
                                   self.embedding_dim, len(fields) - 1, line)
                    continue

                vector = numpy.asarray(fields[1:], dtype='float32')
                embeddings[token] = vector
                self.token_to_idx[token] = len(self.token_to_idx)
                self.idx_to_token[len(self.idx_to_token)] = token
        if not embeddings:
            raise ConfigurationError("No embeddings of correct dimension found; you probably "
                                     "misspecified your embedding_dim parameter, or didn't "
                                     "pre-populate your Vocabulary")

        all_embeddings = numpy.asarray(list(embeddings.values()))
        embeddings_mean = float(numpy.mean(all_embeddings))
        embeddings_std = float(numpy.std(all_embeddings))
        # Now we initialize the weight matrix for an embedding layer, starting with random vectors,
        # then filling in the word vectors we just read.
        vocab_size = len(self.token_to_idx)
        logger.info("Initializing pre-trained embedding layer")
        embedding_matrix = torch.FloatTensor(vocab_size, self.embedding_dim).normal_(embeddings_mean,
                                                                                embeddings_std)
        # Start at 1, since the 0th token is OOV, and fill in the embedding matrix
        for i in range(1, vocab_size):
            embedding_matrix[i] = torch.FloatTensor(embeddings[self.idx_to_token[i]])
        self.weight = torch.nn.Parameter(embedding_matrix, requires_grad=self.trainable)

    def forward(self, sentences: List[List[str]]) -> torch.FloatTensor:
        """
        Parameters
        ----------
        sentences: List[List[str]]
            A batch of sentences. len(sentences) is the batch size, and each sentence
            itself is a list of strings (the constituent words). If the batch is padded,
            the expected padding token in the Python ``None``.

        Returns
        -------
        representations: List[FloatTensor]
            A list with the contextualized representations of all words in an input sentence.
            Each inner FloatTensor is of shape (seq_len, repr_dim), and an outer List
            is used to store the representations for each input sentence.
        """
        batch_representations = []

        for sentence in sentences:
            sentence_tensor = torch.FloatTensor(
                len(sentence), self.embedding_dim)
            for i, word in enumerate(sentence):
                word_index = self.token_to_idx.get(word, 0)
                sentence_tensor[i] = self.weight[word_index]
            batch_representations.append(sentence_tensor)
        return batch_representations


import shutil
import tempfile
import warnings

from .checks import close_to
from .models import simple_categorical, mv_simple, mv_simple_discrete, simple_2model, mv_prior_simple
from pymc3.sampling import assign_step_methods, sample
from pymc3.model import Model, Deterministic
from pymc3.step_methods import (NUTS, BinaryGibbsMetropolis, CategoricalGibbsMetropolis,
                                Metropolis, Slice, CompoundStep, NormalProposal,
                                MultivariateNormalProposal, HamiltonianMC,
                                EllipticalSlice, smc)
from pymc3.theanof import floatX
from pymc3.distributions import (
    Binomial, Normal, Bernoulli, Categorical, Beta, HalfNormal)

from numpy.testing import assert_array_almost_equal
import numpy as np
import numpy.testing as npt
import pytest
import theano
import theano.tensor as tt
from .helpers import select_by_precision

class TestStepMethods(object):  # yield test doesn't work subclassing object
    master_samples = {
        Slice: np.array([
            -8.13087389e-01, -3.08921856e-01, -6.79377098e-01, 6.50812585e-01, -7.63577596e-01,
            -8.13199793e-01, -1.63823548e+00, -7.03863676e-02, 2.05107771e+00, 1.68598170e+00,
            6.92463695e-01, -7.75120766e-01, -1.62296463e+00, 3.59722423e-01, -2.31421712e-01,
            -7.80686956e-02, -6.05860731e-01, -1.13000202e-01, 1.55675942e-01, -6.78527612e-01,
            6.31052333e-01, 6.09012517e-01, -1.56621643e+00, 5.04330883e-01, 3.14824082e-03,
            -1.31287073e+00, 4.10706927e-01, 8.93815792e-01, 8.19317020e-01, 3.71900919e-01,
            -2.62067312e+00, -3.47616592e+00, 1.50335041e+00, -1.05993351e+00, 2.41571723e-01,
            -1.06258156e+00, 5.87999429e-01, -1.78480091e-01, -3.60278680e-01, 1.90615274e-01,
            -1.24399204e-01, 4.03845589e-01, -1.47797573e-01, 7.90445804e-01, -1.21043819e+00,
            -1.33964776e+00, 1.36366329e+00, -7.50175388e-01, 9.25241839e-01, -4.17493767e-01,
            1.85311339e+00, -2.49715343e+00, -3.18571692e-01, -1.49099668e+00, -2.62079621e-01,
            -5.82376852e-01, -2.53033395e+00, 2.07580503e+00, -9.82615856e-01, 6.00517782e-01,
            -9.83941620e-01, -1.59014118e+00, -1.83931394e-03, -4.71163466e-01, 1.90073737e+00,
            -2.08929125e-01, -6.98388847e-01, 1.64502092e+00, -1.19525944e+00, 1.44424109e+00,
            1.52974876e+00, -5.70140077e-01, 5.08633322e-01, -1.70862492e-02, -1.69887948e-01,
            5.19760297e-01, -4.15149647e-01, 8.63685174e-02, -3.66805233e-01, -9.24988952e-01,
            2.33307122e+00, -2.60391496e-01, -5.86271814e-01, -5.01297170e-01, -1.53866195e+00,
            5.71285373e-01, -1.30571830e+00, 8.59587795e-01, 6.72170694e-01, 9.12433943e-01,
            7.04959179e-01, 8.37863464e-01, -5.24200836e-01, 1.28261340e+00, 9.08774240e-01,
            8.80566763e-01, 7.82911967e-01, 8.01843432e-01, 7.09251098e-01, 5.73803618e-01]),
        HamiltonianMC: np.array([
             0.40608634,  0.40608634,  0.04610354, -0.78588609,  0.03773683,
            -0.49373368,  0.21708042,  0.21708042, -0.14413517, -0.68284611,
             0.76299659,  0.24128663,  0.24128663, -0.54835464, -0.84408365,
            -0.82762589, -0.67429432, -0.67429432, -0.57900517, -0.97138029,
            -0.37809745, -0.37809745, -0.19333181, -0.40329098,  0.54999765,
             1.171515  ,  0.90279792,  0.90279792,  1.63830503, -0.90436674,
            -0.02516293, -0.02516293, -0.22177082, -0.28061216, -0.10158021,
             0.0807234 ,  0.16994063,  0.16994063,  0.4141503 ,  0.38505666,
            -0.25936504,  2.12074192,  2.24467132,  0.9628703 , -1.37044749,
             0.32983336, -0.55317525, -0.55317525, -0.40295662, -0.40295662,
            -0.40295662,  0.49076931,  0.04234407, -1.0002905 ,  0.99823615,
             0.99823615,  0.24915904, -0.00965061,  0.48519377,  0.21959942,
            -0.93094702, -0.93094702, -0.76812553, -0.73699981, -0.91834134,
            -0.91834134,  0.79522886, -0.04267669, -0.04267669,  0.51368761,
             0.51368761,  0.02255577,  0.70823409,  0.70823409,  0.73921198,
             0.30295007,  0.30295007,  0.30295007, -0.1300897 ,  0.44310964,
            -1.35839961, -1.55398633, -0.57323153, -0.57323153, -1.15435458,
            -0.17697793, -0.17697793,  0.2925856 , -0.56119025, -0.15360141,
             0.83715916, -0.02340449, -0.02340449, -0.63074456, -0.82745942,
            -0.67626237,  1.13814805, -0.81857813, -0.81857813,  0.26367166]),
        Metropolis: np.array([
            1.62434536, 1.01258895, 0.4844172, -0.58855142, 1.15626034, 0.39505344, 1.85716138,
            -0.20297933, -0.20297933, -0.20297933, -0.20297933, -1.08083775, -1.08083775,
            0.06388596, 0.96474191, 0.28101405, 0.01312597, 0.54348144, -0.14369126, -0.98889691,
            -0.98889691, -0.75448121, -0.94631676, -0.94631676, -0.89550901, -0.89550901,
            -0.77535005, -0.15814694, 0.14202338, -0.21022647, -0.4191207, 0.16750249, 0.45308981,
            1.33823098, 1.8511608, 1.55306796, 1.55306796, 1.55306796, 1.55306796, 0.15657163,
            0.3166087, 0.3166087, 0.3166087, 0.3166087, 0.54670343, 0.54670343, 0.32437529,
            0.12361722, 0.32191694, 0.44092559, 0.56274686, 0.56274686, 0.18746191, 0.18746191,
            -0.15639177, -0.11279491, -0.11279491, -0.11279491, -1.20770676, -1.03832432,
            -0.29776787, -1.25146848, -1.25146848, -0.93630908, -0.5857631, -0.5857631,
            -0.62445861, -0.62445861, -0.64907557, -0.64907557, -0.64907557, 0.58708846,
            -0.61217957, 0.25116575, 0.25116575, 0.80170324, 1.59451011, 0.97097938, 1.77284041,
            1.81940771, 1.81940771, 1.81940771, 1.81940771, 1.95710892, 2.18960348, 2.18960348,
            2.18960348, 2.18960348, 2.63096792, 2.53081269, 2.5482221, 1.42620337, 0.90910891,
            -0.08791792, 0.40729341, 0.23259025, 0.23259025, 0.23259025, 2.76091595, 2.51228118]),
        NUTS: np.array(
            [  1.11832371e+00,   1.11832371e+00,   1.11203151e+00,  -1.08526075e+00,
               2.58200798e-02,   2.03527183e+00,   4.47644923e-01,   8.95141642e-01,
               7.21867642e-01,   8.61681133e-01,   8.61681133e-01,   3.42001064e-01,
              -1.08109692e-01,   1.89399407e-01,   2.76571728e-01,   2.76571728e-01,
              -7.49542468e-01,  -7.25272156e-01,  -5.49940424e-01,  -5.49940424e-01,
               4.39045553e-01,  -9.79313191e-04,   4.08678631e-02,   4.08678631e-02,
              -1.17303762e+00,   4.15335470e-01,   4.80458006e-01,   5.98022153e-02,
               5.26508851e-01,   5.26508851e-01,   6.24759070e-01,   4.55268819e-01,
               8.70608570e-01,   6.56151353e-01,   6.56151353e-01,   1.29968043e+00,
               2.41336915e-01,  -7.78824784e-02,  -1.15368193e+00,  -4.92562283e-01,
              -5.16903724e-02,   4.05389240e-01,   4.05389240e-01,   4.20147769e-01,
               6.88161155e-01,   6.59273169e-01,  -4.28987827e-01,  -4.28987827e-01,
              -4.44203783e-01,  -4.61330842e-01,  -5.23216216e-01,  -1.52821368e+00,
               9.84049809e-01,   9.84049809e-01,   1.02081403e+00,  -5.60272679e-01,
               4.18620552e-01,   1.92542517e+00,   1.12029984e+00,   6.69152820e-01,
               1.56325611e+00,   6.64640934e-01,  -7.43157898e-01,  -7.43157898e-01,
              -3.18049839e-01,   6.87248073e-01,   6.90665184e-01,   1.63009949e+00,
              -4.84972607e-01,  -1.04859669e+00,   8.26455763e-01,  -1.71696305e+00,
              -1.39964174e+00,  -3.87677130e-01,  -1.85966115e-01,  -1.85966115e-01,
               4.54153291e-01,  -8.41705332e-01,  -8.46831314e-01,  -8.46831314e-01,
              -1.57419678e-01,  -3.89604101e-01,   8.15315055e-01,   2.81141081e-03,
               2.81141081e-03,   3.25839131e-01,   1.33638823e+00,   1.59391112e+00,
              -3.91174647e-01,  -2.60664979e+00,  -2.27637534e+00,  -2.81505065e+00,
              -2.24238542e+00,  -1.01648100e+00,  -1.01648100e+00,  -7.60912865e-01,
               1.44384812e+00,   2.07355127e+00,   1.91390340e+00,   1.66559696e+00]),
        smc.SMC: np.array([-0.94129179, -0.24527269, -0.21178402,  0.41638589,  0.98654812,
                           -1.98046225,  1.12342667,  1.6415898 , -1.1200565 ,  0.32953477,
                            1.39889789, -1.73171544,  0.0443656 ,  0.34281177, -0.56480463,
                           -0.73692101, -0.51636731, -0.03384773, -1.38884227,  1.19531008,
                            0.11847074,  0.48935035,  0.19281622, -0.29963799,  1.22910538,
                            0.7410759 , -1.19513787,  1.13016882,  0.0564043 , -0.61605194,
                           -0.0316803 ,  1.05364925,  0.69909548, -0.93454091,  0.52057634,
                            0.56511333,  0.68718385, -0.17977616,  0.0465673 , -0.17018186,
                            0.07703077,  0.75364315,  0.1199398 ,  0.99348863,  0.73182432,
                            1.01110192,  0.07782113,  0.15190815, -0.2146857 ,  0.70445444,
                            0.09726742,  0.47132485, -1.03145464,  0.44136444,  0.17668074,
                           -0.42707508, -0.50563966, -0.39789354, -0.47026749,  0.35764632,
                           -1.12201873, -0.44234106, -0.23588142,  0.94064406, -0.29664115,
                            0.50927203,  0.12289724,  0.37487975,  0.69245379,  0.93506873,
                           -0.38205722,  0.69079024,  0.15645901, -0.40054163, -0.56146871,
                            0.08198217,  1.33145711, -0.28730786,  0.97928876, -0.11001317,
                           -0.3198016 ,  0.29951594, -0.19081766,  0.50318481, -0.20971093,
                           -0.26810202, -0.73819079,  0.5627611 , -0.48295116, -0.1490258 ,
                           -0.25184255, -0.42765883,  1.45981846,  0.36949272,  1.1110567 ,
                            0.4984653 , -0.13902672,  0.18949919, -0.54360687, -0.2740874 ]),
    }

    def setup_class(self):
        self.temp_dir = tempfile.mkdtemp()

    def teardown_class(self):
        shutil.rmtree(self.temp_dir)

    def test_sample_exact(self):
        for step_method in self.master_samples:
            self.check_trace(step_method)

    def check_trace(self, step_method):
        """Tests whether the trace for step methods is exactly the same as on master.

        Code changes that effect how random numbers are drawn may change this, and require
        `master_samples` to be updated, but such changes should be noted and justified in the
        commit.

        This method may also be used to benchmark step methods across commits, by running, for
        example

        ```
        BENCHMARK=100000 ./scripts/test.sh -s pymc3/tests/test_step.py:TestStepMethods
        ```

        on multiple commits.
        """
        n_steps = 100
        with Model():
            x = Normal('x', mu=0, sd=1)
            if step_method.__name__ == 'SMC':
                Deterministic('like', - 0.5 * tt.log(2 * np.pi) - 0.5 * x.T.dot(x))
                trace = smc.ATMIP_sample(n_steps=n_steps, step=step_method(random_seed=1),
                                         n_jobs=1, progressbar=False,
                                         homepath=self.temp_dir)
            else:
                trace = sample(0, tune=n_steps,
                               discard_tuned_samples=False,
                               step=step_method(), random_seed=1)

        assert_array_almost_equal(
            trace.get_values('x'),
            self.master_samples[step_method],
            decimal=select_by_precision(float64=6, float32=4))

    def check_stat(self, check, trace, name):
        for (var, stat, value, bound) in check:
            s = stat(trace[var][2000:], axis=0)
            close_to(s, value, bound)

    def test_step_continuous(self):
        start, model, (mu, C) = mv_simple()
        unc = np.diag(C) ** .5
        check = (('x', np.mean, mu, unc / 10.),
                 ('x', np.std, unc, unc / 10.))
        with model:
            steps = (
                Slice(),
                HamiltonianMC(scaling=C, is_cov=True, blocked=False),
                NUTS(scaling=C, is_cov=True, blocked=False),
                Metropolis(S=C, proposal_dist=MultivariateNormalProposal, blocked=True),
                Slice(blocked=True),
                HamiltonianMC(scaling=C, is_cov=True),
                NUTS(scaling=C, is_cov=True),
                CompoundStep([
                    HamiltonianMC(scaling=C, is_cov=True),
                    HamiltonianMC(scaling=C, is_cov=True, blocked=False)]),
            )
        for step in steps:
            trace = sample(0, tune=8000,
                           discard_tuned_samples=False, step=step,
                           start=start, model=model, random_seed=1)
            self.check_stat(check, trace, step.__class__.__name__)

    def test_step_discrete(self):
        if theano.config.floatX == "float32":
            return  # Cannot use @skip because it only skips one iteration of the yield
        start, model, (mu, C) = mv_simple_discrete()
        unc = np.diag(C) ** .5
        check = (('x', np.mean, mu, unc / 10.),
                 ('x', np.std, unc, unc / 10.))
        with model:
            steps = (
                Metropolis(S=C, proposal_dist=MultivariateNormalProposal),
            )
        for step in steps:
            trace = sample(20000, tune=0, step=step, start=start, model=model, random_seed=1)
            self.check_stat(check, trace, step.__class__.__name__)

    def test_step_categorical(self):
        start, model, (mu, C) = simple_categorical()
        unc = C ** .5
        check = (('x', np.mean, mu, unc / 10.),
                 ('x', np.std, unc, unc / 10.))
        with model:
            steps = (
                CategoricalGibbsMetropolis(model.x, proposal='uniform'),
                CategoricalGibbsMetropolis(model.x, proposal='proportional'),
            )
        for step in steps:
            trace = sample(8000, tune=0, step=step, start=start, model=model, random_seed=1)
            self.check_stat(check, trace, step.__class__.__name__)

    def test_step_elliptical_slice(self):
        start, model, (K, L, mu, std, noise) = mv_prior_simple()
        unc = noise ** 0.5
        check = (('x', np.mean, mu, unc / 10.),
                 ('x', np.std, std, unc / 10.))
        with model:
            steps = (
                EllipticalSlice(prior_cov=K),
                EllipticalSlice(prior_chol=L),
            )
        for step in steps:
            trace = sample(5000, tune=0, step=step, start=start, model=model, random_seed=1)
            self.check_stat(check, trace, step.__class__.__name__)

class TestMetropolisProposal(object):
    def test_proposal_choice(self):
        _, model, _ = mv_simple()
        with model:
            s = np.ones(model.ndim)
            sampler = Metropolis(S=s)
            assert isinstance(sampler.proposal_dist, NormalProposal)
            s = np.diag(s)
            sampler = Metropolis(S=s)
            assert isinstance(sampler.proposal_dist, MultivariateNormalProposal)
            s[0, 0] = -s[0, 0]
            with pytest.raises(np.linalg.LinAlgError):
                sampler = Metropolis(S=s)

    def test_mv_proposal(self):
        np.random.seed(42)
        cov = np.random.randn(5, 5)
        cov = cov.dot(cov.T)
        prop = MultivariateNormalProposal(cov)
        samples = np.array([prop() for _ in range(10000)])
        npt.assert_allclose(np.cov(samples.T), cov, rtol=0.2)

class TestCompoundStep(object):
    samplers = (Metropolis, Slice, HamiltonianMC, NUTS)

    @pytest.mark.skipif(theano.config.floatX == "float32",
                        reason="Test fails on 32 bit due to linalg issues")
    def test_non_blocked(self):
        """Test that samplers correctly create non-blocked compound steps."""
        _, model = simple_2model()
        with model:
            for sampler in self.samplers:
                assert isinstance(sampler(blocked=False), CompoundStep)

    @pytest.mark.skipif(theano.config.floatX == "float32",
                        reason="Test fails on 32 bit due to linalg issues")
    def test_blocked(self):
        _, model = simple_2model()
        with model:
            for sampler in self.samplers:
                sampler_instance = sampler(blocked=True)
                assert not isinstance(sampler_instance, CompoundStep)
                assert isinstance(sampler_instance, sampler)

class TestAssignStepMethods(object):
    def test_bernoulli(self):
        """Test bernoulli distribution is assigned binary gibbs metropolis method"""
        with Model() as model:
            Bernoulli('x', 0.5)
            steps = assign_step_methods(model, [])
        assert isinstance(steps, BinaryGibbsMetropolis)

    def test_normal(self):
        """Test normal distribution is assigned NUTS method"""
        with Model() as model:
            Normal('x', 0, 1)
            steps = assign_step_methods(model, [])
        assert isinstance(steps, NUTS)

    def test_categorical(self):
        """Test categorical distribution is assigned categorical gibbs metropolis method"""
        with Model() as model:
            Categorical('x', np.array([0.25, 0.75]))
            steps = assign_step_methods(model, [])
        assert isinstance(steps, BinaryGibbsMetropolis)
        with Model() as model:
            Categorical('y', np.array([0.25, 0.70, 0.05]))
            steps = assign_step_methods(model, [])
        assert isinstance(steps, CategoricalGibbsMetropolis)

    def test_binomial(self):
        """Test binomial distribution is assigned metropolis method."""
        with Model() as model:
            Binomial('x', 10, 0.5)
            steps = assign_step_methods(model, [])
        assert isinstance(steps, Metropolis)

class TestNutsCheckTrace(object):
    def test_multiple_samplers(self, caplog):
        with Model():
            prob = Beta('prob', alpha=5, beta=3)
            Binomial('outcome', n=1, p=prob)
            with warnings.catch_warnings(record=True) as warns:
                sample(3, tune=2, discard_tuned_samples=False,
                       n_init=None)
            messages = [warn.message.args[0] for warn in warns]
            assert any("contains only 3" in msg for msg in messages)
            assert all('boolean index did not' not in msg for msg in messages)

    def test_bad_init(self):
        with Model():
            HalfNormal('a', sd=1, testval=-1, transform=None)
            with pytest.raises(ValueError) as error:
                sample(init=None)
            error.match('Bad initial')

    def test_linalg(self, caplog):
        with Model():
            a = Normal('a', shape=2)
            a = tt.switch(a > 0, np.inf, a)
            b = tt.slinalg.solve(floatX(np.eye(2)), a)
            Normal('c', mu=b, shape=2)
            with warnings.catch_warnings(record=True) as warns:
                trace = sample(20, init=None, tune=5)
            assert np.any(trace['diverging'])
            assert any('diverging samples after tuning' in str(warn.message)
                       for warn in warns)
            assert any('contains only' in str(warn.message) for warn in warns)

            with pytest.raises(SamplingError):
                sample(20, init=None, nuts_kwargs={'on_error': 'raise'})

import numpy,from numpy,import theano,from pymc3


import argparse
import os
import utils
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from param_stamp import get_param_stamp_from_args
from param_values import set_default_values

description = 'Compare CL strategies using various metrics on each scenario of permuted or split MNIST.'
parser = argparse.ArgumentParser('./create_result.py', description=description)
parser.add_argument('--seed', type=int, default=1, help='[first] random seed (for each random-module used)')
parser.add_argument('--n-seeds', type=int, default=10, help='how often to repeat?')
parser.add_argument('--no-gpus', action='store_false', dest='cuda', help="don't use GPUs")
parser.add_argument('--data-dir', type=str, default='./datasets', dest='d_dir', help="default: %(default)s")
parser.add_argument('--plot-dir', type=str, default='./plots', dest='p_dir', help="default: %(default)s")
parser.add_argument('--results-dir', type=str, default='../benchmark_new', dest='r_dir', help="default: %(default)s")

# expirimental task parameters.
task_params = parser.add_argument_group('Task Parameters')
task_params.add_argument('--experiment', type=str, default='splitMNIST',
                         choices=['rotMNIST', 'permMNIST', 'splitMNIST', 'CIFAR10', 'CIFAR100'])
task_params.add_argument('--scenario', type=str, default='task', choices=['task', 'domain', 'class'])
task_params.add_argument('--tasks', type=int, help='number of tasks')
# specify loss functions to be used
loss_params = parser.add_argument_group('Loss Parameters')
loss_params.add_argument('--loss', type=str, default='none',
                         choices=['otfl', 'fgfl', 'focal', 'ce', 'gbfg', 'none'])
loss_params.add_argument('--bce', action='store_true', help="use binary (instead of multi-class) classication loss")

# model architecture parameters
model_params = parser.add_argument_group('Parameters Main Model')
model_params.add_argument('--fc-layers', type=int, default=3, dest='fc_lay', help="# of fully-connected layers")
model_params.add_argument('--fc-units', type=int, metavar="N", help="# of units in first fc-layers")
model_params.add_argument('--fc-drop', type=float, default=0., help="dropout probability for fc-units")
model_params.add_argument('--fc-bn', type=str, default="no", help="use batch-norm in the fc-layers (no|yes)")
model_params.add_argument('--fc-nl', type=str, default="relu", choices=["relu", "leakyrelu"])
model_params.add_argument('--singlehead', action='store_true', help="for Task-IL: use a 'single-headed' output layer   "
                                                                   " (instead of a 'multi-headed' one)")
model_params.add_argument('--use_teacher', action='store_true', help='Using an offline teacher for distill from memory')
model_params.add_argument('--teacher_epochs', type=int, default=100, help='number of epochs to train teacher')
model_params.add_argument('--teacher_loss', type=str, default='CE', help='teacher loss function')
model_params.add_argument('--teacher_split', type=float, default=0.8, help='split ratio for teacher training')
model_params.add_argument('--teacher_opt', type=str, default='Adam', help='teacher optimizer')
model_params.add_argument('--use_scheduler', action='store_true', help='Using learning rate scheduler for teacher')
model_params.add_argument('--use_augment', action='store_true', help='Using data augmentation for training teacher')
model_params.add_argument('--distill_type', type=str, default='E', choices=['T', 'TS', 'E', 'ET', 'ES', 'ETS'])
model_params.add_argument('--multi_negative', type=utils.str_to_bool, default=False)
model_params.add_argument('--update_teacher_kd', type=utils.str_to_bool, default=True)
model_params.add_argument('--online_kd', type=utils.str_to_bool, default=False)
# training hyperparameters / initialization
train_params = parser.add_argument_group('Training Parameters')
train_params.add_argument('--iters', type=int, help="# batches to optimize solver")
train_params.add_argument('--lr', type=float, help="learning rate")
train_params.add_argument('--batch', type=int, default=128, help="batch-size")
train_params.add_argument('--optimizer', type=str, choices=['adam', 'adam_reset', 'sgd'], default='adam')

# "memory replay" parameters
replay_params = parser.add_argument_group('Replay Parameters')
replay_params.add_argument('--temp', type=float, default=2., dest='temp', help="temperature for distillation")
replay_params.add_argument('--triplet-selection', type=str, default='HP-HN-1', help="Triplet selection strategy")
replay_params.add_argument('--mem_online', type=utils.str_to_bool, default=False, help='icarl using online exemplar mamagement')
# -generative model parameters (if separate model)
genmodel_params = parser.add_argument_group('Generative Model Parameters')
genmodel_params.add_argument('--g-z-dim', type=int, default=100, help='size of latent representation (default: 100)')
genmodel_params.add_argument('--g-fc-lay', type=int, help='[fc_layers] in generator (default: same as classifier)')
genmodel_params.add_argument('--g-fc-uni', type=int, help='[fc_units] in generator (default: same as classifier)')
# - hyper-parameters for generative model (if separate model)
gen_params = parser.add_argument_group('Generator Hyper Parameters')
gen_params.add_argument('--g-iters', type=int, help="# batches to train generator (default: as classifier)")
gen_params.add_argument('--lr-gen', type=float, help="learning rate generator (default: lr)")

# "memory allocation" parameters
cl_params = parser.add_argument_group('Memory Allocation Parameters')
cl_params.add_argument('--lambda', type=float, dest="ewc_lambda", help="--> EWC: regularisation strength")
cl_params.add_argument('--o-lambda', type=float, help="--> online EWC: regularisation strength")
cl_params.add_argument('--fisher-n', type=int, help="--> EWC: sample size estimating Fisher Information")
cl_params.add_argument('--gamma', type=float, help="--> EWC: forgetting coefficient (for 'online EWC')")
cl_params.add_argument('--emp-fi', action='store_true', help="--> EWC: estimate FI with provided labels")
cl_params.add_argument('--c', type=float, dest="si_c", help="--> SI: regularisation strength")
cl_params.add_argument('--epsilon', type=float, default=0.1, dest="epsilon", help="--> SI: dampening parameter")
cl_params.add_argument('--gating-prop', type=float, metavar="PROP", help="--> XdG: prop neurons per layer to gate")

# iCaRL parameters
icarl_params = parser.add_argument_group('iCaRL Parameters')
icarl_params.add_argument('--budget', type=int, default=2000, dest="budget", help="how many exemplars can be stored?")
icarl_params.add_argument('--herding', action='store_true', help="use herding to select exemplars (instead of random)")
icarl_params.add_argument('--use-exemplars', action='store_true', help="use stored exemplars for classification?")
icarl_params.add_argument('--norm-exemplars', action='store_true', help="normalize features/averages of exemplars")
icarl_params.add_argument('--otr-exemplars', type=bool, default=False, help="use otr exemplars instead of random")
# evaluation parameters
eval_params = parser.add_argument_group('Evaluation Parameters')
eval_params.add_argument('--time', action='store_true', help="keep track of total training time")
eval_params.add_argument('--pdf', action='store_true', help="generate pdfs for individual experiments")
eval_params.add_argument('--visdom', action='store_true', help="use visdom for on-the-fly plots")
eval_params.add_argument('--prec-n', type=int, default=1024, help="# samples for evaluating solver's precision")
eval_params.add_argument('--sample-n', type=int, default=64, help="# images to show")

def get_results(args):
    # -get param-stamp
    param_stamp = get_param_stamp_from_args(args)
    # -check whether already run; if not do so
    if os.path.isfile("{}/dict-{}.pkl".format(args.r_dir, param_stamp)):
        print("{}: already run".format(param_stamp))
    else:
        print("{}: ...missing...".format(param_stamp))
        raise Exception('Missing results !!!!')
    # -get results-dict
    dict = utils.load_object("{}/dict-{}".format(args.r_dir, param_stamp))
    # -get average precision
    fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp)
    file = open(fileName)
    ave = float(file.readline())
    file.close()
    # -print average precision on screen
    print("--> average precision: {}".format(ave))
    # -return average precision
    return (dict, ave)

def collect_all(method_dict, seed_list, args, name=None):
    # -print name of method on screen
    if name is not None:
        print("\n------{}------".format(name))
    # -run method for all random seeds
    for seed in seed_list:
        args.seed = seed
        method_dict[seed] = get_results(args)
    # -return updated dictionary with results
    return method_dict

def autolabel(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{:.3f}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 12),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize='small')

if __name__ == '__main__':

    # Load input-arguments
    scenarios = ['task', 'domain', 'class']
    data = []
    errs = []
    for s in scenarios:
        args = parser.parse_args()
        args.r_dir = '{}/{}/{}'.format(args.r_dir, args.experiment, s)
        args.scenario = s
        # -set default-values for certain arguments based on chosen scenario & experiment
        args = set_default_values(args)
        # -set other default arguments
        args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen
        args.g_iters = args.iters if args.g_iters is None else args.g_iters
        args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay
        args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni

        # Add non-optional input argument that will be the same for all runs
        args.metrics = True
        args.feedback = False
        args.log_per_task = True

        # Add input arguments that will be different for different runs
        args.distill = False
        args.agem = False
        args.ewc = False
        args.online = False
        args.si = False
        args.xdg = False
        args.add_exemplars = False
        args.bce_distill= False
        args.icarl = False
        # args.seed could of course also vary!

        #-------------------------------------------------------------------------------------------------#

        #--------------------------#
        #----- RUN ALL MODELS -----#
        #--------------------------#

        seed_list = list(range(args.seed, args.seed + args.n_seeds))

        # Experience Replay + Herding
        args.replay = "exemplars"
        args.herding = True
        args.norm_exemplars = True
        args.mem_online = True
        args.budget = 2000
        ERH = {}
        ERH = collect_all(ERH, seed_list, args, name="ER+herding")
        args.replay = "none"
        args.herding = False
        args.norm_exemplars = False
        args.mem_online = False

        # Experience Replay
        args.replay = "exemplars"
        args.herding = False
        args.mem_online = True
        ER = {}
        ER = collect_all(ER, seed_list, args, name="ER")
        args.replay = "none"
        args.mem_online = False

        # Online Replay
        args.replay = 'online'
        args.budget = 2000
        args.triplet_selection = 'HP-HN-1'
        args.bce = True
        if args.scenario == 'class':
            args.bce_distill = True
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False
        OTR = {}
        OTR = collect_all(OTR, seed_list, args, name='OTR (HP-HN)')
        args.replay = 'none'
        args.bce = False
        args.bce_distill = False
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False

        # Online Replay
        args.replay = 'online'
        args.budget = 2000
        args.triplet_selection = 'HP-SHN-1'
        args.bce = True
        if args.scenario == 'class':
            args.bce_distill = True
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False
        OTR_HPSHN = {}
        OTR_HPSHN = collect_all(OTR_HPSHN, seed_list, args, name='OTR (HP-SHN)')
        args.replay = 'none'
        args.bce = False
        args.bce_distill = False
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False

        # Online Replay
        args.replay = 'online'
        args.budget = 2000
        args.triplet_selection = 'EP-HN-1'
        args.bce = True
        if args.scenario == 'class':
            args.bce_distill = True
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False
        OTR_EPHN = {}
        OTR_EPHN = collect_all(OTR_EPHN, seed_list, args, name='OTR (EP-HN)')
        args.replay = 'none'
        args.bce = False
        args.bce_distill = False
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False

        # Online Replay
        args.replay = 'online'
        args.budget = 2000
        args.triplet_selection = 'EP-SHN-1'
        args.bce = True
        if args.scenario == 'class':
            args.bce_distill = True
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False
        OTR_EPSHN = {}
        OTR_EPSHN = collect_all(OTR_EPSHN, seed_list, args, name='OTR (EP-SHN)')
        args.replay = 'none'
        args.bce = False
        args.bce_distill = False
        args.use_embeddings = False
        args.multi_negative = False
        args.add_exemplars = False

        # Drawing line graph between replay using memory methods
        acc_ER = []
        acc_ERH = []
        acc_OTR = []
        acc_OTR_HPSHN = []
        acc_OTR_EPHN = []
        acc_OTR_EPSHN = []

        for m in seed_list:
            ## AVERAGE TEST ACCURACY
            acc_ER.append(ER[m][1])
            acc_ERH.append(ERH[m][1])
            acc_OTR.append(OTR[m][1])
            acc_OTR_HPSHN.append(OTR_HPSHN[m][1])
            acc_OTR_EPHN.append(OTR_EPHN[m][1])
            acc_OTR_EPSHN.append(OTR_EPSHN[m][1])

        # Calculate the average
        ER_mean = np.mean(acc_ER)
        ERH_mean = np.mean(acc_ERH)
        OTR_mean = np.mean(acc_OTR)
        OTR_HPSHN_mean = np.mean(acc_OTR_HPSHN)
        OTR_EPHN_mean = np.mean(acc_OTR_EPHN)
        OTR_EPSHN_mean = np.mean(acc_OTR_EPSHN)
        print(ER_mean, ERH_mean, OTR_mean, OTR_HPSHN_mean, OTR_EPHN_mean, OTR_EPSHN_mean)
        # Calculate the standard deviation
        ER_std = np.std(acc_ER)
        ERH_std = np.std(acc_ERH)
        OTR_std = np.std(acc_OTR)
        OTR_HPSHN_std = np.std(acc_OTR_HPSHN)
        OTR_EPHN_std = np.std(acc_OTR_EPHN)
        OTR_EPSHN_std = np.std(acc_OTR_EPSHN)

        data.append([ERH_mean, ERH_mean, OTR_mean, OTR_HPSHN_mean, OTR_EPHN_mean, OTR_EPSHN_mean ])
        errs.append([ER_std, ERH_std, OTR_std, OTR_HPSHN_std, OTR_EPHN_std, OTR_EPSHN_std])

    # Create lists for the plot
    methods = ['ER', 'Herding', 'OTR (HP-HN)', 'OTR (HP-SHN)', 'OTR (EP-HN)', 'OTR (EP-SHN)']
    x_pos = np.arange(len(methods))

    # Build the plot
    fig, ax = plt.subplots(figsize=(12, 7))
    task_bar = ax.bar(x_pos - 0.25, data[0], yerr=errs[0], color='#07575B', ecolor='gray',
                      alpha=1, label='Task-IL', width=0.25)
    domain_bar = ax.bar(x_pos, data[1], yerr=errs[1], color='#C4DFE6', ecolor='gray',
                        alpha=1, label='Domain-IL', width=0.25)
    class_bar = ax.bar(x_pos + 0.25, data[2], yerr=errs[2], color='#6FB98F', ecolor='gray',
                       alpha=1, label='Class-IL', width=0.25)

    autolabel(task_bar)
    autolabel(domain_bar)
    autolabel(class_bar)

    ax.set_ylabel('Test accuracy')
    ax.set_xticks(x_pos)
    ax.set_xticklabels(methods)
    # ax.set_title('Memory sampling methods comparison')
    ax.legend(loc='lower center', ncol=3, bbox_to_anchor=(0.5, -0.2))

    plt.xticks(rotation=30, ha='right')
    plot_margin = 0.1
    x0, x1, y0, y1 = plt.axis()
    plt.axis((x0, x1, y0, y1 + plot_margin))
    plt.tight_layout()
    # Save the figure and show
    # plt.show()
    plt.savefig('{}_selection.png'.format(args.experiment))


# -*- coding: utf-8 -*-

import pandas as pd
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import metrics
from sklearn.neighbors import NearestCentroid
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from nltk.stem import *
from numpy import concatenate
from sklearn.ensemble import RandomForestClassifier
import HTMLParser
from sklearn.externals import joblib
#from bs4 import BeautifulSoup
#import Stemmer

'''
1 = LOC
2 = ORG
3 = PER
'''
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression as LR
#N = 10
#a = np.random.rand(N,N)
#b = np.zeros((N,N+1))
#b[:,:-1] = a

from horus.core.config import HorusConfig
config = HorusConfig()

#CONST_WIKI_PER_FILE = config.dataset_path + '/Wikipedia/wiki_PER2.csv'
#CONST_WIKI_LOC_FILE = config.dataset_path + '/Wikipedia/wiki_LOC2.csv'
#CONST_WIKI_ORG_FILE = config.dataset_path + '/Wikipedia/wiki_ORG2.csv'
CONST_WIKI_ALL = config.dataset_path + '/Wikipedia/wiki_3classes2.csv'

#train_ds = pd.read_csv(CONST_WIKI_PER_FILE, header=0, quoting=3) #engine='python' sep='\t',
#train_ds_loc = np.genfromtxt(CONST_WIKI_LOC_FILE, delimiter="|\-/|", skip_header=1, dtype={'names': ('klass', 'text'), 'formats': (np.int, '|S1000')})
#train_ds_org = np.genfromtxt(CONST_WIKI_ORG_FILE, delimiter="|\-/|", skip_header=1, dtype={'names': ('klass', 'text'), 'formats': (np.int, '|S1000')})
#train_ds_per = np.genfromtxt(CONST_WIKI_PER_FILE, delimiter="|\-/|", skip_header=1, dtype={'names': ('klass', 'text'), 'formats': (np.int, '|S1000')})
train_ds = np.genfromtxt(CONST_WIKI_ALL, delimiter="|\-/|", skip_header=1, dtype={'names': ('klass', 'text'), 'formats': (np.int, '|S1000')})

html_parser = HTMLParser.HTMLParser()
train_ds_clean = html_parser.unescape(train_ds)
print train_ds_clean.shape

Xtrain = np.array(train_ds)

#english_stemmer = Stemmer.Stemmer('en')
#Xtrain = concatenate((train_ds_loc, train_ds_org, train_ds_per),axis=0)
#print ds

#X = np.empty(shape=(len(train_ds), 1)) #numpy.zeros or numpy.ones or numpy.empty
#X = []
#print X.shape
#for example in ds:
#    #temp = example[0].split("|-|")
#    #X.append(([temp[0]], [temp[1]]))
#    #np.append(X, [temp[0], temp[1]], axis=0)
#    X.append((example[0], example[1]))

#loc_data = [x + [1] for x in loc_data]
#for row in loc_data:
#    row.append(1)

''' count words '''

from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

if 1==2:
    #count_vect = CountVectorizer(stop_words='english', lowercase=True, strip_accents='unicode', encoding='utf-8',
    #                             decode_error='ignore')

    #X_train_counts = count_vect.fit_transform(Xtrain['text'])
    #tfidf_transformer = TfidfTransformer()
    #X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

    vectorizer = TfidfVectorizer(stop_words='english', lowercase=True, strip_accents='unicode', encoding='utf-8',
                                 decode_error='ignore')
    nb = BernoulliNB()
    model = Pipeline([('vectorizer', vectorizer), ('nb', nb)])
    clf = model.fit(Xtrain['text'], Xtrain['klass'])
    _ = joblib.dump(clf, 'nbpipelinetfidf.pkl', compress=3)

docs_per2 = [['A Última Vez Que Vi Richard ilustração. quadrinhos Diego Esteves . Portfólio', 3],
             ['Diego Esteves’ berufliches Profil anzeigen LinkedIn ist das weltweit größte berufliche Netzwerk, das Fach- und Führungskräften wie Diego Esteves ... Diego Esteves | LinkedIn', 3],
             ['Diego Esteves. Systems Engineer at Dell. Location Austin, Texas Area Industry Information Technology and Services Diego Esteves | LinkedIn', 3],
             ['The latest Tweets from Diego Esteves (@estevesdiego): "La Chiqui nada dijo hasta ahora del contrato de $1,5 millones de su productor general y nieto, Nacho ... Diego Esteves (@estevesdiego) | Twitter', 3],
             ['View the profiles of professionals named Diego Esteves on LinkedIn. There are 57 professionals named Diego Esteves, who use LinkedIn to exchange ... Top 10 Diego Esteves profiles | LinkedIn', 3],
             ['The latest Tweets from Diego Esteves (@DiegoEsteves_). Editor de vídeo, Videomaker e Fotografo em Rossini s Imagens http://t.co/nRf00hc4o5 apaixonado por ... Diego Esteves (@DiegoEsteves_) | Twitter', 3],
             ['Diego Esteves is a PhD Student at the University of Leipzig. Diego’s research interests are in the area of Fact Validation on the Web. Research Interests Diego Esteves – Smart Data Analytics - sda.cs.uni-bonn.de', 3],
             ['Email: diegoesteves3d@gmail.com Phone: +55 011-96971-1500 Brazil, Santo André - SP Diego Esteves', 3],
             ['Diego Esteves is on Facebook. Join Facebook to connect with Diego Esteves and others you may know. Facebook gives people the power to share and makes the... Diego Esteves | Facebook', 3],
             ['Ähnliche XING-Profile wie das von DIEGO ESTEVES ALMANZA. Martin Reyes Rico Gerente Comercial Mexico adrian soriano carrillo ... DIEGO ESTEVES ALMANZA - GERENTE - xing.com', 3]]

clf = joblib.load(config.models_text_root + 'nbpipelinetfidf.pkl')
print clf.predict('A Última Vez Que Vi Richard ilustração. quadrinhos Diego Esteves')
#exit(0)

count_vect = CountVectorizer(stop_words='english', lowercase=True, strip_accents='unicode',
                             encoding='utf-8', decode_error='ignore')
X_train_counts = count_vect.fit_transform(Xtrain['text'])
print X_train_counts.shape
joblib.dump(X_train_counts, config.models_text_root + 'xtrain-counts.data', compress=3)
#print count_vect.vocabulary_.get(u'algorithm')

''' tf-idf '''

#tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)
#X_train_tf = tf_transformer.transform(X_train_counts)
#print X_train_tf.shape
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
print X_train_tfidf.shape
joblib.dump(X_train_counts, config.models_text_root + 'xtrain-counts-tfidf.data', compress=3)

''' feature_extraction '''

clf = MultinomialNB().fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf, config.models_text_root + 'multiNB.pkl', compress=3)

clf2 = BernoulliNB().fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf2, config.models_text_root + 'bernoulliNB.pkl', compress=3)

clf3 = SGDClassifier(alpha=.0001, n_iter=50, penalty="elasticnet").fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf3, config.models_text_root + 'sgd.pkl', compress=3)

clf4 = LinearSVC(loss='l2', dual=False, tol=1e-3).fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf4, config.models_text_root + 'linearSVCloss.pkl', compress=3)

clf5 = MultinomialNB(alpha=.01).fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf5, config.models_text_root + 'multiNBa01.pkl', compress=3)

clf6 = BernoulliNB(alpha=.01).fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf6, config.models_text_root + 'bernoulliNBa01.pkl', compress=3)

clf7 = NearestCentroid().fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf7, config.models_text_root + 'NC.pkl', compress=3)

clf8 = LinearSVC().fit(X_train_tfidf, np.array(Xtrain['klass'], dtype=np.int32))
joblib.dump(clf8, config.models_text_root + 'linearSVC.pkl', compress=3)

''' testing DBPedia '''
colnames = ['object', 'abstract']
df1 = pd.read_csv((config.dataset_path + "/DBPedia/dbo_LOC.csv"), sep=',',
                       names=colnames, header=0,
                       dtype={"object": str, "abstract": str})
df2 = pd.read_csv((config.dataset_path + "/DBPedia/dbo_ORG.csv"), sep=',',
                       names=colnames, header=0,
                       dtype={"object": str, "abstract": str})
df3 = pd.read_csv((config.dataset_path + "/DBPedia/dbo_PER.csv"), sep=',',
                       names=colnames, header=0,
                       dtype={"object": str, "abstract": str})
#loc_data = data.abstract.tolist()

df1['klass'] = 1
df2['klass'] = 2
df3['klass'] = 3

frames = [df1, df2, df3]
dffinal = pd.concat(frames)

Y_test = dffinal.klass.tolist()

X_new_counts = count_vect.transform(dffinal.abstract.tolist())
X_new_tfidf = tfidf_transformer.transform(X_new_counts)

predicted = clf.predict(X_new_tfidf)
predicted2 = clf2.predict(X_new_tfidf)
predicted3 = clf3.predict(X_new_tfidf)
predicted4 = clf4.predict(X_new_tfidf)
predicted5 = clf5.predict(X_new_tfidf)
predicted6 = clf6.predict(X_new_tfidf)
predicted7 = clf7.predict(X_new_tfidf)
predicted8 = clf8.predict(X_new_tfidf)

target_names = ['LOC', 'ORG', 'PER']
print(classification_report(Y_test, predicted, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted2, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted3, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted4, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted5, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted6, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted7, target_names=target_names, digits=3))
print(classification_report(Y_test, predicted8, target_names=target_names, digits=3))

''' testing '''

docs_loc = [['Belo Horizonte is the capital of the state of Minas Gerais, Brazil s second most populous state. ... The city features a mixture of contemporary and classical buildings, and is home to several modern Brazilian architectural icons', 1],
            ['Angola is a country in Southern Africa. It is the seventh-largest country in Africa, and is bordered by Namibia to the', 1],
            ['officially the Macao Special Administrative Region of the People s Republic of China, is an autonomous territory on', 1],
            ['The Amazon rainforest also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South', 1],
            ['New Zealand is an island nation in the southwestern Pacific Ocean. The country geographically ', 1],
            ['Japan is an island country in East Asia. Located in the Pacific Ocean, it lies to the east of the ... Japan is pronounced Nippon or Nihon. The English word Japan possibly derives from the historical', 1]]

docs_org = [['Bayern Munich is a German sports club based in Munich, Bavaria, Germany. It is best known for its professional football team, which plays in the Bundesliga ...', 2],
            ['Microsoft Corporation is an American multinational technology company headquartered in Redmond, Washington, that ...', 2],
            ['Burger King (often abbreviated as BK) is an American global chain of hamburger fast food restaurants, headquartered in the unincorporated area of Miami-Dade ...', 2],
            ['Starbucks Corporation is an American coffee company and coffeehouse chain. Starbucks was founded in Seattle, Washington in 1971. Today it operates 23,768 ...', 2],
            ['SC Victoria Hamburg is a German association football club from the city of Hamburg. ... It was one of the founding members of the DFB', 2],
            ['A strip mall is an open-air shopping mall where the stores are arranged in a row, with a sidewalk in front. Strip malls are typically developed as a unit and have...', 2]]

docs_per = [['Michael Hoffman, Director: One Fine Day. Michael Hoffman was born on November 30, 1956 in Honolulu, Hawaii, USA. He is a director and writer, known for ...', 3],
            ['By now you must have seen Michael Hoffman, all of Michael Hoffman, every nook and cranny of Michael Hoffman. The bodybuilder, tumblr star, ', 3],
            ['Michael Lynn Hoffman (born November 30, 1956) is an American film director. Contents. [hide]. 1 Early life and education; 2 Career; 3 Personal life ...', 3],
            ['Revisionist historian Michael Hoffman "A scrupulously liberating desire for truth-telling." Independent History & Research Box 849  Coeur d', 3],
            ['Remember Michael Hoffman? He is the tattooed beefcake who made ripples across the web when a video of him pleasuring the muscle ...', 3],
            [' Here is a transcript of Hoffman s apology video: Fuuuccckkk! Hey everyone, it is Michael. It is been quite a while since I made a video', 3]]

docs_per2 = [['A Última Vez Que Vi Richard ilustração. quadrinhos Diego Esteves . Portfólio', 3],
             ['Diego Esteves’ berufliches Profil anzeigen LinkedIn ist das weltweit größte berufliche Netzwerk, das Fach- und Führungskräften wie Diego Esteves ... Diego Esteves | LinkedIn', 3],
             ['Diego Esteves. Systems Engineer at Dell. Location Austin, Texas Area Industry Information Technology and Services Diego Esteves | LinkedIn', 3],
             ['The latest Tweets from Diego Esteves (@estevesdiego): "La Chiqui nada dijo hasta ahora del contrato de $1,5 millones de su productor general y nieto, Nacho ... Diego Esteves (@estevesdiego) | Twitter', 3],
             ['View the profiles of professionals named Diego Esteves on LinkedIn. There are 57 professionals named Diego Esteves, who use LinkedIn to exchange ... Top 10 Diego Esteves profiles | LinkedIn', 3],
             ['The latest Tweets from Diego Esteves (@DiegoEsteves_). Editor de vídeo, Videomaker e Fotografo em Rossini s Imagens http://t.co/nRf00hc4o5 apaixonado por ... Diego Esteves (@DiegoEsteves_) | Twitter', 3],
             ['Diego Esteves is a PhD Student at the University of Leipzig. Diego’s research interests are in the area of Fact Validation on the Web. Research Interests Diego Esteves – Smart Data Analytics - sda.cs.uni-bonn.de', 3],
             ['Email: diegoesteves3d@gmail.com Phone: +55 011-96971-1500 Brazil, Santo André - SP Diego Esteves', 3],
             ['Diego Esteves is on Facebook. Join Facebook to connect with Diego Esteves and others you may know. Facebook gives people the power to share and makes the... Diego Esteves | Facebook', 3],
             ['Ähnliche XING-Profile wie das von DIEGO ESTEVES ALMANZA. Martin Reyes Rico Gerente Comercial Mexico adrian soriano carrillo ... DIEGO ESTEVES ALMANZA - GERENTE - xing.com', 3]]

docs_new = np.append(docs_loc, docs_org, axis=0)
docs_new = np.append(docs_new, docs_per, axis=0)
docs_new = np.append(docs_new, docs_per2, axis=0)

X_new_counts = count_vect.transform(docs_new[:, 0])
X_new_tfidf = tfidf_transformer.transform(X_new_counts)

predicted = clf.predict(X_new_tfidf)
predicted2 = clf2.predict(X_new_tfidf)
predicted3 = clf3.predict(X_new_tfidf)
predicted4 = clf4.predict(X_new_tfidf)
predicted5 = clf5.predict(X_new_tfidf)
predicted6 = clf6.predict(X_new_tfidf)
predicted7 = clf7.predict(X_new_tfidf)
predicted8 = clf8.predict(X_new_tfidf)

hit1, hit2, hit3, hit4, hit5, hit6, hit7, hit8 = 0, 0, 0, 0, 0, 0, 0, 0
tot = len(docs_new)

print '*********************************'
print 'predictions model 1'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit1 = hit1 + 1
print 'accuracy is %s' % str(float(hit1) / tot)

print '*********************************'
print 'predictions model 2'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted2):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit2 = hit2 + 1
print 'accuracy is %s' % str(float(hit2) / tot)

print '*********************************'
print 'predictions model 3'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted3):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit3 = hit3 + 1
print 'accuracy is %s' % str(float(hit3) / tot)

print '*********************************'
print 'predictions model 4'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted4):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit4 = hit4 + 1
print 'accuracy is %s' % str(float(hit4) / tot)

print '*********************************'
print 'predictions model 5'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted5):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit5 = hit5 + 1
print 'accuracy is %s' % str(float(hit5) / tot)

print '*********************************'
print 'predictions model 6'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted6):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit6 = hit6 + 1
print 'accuracy is %s' % str(float(hit6) / tot)

print '*********************************'
print 'predictions model 7'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted7):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit7 = hit7 + 1
print 'accuracy is %s' % str(float(hit7) / tot)

print '*********************************'
print 'predictions model 8'
for doc, real, category in zip(docs_new[:, 0], docs_new[:][:,1], predicted8):
    #print('%r = %s => %s' % (doc[:20], real, category))
    if int(real) == int(category):
        hit8 = hit8 + 1
print 'accuracy is %s' % str(float(hit8) / tot)


import torch
import torch.nn as nn
import os
import os.path as osp
import numpy as np
from . import networks
import torch.optim as optim
from models.optimizer import adabound
from torch_geometric.utils import remove_self_loops, contains_self_loops, contains_isolated_nodes
import hiddenlayer as hl
import cv2

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUTPUT_DIR = os.path.join(ROOT_DIR, "feature_img")
print(OUTPUT_DIR)
class mesh_graph:
    def __init__(self, opt):
        self.opt = opt
        self.cuda = opt.cuda
        self.is_train = opt.is_train
        self.device = torch.device('cuda:{}'.format(
            self.cuda[0]) if self.cuda else 'cpu')
        self.save_dir = osp.join(opt.ckpt_root, opt.name)
        self.optimizer = None
        self.loss = None

        # init mesh data
        self.nclasses = opt.nclasses

        # init network
        self.net = networks.get_net(opt)
        self.net.train(self.is_train)

        # criterion
        self.loss = networks.get_loss(self.opt).to(self.device)

        if self.is_train:
            # self.optimizer = adabound.AdaBound(
            #     params=self.net.parameters(), lr=self.opt.lr, final_lr=self.opt.final_lr)
            self.optimizer = optim.SGD(self.net.parameters(
            ), lr=opt.lr, momentum=opt.momentum, weight_decay=opt.weight_decay)
            self.scheduler = networks.get_scheduler(self.optimizer, self.opt)
        if not self.is_train or opt.continue_train:
            self.load_state(opt.last_epoch)

        # A History object to store metrics
        self.history = hl.History()
        # A Canvas object to draw the metrics
        self.canvas = hl.Canvas()

    def test(self):
        """tests model
        returns: number correct and total number
        """
        with torch.no_grad():
            out, fea = self.forward()
            # compute number of correct
            pred_class = torch.max(out, dim=1)[1]
            # print(pred_class)
            label_class = self.labels
            correct = self.get_accuracy(pred_class, label_class)
        return correct, len(label_class)

    def get_accuracy(self, pred, labels):
        """computes accuracy for classification / segmentation """
        if self.opt.task == 'cls':
            correct = pred.eq(labels).sum()
        return correct

    def load_state(self, last_epoch):
        ''' load epoch '''
        load_file = '%s_net.pth' % last_epoch
        load_path = osp.join(self.save_dir, load_file)
        net = self.net
        if isinstance(net, torch.nn.DataParallel):
            net = net.module
        print('loading the model from %s' % load_path)
        state_dict = torch.load(load_path, map_location=str(self.device))
        if hasattr(state_dict, '_metadata'):
            del state_dict._metadata
        net.load_state_dict(state_dict)

    def set_input_data(self, data):
        '''set input data'''

        gt_label = data.y
        edge_index = data.edge_index
        if self.opt.batch_size == 1:
            nodes_features = data.x.unsqueeze(0)
            # neigbour_index = data.pos.unsqueeze(0)
        else:
            nodes_features = data.x.view(self.opt.batch_size, 1024, -1)
            neigbour_index = data.pos.view(self.opt.batch_size, -1, 3)

        self.labels = gt_label.to(self.device).long()
        self.edge_index = edge_index.to(self.device).long()

        self.neigbour_index = neigbour_index.to(self.device).long()
        self.centers = nodes_features[:, :, :3].transpose(1, 2)
        self.corners = nodes_features[:, :, 3:12].transpose(1, 2)
        self.normals = nodes_features[:, :, 12:].transpose(1, 2)
        self.x = data.x[:, -3:].to(self.device).float()

    def forward(self):
        out = self.net(self.x, self.edge_index, self.centers,
                       self.corners, self.normals, self.neigbour_index)
        return out

    def backward(self, out, fea):
        self.loss_val = self.loss(out, self.labels)
        self.loss_val.backward()

    def optimize(self):
        '''
        optimize paramater
        '''
        self.optimizer.zero_grad()
        out, fea = self.forward()
        # print(self.net.module.concat_mlp[0].weight)
        self.backward(out, fea)
        nn.utils.clip_grad_norm_(self.net.parameters(), self.opt.grad_clip)
        self.optimizer.step()

    def save_network(self, which_epoch):
        '''
        save network to disk
        '''
        save_filename = '%s_net.pth' % (which_epoch)
        save_path = osp.join(self.save_dir, save_filename)
        if len(self.cuda) > 0 and torch.cuda.is_available():
            torch.save(self.net.module.cpu().state_dict(), save_path)
            self.net.cuda(self.cuda[0])
        else:
            torch.save(self.net.cpu().state_dict(), save_path)

    def log_history_and_plot(self, writer, epoch, batch):
        writer.history_log(epoch, batch, self.net.module.concat_mlp[0].weight)
        writer.draw_hist()

    def log_features_and_plot(self, epoch, batch):
        out, fea = self.forward()
        labels = self.labels
        fea_norm_numpy = fea.cpu().detach().numpy()
        cv_img = deprocess_image(fea_norm_numpy)
        im_color = cv2.applyColorMap(cv_img, cv2.COLORMAP_JET)
        cv2.imwrite(os.path.join(OUTPUT_DIR, "%d_epoch1.png" % epoch),im_color)
        # add canvas

        # self.history.log(epoch, image=im_color)
        # # self.canvas.draw_image(self.history["image"])
        # self.canvas.save(os.path.join(OUTPUT_DIR, "%d_epoch.png" % epoch))

def deprocess_image(img):
    """ see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """
    img = img - np.mean(img)
    img = img / (np.std(img) + 1e-5)
    img = img * 0.1
    img = img + 0.5
    img = np.clip(img, 0, 1)
    return np.uint8(img*255)


import torch
import torch.nn as nn
import glob
import os
import tqdm
import numpy as np
import argparse
import json
import random
import timeit
from tensorboardX import SummaryWriter
import pase
from random import shuffle
from pase.dataset import *
from pase.models.frontend import wf_builder
import pase.models.classifiers as pmods
import pase.dataset as pdsets
from pase.transforms import SingleChunkWav
from pase.utils import kfold_data, split_train_valid
from torch.utils.data import DataLoader
import torch.optim as optim
from torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR
#torch.backends.cudnn.benchmark = False

def retrieve_model_and_datasets(encoder_cfg,
                                model_cfg, data_cfg,
                                train_list, valid_list,
                                test_list):

    with open(model_cfg, 'r') as cfg_f:
        model_cfg = json.load(cfg_f)

    if encoder_cfg is not None:
        with open(encoder_cfg, 'r') as cfg_f:
            encoder_cfg = json.load(cfg_f)

    with open(data_cfg, 'r') as cfg_f:
        data_cfg = json.load(cfg_f)

    # prepare the three datasets; train, valid and test
    splits = [train_list, valid_list, test_list]
    cls_name = model_cfg.pop('name')
    dset_name = data_cfg.pop('name')
    if 'chunk_cfg' in data_cfg:
        chunker = SingleChunkWav(**data_cfg.pop('chunk_cfg'))
        data_cfg['chunker'] = chunker

    if encoder_cfg is not None:
        name = encoder_cfg.pop('name')

        if name == 'pase' or name == 'PASE':
            if 'ckpt' in encoder_cfg:
                ckpt = encoder_cfg.pop('ckpt')
            else:
                ckpt = None
            encoder = wf_builder(encoder_cfg)
            if ckpt is not None:
                encoder.load_pretrained(ckpt,
                                        load_last=True,
                                        verbose=True)
            model_cfg['frontend'] = encoder
        elif name == 'tdnn' or name == 'TDNN':
            model_cfg['xvector'] = True
            encoder = TDNN(**encoder_cfg)
            model_cfg['frontend'] = encoder
        else:
            raise ValueError('Unrecognized encoder: ', name)

    model = getattr(pmods, cls_name)(**model_cfg)
    datasets = []
    for si, split in enumerate(splits, start=1):
        if split is None:
            # skip this split (validation for instance)
            datasets.append(None)
        else:
            data_cfg['split_list'] = split
            if si >= len(splits) - 1 and 'chunker' in data_cfg:
                # remove the chunker for test split
                del data_cfg['chunker']
            datasets.append(getattr(pdsets, dset_name)(**data_cfg))
    return model, datasets

def accuracy(Y_, Y):
    # Get rid of temporal resolution here,
    # average likelihood in time and then
    # compute argmax and accuracy
    Y__avg = torch.mean(Y_, 2)
    pred = Y__avg.max(1, keepdim=True)[1]
    acc = pred.eq(Y[:, 0].view_as(pred)).float().mean().item()
    return acc

def valid_round(dloader, model, writer, epoch, device='cpu', tag='eval'):
    model.eval()
    with torch.no_grad():
        val_loss = []
        val_acc = []
        for bi, batch in enumerate(dloader, start=1):
            X, Y = batch
            if len(X.shape) == 2:
                X = X.unsqueeze(1)
            X = X.to(device)
            Y = Y.to(device)
            y = model(X)
            loss = F.nll_loss(y, Y)
            acc = accuracy(y, Y)
            val_loss.append(loss.item())
            val_acc.append(acc)
        mval = np.mean(val_loss)
        macc = np.mean(val_acc)
        print('{}: Epoch {} mloss: {:.2f}, macc: {:.2f}'.format(tag, epoch,
                                                                mval, macc))
        writer.add_scalar('{}/loss'.format(tag), mval, epoch)
        writer.add_scalar('{}/acc'.format(tag), macc, epoch)
        return mval, macc

def main(opts):
    CUDA = True if torch.cuda.is_available() and not opts.no_cuda else False
    device = 'cuda' if CUDA else 'cpu'
    num_devices = 1
    np.random.seed(opts.seed)
    random.seed(opts.seed)
    torch.manual_seed(opts.seed)
    if CUDA:
        torch.cuda.manual_seed_all(opts.seed)
        num_devices = torch.cuda.device_count()
        print('[*] Using CUDA {} devices'.format(num_devices))
    else:
        print('[!] Using CPU')
    print('Seeds initialized to {}'.format(opts.seed))
    # check num of classes by loading the utt2class file
    with open(opts.utt2class, 'r') as u2c_f:
        utt2class = json.load(u2c_f)
        num_classes = len(set(utt2class.values()))
        data_list = list(utt2class.keys())
    save_path = opts.save_path
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    train_list = [l.rstrip() for l in open(opts.train_list, 'r')]
    test_list = [l.rstrip() for l in open(opts.test_list, 'r')]
    if opts.valid_p > 0:
        train_list, valid_list = split_train_valid(train_list, valid_p=opts.valid_p)
    else:
        valid_list = None
    model, dset = retrieve_model_and_datasets(opts.encoder_cfg,
                                              opts.model_cfg,
                                              opts.data_cfg,
                                              train_list, valid_list,
                                              test_list)
    model.describe_params()
    if opts.ckpt is not None:
        # load ckpt for the model on previous state
        model.load_pretrained(opts.ckpt)
    if num_devices > 1:
        model_dp = nn.DataParallel(model)
    else:
        model_dp = model
    model_dp.to(device)
    if not opts.no_train:
        # Build DataLoaders: train, valid and test
        dloader = DataLoader(dset[0], opts.batch_size,
                             shuffle=True,
                             pin_memory=CUDA)
        if dset[1] is not None:
            va_dloader = DataLoader(dset[1], 1, shuffle=False,
                                    pin_memory=CUDA)
        else:
            va_dloader = None
        VALID = va_dloader is not None
        te_dloader = DataLoader(dset[2], 1, shuffle=False,
                                pin_memory=CUDA)
        writer = SummaryWriter(save_path)
        # Build optimizer and scheduler
        if opts.opt == 'adam':
            opt = optim.Adam(model.parameters(), opts.lr)
        else:
            opt = optim.SGD(model.parameters(), opts.lr)
        if opts.sched == 'plateau' and VALID:
            sched = ReduceLROnPlateau(opt, 'max', factor=0.5,
                                      patience=opts.plateau_patience)
        else:
            sched = StepLR(opt, step_size=opts.decay_step,
                           gamma=0.5)
        best_val_acc = 0
        estop_pat = opts.early_stop_patience
        beg_t = timeit.default_timer()
        for giter in range(1, opts.iters + 1):
            X, Y = next(dloader.__iter__())
            if len(X.shape) == 2:
                X = X.unsqueeze(1)
            #X = random_slice_X(X)
            X = X.to(device)
            Y = Y.to(device)
            y = model_dp(X)
            loss = F.nll_loss(y, Y)
            loss.backward()
            acc = accuracy(y, Y)
            opt.step()
            opt.zero_grad()
            end_t = timeit.default_timer()
            if giter % opts.log_freq == 0:
                print('Iter {:5d}/{:5d} loss: {:.2f}, acc: {:.2f} '
                      'btime: {:.1f} s'.format(giter, opts.iters,
                                               loss.item(),
                                               acc,
                                               end_t - beg_t))
                writer.add_scalar('train/loss', loss.item(), giter)
                writer.add_scalar('train/acc', acc, giter)
            beg_t = timeit.default_timer()
            if not isinstance(sched, ReduceLROnPlateau):
                sched.step()
            if giter % opts.save_freq == 0:
                best_val = False
                if VALID:
                    val, acc = valid_round(va_dloader, model, writer, giter, device)
                    if best_val_acc < acc:
                        best_val_acc = acc
                        estop_pat = opts.early_stop_patience
                        best_val = True
                    else:
                        estop_pat -= 1
                        if estop_pat <= 0:
                            print('BREAKING TRAINING LOOP AFTER {} VAL STEPS '
                                  'WITHOUT '
                                  'IMPROVEMENT'.format(opts.early_stop_patience))
                            break
                tes, teacc = valid_round(te_dloader, model, writer, giter,
                                         device, 'test')
                model.save(save_path,
                           giter, best_val=best_val)
                if isinstance(sched, ReduceLROnPlateau):
                    sched.step(acc)
                model.train()
    # TEST THIS FOLD AND STORE THE VALUE IN A LOG IN ROOT SAVE_PATH
    te_files = dset[2].split_list
    data_root = dset[2].data_root
    # TODO: Load last best ckpt
    best_ckpt = get_best_ckpt(save_path)
    if best_ckpt is None:
        print('ERROR in test: skipping this fold for '
              'did not find a ckpt')
        raise NotImplementedError
    print('Loading ckpt for test: ', best_ckpt)
    model.load_pretrained(best_ckpt, load_last=True)
    with torch.no_grad():
        model.eval()
        res = {}
        for test_file in tqdm.tqdm(te_files, total=len(te_files)):
            bname = os.path.splitext(test_file)[0]
            lab = utt2class[bname]
            if isinstance(dset[2], WavClassDataset):
                wav, rate = sf.read(os.path.join(data_root, bname + ".wav"))
                wav = torch.FloatTensor(wav).view(1, -1)
                wav = dset[2].z_norm(wav)
            else:
                wav = np.load(os.path.join(data_root, bname + ".npy"))
                wav = torch.FloatTensor(wav).view(wav.shape[0], -1)
                wav = dset[2].z_norm(wav.transpose(0, 1)).transpose(0, 1)
            wav = wav.unsqueeze(0)
            wav = wav.to(device)
            y = model(wav)
            res[bname] = {'prediction':y.max(1, keepdim=True)[1].item(),
                              'lab':lab}
        with open(os.path.join(save_path, 'predictions.json'), 'w') as f:
            f.write(json.dumps(res, indent=2))

def get_best_ckpt(load_path):
    ckpt_track = glob.glob(os.path.join(load_path, '*-checkpoints'))
    if len(ckpt_track) == 0:
        return None
    ckpt_track = ckpt_track[0]
    with open(ckpt_track, 'r') as f:
        ckpts = json.load(f)
        curr_ckpt = 'weights_{}'.format(ckpts['current'])
        return os.path.join(load_path, curr_ckpt)

def random_slice_X(X, lens=[32000, 96000]):
    sz = random.choice(list(range(lens[0], lens[1])))
    idxs = list(range(X.shape[-1] - sz))
    beg_i = random.choice(idxs)
    X = X[:, :, beg_i:beg_i + sz]
    return X

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_root', type=str,
                        default='data/IEMOCAP_ahsn_leave-two-speaker-out')
    parser.add_argument('--train_list', type=str,
                        default='data/sess4train.guia')
    parser.add_argument('--test_list', type=str, default='data/sess4test.guia')
    parser.add_argument('--ckpt', type=str, default=None)
    parser.add_argument('--valid_split', type=float, default=0.1,
                        help='Validation split inside the training fold' \
                             ' (Def: 0.1).')
    parser.add_argument('--utt2class', type=str,
                        default='data/utt2class.json')
    parser.add_argument('--batch_size', type=int, default=32)
    parser.add_argument('--iters', type=int, default=200000)
    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--no-cuda', action='store_true', default=False)
    parser.add_argument('--save_path', type=str, default='IEMOCAP_ckpt')
    parser.add_argument('--encoder_cfg', type=str, default=None)
    parser.add_argument('--model_cfg', type=str, default=None)
    parser.add_argument('--data_cfg', type=str, default=None)
    parser.add_argument('--plateau_patience', type=int, default=5)
    parser.add_argument('--log_freq', type=int, default=200)
    parser.add_argument('--save_freq', type=int, default=2500)
    parser.add_argument('--early_stop_patience', type=int, default=20)
    parser.add_argument('--decay_step', type=int, default=20000,
                        help='Number of iters til LR decay (Def: 20k).')
    parser.add_argument('--sched', type=str, default='plateau')
    parser.add_argument('--lr', type=float, default=0.1)
    parser.add_argument('--valid_p', type=float, default=0)
    parser.add_argument('--no-train', action='store_true', default=False)
    parser.add_argument('--opt', type=str, default='adam')
    parser.add_argument('--iter_sched', action='store_true', default=False)

    opts = parser.parse_args()

    main(opts)


#!/usr/bin/python3

"""
Copyright 2018-2019  Firmin.Sun (fmsunyh@gmail.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# -----------------------------------------------------
# @Time    : 11/8/2018 4:54 PM
# @Author  : Firmin.Sun (fmsunyh@gmail.com)
# @Software: ZJ_AI
# -----------------------------------------------------
# -*- coding: utf-8 -*-
import numpy as np
import random
import threading
import warnings

import keras

from core.preprocessing.utils import bbox_iou
from core.preprocessing.utils import BoundBox
from core.utils.image import (
    resize_image,
    preprocess_image)
from core.utils.image_transform import imageTransfor

class Generator(object):
    def __init__(
        self,
        transform_generator = None,
        batch_size=1,
        group_method='random',  # one of 'none', 'random', 'ratio'
        shuffle_groups=True,
        image_min_side=416,
        image_max_side=416,
        cell_size=13,
        anchors=list([2.56,2.97, 2.77,4.63, 3.71,3.76, 3.93,5.22, 5.13,5.62]),
        gt_box_max_buffer = 100,
        transform_parameters=None,
    ):
        self.transform_generator    = transform_generator
        self.batch_size             = int(batch_size)
        self.group_method           = group_method
        self.shuffle_groups         = shuffle_groups
        self.image_min_side         = image_min_side
        self.image_max_side         = image_max_side
        self.cell_size              = cell_size
        self.transform_parameters   = transform_parameters

        self.anchors                = anchors
        self.num_boxes              = len(self.anchors)//2
        self.anchor_boxes = [BoundBox(0, 0, self.anchors[2*i], self.anchors[2*i+1]) for i in range(len(self.anchors)//2)]

        self.gt_box_max_buffer = gt_box_max_buffer
        self.group_index = 0
        self.lock = threading.Lock()

        self.group_images()

    def get_anchors(self):
        return self.anchors

    def get_batch_size(self):
        return self.batch_size

    def size(self):
        raise NotImplementedError('size method not implemented')

    def num_classes(self):
        raise NotImplementedError('num_classes method not implemented')

    def name_to_label(self, name):
        raise NotImplementedError('name_to_label method not implemented')

    def label_to_name(self, label):
        raise NotImplementedError('label_to_name method not implemented')

    def image_aspect_ratio(self, image_index):
        raise NotImplementedError('image_aspect_ratio method not implemented')

    def load_image(self, image_index):
        raise NotImplementedError('load_image method not implemented')

    def load_annotations(self, image_index):
        raise NotImplementedError('load_annotations method not implemented')

    def load_annotations_group(self, group):
        return [self.load_annotations(image_index) for image_index in group]

    def filter_annotations(self, image_group, annotations_group, group):
        # test all annotations
        for index, (image, annotations) in enumerate(zip(image_group, annotations_group)):
            assert(isinstance(annotations, np.ndarray)), '\'load_annotations\' should return a list of numpy arrays, received: {}'.format(type(annotations))

            # test x2 < x1 | y2 < y1 | x1 < 0 | y1 < 0 | x2 <= 0 | y2 <= 0 | x2 >= image.shape[1] | y2 >= image.shape[0]
            invalid_indices = np.where(
                (annotations[:, 2] <= annotations[:, 0]) |
                (annotations[:, 3] <= annotations[:, 1]) |
                (annotations[:, 0] < 0) |
                (annotations[:, 1] < 0) |
                (annotations[:, 2] > image.shape[1]) |
                (annotations[:, 3] > image.shape[0])
            )[0]

            # delete invalid indices
            if len(invalid_indices):
                warnings.warn('Image with id {} (shape {}) contains the following invalid boxes: {}.'.format(
                    group[index],
                    image.shape,
                    [annotations[invalid_index, :] for invalid_index in invalid_indices]
                ))
                annotations_group[index] = np.delete(annotations, invalid_indices, axis=0)

        return image_group, annotations_group

    def load_image_group(self, group):
        return [self.load_image(image_index) for image_index in group]

    def random_transform_group_entry(self, image, annotations):
        # randomly transform both image and annotations
        if self.transform_generator:
            image, annotations = imageTransfor(self.transform_generator)([image, annotations])

        return image, annotations

    def resize_image(self, image):
        return resize_image(image, min_side=self.image_min_side, max_side=self.image_max_side)

    def preprocess_image(self, image):
        return preprocess_image(image)

    def preprocess_group_entry(self, image, annotations):
        # preprocess the image
        image = self.preprocess_image(image)

        # randomly transform image and annotations
        image, annotations = self.random_transform_group_entry(image, annotations)

        # resize image
        image, image_scale = self.resize_image(image)

        # apply resizing to annotations too
        annotations[:, 0:4:2] *= image_scale[1]
        annotations[:, 1:4:2] *= image_scale[0]

        return image, annotations

    def preprocess_group(self, image_group, annotations_group):
        for index, (image, annotations) in enumerate(zip(image_group, annotations_group)):
            # preprocess a single group entry
            image, annotations = self.preprocess_group_entry(image, annotations)

            # copy processed data back to group
            image_group[index]       = image
            annotations_group[index] = annotations

        return image_group, annotations_group

    def group_images(self):
        # determine the order of the images
        order = list(range(self.size()))
        if self.group_method == 'random':
            random.shuffle(order)
        elif self.group_method == 'ratio':
            order.sort(key=lambda x: self.image_aspect_ratio(x))

        # divide into groups, one group = one batch
        self.groups = [[order[x % len(order)] for x in range(i, i + self.batch_size)] for i in range(0, len(order), self.batch_size)]

    def compute_inputs(self, image_group):
        # get the max image shape
        max_shape = tuple(max(image.shape[x] for image in image_group) for x in range(3))

        # construct an image batch object
        image_batch = np.zeros((self.batch_size,) + max_shape, dtype=keras.backend.floatx())

        # c opy all images to the upper left part of the image batch object
        for image_index, image in enumerate(image_group):
            image_batch[image_index, :image.shape[0], :image.shape[1], :image.shape[2]] = image

        return image_batch

    def compute_targets(self, image_group, annotations_group):
        # same size for all batch image
        h, w, _ = image_group[0].shape

        gt_batch = np.zeros((self.batch_size, 1, 1, 1, self.gt_box_max_buffer, 4))
        targets = np.zeros(
            (self.batch_size, self.cell_size, self.cell_size, self.num_boxes, 4 + 1 + self.num_classes()))
        for annotation_index, annotation in enumerate(annotations_group):
            label = np.zeros((self.cell_size, self.cell_size, self.num_boxes, 4 + 1 + self.num_classes()))

            box_chw = np.stack([
                (annotation[:, 0] + annotation[:, 2]) / 2,
                (annotation[:, 1] + annotation[:, 3]) / 2,
                (annotation[:, 2] - annotation[:, 0]),
                (annotation[:, 3] - annotation[:, 1])], axis=1)
            # cls_ind = [self.label_to_name(label) for label in annotation[:, 4]]

            box_chw[:, 0] = box_chw[:, 0] / w * self.cell_size
            box_chw[:, 1] = box_chw[:, 1] / h * self.cell_size
            box_chw[:, 2] = box_chw[:, 2] / w * self.cell_size
            box_chw[:, 3] = box_chw[:, 3] / h * self.cell_size

            cls_ind = np.int32(annotation[:, 4])

            for index, (class_index, box) in enumerate(zip(cls_ind, box_chw)):
                best_anchor = -1
                max_iou = -1

                shifted_box = BoundBox(0,
                                       0,
                                       box[2],
                                       box[3])
                for i in range(self.num_boxes):
                    anchor_box = self.anchor_boxes[i]
                    iou = bbox_iou(shifted_box, anchor_box)

                    if max_iou < iou:
                        best_anchor = i
                        max_iou = iou

                # x_ind， y_ind
                x_ind = np.int32(box[0])
                y_ind = np.int32(box[1])

                # Each grid cell predicts five object
                if label[y_ind, x_ind, best_anchor, 4] == 1:
                    # alreadly existed anchor, skill it
                    continue

                label[y_ind, x_ind, best_anchor, 0:4] = box
                label[y_ind, x_ind, best_anchor, 4] = 1
                label[y_ind, x_ind, best_anchor, 5 + class_index] = 1

                # index must less than self.gt_box_max_buffer
                gt_batch[annotation_index, 0, 0, 0, index] = box
            targets[annotation_index] = label

        return np.asarray(gt_batch), np.asarray(targets)

    def compute_input_output(self, group):
        # load images and annotations
        image_group       = self.load_image_group(group)
        annotations_group = self.load_annotations_group(group)

        # check validity of annotations
        image_group, annotations_group = self.filter_annotations(image_group, annotations_group, group)

        # perform preprocessing steps
        image_group, annotations_group = self.preprocess_group(image_group, annotations_group)

        # compute network inputs
        inputs = self.compute_inputs(image_group)

        # compute network targets
        gt_boxes, targets = self.compute_targets(image_group, annotations_group)

        return [inputs, gt_boxes, targets], None

    def __next__(self):
        return self.next()

    def next(self):
        # advance the group index
        with self.lock:
            if self.group_index == 0 and self.shuffle_groups:
                # shuffle groups at start of epoch
                random.shuffle(self.groups)
            group = self.groups[self.group_index]
            self.group_index = (self.group_index + 1) % len(self.groups)

        return self.compute_input_output(group)


'''
As of 6/22/17
Operation Resurrect-George (just get him to drive, dammit!)
This should be tested on George.
'''

import cv2
import keras.models
import numpy as np
import os
import picamera
import RPi.GPIO as GPIO
import socket
import threading
import time
from imutils.object_detection import non_max_suppression

SIGMA = 0.33
stop_classifier = cv2.CascadeClassifier('cascade_xml/stop_sign.xml')
timestr = time.strftime('%Y%m%d_%H%M%S')

left    = 11
right   = 12
forward = 13
reverse = 15

GPIO.setmode(GPIO.BOARD)       # Numbers pins by physical location

GPIO.setup(left, GPIO.OUT)
GPIO.output(left, GPIO.HIGH)

GPIO.setup(right, GPIO.OUT)
GPIO.output(right, GPIO.HIGH)

GPIO.setup(forward, GPIO.OUT)
GPIO.output(forward, GPIO.HIGH)

GPIO.setup(reverse, GPIO.OUT)
GPIO.output(reverse, GPIO.HIGH)

class CarCamera(object):

    def __init__(self):
        self.camera = picamera.PiCamera()
        self.set_camera_resolution()
        self.frame = None
        self.frame_capture()
        self.start()
        self.

    def set_camera_resolution(self):
        self.camera.resolution = (320, 240)      # pi camera resolution

    def frame_capture(self):
        self.frame = camera.capture('jpeg')      # CONFIRM PARAMETERS
        print 'Captured frame'

    def start(self):
        print 'Starting neural network...'
        NeuralNetwork(receiving=True, CarCameraObject=self)

	def read(self):
		# return the frame most recently read
        print 'Starting frame capture...'
        print ' \"Hold on to your butts!\" '
		return self.frame

class NeuralNetwork(object):

    # global stop_classifier
    global timestr

    def __init__(self, receiving=False, CarCameraObject=None):
        self.receiving = receiving
        self.model = keras.models.load_model('nn_h5/nn.h5')

        # CarCamera class object is now here.
        self.CarCameraObject = CarCameraObject
        self.rcdriver = RCDriver()

        while self.receiving:
            self.fetch()

    def auto_canny(self, blurred):
        # Compute the median of the single channel pixel intensities
        global SIGMA
        v = np.median(blurred)

        # Apply automatic Canny edge detection using the computed median of the image
        lower = int(max(0,   (1.0 - SIGMA) * v))
        upper = int(min(255, (1.0 + SIGMA) * v))
        edged = cv2.Canny(blurred, lower, upper)
        return edged

    def preprocess(self, frame):
        image_array = frame.reshape(1, 38400).astype(np.float32)
        image_array = image_array / 255.    # Normalizing
        return image_array

    def predict(self, image):
        image_array = self.preprocess(image)
        y_hat       = self.model.predict(image_array)
        i_max       = np.argmax(y_hat)
        y_hat_final = np.zeros((1,3))
        np.put(y_hat_final, i_max, 1)
        return y_hat_final[0], y_hat

    def fetch(self):

        frame = 0

        while self.receiving:

            # There's a chance that the script can get to this point before the New thread begins streaming images.
            # To account for this, we create the jpg variable but set to None, and keep checking until it actually has something.
            jpg = None
            while jpg is None:
                jpg = self.CarCameraObject.frame

            gray  = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_GRAYSCALE)
            image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)

            # Object detection
            obj_detection.detect(stop_classifier, gray, image)

            # Lower half of the grayscale image
            roi = gray[120:240, :]

            # Apply GuassianBlur (reduces noise)
            blurred = cv2.GaussianBlur(roi, (3, 3), 0)

            # Apply Canny filter
            auto = self.auto_canny(blurred)

            # Show streaming images
            cv2.imshow('Original', image)
            cv2.imshow('What the model sees', auto)

            # Neural network model makes prediciton
            # prediction = self.model.predict(auto)
            prediction, probas = self.predict(auto)

            # Save frame and prediction record for debugging research
            prediction_english = None
            prediction_english_proba = None

            proba_left, proba_right, proba_forward = probas[0]

            if np.all(prediction   == [ 0., 0., 1.]):
                prediction_english = 'FORWARD'
                prediction_english_proba = proba_forward

            elif np.all(prediction == [ 1., 0., 0.]):
                prediction_english = 'LEFT'
                prediction_english_proba = proba_left

            elif np.all(prediction == [ 0., 1., 0.]):
                prediction_english = 'RIGHT'
                prediction_english_proba = proba_right

            # cv2.putText(gray, "Model prediction: {}".format(prediction_english), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, .45, (255, 255, 0), 1)
            cv2.putText(gray, "Prediction (sig={}): {}, {:>05}".format(SIGMA, prediction_english, prediction_english_proba), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, .45, (255, 255, 0), 1)
            cv2.imwrite('test_frames_temp/frame{:>05}.jpg'.format(frame), gray)
            frame += 1

            # Send prediction to driver to tell it how to steer
            self.rcdriver.steer(prediction)

            if cv2.waitKey(1) & 0xFF == ord('q'):
                self.receiving = False
                cv2.destroyAllWindows()

class RCDriver(object):

    def __init__(self):
        # self.frame

    def steer(self, prediction):

        # FORWARD
        if np.all(prediction   == [ 0., 0., 1.]):
            print 'forward'
        	GPIO.output(forward, GPIO.LOW)
        	time.sleep(0.3)
            GPIO.output(forward, GPIO.HIGH)

        # FORWARD-LEFT
        elif np.all(prediction == [ 1., 0., 0.]):
            print 'left'
    		GPIO.output(left, GPIO.LOW)
    		GPIO.output(forward, GPIO.LOW)
    		time.sleep(0.4)
    		GPIO.output(left, GPIO.HIGH)
    		GPIO.output(forward, GPIO.HIGH)

        # FORWARD-RIGHT
        elif np.all(prediction == [ 0., 1., 0.]):
            print 'right'
    		GPIO.output(right, GPIO.LOW)
    		GPIO.output(forward, GPIO.LOW)
    		time.sleep(2.0)
    		GPIO.output(right, GPIO.HIGH)
    		GPIO.output(forward, GPIO.HIGH)

    def stop(self):
        print '* * * STOPPING! * * *'
        time.sleep(4.0)

rcdriver = RCDriver()

if __name__ == '__main__':
    try:
        RCDriver()

    except KeyboardInterrupt:
        # Rename the folder that collected all of the test frames. Then make a new folder to collect next round of test frames.
        os.rename(  './test_frames_temp', './test_frames_SAVED/test_frames_{}'.format(timestr))
        os.makedirs('./test_frames_temp')
        print '\nTerminating...\n'

        GPIO.output(left, GPIO.HIGH)
        GPIO.output(right, GPIO.HIGH)
        GPIO.output(forward, GPIO.HIGH)
        GPIO.output(reverse, GPIO.HIGH)
    	GPIO.cleanup()                     # Release resource

        print '\nDone.\n'