import torch

from COIGAN.training.data.augmentation.noise_generators.base_noise_generator import BaseNoiseGenerator

class GaussianNoiseGenerator(BaseNoiseGenerator):
    """
    Applicator of gaussian noise to masks.
    """

    strategy_dict = {
        "additive": 0,
        "multiplicative": 1,
        "replace": 2
    }

    def __init__(
        self,
        mean: float = 0.0,
        std: float = 1.0,
        strategy: str = "additive"
    ):
        """
        Args:
            mean (float): mean of the gaussian noise
            std (float): standard deviation of the gaussian noise
            strategy (str): strategy to apply the noise:
                - additive: add the noise to the original masks (default).      es. (masks + noise)[masks == 0] = 0
                - multiplicative: multiply the original masks by the noise.     es. masks * noise
                - replace: replace the original masks with the noise.           es. noise[masks == 0] = 0
        """
        # check if the strategy is valid
        if strategy not in ["additive", "multiplicative", "replace"]:
            raise ValueError(f"Invalid strategy {strategy}. Valid strategies are: additive, multiplicative, replace.")

        self.mean = mean
        self.std = std
        self.strategy = self.strategy_dict[strategy]

    def __call__(self, masks: torch.Tensor) -> torch.Tensor:
        """
        Add gaussian noise to the masks, only where the original masks are > 0.

        Args:
            masks (torch.Tensor): a new tensor with noise added

        Returns:
            torch.Tensor: masks with noise, only where the original masks are > 0
        """
        noise = torch.normal(mean=self.mean, std=self.std, size=masks.shape, device=masks.device)

        if self.strategy == 0: # additive
            return (masks + noise) * (masks > 0)

        elif self.strategy == 1: # multiplicative
            return masks * noise

        elif self.strategy == 2: # replace
            #noise[masks == 0] = 0
            return noise * (masks > 0)

    def get_noise(self, shape: tuple, device = "cpu") -> torch.Tensor:
        """
        Get the noise tensor.

        Args:
            shape (tuple): shape of the noise tensor

        Returns:
            torch.Tensor: noise tensor
        """
        return torch.normal(mean=self.mean, std=self.std, size=shape, device=device)

##########################################
### DEBUG SECTION

if __name__ == "__main__":

    from time import time

    masks = torch.zeros((1, 4, 256, 256)).to("cuda")

    # create a cube of 1s
    masks[:, :, 100:150, 100:150] = 1

    orig = masks[0, 0].cpu().numpy()

    # add noise and time it
    noise_gen = GaussianNoiseGenerator(strategy="additive")
    t0 = time()
    noise_masks = noise_gen(masks)
    tadd = time() - t0

    test_0 = noise_masks[0, 0].cpu().numpy()

    # multiply noise and time it
    noise_gen = GaussianNoiseGenerator(strategy="multiplicative")
    t0 = time()
    noise_masks = noise_gen(masks)
    tmul = time() - t0

    test_1 = noise_masks[0, 0].cpu().numpy()

    # replace noise and time it
    noise_gen = GaussianNoiseGenerator(strategy="replace")
    t0 = time()
    noise_masks = noise_gen(masks)
    trep = time() - t0

    test_2 = noise_masks[0, 0].cpu().numpy()

    print(f"Additive noise: {tadd:.6f}s")
    print(f"Multiplicative noise: {tmul:.6f}s")
    print(f"Replace noise: {trep:.6f}s")


#!/usr/bin/env python3
# -*- mode: python; py-indent-offset: 4; py-continuation-offset: 4 -*-
'''
Tests for the Merge chunk of the Driver script
'''
from __future__ import print_function
import sys
sys.dont_write_bytecode = True

import os
sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import unittest
from unittest.mock import patch

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

try:
    import unittest.mock as mock
except ImportError:  # pragma nocover
    import mock

from argparse import Namespace
import contextlib
from subprocess import CalledProcessError

import PullRequestLinuxDriverMerge as PRMerge

class Test_header(unittest.TestCase):
    '''Test that we can properly echo the header information'''

    def test_writeHeader(self):
        print("BEGIN:  Test PullRequestLinuxDriverMerge.py :: `write_header()`:")

        #m_stdout = StringIO()
        #with contextlib.redirect_stdout(m_stdout):
        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
            PRMerge.write_header()
            self.assertIn("Begin: PullRequestLinuxDriverMerge.py", m_stdout.getvalue())

        print("FINISH: Test PullRequestLinuxDriverMerge.py :: `write_header()`:")
        return

class Test_EchoJenkinsVars(unittest.TestCase):
    '''Test that the Jenkins environment is echoed properly'''

    def setUp(self):
        tmp_environ = {}
        tmp_environ['JOB_BASE_NAME'] = 'TEST_JOB_BASE_NAME'
        tmp_environ['JOB_NAME']      = 'TEST_JOB_NAME'
        tmp_environ['WORKSPACE']     = os.path.join(os.sep, 'dev', 'null', 'TEST_WORKSPACE')
        tmp_environ['NODE_NAME']     = 'TEST_NODE_NAME'
        self.m_environ = mock.patch.dict(os.environ, tmp_environ, clear=True)
        return

    def test_echoJenkinsVars(self):
        print("BEGIN:  Test PullRequestLinuxDriverMerge.py :: `echoJenkinsVars()`:")
        with self.m_environ:
            env_string_io = StringIO()
            for key in os.environ:
                print(key + ' = ' + os.environ[key],
                      file=env_string_io)

        tmp_path = os.path.join(os.sep, 'dev', 'null', 'TEST_WORKSPACE')

        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout, self.m_environ:
            PRMerge.echoJenkinsVars( tmp_path )

        stdout_actual = m_stdout.getvalue()
        self.assertIn("JOB_BASE_NAME = TEST_JOB_BASE_NAME", stdout_actual)
        self.assertIn("JOB_NAME = TEST_JOB_NAME", stdout_actual)
        self.assertIn("WORKSPACE = /dev/null/TEST_WORKSPACE", stdout_actual)
        self.assertIn("NODE_NAME = TEST_NODE_NAME", stdout_actual)
        print("FINISH: Test PullRequestLinuxDriverMerge.py :: `echoJenkinsVars()`:")
        return

class Test_parsing(unittest.TestCase):
    '''Finally given a decent parser I want to now pass
       all parameters and check them'''

    def test_parseArgs(self):
        test_namespace = Namespace()
        setattr(test_namespace, 'sourceRepo', '/dev/null/source/Trilinos.git')
        setattr(test_namespace, 'targetRepo', '/dev/null/target/Trilinos.git')
        setattr(test_namespace, 'targetBranch', 'fake_develop')
        setattr(test_namespace, 'sourceSHA', '0123456789abcdef')
        setattr(test_namespace, 'workspaceDir', '/dev/null/workspace')

        with mock.patch.object(sys, 'argv', ['programName',
                                             os.path.join(os.path.sep,
                                                          'dev',
                                                          'null',
                                                          'source',
                                                          'Trilinos.git'),
                                             os.path.join(os.path.sep,
                                                          'dev',
                                                          'null',
                                                          'target',
                                                          'Trilinos.git'),
                                             'fake_develop',
                                             '0123456789abcdef',
                                             os.path.join(os.path.sep,
                                                          'dev',
                                                          'null',
                                                          'workspace')]):

            args = PRMerge.parseArgs()
        self.assertEqual(test_namespace, args)
        return

    def test_parseInsufficientArgs_fails(self):
        test_namespace = Namespace()
        setattr(test_namespace, 'sourceRepo', '/dev/null/source/Trilinos.git')
        expected_output = '''usage: programName [-h]
                   sourceRepo targetRepo targetBranch sourceSHA workspaceDir
programName: error: the following arguments are required: sourceRepo, \
targetRepo, targetBranch, sourceSHA, workspaceDir
'''
        if sys.version_info.major != 3:
            expected_output = '''usage: programName [-h]
                   sourceRepo targetRepo targetBranch sourceSHA
                   workspaceDir\nprogramName: error: too few arguments
'''
        with mock.patch.object(sys, 'argv', ['programName']), \
                mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
            if sys.version_info.major != 3:
                with self.assertRaisesRegexp(SystemExit, '2'):
                    PRMerge.parseArgs()
            else:
                with self.assertRaisesRegex(SystemExit, '2'):
                    PRMerge.parseArgs()
        self.assertEqual(expected_output, m_stderr.getvalue())
        return

class Test_mergeBranch(unittest.TestCase):
    '''Verify that we call the correct sequence to merge the source branch/SHA
       into the target branch'''

    def test_mergeBranch_without_source_remote(self):
        with mock.patch('subprocess.check_output', side_effect=['origin /dev/null/target/Trilinos', '']) as m_check_out, \
            mock.patch('subprocess.check_call') as m_check_call:
            PRMerge.merge_branch(os.path.join(os.path.sep,
                                                                  'dev',
                                                                  'null',
                                                                  'source',
                                                                  'Trilinos.git'),
                                                    'fake_develop',
                                                    'df324ae')
        m_check_out.assert_has_calls([mock.call(['git', 'remote', '-v'])])

        m_check_call.assert_has_calls([mock.call(['git', 'remote', 'add', 'source_remote', '/dev/null/source/Trilinos.git']),
                                       mock.call(['git', 'prune']),
                                       mock.call(['git', 'gc']),
                                       mock.call(['git', 'fetch', 'source_remote', 'df324ae']),
                                       mock.call(['git', 'fetch', 'origin', 'fake_develop']),
                                       mock.call(['git', 'reset', '--hard', 'HEAD']),
                                       mock.call(['git', 'checkout', '-B', 'fake_develop', 'origin/fake_develop']),
                                       mock.call(['git', 'merge', '--no-edit', 'df324ae']),
                                       ])
        return

    @patch('subprocess.check_call')
    def test_mergeBranch_with_source_remote(self, m_check_call):
        """
        """
        tmp_path = os.path.join(os.path.sep, 'dev', 'null', 'source', 'Trilinos.git')

        side_effect_list = ["origin /dev/null/target/Trilinos\nsource_remote /dev/null/source12/Trilinos.git", '']

        with mock.patch('subprocess.check_output', side_effect=side_effect_list) as m_check_out:
            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                PRMerge.merge_branch(tmp_path, 'fake_develop', 'df324ae')

        expected_calls = []
        expected_calls.append(mock.call(['git', 'remote', '-v']))
        expected_calls.append(mock.call('git rev-parse --verify --quiet source_remote/df324ae || true', shell=True))
        m_check_out.assert_has_calls(expected_calls)

        m_check_call.assert_has_calls([mock.call(['git', 'remote', 'rm', 'source_remote']),
                                       mock.call(['git', 'remote', 'add', 'source_remote', '/dev/null/source/Trilinos.git']),
                                       mock.call(['git', 'prune']),
                                       mock.call(['git', 'gc']),
                                       mock.call(['git', 'fetch', 'source_remote', 'df324ae']),
                                       mock.call(['git', 'fetch', 'origin', 'fake_develop']),
                                       mock.call(['git', 'reset', '--hard', 'HEAD']),
                                       mock.call(['git', 'checkout', '-B', 'fake_develop', 'origin/fake_develop']),
                                       mock.call(['git', 'merge', '--no-edit', 'df324ae']),
                                       ])
        self.assertIn("git remote exists, removing it", m_stdout.getvalue())
        return

    @patch('subprocess.check_call')
    def test_mergeBranch_ref_is_remote_branch(self, m_check_call):
        """
        """
        tmp_path = os.path.join(os.path.sep, 'dev', 'null', 'source', 'Trilinos.git')

        side_effect_list = ["origin /dev/null/target/Trilinos\nsource_remote /dev/null/source12/Trilinos.git", 'df324ae']

        with mock.patch('subprocess.check_output', side_effect=side_effect_list) as m_check_out:
            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                PRMerge.merge_branch(tmp_path, 'fake_develop', 'some_ref')

        expected_calls = []
        expected_calls.append( mock.call(['git', 'remote', '-v']) )
        m_check_out.assert_has_calls(expected_calls)

        m_check_call.assert_has_calls([mock.call(['git', 'remote', 'rm', 'source_remote']),
                                       mock.call(['git', 'remote', 'add', 'source_remote', '/dev/null/source/Trilinos.git']),
                                       mock.call(['git', 'prune']),
                                       mock.call(['git', 'gc']),
                                       mock.call(['git', 'fetch', 'source_remote', 'some_ref']),
                                       mock.call(['git', 'fetch', 'origin', 'fake_develop']),
                                       mock.call(['git', 'reset', '--hard', 'HEAD']),
                                       mock.call(['git', 'checkout', '-B', 'fake_develop', 'origin/fake_develop']),
                                       mock.call(['git', 'merge', '--no-edit', 'source_remote/some_ref']),
                                       ])
        self.assertIn("git remote exists, removing it", m_stdout.getvalue())
        return

    @patch('subprocess.check_output', side_effect=['origin /dev/null/target/Trilinos', 'df324ae'])
    def test_mergeBranch_fails_on_source_fetch(self, m_check_out):
        """
        Tests that ``merge_branch`` should fail on source fetch if a failure
        occurrs in the *fetch* operation
        """
        tmp_path = os.path.join(os.path.sep, 'dev', 'null', 'source', 'Trilinos.git')

        side_effect_list = [
            None,
            None,
            None,
            CalledProcessError(-1, 'cmd'),
            CalledProcessError(-2, 'cmd'),
            CalledProcessError(-3, 'cmd')
        ]

        with self.assertRaises(SystemExit):
            with mock.patch('subprocess.check_call', side_effect=side_effect_list) as m_check_call:
                PRMerge.merge_branch(tmp_path, 'fake_develop', 'df324ae')

        expected_calls = []
        expected_calls.append(mock.call(['git', 'remote', '-v']))
        m_check_out.assert_has_calls(expected_calls)

        expected_calls = []
        expected_calls.append(mock.call(['git', 'remote', 'add', 'source_remote', '/dev/null/source/Trilinos.git']))
        expected_calls.append(mock.call(['git', 'prune']))
        expected_calls.append(mock.call(['git', 'gc']))
        expected_calls.append(mock.call(['git', 'fetch', 'source_remote', 'df324ae']))
        expected_calls.append(mock.call(['git', 'fetch', 'source_remote', 'df324ae']))
        expected_calls.append(mock.call(['git', 'fetch', 'source_remote', 'df324ae']))
        m_check_call.assert_has_calls(expected_calls)

        return

class Test_run(unittest.TestCase):
    '''This is the main function that ties everything together in order'''

    @patch('PullRequestLinuxDriverMerge.parseArgs')
    @patch('os.chdir')
    @patch('PullRequestLinuxDriverMerge.write_header')
    @patch('PullRequestLinuxDriverMerge.echoJenkinsVars')
    @patch('PullRequestLinuxDriverMerge.merge_branch')
    @patch('os.path.join')
    def test_run(self, m_join, m_mergeBranch, m_echoJenkins, m_writeHeader, m_chdir, m_parser):
        """
        TODO: Add docstring that explains this test's purpose
        """
        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
            self.assertTrue(PRMerge.run())

        m_parser.assert_called_once_with()
        m_chdir.assert_called_once_with(m_join(m_parser().workspaceDir, 'Trilinos'))
        m_writeHeader.assert_called_once_with()
        m_echoJenkins.assert_called_once_with(m_parser().workspaceDir)
        m_mergeBranch.assert_called_once_with(m_parser().sourceRepo,
                                              m_parser().targetBranch,
                                              m_parser().sourceSHA)

        self.assertIn('Set CWD = ' + str(m_join()) + '\n', m_stdout.getvalue())
        return

    def test_run_fails_on_bad_parse(self):
        with mock.patch('PullRequestLinuxDriverMerge.parseArgs', side_effect=SystemExit(2)):
            self.assertFalse(PRMerge.run())
        return

    @patch('os.path.join')
    @patch('os.chdir')
    @patch('PullRequestLinuxDriverMerge.parseArgs')
    def test_run_fails_on_bad_fetch(self, m_parseArgs, m_chdir, m_join):
        """
        TODO: Add a docstring explaining this test and its purpose.
        """
        side_effect_check_output = ['origin /dev/null/target/Trilinos', 'df324ae']

        side_effect_check_call = []
        side_effect_check_call.append( None )
        side_effect_check_call.append( CalledProcessError(-1, 'cmd') )
        side_effect_check_call.append( CalledProcessError(-2, 'cmd') )
        side_effect_check_call.append( CalledProcessError(-3, 'cmd') )

        with mock.patch('subprocess.check_output', side_effect=side_effect_check_output):
            with mock.patch('subprocess.check_call', side_effect=side_effect_check_call):
                # I'm not sure if we really needed to mock out stdio, we don't check it
                # so would it be more useful to provide that to the output if the test
                # should fail? Otherwise, are we flying blin?
                with mock.patch('sys.stdout', new_callable=StringIO):
                    self.assertFalse(PRMerge.run())

        return

    @patch('os.path.join')
    @patch('os.chdir')
    @patch('PullRequestLinuxDriverMerge.parseArgs')
    @patch('sys.stdout', new_callable=StringIO)
    def test_run_fails_on_bad_remote_add(self, m_stdout, m_parseArgs, m_chdir, m_path_join):

        expected_string = '''Recieved subprocess.CalledProcessError - returned -1
  from command test_cmd
  output None
  stdout None
  stderr None\n'''

        if sys.version_info.major != 3:
            expected_string = '''Recieved subprocess.CalledProcessError - returned -1
  from command test_cmd
  output None\n'''

        side_effect_check_output = ['origin /dev/null/target/Trilinos', 'df324ae']
        side_effect_check_call   = CalledProcessError(-1, 'test_cmd')

        with mock.patch('subprocess.check_output', side_effect=side_effect_check_output), \
            mock.patch('subprocess.check_call', side_effect=side_effect_check_call):
            self.assertFalse(PRMerge.run())

        expected_string_list = []
        expected_string_list.append("Received subprocess.CalledProcessError - returned -1")
        expected_string_list.append("from command test_cmd")
        expected_string_list.append("output None")

        if sys.version_info.major == 3:
            expected_string_list.append("output None")
            expected_string_list.append("stdout None")
            expected_string_list.append("stderr None")

        for expected_string_i in expected_string_list:
            self.assertIn(expected_string_i, m_stdout.getvalue())

        return

if __name__ == '__main__':
    unittest.main()  # pragma nocover


from __future__ import annotations

import itertools
from collections.abc import Iterator
from typing import Generator, Iterable, Optional, Sequence

import numpy as np
import tensorflow as tf

from pararealml.constrained_problem import ConstrainedProblem
from pararealml.initial_condition import VectorizedInitialConditionFunction
from pararealml.initial_value_problem import TemporalDomainInterval
from pararealml.operators.ml.physics_informed.collocation_point_sampler import (  # noqa: 501
    CollocationPointSampler,
)

class Dataset:
    """
    A generator and container of all the data necessary to train a
    physics-informed regresion model with variable initial conditions.
    """

    def __init__(
        self,
        cp: ConstrainedProblem,
        t_interval: TemporalDomainInterval,
        y_0_functions: Iterable[VectorizedInitialConditionFunction],
        point_sampler: CollocationPointSampler,
        n_domain_points: int,
        n_boundary_points: int = 0,
        vertex_oriented: bool = False,
    ):
        """
        :param cp: the constrained problem to generate the spatial data about
        :param t_interval: the bounds of the temporal domain to generate data
            from
        :param y_0_functions: the initial condition functions
        :param point_sampler: a sampler instance for sampling the collocation
            points
        :param n_domain_points: the number of domain points to sample
        :param n_boundary_points: the number of boundary points to sample; if
            the constrained problem is an ODE, it should be 0
        :param vertex_oriented: whether the initial condition collocation
            points should be the vertices or the cell centers of the mesh
        """
        x_dimension = cp.differential_equation.x_dimension

        if n_domain_points <= 0:
            raise ValueError(
                f"number of domain points ({n_domain_points}) must be greater "
                f"than 0"
            )
        if n_boundary_points < 0:
            raise ValueError(
                f"number of boundary points ({n_boundary_points}) must be "
                f"non-negative"
            )
        if not x_dimension and n_boundary_points:
            raise ValueError("number of boundary points must be 0 for ODEs")

        self._cp = cp
        self._t_interval = t_interval
        self._y_0_functions = y_0_functions
        self._point_sampler = point_sampler
        self._n_domain_points = n_domain_points
        self._n_boundary_points = n_boundary_points
        self._vertex_oriented = vertex_oriented

        self._initial_value_data = self._create_initial_value_data()
        self._domain_collocation_data = self._create_domain_collocation_data()
        self._initial_collocation_data = (
            self._create_initial_collocation_data()
        )
        self._boundary_collocation_data = (
            self._create_boundary_collocation_data()
        )

    @property
    def constrained_problem(self) -> ConstrainedProblem:
        """
        The constrained problem the dataset is built around.
        """
        return self._cp

    @property
    def initial_value_data(self) -> np.ndarray:
        """
        The initial value data where each row is a different initial condition
        function and each column represents a component of the initial
        condition function evaluated over a point of the constrained problem's
        mesh.
        """
        return self._initial_value_data

    @property
    def domain_collocation_data(self) -> np.ndarray:
        """
        The domain collocation data points where the first column is t and all
        other columns are x.
        """
        return self._domain_collocation_data

    @property
    def initial_collocation_data(self) -> np.ndarray:
        """
        The initial collocation data points where the first column is t and all
        other columns are x.
        """
        return self._initial_collocation_data

    @property
    def boundary_collocation_data(self) -> Optional[np.ndarray]:
        """
        The boundary collocation data points where the columns represent t, x,
        y, the derivative of y with respect to the unit normal vector of the
        boundary, and finally the axis denoting the direction of the normal
        vector.
        """
        return self._boundary_collocation_data

    def get_iterator(
        self, n_batches: int, n_ic_repeats: int = 1, shuffle: bool = True
    ) -> DatasetIterator:
        """
        Returns an iterator over the dataset to enable iterating over the
        Cartesian product of the initial value data and the collocation data
        batch by batch.

        :param n_batches: the number of batches to map the underlying data to
        :param n_ic_repeats: the number of times to repeat the initial
            collocation data samples in an epoch
        :param shuffle: whether to shuffle the data behind the iterator
        :return: the iterator over the dataset
        """
        return DatasetIterator(self, n_batches, n_ic_repeats, shuffle)

    def _create_initial_value_data(self) -> np.ndarray:
        """
        Creates the initial value data by evaluating the initial condition
        functions (over the vertices or the cell centers of the mesh in case
        the constrained problem is a PDE).
        """
        if self._cp.differential_equation.x_dimension:
            x = self._cp.mesh.all_index_coordinates(
                self._vertex_oriented, flatten=True
            )
            initial_value_data = np.vstack(
                [y_0_func(x).flatten() for y_0_func in self._y_0_functions]
            )
        else:
            initial_value_data = np.array(
                [y_0_func(None) for y_0_func in self._y_0_functions]
            )

        initial_value_data.setflags(write=False)
        return initial_value_data

    def _create_domain_collocation_data(self) -> np.ndarray:
        """
        Creates the domain collocation data by sampling collocation points from
        the space-time domain of the constrained problem combined with the time
        interval.
        """
        domain_points = self._point_sampler.sample_domain_points(
            self._n_domain_points, self._t_interval, self._cp.mesh
        )
        domain_collocation_data = (
            np.concatenate((domain_points.t, domain_points.x), axis=1)
            if self._cp.differential_equation.x_dimension
            else domain_points.t
        )

        domain_collocation_data.setflags(write=False)
        return domain_collocation_data

    def _create_initial_collocation_data(self) -> np.ndarray:
        """
        Creates the initial collocation data by combining the coordinates of
        the vertices or the cell centers of the constrained problem's mesh (if
        the constrained problem is a PDE) with an array of zeros representing
        the time points.
        """
        if self._cp.differential_equation.x_dimension:
            x = self._cp.mesh.all_index_coordinates(
                self._vertex_oriented, flatten=True
            )
            t = np.zeros((len(x), 1))
            initial_collocation_data = np.hstack((t, x))
        else:
            initial_collocation_data = np.zeros((1, 1))

        initial_collocation_data.setflags(write=False)
        return initial_collocation_data

    def _create_boundary_collocation_data(self) -> Optional[np.ndarray]:
        """
        Creates the boundary collocation data by sampling collocation points
        from the spatial boundaries of the space-time domain of the constrained
        problem combined with the time interval; if the constrained problem is
        a PDE, it also evaluates the boundary conditions (both Dirichlet and
        Neumann) and includes the constraints in the dataset.
        """
        diff_eq = self._cp.differential_equation
        x_dimension = diff_eq.x_dimension
        y_dimension = diff_eq.y_dimension
        if not x_dimension:
            return None

        all_boundary_points = self._point_sampler.sample_boundary_points(
            self._n_boundary_points, self._t_interval, self._cp.mesh
        )

        t = []
        x = []
        y = []
        d_y_over_d_n = []
        axis = []
        for ax, (bc_pair, boundary_points_pair) in enumerate(
            zip(self._cp.boundary_conditions, all_boundary_points)
        ):
            for bc, boundary_points in zip(bc_pair, boundary_points_pair):
                if boundary_points is None:
                    continue

                for i in range(boundary_points.t.shape[0]):
                    t_i = boundary_points.t[i]
                    x_i = boundary_points.x[i : i + 1]
                    y_i = (
                        bc.y_condition(x_i, t_i)[0]
                        if bc.has_y_condition
                        else [None] * y_dimension
                    )
                    d_y_over_d_n_i = (
                        bc.d_y_condition(x_i, t_i)[0]
                        if bc.has_d_y_condition
                        else [None] * y_dimension
                    )

                    t.append(t_i)
                    x.append(x_i[0])
                    axis.append([ax])
                    y.append(y_i)
                    d_y_over_d_n.append(d_y_over_d_n_i)

        boundary_collocation_data = np.concatenate(
            [
                np.array(t),
                np.array(x),
                np.array(y),
                np.array(d_y_over_d_n),
                np.array(axis),
            ],
            axis=1,
        )
        boundary_collocation_data.setflags(write=False)
        return boundary_collocation_data

class DatasetIterator(Iterator):
    """
    An iterator over a dataset that computes the Cartesian products of the
    initial value data with the domain, initial, and boundary collocation
    data.
    """

    def __init__(
        self,
        dataset: Dataset,
        n_batches: int,
        n_ic_repeats: int = 1,
        shuffle: bool = True,
    ):
        """
        :param dataset: the dataset to iterate over
        :param n_batches: the number of batches per epoch
        :param n_ic_repeats: the number of times to repeat the initial
            collocation data samples in an epoch
        :param shuffle: whether to shuffle the Cartesian product of the initial
            condition data and collocation data.
        """
        self._dataset = dataset
        self._n_batches = n_batches
        self._n_ic_repeats = n_ic_repeats
        self._shuffle = shuffle

        self._iv_data_size = dataset.initial_value_data.shape[0]
        self._domain_collocation_data_size = (
            dataset.domain_collocation_data.shape[0]
        )
        self._initial_collocation_data_size = (
            dataset.initial_collocation_data.shape[0]
        )
        self._boundary_collocation_data_size = (
            0
            if dataset.boundary_collocation_data is None
            else dataset.boundary_collocation_data.shape[0]
        )

        self._total_domain_data_size = (
            self._iv_data_size * self._domain_collocation_data_size
        )
        self._total_initial_data_size = (
            n_ic_repeats
            * self._iv_data_size
            * self._initial_collocation_data_size
        )
        self._total_boundary_data_size = (
            self._iv_data_size * self._boundary_collocation_data_size
        )

        if (
            self._total_domain_data_size % n_batches != 0
            or self._total_initial_data_size % n_batches != 0
            or self._total_boundary_data_size % n_batches != 0
        ):
            raise ValueError(
                "number of batches must be a common divisor of total domain "
                f"data size ({self._total_domain_data_size}), total initial "
                f"data size ({self._total_initial_data_size}), and total "
                f"boundary data size ({self._total_boundary_data_size})"
            )

        self._domain_batch_size = self._total_domain_data_size // n_batches
        self._initial_batch_size = self._total_initial_data_size // n_batches
        self._boundary_batch_size = self._total_boundary_data_size // n_batches

        self._domain_indices = self._create_cartesian_product_indices(
            self._iv_data_size, self._domain_collocation_data_size
        )
        self._initial_indices = np.tile(
            self._create_cartesian_product_indices(
                self._iv_data_size, self._initial_collocation_data_size
            ),
            (n_ic_repeats, 1),
        )
        self._boundary_indices = (
            self._create_cartesian_product_indices(
                self._iv_data_size, self._boundary_collocation_data_size
            )
            if self._boundary_collocation_data_size
            else None
        )

        self._batch_index = 0

    def __len__(self) -> int:
        return self._n_batches

    def __getitem__(self, index: int) -> Sequence[Sequence[tf.Tensor]]:
        return (
            self._get_domain_batch(index),
            self._get_initial_batch(index),
            self._get_boundary_batch(index),
        )

    def __next__(self) -> Sequence[Sequence[tf.Tensor]]:
        if self._batch_index >= self._n_batches:
            raise StopIteration

        batch = self[self._batch_index]
        self._batch_index += 1
        return batch

    def __iter__(self) -> DatasetIterator:
        self._batch_index = 0
        if self._shuffle:
            np.random.shuffle(self._domain_indices)
            np.random.shuffle(self._initial_indices)
            if self._boundary_collocation_data_size:
                np.random.shuffle(self._boundary_indices)

        return self

    @property
    def dataset(self) -> Dataset:
        """
        The dataset behind the iterator.
        """
        return self._dataset

    @property
    def domain_batch_size(self) -> int:
        """
        The domain data batch size used by the iterator.
        """
        return self._domain_batch_size

    @property
    def initial_batch_size(self) -> int:
        """
        The initial data batch size used by the iterator.
        """
        return self._initial_batch_size

    @property
    def boundary_batch_size(self) -> int:
        """
        The boundary data batch size used by the iterator.
        """
        return self._boundary_batch_size

    def to_infinite_generator(
        self,
    ) -> Generator[Sequence[Sequence[tf.Tensor]], None, None]:
        """
        Returns a generator that cycles over this iterator infinitely.
        """
        return (batch for _ in itertools.count(0) for batch in self)

    def _get_domain_batch(self, index: int) -> Sequence[tf.Tensor]:
        """
        Returns a domain data batch.

        :param index: the domain data batch index
        :return: the indexed domain data batch
        """
        start_index = index * self._domain_batch_size
        domain_indices = self._domain_indices[
            start_index : start_index + self._domain_batch_size, :
        ]
        domain_iv_data_indices = domain_indices[:, 0]
        domain_collocation_data_indices = domain_indices[:, 1]
        domain_iv_data = self._dataset.initial_value_data[
            domain_iv_data_indices
        ]
        domain_collocation_data = self._dataset.domain_collocation_data[
            domain_collocation_data_indices
        ]

        diff_eq = self._dataset.constrained_problem.differential_equation
        return (
            tf.convert_to_tensor(domain_iv_data, tf.float32),
            tf.convert_to_tensor(domain_collocation_data[:, :1], tf.float32),
            tf.convert_to_tensor(domain_collocation_data[:, 1:], tf.float32)
            if diff_eq.x_dimension
            else None,
        )

    def _get_initial_batch(self, index: int) -> Sequence[tf.Tensor]:
        """
        Returns an initial data batch.

        :param index: the initial data batch index
        :return: the indexed initial data batch
        """
        start_index = index * self._initial_batch_size
        initial_indices = self._initial_indices[
            start_index : start_index + self._initial_batch_size, :
        ]
        initial_iv_data_indices = initial_indices[:, 0]
        initial_collocation_data_indices = initial_indices[:, 1]
        initial_iv_data = self._dataset.initial_value_data[
            initial_iv_data_indices
        ]
        initial_collocation_data = self._dataset.initial_collocation_data[
            initial_collocation_data_indices
        ]

        initial_u_tensor = tf.convert_to_tensor(initial_iv_data, tf.float32)
        initial_t_tensor = tf.convert_to_tensor(
            initial_collocation_data[:, :1], tf.float32
        )

        diff_eq = self._dataset.constrained_problem.differential_equation
        if diff_eq.x_dimension:
            initial_x_tensor = tf.convert_to_tensor(
                initial_collocation_data[:, 1:], tf.float32
            )
            initial_y_tensor = tf.convert_to_tensor(
                initial_iv_data.reshape(
                    (self._initial_batch_size, -1, diff_eq.y_dimension)
                )[
                    np.arange(self._initial_batch_size),
                    initial_collocation_data_indices,
                    :,
                ],
                tf.float32,
            )
        else:
            initial_x_tensor = None
            initial_y_tensor = initial_u_tensor

        return (
            initial_u_tensor,
            initial_t_tensor,
            initial_x_tensor,
            initial_y_tensor,
        )

    def _get_boundary_batch(self, index: int) -> Optional[Sequence[tf.Tensor]]:
        """
        Returns a boundary data batch.

        :param index: the boundary data batch index
        :return: the indexed boundary data batch
        """
        if self._boundary_batch_size == 0:
            return None

        start_index = index * self._boundary_batch_size
        boundary_indices = self._boundary_indices[
            start_index : start_index + self._boundary_batch_size, :
        ]
        boundary_iv_data_indices = boundary_indices[:, 0]
        boundary_collocation_data_indices = boundary_indices[:, 1]
        boundary_iv_data = self._dataset.initial_value_data[
            boundary_iv_data_indices
        ]
        boundary_collocation_data = self._dataset.boundary_collocation_data[
            boundary_collocation_data_indices
        ]

        diff_eq = self._dataset.constrained_problem.differential_equation
        x_offset = 1
        y_offset = x_offset + diff_eq.x_dimension
        d_y_over_d_n_offset = y_offset + diff_eq.y_dimension
        axis_offset = d_y_over_d_n_offset + diff_eq.y_dimension

        return (
            tf.convert_to_tensor(boundary_iv_data, tf.float32),
            tf.convert_to_tensor(
                boundary_collocation_data[:, :x_offset], tf.float32
            ),
            tf.convert_to_tensor(
                boundary_collocation_data[:, x_offset:y_offset], tf.float32
            ),
            tf.convert_to_tensor(
                boundary_collocation_data[:, y_offset:d_y_over_d_n_offset],
                tf.float32,
            ),
            tf.convert_to_tensor(
                boundary_collocation_data[:, d_y_over_d_n_offset:axis_offset],
                tf.float32,
            ),
            tf.convert_to_tensor(
                boundary_collocation_data[:, axis_offset], tf.int32
            ),
        )

    @staticmethod
    def _create_cartesian_product_indices(
        first_set_size: int, second_set_size: int
    ) -> np.ndarray:
        """
        Creates a 2D array of indices for the Cartesian product of two sets of
        data rows.

        The first column of the returned array is the first set's row indices
        while the second column is the second set's row indices.

        :param first_set_size: the number of rows in the first set
        :param second_set_size: the number of rows in the second set
        :return: a 2D array with two columns of data row indices
        """
        first_set_indices = np.arange(0, first_set_size)
        second_set_indices = np.arange(0, second_set_size)
        cartesian_product_first_set_indices = np.repeat(
            first_set_indices, second_set_size, axis=0
        )
        cartesian_product_second_set_indices = np.tile(
            second_set_indices, (first_set_size,)
        )
        return np.stack(
            (
                cartesian_product_first_set_indices,
                cartesian_product_second_set_indices,
            ),
            axis=1,
        )


# Configure spark variables
from pyspark import SparkConf
from pyspark.context import SparkContext

conf = SparkConf().setAppName("appName").setMaster("local")
sc = SparkContext(conf=conf)

readRdd = sc.textFile("C:\\Project\\Files\\Input\\csv\\order_detail.csv")
# print(readRdd.collect())

header = readRdd.first()
print(header)

orderDetail = readRdd.filter(lambda x: x != header)
# print(orderDetail.collect())

orderToken = orderDetail.map(lambda x: x.split(","))
# print(orderToken.collect())

# Find the unique product code in the RDD
product_code = orderToken.map(lambda x: x[1]).distinct()
print(product_code.collect())

# Pair RDD Function
# Find the total sales amount for each product
productSalesPairRDD = orderToken.map(lambda x: (x[1], int(x[2]) * float(x[3])))
print(productSalesPairRDD.count())
productSalesRDD = productSalesPairRDD.reduceByKey(lambda acc, value: acc + value)
print(productSalesRDD.collect())

# Archive the same result using groupByKey()
productSalesRDD_1 = productSalesPairRDD.groupByKey().mapValues(lambda x: sum(x))
print(productSalesRDD_1.collect())


import datetime
from django import forms
from .models import  *
from account.models import Student , Teacher
from socle.models import Knowledge, Skill , Level
from group.models import Group
from bootstrap_datepicker_plus import DatePickerInput, DateTimePickerInput
from django.forms import MultiWidget, TextInput , CheckboxInput
from django.template.defaultfilters import filesizeformat
from django.conf import settings

from itertools import groupby
from django.forms.models import inlineformset_factory, BaseInlineFormSet , ModelChoiceIterator, ModelChoiceField, ModelMultipleChoiceField

def validation_file(content):
	if content :
		content_type = content.content_type.split('/')[0]
		if content_type in settings.CONTENT_TYPES:
			if content._size > settings.MAX_UPLOAD_SIZE:
				raise forms.ValidationError("Taille max : {}. Taille trop volumineuse {}".format(filesizeformat(settings.MAX_UPLOAD_SIZE), filesizeformat(content._size)))
		else:
			raise forms.ValidationError("Type de fichier non accepté")
		return content

class ToolForm(forms.ModelForm):

	class Meta:
		model = Tool
		fields = '__all__'

	def __init__(self, *args, **kwargs):
		super(ToolForm, self).__init__(*args, **kwargs)
		self.fields['levels'] = forms.ModelMultipleChoiceField(queryset=Level.objects.exclude(pk=13), required=False)

class QuestionForm(forms.ModelForm):

	class Meta:
		model = Question
		fields = '__all__'
		widgets = {
            'is_correct' : CheckboxInput(),
        }

	def __init__(self, *args, **kwargs):
		quizz = kwargs.pop('quizz')
		super(QuestionForm, self).__init__(*args, **kwargs)

		levels = quizz.levels.all()
		themes = quizz.themes.all()
		subject = quizz.subject
		knowledges = []
		if len(levels) > 0 and len(themes) > 0  :
			knowledges = Knowledge.objects.filter(theme__subject = subject ,level__in=levels, theme__in=themes )
		elif len(levels) > 0 :
			knowledges = Knowledge.objects.filter(theme__subject = subject ,level__in=levels)
		elif len(themes) > 0 :
			knowledges = Knowledge.objects.filter(theme__subject = subject ,theme__in=themes)
		self.fields['knowledge'] = forms.ModelChoiceField(queryset=knowledges, required=False)

	def clean_content(self):
		content = self.cleaned_data['imagefile']
		validation_file(content)
		audio_ = self.cleaned_data['audio']
		validation_file(audio_)
		video_ = self.cleaned_data['video']
		validation_file(video_)

class QuestionPositionnementForm(forms.ModelForm):

	class Meta:
		model = Question
		fields = '__all__'
		widgets = {
            'is_correct' : CheckboxInput(),
        }

	def __init__(self, *args, **kwargs):
		positionement = kwargs.pop('positionnement')
		super(QuestionPositionnementForm, self).__init__(*args, **kwargs)

		level_id = int(positionement.level.id)
		if 1<level_id < 13 :
			level_id = level_id - 1
			level    = Level.objects.get(pk=level_id)
		elif level_id == 1 : level    = Level.objects.get(pk=1)
		elif level_id == 14 : level    = Level.objects.get(pk=14)

		subject = positionement.subject
		knowledges = []

		if subject and level :
			knowledges = Knowledge.objects.filter(theme__subject = subject ,level = level)
			self.fields['knowledge'] = forms.ModelChoiceField(queryset=knowledges, required=False)

#################################################################################################################################################################################
###################################        FORMULAIRES GROUPES ET SOUS FORMULAIRES       ########################################################################################
#################################################################################################################################################################################
def is_empty_form(form):
    if form.is_valid() and not form.cleaned_data:
        return True
    else:
        return False

def is_form_persisted(form):
    if form.instance and not form.instance._state.adding:
        return True
    else:
        return False

class BaseSupportchoiceFormset(BaseInlineFormSet):

    def add_fields(self, form, index):
        super().add_fields(form, index)
        # save the formset in the 'nested' property
        form.nested = formSubSet(
                        instance=form.instance,
                        data=form.data if form.is_bound else None,
                        files=form.files if form.is_bound else None,
                        prefix='choice-{}-subchoice'.format( form.prefix ) )

    def is_valid(self):
        result = super(BaseSupportchoiceFormset, self).is_valid()
        if self.is_bound:
            for form in self.forms:
                if hasattr(form, 'nested'):
                    result = result and form.nested.is_valid()
        return result

    def clean(self):
        super(BaseSupportchoiceFormset, self).clean()
        for form in self.forms:
            if not hasattr(form, "nested") or self._should_delete_form(form):
                continue

            if self._is_adding_nested_inlines_to_empty_form(form):
                form.add_error(
                    field=None,
                    error=_(
                        "You are trying to add image(s) to a book which "
                        "does not yet exist. Please add information "
                        "about the book and choose the image file(s) again."
                    ),
                )

    def save(self, commit=True):
        result = super(BaseSupportchoiceFormset, self).save(commit=commit)
        for form in self.forms:
            if hasattr(form, 'nested'):
                if not self._should_delete_form(form):
                    form.nested.save(commit=commit)
        return result

    def _is_adding_nested_inlines_to_empty_form(self, form):
        if not hasattr(form, "nested"):
            return False

        if is_form_persisted(form):
            return False

        if not is_empty_form(form):
            return False

        non_deleted_forms = set(form.nested.forms).difference(set(form.nested.deleted_forms))

        return any(not is_empty_form(nested_form) for nested_form in non_deleted_forms)

formSetNested = inlineformset_factory(Question, Choice, fields=('answer','imageanswer','answerbis','imageanswerbis','is_correct','retroaction','xmin','xmax','tick','subtick','precision','is_written') , formset=BaseSupportchoiceFormset,   extra=1)
formSubSet    = inlineformset_factory(Choice, Subchoice, fields=('answer','imageanswer','label','is_correct','retroaction') , extra=2)

class BaseSupportchoiceUpdateFormset(BaseInlineFormSet):

    def add_fields(self, form, index):
        super().add_fields(form, index)
        # save the formset in the 'nested' property
        form.nested = formSubSetUpdate(
                        instance=form.instance,
                        data=form.data if form.is_bound else None,
                        files=form.files if form.is_bound else None,
                        prefix='choice-{}-subchoice'.format( form.prefix ) )

    def is_valid(self):
        result = super().is_valid()
        if self.is_bound:
            for form in self.forms:
                if hasattr(form, 'nested'):
                    result = result and form.nested.is_valid()
        return result

    def save(self, commit=True):
        result = super().save(commit=commit)
        for form in self.forms:
            if hasattr(form, 'nested'):
                if not self._should_delete_form(form):
                    form.nested.save(commit=commit)

        return result

formSetUpdateNested = inlineformset_factory(Question, Choice, fields=('answer','imageanswer','answerbis','imageanswerbis','is_correct','retroaction','xmin','xmax','tick','subtick','precision','is_written')  , formset=BaseSupportchoiceUpdateFormset,   extra=0)
formSubSetUpdate    = inlineformset_factory(Choice, Subchoice, fields=('answer','imageanswer','label','is_correct','retroaction') , extra=0)
#################################################################################################################################################################################
###################################    UPDATE FORMULAIRES GROUPES ET SOUS FORMULAIRES       #####################################################################################
#################################################################################################################################################################################

class QuizzForm(forms.ModelForm):

	class Meta:
		model = Quizz
		fields = '__all__'

	def __init__(self, *args, **kwargs):
		teacher = kwargs.pop('teacher')
		folder  = kwargs.pop('folder')
		group   = kwargs.pop('group')

		super(QuizzForm, self).__init__(*args, **kwargs)

		if group : all_folders = group.group_folders.filter(is_archive=0,is_trash=0)
		else : all_folders = teacher.teacher_folders.filter(is_archive=0,is_trash=0)

		if folder : parcours = folder.parcours.filter(is_archive=0,is_trash=0)
		else : parcours =  teacher.teacher_parcours.filter(is_archive=0,is_trash=0)

		coteacher_parcours = teacher.coteacher_parcours.filter(is_archive=0,is_trash=0)
		all_parcours = parcours|coteacher_parcours

		groups =  teacher.groups.all()
		teacher_groups = teacher.teacher_group.all()
		all_groups = groups|teacher_groups

		self.fields['levels']   = forms.ModelMultipleChoiceField(queryset=teacher.levels.all(), required=False)
		self.fields['subject']  = forms.ModelChoiceField(queryset=teacher.subjects.all(), required=False)
		self.fields['groups']   = forms.ModelMultipleChoiceField(queryset=all_groups.order_by("teachers","level"), widget=forms.CheckboxSelectMultiple, required=True)
		self.fields['parcours'] = forms.ModelMultipleChoiceField(queryset = all_parcours.order_by("level"), widget=forms.CheckboxSelectMultiple,  required=False)
		self.fields['folders']  = forms.ModelMultipleChoiceField(queryset = all_folders.order_by("level"), widget=forms.CheckboxSelectMultiple,  required=False)

	def clean_content(self):
		content = self.cleaned_data['imagefile']
		validation_file(content)

class PositionnementForm(forms.ModelForm):

	class Meta:
		model = Positionnement
		fields = '__all__'
	def __init__(self, *args, **kwargs):
		super(PositionnementForm, self).__init__(*args, **kwargs)
		self.fields['level']   = forms.ModelChoiceField( queryset=Level.objects.exclude(pk=13).order_by("ranking") , required=False)

class ChoiceForm(forms.ModelForm):
	class Meta:
		model = Choice
		fields = '__all__'

	def clean_content(self):
		content = self.cleaned_data['imagefile']
		validation_file(content)

class DiaporamaForm(forms.ModelForm):

	class Meta:
		model = Diaporama
		fields = '__all__'

	def __init__(self, *args, **kwargs):
		teacher = kwargs.pop('teacher')
		super(DiaporamaForm, self).__init__(*args, **kwargs)
		groups = teacher.groups.order_by("level") | teacher.teacher_group.order_by("group__level")

		self.fields['levels'] = forms.ModelMultipleChoiceField(queryset=teacher.levels.all(), required=False)
		self.fields['subject'] = forms.ModelChoiceField(queryset=teacher.subjects.all(), required=False)
		self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=teacher.groups.all(), required=False)

	def clean_content(self):
		content = self.cleaned_data['imagefile']
		validation_file(content)

class SlideForm(forms.ModelForm):

	class Meta:
		model = Slide
		fields = '__all__'

class QrandomForm(forms.ModelForm):

	class Meta:
		model = Qrandom
		fields = '__all__'

	def clean_content(self):
		content = self.cleaned_data['imagefile']
		validation_file(content)

class VariableForm(forms.ModelForm):

	class Meta:
		model = Variable
		fields = '__all__'

class AnswerplayerForm(forms.ModelForm):

	class Meta:
		model = Answerplayer
		fields = '__all__'

class VideocopyForm(forms.ModelForm):

	class Meta:
		model = Videocopy
		fields = '__all__'

	def clean_content(self):
		content = self.cleaned_data['image']
		validation_file(content)


import os
from functools import wraps

from commons.data.savers import DATASETS_DIR_NAME, DICTIONARY_NAME, NEGATIVITY_BIPART_DICT_NAME, DISCORD_BIPART_DICT_NAME, NUMERICAL_SEPARABILITY_BIPART_DICT_NAME, NUM_NEAR_ZERO_EIGVALS_BIPART_DICT_NAME

def generate_with_assertion(batch_size=None):
    def _decorator(generation_function, batch_size=batch_size):
        @wraps(generation_function)
        def _wrapper(self, *args, batch_size=batch_size, **kwargs):
            try:
                save_data_dir = args[2]
            except:
                save_data_dir = kwargs['save_data_dir']
            if save_data_dir == None or kwargs.get('return_matrices', False):
                return generation_function(self, *args, **kwargs)

            try:
                qubits_num = args[0]
            except:
                qubits_num = kwargs['qubits_num']
            try:
                desired_examples = args[1]
                if batch_size is not None and desired_examples > batch_size:
                    args[1] = batch_size
                    if 'base_size' in kwargs:
                        kwargs['base_size'] = batch_size
            except:
                desired_examples = kwargs['examples']
                if batch_size is not None and desired_examples > batch_size:
                    kwargs['examples'] = batch_size
                    if 'base_size' in kwargs:
                        kwargs['base_size'] = batch_size

            if batch_size is None or desired_examples < batch_size:
                batch_size = desired_examples

            filepath = f"./{DATASETS_DIR_NAME}/{qubits_num}qbits/{save_data_dir}/"
            if not os.path.isdir(filepath):
                os.makedirs(filepath)
            fps = [filepath + DICTIONARY_NAME]
            if kwargs.get('encoded', False):
                fps.append(filepath + NEGATIVITY_BIPART_DICT_NAME)
                fps.append(filepath + NUMERICAL_SEPARABILITY_BIPART_DICT_NAME)
                fps.append(filepath + NUM_NEAR_ZERO_EIGVALS_BIPART_DICT_NAME)
            if kwargs.get('discord', False):
                fps.append(filepath + DISCORD_BIPART_DICT_NAME)

            try:
                with open(fps[0], 'r') as f:
                    start_num_examples = len(f.readlines())
            except:
                start_num_examples = 0

            generated_examples = 0
            start_indx = kwargs.get('start_index', 0)
            kwargs['start_index'] = start_indx

            while generated_examples < desired_examples:
                generation_function(self, *args, **kwargs)
                for fp in fps:
                    try:
                        with open(fp, 'r+') as dictionary:
                            data = dictionary.readlines()
                            generated_examples = len(data) - start_num_examples
                            if generated_examples > desired_examples:
                                extra_lines = generated_examples - desired_examples
                                dictionary.seek(0)
                                dictionary.truncate()
                                dictionary.writelines(data[:-extra_lines])
                                generated_examples -= extra_lines
                    except:
                        pass

                kwargs['start_index'] += batch_size
            return kwargs['start_index']

        return _wrapper
    return _decorator


from django.http import JsonResponse
from urllib.parse import quote
from random import randint
from django.core.cache import cache

def get_state():
    res = ""
    for i in range(8):
        res += str(randint(0, 9))
    return res

def apply_code(request):
    response_type = "code"
    client_id = "102024822"
    redirect_uri = quote("https://aigame.zzqahm.top/player/qq/receive_code")
    scope = "get_user_info"
    state = get_state()
    cache.set(state, True, 7200)

    apply_code_url = "https://graph.qq.com/oauth2.0/authorize"
    return JsonResponse({
        'result': "success",
        'apply_code_url': apply_code_url + "?response_type=%s&client_id=%s&redirect_uri=%s&scope=%s&state=%s" % (response_type, client_id, redirect_uri, scope, state)
    })


#!/usr/bin/python3

# python3 mg30.py [ticker]
# return 0: success

import sys, requests, time, re, os, webbrowser
import urllib.request
from datetime import timedelta, datetime
from dateutil.relativedelta import relativedelta
from bs4 import BeautifulSoup
from pprint import pprint

tbl = []
title = []
net_m   = [ 0, 0, 0, 0, 0 ]
net_d   = [ 0, 0, 0, 0, 0 ]
latch   = [ 0, 0, 0, 0, 0 ]
sample = 0
MAX_SAMPLE = 7
def latch_consequtive_bs(amount, index):
    if not latch[index]:
        # amount = int( tds[index].text.strip() )
        if ( 0 < amount and 0 <= net_m[index] and net_d[index] <= 0 ):
            net_m[index] += 1
        elif ( amount < 0 and 0 <= net_d[index] and net_m[index] <= 0 ):
            net_d[index] += 1
        else:
            latch[index] = 1

# @see https://stackoverflow.com/a/66187139
def say(msg = "Finish", voice = "Victoria"):
    os.system(f'say -v {voice} {msg}')

ticker = sys.argv[1]

mg30 = "datafiles/mg30.txt" # download html
with open(mg30) as fp:
    soup = BeautifulSoup(fp, 'html.parser')

'''
url = "https://histock.tw/stock/chips.aspx?no="+ticker+"&m=mg"
# print(url)
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
'''

if ( soup is None ):
    print("n\a")
# print(soup.prettify()) # be aware, less frequent access

tr1 = soup \
    .findAll("table", class_="tb-stock tbChip nofade")[0] \
    .find_all('tr')

tr2 = soup \
    .findAll("table", class_="tb-stock tbChip fade")[0] \
    .find_all('tr')

tr3 = soup \
    .findAll("table", class_="tb-stock tbChip fade")[1] \
    .find_all('tr')

if ( len(tr1) <=0 or len(tr2) <=0 or len(tr3) <=0 ):
    print("tables not found")

ths = tr1[0].find_all('th')
title.append( ths[0].text.replace('\n','').strip().center(6," ") )
title.append( ths[1].text.replace('\n','').strip().center(6," ") )
title.append( ths[2].text.replace('\n','').strip().center(6," ") )

ths2 = tr2[0].find_all('th')
title.append( ths2[2].text.replace('\n','').strip().center(6," ") )

ths3 = tr3[0].find_all('th')
title.append( ths3[1].text.replace('\n','').strip().center(6," ") )
tbl.append(title)
tbl.append(net_m)
tbl.append(net_d)
tbl.append(latch)

for i in range(2, 2+MAX_SAMPLE):
    tds1 = tr1[i].find_all('td')
    tds2 = tr2[i].find_all('td')

    row = []
    row.append(tds1[0].text.strip())
    row.append(tds1[1].text.strip().rjust(6," "))
    row.append(tds1[4].text.strip().rjust(6," "))
    row.append(tds2[11].text.strip().rjust(6," "))
    tbl.append(row)

for i in range(1, 1+MAX_SAMPLE):
    tds3 = tr3[i].find_all('td')
    tbl[3+i].append(tds3[1].text.strip().rjust(6," "))

for i in range(4, 4+MAX_SAMPLE):
    for j in range(1, len(tbl[4])):
        latch_consequtive_bs( \
            int( tbl[i][j].replace(',','').strip() ), j)

pprint(tbl)

for i in range(0, len(tbl[0])):
    if ( 3 <= tbl[1][i] ):
        msg = title[i] + "連" + str(net_m[i]) + "增"
        say(msg, "Mei-Jia")
    if ( 3 <= tbl[2][i] ):
        msg = title[i] + "連" + str(net_d[i]) + "減"
        say(msg, "Mei-Jia")

# // TODO: check if in the list
# https://www.wantgoo.com/stock/public-bank/buy-sell?market=-1&orderBy=count&during=5&industry=
#

sys.exit(0)


# eng_network_single_b29.py
# Version b29
# by jmg - j.gagen*AT*gold*DOT*ac*DOT*uk
# May 10th 2016

# Licence: http://creativecommons.org/licenses/by-nc-sa/3.0/
# Source code at: https://github.com/pha5echange/eng-tools

# Name changed to 'eng_network_single'

# Plots network graph from edgelist 'data\OMEGAYEAR_wuGraphData.txt'
# Adds dates from 'data\first_clusters.txt' as node attributes
# Adds artist numbers from 'data\OMEGAYEAR_artistNums.txt' as node attributes
# Removes nodes where date-cluster information is not available
# Removes self-loop edges and zero-degree nodes if required
# Removes nodes based upon 'incepDate' if required - all nodes NEWER than Omega Year are removed
# Displays using parameters from 'config/config_nw.txt'
# Writes 'data\eng_network_wd_nodeList.txt' with nodes and degrees (k), '...edgeList.txt' with neighbours
# Writes analysis files to 'results\'
# Writes gexf files to 'gexf\', including 'gexf\YEAR.gexf' for use by 'nhm.py'
# Writes image to 'networks\'
# This version incorporates PageRank
# Incorporates Artist Time Slicing
# Added 'maxDeg' metric and 'isolated nodes' counter
# This version writes a list of nodes in the Largest Connected Component to 'ts_data' and a GEXF of the LCC

# Run AFTER 'eng_nodesets.py'

# Import packages
import os
import resource
import numpy as np
import networkx as nx
import community
from networkx.algorithms.approximation import clique
import matplotlib.pyplot as plt
from collections import OrderedDict
from datetime import datetime

versionNumber = ("b29")

# Initiate timing of run
runDate = datetime.now()
startTime = datetime.now()

# Create 'logs' subdirectory if necessary
if not os.path.exists("logs"):
    os.makedirs("logs")

# Create 'data' subdirectories if necessary
if not os.path.exists("data"):
	os.makedirs("data")

if not os.path.exists("data/node-edge-lists"):
	os.makedirs("data/node-edge-lists")

# Create 'gexf' subdirectories if necessary
if not os.path.exists("gexf"):
	os.makedirs("gexf")

if not os.path.exists("gexf/initial"):
	os.makedirs("gexf/initial")

if not os.path.exists("gexf/directed"):
	os.makedirs("gexf/directed")

if not os.path.exists("gexf/final"):
	os.makedirs("gexf/final")

if not os.path.exists("gexf/lcc"):
	os.makedirs("gexf/lcc")

# Create 'results' subdirectories if necessary
if not os.path.exists("results"):
	os.makedirs("results")

if not os.path.exists("results/analysis"):
	os.makedirs("results/analysis")

# Create 'networks' subdirectory if necessary
if not os.path.exists("networks"):
    os.makedirs("networks")

# Open file for writing log
logPath = os.path.join("logs", 'eng_network_single_' + versionNumber + '_log.txt')
runLog = open(logPath, 'a')

# Begin
print ('\n' + "Single-Network Thing | Version " + versionNumber + " | Starting...")
runLog.write ("==========================================================================" + '\n' + '\n')
runLog.write ("Single-Network Thing | Version " + versionNumber + '\n' + '\n')

# Get user input
print
dateIP = int(input ("Enter an omega year for this graph (or 0 to leave the network intact): "))
omegaYear = str(dateIP)

# Uncomment the line below to facilitate optional self-loop removal
#selfLoopIP = int(input ("Enter 1 here to remove self-loop edges: "))
# Auto-remove self-loops - comment out the line below if self-loop removal is optional
selfLoopIP = 1

# Uncomment the line below to facilitate optional isolate removal
#isolatedIP = int(input ("Enter 1 here to remove isolated nodes: "))
# Do not remove isolated nodes - comment out the line below if isolate removal is optional
isolatedIP = 0

# Generate dictionary containing nodenames and dates (from 'data\first_cluster.txt')
dateInputPath = os.path.join("data", 'first_cluster.txt')
dateInput = open(dateInputPath, 'r')

dateDict = {}

for line in dateInput:
	genreInput, genreDate, newline = line.split(",")

	# Remove genres AFTER the Omega Year
	intGenreDate = int(genreDate)

	if intGenreDate <= dateIP:
		genreName = str(omegaYear + '_' + genreInput).replace(" ", "")
		dateDict[genreName] = genreDate

sortDates = OrderedDict(sorted(dateDict.items()))

# Generate dictionary containing nodenames and artist numbers (from 'data\OMEGAYEAR\_artistNums.txt')
artistInputPath = os.path.join("ts_data", omegaYear, omegaYear + '_artistNums.txt')
artistInput = open(artistInputPath, 'r').readlines()

firstLine = artistInput.pop(0)

artistDict = {}

for line in artistInput:
	genreInput, artistNum, newline = line.split(",")
	genreName = str(genreInput).replace(" ", "")
	artistDict[genreName] = artistNum

sortArtists = OrderedDict(sorted(artistDict.items()))

# Open file to write list of nodes
nodeListPath = os.path.join("ts_data", omegaYear, 'eng_network_' + versionNumber + '_' + omegaYear + '_nodeList.txt')
nodeListOP = open (nodeListPath, 'w')

# Open file for writing LCC nodes
lccPath = os.path.join("ts_data", omegaYear, 'eng_network_' + versionNumber + '_' + omegaYear + '_lcc.txt')
lccOP = open(lccPath, 'w')

# Open file to write list of edges
edgeListPath = os.path.join("ts_data", omegaYear, 'eng_network_' + versionNumber + '_' + omegaYear + '_edgeList.txt')
edgeListOP = open (edgeListPath, 'w')

# Open file for writing initial gexf
gexfPath = os.path.join("gexf/initial", 'eng_network_' + versionNumber + '_' + omegaYear + '.gexf')
gexfFile = open(gexfPath, 'w')

# Open file for writing digraph gexf
gexfDPath = os.path.join("gexf/directed", omegaYear + '.gexf')
gexfDFile = open(gexfDPath, 'w')

# Open file for writing final gexf
gexfFinPath = os.path.join("gexf/final", 'eng_network_final_' + versionNumber + '_' + omegaYear + '.gexf')
gexfFinFile = open(gexfFinPath, 'w')

# OPen file for writing LCC gexf
gexfLccPath = os.path.join("gexf/lcc", 'eng_lcc_' + versionNumber + '_' + omegaYear + '.gexf')
gexfLccFile = open(gexfLccPath, 'w')

# Open file for Page Rank data
prPath = os.path.join("ts_data", omegaYear, omegaYear +'_pagerank.txt')
prFile = open(prPath, 'w')

# Open file for analysis results
anPath = os.path.join("results/analysis", 'eng_network_' + versionNumber + '_' + omegaYear + '_analysis.txt')
anFile = open(anPath, 'w')

# Open file to write image
nwImgPath = os.path.join("networks", 'eng_network_' + versionNumber + '_' + omegaYear + '_nw.eps')

anFile.write ('\n' + "==========================================================================" + '\n' + '\n')
anFile.write ("Single-Network Thing | Version " + versionNumber + '\n' + '\n')

# Read the edgelist and generate undirected graph
print ('\n' + "Importing Weighted Edge List... ")
inputPath = os.path.join("ts_data", omegaYear, omegaYear + '_wuGraph_data.txt')
edgeList = open (inputPath, 'r')
enGraph = nx.read_weighted_edgelist(edgeList, delimiter=',')

# Calculate basic graph statistics
print ('\n' + "Calculating various things... " + '\n')
nodes = nx.number_of_nodes(enGraph)
edges = nx.number_of_edges(enGraph)
density = nx.density(enGraph)
nodeList = nx.nodes(enGraph)
nodeList.sort()
selfLoopTotal = enGraph.number_of_selfloops()
#selfLoops = enGraph.selfloop_edges()
connections = edges - selfLoopTotal

print ('Nodes: ' + str(nodes))
print ('Edges: ' + str(edges))
print ('Number of self-loops: ' + str(selfLoopTotal))
#print ('Self-loop edges: ' + str(selfLoops))
print ('Connections (edges minus self-loops): ' + str(connections))
print ('Density: ' + str(density))
print

runLog.write ('Initial data: ' + '\n' + '\n')
runLog.write ('User-entered date: ' + str(dateIP) + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
#runLog.write ('Self-loop edges: ' + str(selfLoops) + '\n')
runLog.write ('Connections (edges minus self-loops): ' + str(connections) + '\n')
runLog.write ('Density: ' + str(density) + '\n')

# Apply artist numbers of nodes as attributes
print ('Applying total artist number attribute to nodes...' + '\n')
runLog.write ('\n' + 'Applying total artist number attribute to nodes...' + '\n')
nx.set_node_attributes (enGraph, 'totalArtists', sortArtists)

# Discard nodes without first-cluster dates
noDateNode = 0
print ('Checking genre inception dates, and removing if none...' + '\n')
for i in nodeList:
	if not i in sortDates.keys():
		enGraph.remove_node(i)
		print ('Removed node ' + str(i))
		noDateNode += 1

if noDateNode == 1:
	print ('\n' + 'Removed ' + str(noDateNode) + ' node due to no inception date.')
	runLog.write ('\n' + 'Removed ' + str(noDateNode) + ' node due to no inception date.' + '\n')

if noDateNode > 1:
	print ('\n' + 'Removed ' + str(noDateNode) + ' nodes due to no inception dates.')
	runLog.write ('\n' + 'Removed ' + str(noDateNode) + ' nodes due to no inception dates.' + '\n')

# Apply dates of nodes as attributes
print ('Applying inception date attribute to nodes...' + '\n')
runLog.write ('\n' + 'Applying inception date attribute to nodes...' + '\n')
nx.set_node_attributes (enGraph, 'incepDate', sortDates)

# Recalculate basic graph statistics
print ('\n' + 'Recalculating various things...' + '\n')
nodes = nx.number_of_nodes(enGraph)
edges = nx.number_of_edges(enGraph)
density = nx.density(enGraph)
nodeList = nx.nodes(enGraph)
nodeList.sort()
selfLoopTotal = enGraph.number_of_selfloops()
connections = edges - selfLoopTotal

print ('Nodes: ' + str(nodes))
print ('Edges: ' + str(edges))
print ('Self-loops: ' + str(selfLoopTotal))
print ('Connections (edges minus self-loops): ' + str(connections))
print ('Density: ' + str(density))
print

runLog.write ('\n' + 'Stage 1 data: ' + '\n' + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
runLog.write ('Connections (edges minus self-loops): ' + str(connections) + '\n')
runLog.write ('Density: ' + str(density) + '\n')

# Remove nodes where 'incepDate' attribute is NEWER than user input ('dateIP')
if dateIP != 0:
	runLog.write('\n' + 'Removing nodes based upon user date input...' + '\n')
	for i in nodeList:
		incepDate = nx.get_node_attributes (enGraph, 'incepDate')
		intIncep = int(incepDate[i])
		if intIncep > dateIP:
			enGraph.remove_node(i)
			print ('Removed newer node ' + str(i) + ' based upon user date input.')
else:
	print ('No nodes removed based upon user date input.')
	runLog.write ('\n' + 'No nodes removed based upon user date input.' + '\n')

# Recalculate basic graph statistics
print ('\n' + 'Recalculating various things...' + '\n')
nodes = nx.number_of_nodes(enGraph)
edges = nx.number_of_edges(enGraph)
density = nx.density(enGraph)
nodeList = nx.nodes(enGraph)
nodeList.sort()
selfLoopTotal = enGraph.number_of_selfloops()
connections = edges - selfLoopTotal

print ('Nodes: ' + str(nodes))
print ('Edges: ' + str(edges))
print ('Self-loops: ' + str(selfLoopTotal))
print ('Connections (edges minus self-loops): ' + str(connections))
print ('Density: ' + str(density))
print

runLog.write ('\n' + 'Stage 2 data: ' + '\n' + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
runLog.write ('Connections (edges minus self-loops): ' + str(connections) + '\n')
runLog.write ('Density: ' + str(density) + '\n')

# Remove self-loops
selfLoopCount = 0
if selfLoopIP == 1:
	print ('\n' + 'Checking for and removing self-loops...' + '\n')
	runLog.write('\n' + 'Checking for and removing self-loops...' + '\n')
	for u,v,data in enGraph.edges(data=True):
		if u == v:
			enGraph.remove_edge(u,v)
			print ('removed self-loop ' + str(u))
			selfLoopCount += 1

	if selfLoopCount == 1:
		print ('\n' + 'Removed ' + str(selfLoopCount) + ' self-loop edge.')
		runLog.write ('\n' + 'Removed ' + str(selfLoopCount) + ' self-loop edge.' + '\n')

	if selfLoopCount > 1:
		print ('\n' + 'Removed ' + str(selfLoopCount) + ' self-loop edges.')
		runLog.write ('\n' + 'Removed ' + str(selfLoopCount) + ' self-loop edges.' + '\n')
else:
	print ('Self-loops intact.' + '\n')
	runLog.write('\n' + 'Self-loops intact.' + '\n')

# Remove zero degree nodes
isolateCount = 0
if isolatedIP == 1:

	print ('\n' + 'Checking for and removing isolated (zero degree) nodes...' +'\n')
	runLog.write('\n' + 'Checking for and removing isolated (zero degree) nodes...' +'\n' + '\n')
	for i in nodeList:
		if nx.is_isolate(enGraph,i):
			enGraph.remove_node(i)
			print ('Removed isolated node ' + str(i))
			runLog.write('Removed isolated node ' + str(i) + '\n')
			isolateCount += 1

	if isolateCount == 1:
		print ('\n' + "Removed " + str(isolateCount) + " isolated node. " + '\n')
		runLog.write ('\n' + "Removed " + str(isolateCount) + " isolated node. " + '\n')

	if isolateCount > 1:
		print ('\n' + "Removed " + str(isolateCount) + " isolated nodes. " + '\n')
		runLog.write ('\n' + "Removed " + str(isolateCount) + " isolated nodes. " + '\n')

else:

	for i in nodeList:
		if nx.is_isolate(enGraph,i):
			isolateCount += 1

	if isolateCount == 1:
		print ('\n' + str(isolateCount) + ' isolated node intact.')
		runLog.write('\n' + str(isolateCount) + ' isolated node intact.' + '\n')

	if isolateCount > 1:
		print ('\n' + str(isolateCount) + ' isolated nodes intact.')
		runLog.write('\n' + str(isolateCount) + ' isolated nodes intact.' + '\n')

# Recalculate basic graph statistics
print ('Recalculating various things...' + '\n')
nodes = nx.number_of_nodes(enGraph)
edges = nx.number_of_edges(enGraph)
density = nx.density(enGraph)
nodeList = nx.nodes(enGraph)
nodeList.sort()
selfLoopTotal = enGraph.number_of_selfloops()

print ('Nodes: ' + str(nodes))
print ('Isolated nodes:' + str(isolateCount))
print ('Edges: ' + str(edges))
print ('Self-loops: ' + str(selfLoopTotal))
print ('Density: ' + str(density))
print

runLog.write ('\n' + 'Stage 3 data: ' + '\n' + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Isolated nodes:' + str(isolateCount) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
runLog.write ('Density: ' + str(density) + '\n')

# Clean edge-labels
print ('Cleaning edge labels...')
labels = {}
for u,v,data in enGraph.edges(data=True):
	labels[(u,v)] = int(data['weight'])

# Write file with nodes and degree,for reference
print ('\n' + 'Writing node list with degree and neighbours...' + '\n')
for i in nodeList:
	nodeDegree = enGraph.degree(i)
	neighboursList = list(nx.all_neighbors(enGraph, i))
	nodeListOP.write(str(i) + ',' + str(nodeDegree) + ',' + str(neighboursList) +'\n')
	print ("Node " + str(i) + " degree: " + str(nodeDegree))
	print ("Neighbours: " + str(neighboursList))

nodeListOP.close()

# Calculate new graph-wide total artists value
# Use artist-total node attributes
totalArtists = 0
totalEdgeWeight = 0
edgeList = enGraph.edges(data=True)

for i in nodeList:
	nodeArtists = int(sortArtists[i])
	totalArtists += nodeArtists

for i in edgeList:
	nodesStr = str(i).replace("u'","").replace("'","").replace("(","").replace(")","").replace(" ","")
	nodeU, nodeV, edgeData = nodesStr.split(",")
	edgeWeight = float(str(edgeData).replace("{weight:","").replace("}",""))
	totalEdgeWeight += edgeWeight

# Analysis
print ('\n' + 'Analysing undirected graph...' + '\n')
print ('Average clustering coefficient...' + '\n')
avClustering = nx.average_clustering(enGraph)
print ('Connected components...' + '\n')
connectComp = [len(c) for c in sorted(nx.connected_components(enGraph), key=len, reverse=True)]
print ('Find cliques...' + '\n')
cl = nx.find_cliques(enGraph)
cl = sorted(list(cl), key = len, reverse = True)
print ('Number of cliques: ' + str(len(cl)) + '\n')
cl_sizes = [len(c) for c in cl]
#print ('Size of cliques: ' + str(cl_sizes))

print ('\n' + 'Undirected graph data: ' + '\n')
print ('Nodes: ' + str(nodes))
print ('Edges: ' + str(edges))
print ('Self-loops: ' + str(selfLoopTotal))
print ('Density: ' + str(density))
print ('Average Clustering Coefficient: ' + str(avClustering))
print ('Number of cliques: ' + str(len(cl)))
print ('Connected Components: ' + str(connectComp))
print
print (str(nx.info(enGraph)))
print

runLog.write ('\n' + 'Undirected graph data: ' + '\n' + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
runLog.write ('Density: ' + str(density) + '\n')
runLog.write ('Average Clustering Coefficient: ' + str(avClustering) + '\n')
runLog.write ('Number of cliques: ' + str(len(cl)) + '\n')
runLog.write ('Connected Components: ' + str(connectComp) + '\n')
runLog.write ('\n' + str(nx.info(enGraph)) + '\n')

# Write undirected gexf file for use in Gephi
print ("Writing undirected gexf file... " + '\n')
runLog.write('\n' + "Writing undirected gexf file... " + '\n')
nx.write_gexf(enGraph, gexfFile)
gexfFile.close()

# Direct graph and write gexf of this
print ("Directing graph... " + '\n')
runLog.write('\n' + "Directing graph..." + '\n')
diEnGraph = nx.DiGraph()
diEnGraph.add_nodes_from(enGraph.nodes(data=True))
diEnGraph.add_edges_from(enGraph.edges(data=True))

edgeList = diEnGraph.edges(data=True)
removedNodes = 0
for i in edgeList:
	# clean up fraking unicode string
	nodesStr = str(i).replace("u'","").replace("'","").replace("(","").replace(")","").replace(" ","")

	# display / write a list of edges
	print(nodesStr)
	edgeListOP.write(nodesStr + '\n')

	# implement calculation for edge-direction
	nodeU, nodeV, edgeData = nodesStr.split(",")
	edgeWeight = float(str(edgeData).replace("{weight:","").replace("}",""))
	nodeUdate = int(sortDates[nodeU])
	print(nodeUdate)
	nodeVdate = int(sortDates[nodeV])
	print (nodeVdate)

	if nodeUdate > nodeVdate:
		# direct from v to u
		print("Directing edge from V to U" + '\n')
		# remove the edge
		diEnGraph.remove_edge(nodeU,nodeV)
		# add a directed edge
		diEnGraph.add_edge(nodeV, nodeU, weight=edgeWeight)

	elif nodeVdate > nodeUdate:
		# direct from u to v
		print("Direct from U to V" + '\n')
		# remove the edge
		diEnGraph.remove_edge(nodeU,nodeV)
		# add a directed edge
		diEnGraph.add_edge(nodeU, nodeV,  weight=edgeWeight)

	elif nodeUdate == nodeVdate:
		# remove the edge
		diEnGraph.remove_edge(nodeU,nodeV)
		# write to a list of removed edges
		removedNodes += 1
		print("Removed edge between nodes " + str(nodeU) + " and " + str(nodeV) + " due to identical node inception dates. " + '\n')
		edgeListOP.write("Removed edge between nodes " + str(nodeU) + " and " + str(nodeV) + " due to identical node inception dates. " + '\n')

print (str(removedNodes) + " edges removed due to identical node inception dates. " + '\n')
runLog.write ('\n' + str(removedNodes) + " edges removed due to identical node inception dates. " + '\n')
edgeListOP.close()

print ("Writing directed gexf file... " + '\n')
runLog.write('\n' + "Writing directed gexf file... " + '\n')
nx.write_gexf(diEnGraph, gexfDFile)
gexfDFile.close()

# Recheck zero degree nodes
print ('Rechecking isolated (zero degree) nodes...' +'\n')
runLog.write('\n' + 'Rechecking isolated (zero degree) nodes...' +'\n' + '\n')

diNodeList = nx.nodes(diEnGraph)
diNodeList.sort()

isolateCount = 0
if isolatedIP == 1:

	for i in diNodeList:
		if nx.is_isolate(diEnGraph,i):
			diEnGraph.remove_node(i)
			print ('Removed isolated node ' + str(i))
			runLog.write('Removed isolated node ' + str(i) + '\n')
			isolateCount += 1

	if isolateCount == 1:
		print ("Removed " + str(isolateCount) + " isolated node. " + '\n')
		runLog.write ('\n' + "Removed " + str(isolateCount) + " isolated node. " + '\n')

	if isolateCount > 1:
		print ("Removed " + str(isolateCount) + " isolated nodes. " + '\n')
		runLog.write ('\n' + "Removed " + str(isolateCount) + " isolated nodes. " + '\n')

else:

	for i in diNodeList:
		if nx.is_isolate(diEnGraph,i):
			isolateCount += 1

	if isolateCount == 1:
		print (str(isolateCount) + ' isolated node intact.' +'\n')
		runLog.write('\n' + str(isolateCount) + ' isolated node intact.' + '\n')

	if isolateCount > 1:
		print (str(isolateCount) + ' isolated nodes intact.' + '\n')
		runLog.write('\n' + str(isolateCount) + ' isolated nodes intact.' + '\n')

# Plot and display graph
# Graph plotting parameters - moved to config file 'config_nw.txt'
print ('Reading layout config file...' + '\n')

# Open and read 'config_nw.txt'
nwConfigPath = os.path.join ("config", 'config_nw.txt')
nwConfig = open(nwConfigPath, 'r').readlines()

# Remove the first line
firstLine = nwConfig.pop(0)

for line in nwConfig:
	n_size, n_alpha, node_colour, n_text_size, text_font, e_thickness, e_alpha, edge_colour, l_pos, e_text_size, edge_label_colour = line.split(",")

node_size = int(n_size)
node_alpha = float(n_alpha)
node_text_size = int(n_text_size)
edge_thickness = int(e_thickness)
edge_alpha = float(e_alpha)
label_pos = float(l_pos)
edge_text_size = int(e_text_size)

print ('Laying out graph...' + '\n')

#nx.draw(enGraph)
graph_pos = nx.spring_layout(diEnGraph)
nx.draw_networkx_nodes(diEnGraph, graph_pos, node_size = node_size, alpha = node_alpha, node_color=node_colour)
nx.draw_networkx_edges(diEnGraph, graph_pos, width = edge_thickness, alpha = edge_alpha, color = edge_colour)
#nx.draw_networkx_labels(diEnGraph, graph_pos, font_size = node_text_size, font_family = text_font)
#nx.draw_networkx_edge_labels(diEnGraph, graph_pos, edge_labels = labels, label_pos = label_pos, font_color = edge_label_colour, font_size = edge_text_size, font_family = text_font)

# write image file
#print ('Writing image file...' + '\n')
plt.savefig(nwImgPath, format = 'eps', bbox_inches='tight')

# display graph
print ('Displaying graph...' + '\n')
plt.show()

# Clear plot
plt.clf()

# Recalculate basic graph statistics
nodes = nx.number_of_nodes(diEnGraph)
edges = nx.number_of_edges(diEnGraph)
nodeList = nx.nodes(diEnGraph)
nodeList.sort()
density = nx.density(diEnGraph)
isDag = nx.is_directed_acyclic_graph(diEnGraph)

# Count sources and sinks
sourceCount = 0
sinkCount = 0
for i in nodeList:
	outDeg = diEnGraph.out_degree(i)
	inDeg = diEnGraph.in_degree(i)
	if outDeg == 0 and inDeg >= 1:
		sinkCount += 1

	elif inDeg == 0 and outDeg >= 1:
		sourceCount += 1

print ('Final Directed Graph Information' + '\n')
print ('User-entered date: ' + str(dateIP))
print ('Nodes: ' + str(nodes))
print ('Isolated nodes: ' + str(isolateCount))
print ('Source nodes: ' + str(sourceCount))
print ('Sink nodes: ' + str(sinkCount))
print ('Edges: ' + str(edges))
print ('Density: ' + str(density))
print ('Is DAG? ' + str(isDag))
print
print (str(nx.info(diEnGraph)))

anFile.write ('Nodes: ' + str(nodes) + '\n')
anFile.write ('Isolated nodes: ' + str(isolateCount) + '\n')
anFile.write ('Source nodes: ' + str(sourceCount) + '\n')
anFile.write ('Sink nodes: ' + str(sinkCount) + '\n')
anFile.write ('Edges: ' + str(edges) + '\n')
anFile.write ('Density: ' + str(density) + '\n')
anFile.write ('Is DAG? ' + str(isDag) + '\n')
anFile.write ('\n' + str(nx.info(diEnGraph)) + '\n')

runLog.write ('\n' + 'Final Directed Graph Information' + '\n' + '\n')
runLog.write ('User-entered date: ' + str(dateIP) + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Isolated nodes:' + str(isolateCount) + '\n')
runLog.write ('Source nodes: ' + str(sourceCount) + '\n')
runLog.write ('Sink nodes: ' + str(sinkCount) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Density: ' + str(density) + '\n')
runLog.write ('Is DAG? ' + str(isDag) + '\n')
runLog.write ('\n' + str(nx.info(diEnGraph)) + '\n')

prFile.write(str(nx.pagerank(diEnGraph)))
prFile.close()

# Render undirected version of diEnGraph to facilitate final analysis of graph characteristics
print('\n' + "Rendering undirected version to facilitate final analysis of graph characteristics... ")
runLog.write('\n' + "Rendering undirected version to facilitate final analysis of graph characteristics... " + '\n')
newEnGraph = diEnGraph.to_undirected()

# Analysis
print ('\n' + 'Analysing final undirected graph...' + '\n')
print ('Maximal degree... ' + '\n')
degreeSeq = sorted(nx.degree(newEnGraph).values(),reverse=True)
maxDeg = max(degreeSeq)
print ('Maximal degree: ' + str(maxDeg) + '\n')
anFile.write ('Maximal degree: ' + str(maxDeg) + '\n')
print ('Average clustering coefficient...' + '\n')
avClustering = nx.average_clustering(newEnGraph)
print ('Connected components...' + '\n')
connectComp = [len(c) for c in sorted(nx.connected_components(newEnGraph), key=len, reverse=True)]
print ('Find cliques...' + '\n')
cl = nx.find_cliques(newEnGraph)
cl = sorted(list(cl), key = len, reverse = True)
print ('Number of cliques: ' + str(len(cl)) + '\n')
anFile.write ('Number of cliques: ' + str(len(cl)) + '\n')
cl_sizes = [len(c) for c in cl]
#print ('Size of cliques: ' + str(cl_sizes))
anFile.write ('Size of cliques: ' + str(cl_sizes) + '\n' + '\n')

# Find LCC nodes, print to screen and and write to file
largeCC = max(nx.connected_components(newEnGraph), key=len)
print
print ("Nodes in LCC: ")
print
print (largeCC)
lccOP.write(str(largeCC))
lccOP.write('\n' + '\n' + "There are " + str(len(largeCC)) + " nodes in the LCC of this graph." + '\n')
lccOP.close()
print
print ("There are " + str(len(largeCC)) + " nodes in the LCC of this graph.")
print

# Render GEXF file of LCC
print
print ("Writing gexf file of LCC... " + '\n')
runLog.write('\n' + "Writing gexf file of LCC... " + '\n')
lccSub = max(nx.connected_component_subgraphs(newEnGraph), key=len)
nx.write_gexf(lccSub, gexfLccFile)
gexfLccFile.close()

# Analysis of LCC
print ('\n' + 'Analysing LCC...' + '\n')
lccNodes = nx.number_of_nodes(lccSub)
lccEdges = nx.number_of_edges(lccSub)
lccDegreeSeq = sorted(nx.degree(lccSub).values(),reverse=True)
lccMaxDeg = max(lccDegreeSeq)
lccDensity = nx.density(lccSub)
lccAvClustering = nx.average_clustering(lccSub)
lccCl = nx.find_cliques(lccSub)
lccCl = sorted(list(lccCl), key = len, reverse = True)
lccCl_sizes = [len(c) for c in lccCl]

print ('LCC Nodes: ' + str(lccNodes))
anFile.write('LCC Nodes: ' + str(lccNodes) + '\n')
print ('LCC Edges: ' + str(lccEdges))
anFile.write('LCC Edges: ' + str(lccEdges) + '\n')
print ('LCC Density: ' + str(lccDensity))
anFile.write('LCC Density: ' + str(lccDensity) + '\n')
print ('Maximal degree of LCC: ' + str(lccMaxDeg) + '\n')
anFile.write ('Maximal degree of LCC: ' + str(lccMaxDeg) + '\n')
print ('ACC of LCC: ' + str(lccAvClustering))
anFile.write('ACC of LCC: ' + str(lccAvClustering) + '\n')
print ('Number of cliques in LCC: ' + str(len(lccCl)) + '\n')
anFile.write ('Number of cliques in LCC: ' + str(len(lccCl)) + '\n' + '\n')
print ('Size of LCC cliques: ' + str(lccCl_sizes))
anFile.write ('Size of LCC cliques: ' + str(lccCl_sizes) + '\n')
print

# Write undirected gexf file for use in Gephi
print ('\n' + 'Writing final undirected gexf file...' + '\n')
nx.write_gexf(newEnGraph, gexfFinFile)
gexfFinFile.close()

# End timing of run
endTime = datetime.now()

anFile.write ('\n' + 'Final Undirected Graph Information' + '\n' + '\n')
anFile.write ('Date of run: {}'.format(runDate) + '\n')
anFile.write ('Omega Year: ' + omegaYear + '\n')
#anFile.write ("Total artists in all genres: " + str(totalArtists) + '\n')
#anFile.write ("Total edge-weighting: " + str(int(totalEdgeWeight)) + '\n')
anFile.write ('Nodes: ' + str(nodes) + '\n')
anFile.write ('Isolated nodes: ' + str(isolateCount) + '\n')
anFile.write ('Edges: ' + str(edges) + '\n')
anFile.write ('Density: ' + str(density) + '\n')
anFile.write ('Maximal degree: ' + str(maxDeg) + '\n')
anFile.write ('Average Clustering Coefficient: ' + str(avClustering) + '\n')
anFile.write ('Number of cliques: ' + str(len(cl)) + '\n')
anFile.write ('Connected Components: ' + str(connectComp) + '\n')
anFile.write ('\n' + str(nx.info(newEnGraph)))
anFile.close()

print ('Final Undirected Graph Information' + '\n')
print ('User-entered date: ' + str(dateIP))
print ('Omega Year: ' + omegaYear)
#print ("Total artists in all genres: " + str(totalArtists))
#print ("Total edge-weighting: " + str(int(totalEdgeWeight)))
print ('Nodes: ' + str(nodes))
print ('Isolated nodes: ' + str(isolateCount))
print ('Edges: ' + str(edges))
print ('Self-loops: ' + str(selfLoopTotal))
print ('Density: ' + str(density))
print ('Maximal degree: ' + str(maxDeg))
print ('Average Clustering Coefficient: ' + str(avClustering))
print ('Number of cliques: ' + str(len(cl)))
print ('Connected Components: ' + str(connectComp))
print
print (str(nx.info(newEnGraph)))
print
print ('Date of run: {}'.format(runDate))
print ('Duration of run : {}'.format(endTime - startTime))

runLog.write ('\n' + 'Final Undirected Graph Information' + '\n' + '\n')
runLog.write ('Omega Year: ' + omegaYear + '\n')
#runLog.write ("Total artists in all genres: " + str(totalArtists) + '\n')
#runLog.write ("Total edge-weighting: " + str(int(totalEdgeWeight)) + '\n')
runLog.write ('Nodes: ' + str(nodes) + '\n')
runLog.write ('Isolated nodes:' + str(isolateCount) + '\n')
runLog.write ('Edges: ' + str(edges) + '\n')
runLog.write ('Self-loops: ' + str(selfLoopTotal) + '\n')
runLog.write ('Density: ' + str(density) + '\n')
runLog.write ('Maximal degree: ' + str(maxDeg) + '\n')
runLog.write ('Average Clustering Coefficient: ' + str(avClustering) + '\n')
runLog.write ('Number of cliques: ' + str(len(cl)) + '\n')
runLog.write ('Connected Components: ' + str(connectComp) + '\n' + '\n')
runLog.write (str(nx.info(newEnGraph)))
runLog.write ('\n' + '\n' + 'Date of run: {}'.format(runDate) + '\n')
runLog.write ('Duration of run : {}'.format(endTime - startTime) + '\n')
runLog.close()


# -*- coding: utf-8 -*-
import logging
from urllib.parse import quote
from django.urls import reverse
from django.contrib.sites.models import Site

import requests
from django.conf import settings
from jira import JIRA, JIRAError
from rdmo.projects.models import Project

from config.settings.base import (
    JIRA_ACCOUNT_SERVICE_USER,
    JIRA_ACCOUNT_SERVICE_PASSWORD,
)
from .configuration.settings import JIRA_FALLBACK_USERNAME
from .configuration.settings import (
    JIRA_USERNAME_URL_TEMPLATE,
    JIRA_USERNAME_URL_FULLNAME_TEMPLATE,
)
from .models import DmptIssue
from ..users.models import User

logger = logging.getLogger(__name__)

def _get_gfbio_helpdesk_username(user_name, email, fullname=""):
    url = JIRA_USERNAME_URL_TEMPLATE.format(user_name, email)
    if len(fullname):
        url = JIRA_USERNAME_URL_FULLNAME_TEMPLATE.format(
            user_name, email, quote(fullname)
        )
    return requests.get(
        url=url, auth=(JIRA_ACCOUNT_SERVICE_USER, JIRA_ACCOUNT_SERVICE_PASSWORD)
    )

def get_rdmo_project_for_project_id(rdmo_project_id):
    rdmo_project = None
    try:
        rdmo_project = Project.objects.get(id=rdmo_project_id)
    except Project.DoesNotExist as e:
        logger.error(
            f"jira_utils.py | get_rdmo_project_for_project_id | "
            f"error getting rdmo project | {e}"
        )
    return rdmo_project

def get_issue_reporter(email, user_id):
    user = None
    if user_id:
        try:
            user = User.objects.get(pk=user_id)
        except User.DoesNotExist as e:
            logger.warning(
                f"jira_utils.py | get_issue_reporter | "
                f"error getting user by user_id | {e}"
            )
    user_name = email if user is None else user.username
    response = _get_gfbio_helpdesk_username(
        user_name=user_name,
        email=email,
    )
    logger.info(
        f"jira_utils.py | get_issue_reporter | "
        f"processed _get_gfbio_helpdesk_username with | user_name={user_name} | "
        f"email={email} | response.content={response.content}"
    )
    reporter = f'{response.content.decode("utf-8")}'
    return reporter

def create_support_issue(rdmo_project, reporter, message=""):
    current_site = Site.objects.get_current()
    rdmo_project_url = "https://{}{}".format(
        current_site.domain, f"/rdmo/projects/{rdmo_project.id}"
    )

    jira = JIRA(
        server=settings.JIRA_URL,
        basic_auth=(settings.JIRA_USERNAME, settings.JIRA_PASS),
        get_server_info=False,
    )
    try:
        issue = jira.create_issue(
            fields={
                "project": {"key": settings.JIRA_PROJECT},
                "summary": "DMP Support Request",
                "description": f"Help request for"
                f'data management plan "{rdmo_project.title}" which you can find here {rdmo_project_url} '
                f'using the "{rdmo_project.catalog}" catalogue\n\n{message}',
                "reporter": {"name": reporter},
                "issuetype": {"name": "DMP"},
            }
        )
        dmpt_project = None
        if hasattr(rdmo_project, "dmptproject"):
            dmpt_project = rdmo_project.dmptproject
        DmptIssue.objects.create(
            rdmo_project=rdmo_project, issue_key=issue.key, dmpt_project=dmpt_project
        )
        logger.info(
            f"jira_utils.py | create_support_issue | " f"issue created | issue={issue}"
        )
        return issue
    except JIRAError as e:
        logger.error(
            f"jira_utils.py | create_support_issue | error creating issue | {e.text}"
        )
        return None

def _add_interests_to_message(form_data, message):
    if form_data.get("data_collection_and_assurance", False):
        message += """- Data Collection and Assurance
        """
    if form_data.get("data_curation", False):
        message += """- Data Curation
        """
    if form_data.get("data_archiving", False):
        message += """- Data Archiving
        """
    if form_data.get("terminology_service", False):
        message += """- Terminology Service
        """
    if form_data.get("data_visualization_and_analysis", False):
        message += """- Data Visualization and Analysis
        """
    if form_data.get("data_publication", False):
        message += """- Data Publication
        """
    if form_data.get("data_management_training", False):
        message += """- Data Management Training
           """
    return message

def create_support_issue_in_view(form_data={}):
    logger.info(
        f"jira_utils.py | create_support_issue_in_view | " f"start | {form_data}"
    )

    rdmo_project_id = form_data.get("rdmo_project_id")
    rdmo_project = get_rdmo_project_for_project_id(rdmo_project_id)
    if rdmo_project is None:
        logger.error(
            f"jira_utils.py | create_support_issue_in_view | "
            f"error getting rdmo project | rdmo_project_id={rdmo_project_id}"
        )
        return {"error": f"no rdmo project found with id: {rdmo_project_id}"}

    reporter = get_issue_reporter(form_data.get("email"), form_data.get("user_id"))
    message = f"""
    Message:

    {form_data.get('message')}

    I am interested in:

    """
    message = _add_interests_to_message(form_data, message)

    issue = create_support_issue(rdmo_project, reporter, message)
    if issue is None:
        return {"error": f"no issue could be created"}
    return {
        "issue_key": issue.key,
        "issue_url": f"https://helpdesk.gfbio.org/browse/{issue.key}",
    }
    # TODO: only for testing, to avoid creating too many real issues in the helpdesk
    # return {"issue_key": "SAND-1661", "issue_url": "https://www.example.com/rest/api/2/issue/16814"}