"""Tests of the FFT classes.

The fundemental building block of PsDNS are the FFT classes defined in
:mod:`~psdns.bases`.  These are tested in this module.  Because the
space of potential inputs to the FFT functions is so large, it is
necessary to identify specific tests which will be sufficiently
comprehensive to provide confidence for all possible FFTs.  Currently
there are three main tests:

  1. The :class:`TestSingleMode` class tests forward and backward
     transforms of a single Fourier mode.
  2. The classes :class:`TestSymmetries` and :class:`TestProperties` test
     whether the transforms obey all the properties they should.
  3. :class:`TestMPI4PyFFT` performs a code-to-code comparison between
     PsDNS and the `mpi4py <https://mpi4py-fft.readthedocs.io>`_
     package.

Each of these tests is performed using :attr:`domains` of several
different sizes, intended to exercise the various different
scenarios for domain truncation.

In principle, testing each mode individually, along with linearity,
should be enough to guarantee all possible FFTs, however, additional
tests are provided to provide redundant testing.

Although it is usually preferrable to make unit tests purely
deterministic, in order to avoid missing errors due to particular
choice of array values, several tests make use of randomized arrays
provided by the helper routines :func:`random_spectral_array` and
:func:`random_physical_array`.

The tests in this module do not automatically run on different numbers
of MPI ranks.  The tests all use the default communicator, that is,
they perform the FFTs on all the available MPI ranks.  Testing for
different numbers of MPI ranks must be managed manually by the user by
running these tests with different arguments to :program:`mpirun` (as
described in :mod:`psdns.tests`).  Note that the scaling tests
(:mod:`psdns.tests.test_scaling`) do not cover this, since they test
only run-times, not whether the results are correct.
"""
import unittest

import numpy
from numpy import testing as nptest

from mpi4py import MPI
from mpi4py_fft import PFFT, newDistArray

from psdns import *

#: :meta hide-value:
#:
#: A list of domains on which to run individual tests.  Each domain
#: is a 2-tuple which is passed as the arguments to the
#: :class:`~psdns.bases.SpectralGrid` constructor.  The domains sizes
#: are choses to be small enough to keep test times short, and to
#: inlude all the important combinations of truncanted (anti-aliased)
#: and non-truncated, as well as odd and even length, transforms in each
#: direction.
domains = [
    ((9, 9, 9), (9, 9, 9)),
    ((8, 9, 9), (8, 9, 9)),
    ((9, 8, 9), (9, 8, 9)),
    ((8, 8, 9), (8, 8, 9)),
    ((9, 9, 8), (9, 9, 8)),
    ((8, 9, 8), (8, 9, 8)),
    ((9, 8, 8), (9, 8, 8)),
    ((8, 8, 8), (8, 8, 8)),
    ((9, 9, 9), (12, 12, 12)),
    ((8, 9, 9), (12, 12, 12)),
    ((9, 8, 9), (12, 12, 12)),
    ((8, 8, 9), (12, 12, 12)),
    ((9, 9, 8), (12, 12, 12)),
    ((8, 9, 8), (12, 12, 12)),
    ((9, 8, 8), (12, 12, 12)),
    ((8, 8, 8), (12, 12, 12)),
    ((8, 8, 8), (12, 12, 8)),
    ]

def random_spectral_array(grid, shape=()):
    """Return a random :class:`~psdns.bases.SpectralArray`.

    Return a :class:`~psdns.bases.SpectralArray` on the specified
    *grid* with the specified *shape*, filled with random data.

    In order to assure that this spectral array has the approriate
    symmetries, it is obtained by first creating a random physical
    array, and then transforming to spectral space.
    """
    return random_physical_array(grid, shape).to_spectral()

def random_physical_array(grid, shape=()):
    """Return a random :class:`~psdns.bases.PhysicalArray`.

    Return a :class:`~psdns.bases.PhysicalArray` on the specified
    *grid* with the specified *shape*, filled with random data.

    Note, if the specified *grid* is anti-aliased, then this physical
    array will contain higher mode data that will be lost when
    transformed.  In particular, transforming this array to spectral
    space, and then back to physical space, will not recover the
    original array.  For tests that require a physical array that does
    not contain truncated spectral modes, the data can be filtered
    using a transform to spectral and back to physical, i.e.::

        p = random_physical_array(grid).to_spectral().to_physical()
    """
    return PhysicalArray(grid, numpy.random.random(shape+grid.x.shape[1:]))

class TestSingleMode(tests.TestCase):
    """Test transform of a single spectral mode
    """
    def initialize_transform_pair(self, klm, sdims, pdims):
        r"""Return a physical and spectral representation of a single mode

        In order to test transforms of a single mode, this method
        returns a tuple containing a
        :class:`~psdns.bases.PhysicalArray`, and
        :class:`~psdns.bases.SpectralArray`, and a string describing the
        type of mode (see below for descriptions of the mode types).
        The amplitude is always set to one, and the phase is random.
        The test method can then use these fields to test that one
        transforms to the other.  The index of the spectral mode is
        given by the tuple *klm*; *sdims* and *pdims* are the size of the
        array to create.

        As described in :ref:`Real Transforms` the
        :class:`~psdns.bases.SpectralArray` class retains slightly over
        half the Hermitian symmetric modes.  These can be divided into
        different groups, which transform differently.  Note that the
        various cases could be broken out into different tests methods.
        However, this implementation, that iterates over all modes in
        one routine, was chosen to reduce the likelihood that tests of
        certain modes were inadvertantly omitted.

        #. **Interior modes.** For most modes, their is also a Hermitian
           symmetric mode which is not stored.  The physical
           representation is given by equation
           :eq:`3d-transform`.

        #. **Edge modes.**  These are modes for which both of the
           Hermitian symmetric mode are part of the array, which happens
           when :math:`m=0` or :math:`2 m = N_z`
           (eqs. :eq:`edge-zero`-:eq:`edge-nz2`) .  In this case, both
           modes must be initialized consistently.

        #. **Corner modes.**  Hermitian symmetry requires that certain
           modes (see eq. :eq:`corner`)  be purely real, and the
           physical space representation is given by equation
           :eq:`corner-transform`.

        #. **Truncated modes.** When the physical dimensions are larger
           than the spectral dimensions, certain modes are truncated.
           For these modes, the physical-to-spectral transform will
           return zero.  There is no test for the spectral-to-physical
           transform, since these modes are not representable in
           spectral space.

           Also, with ``aliasing_strategy=truncate`` (see :ref:`Keeping
           it real`), which is the setting tested in this test, for
           truncation to an even number of modes, the extra negative
           mode, :math:`-N_x/2` or :math:`-N_y/2`, is zeroed out, and
           therefore is tested identically to the truncated mode.

           .. note::

               This test does not check the result of a
               spectral-to-physical transform of a zeroed out mode set
               to a non-zero value by the user, since this behavior is
               not defined, and should be avoided.
        """
        k, l, m = klm
        grid = SpectralGrid(sdims, pdims)
        s = SpectralArray(grid)
        p = PhysicalArray(grid)
        theta = grid.comm.bcast(numpy.random.rand())
        if ((grid.pdims[0] > grid.sdims[0] and abs(2*k) >= grid.sdims[0]) or
            (grid.pdims[1] > grid.sdims[1] and abs(2*l) >= grid.sdims[1]) or
            2*m > sdims[2]):
            p[...] = 2*numpy.cos(
                k*p.grid.x[0]+l*p.grid.x[1]+m*p.grid.x[2]+theta
                )
            typ = 'trunc'
        elif ((k == 0 or -2*k == pdims[0]) and
              (l == 0 or -2*l == pdims[1]) and
              (m == 0 or 2*m == pdims[2] == sdims[2])):
            s.set_mode([k, l, m], 1)
            p[...] = numpy.cos(k*p.grid.x[0]+l*p.grid.x[1]+m*p.grid.x[2])
            typ = 'corner'
        elif m == 0 or 2*m == pdims[2]:
            s.set_mode([k, l, m], numpy.exp(1j*theta))
            s.set_mode([-k, -l, m], numpy.exp(-1j*theta))
            p[...] = 2*numpy.cos(k*p.grid.x[0]+l*p.grid.x[1]+m*p.grid.x[2]+theta)
            typ = 'edge'
        else:
            s.set_mode([k, l, m], numpy.exp(1j*theta))
            p[...] = 2*numpy.cos(k*p.grid.x[0]+l*p.grid.x[1]+m*p.grid.x[2]+theta)
            typ = 'interior'
        return p, s, typ

    def test_single_mode(self):
        r"""Forward and backward transforms of a single mode are correct.

        Test that a single spectral mode transforms correctly, both
        physical-to-spectral and spectral-to-physical.  This test loops
        over all modes supported on the physical space grid, and uses
        the method :meth:`initialize_transform_pair` to generate the
        physical and spectral space representations.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                for k in range(-(pdims[0]//2), (pdims[0]+1)//2):
                    for l in range(-(pdims[1]//2), (pdims[1]+1)//2):
                        for m in range(pdims[2]//2+1):
                            with self.subTest(k=k, l=l, m=m):
                                p, s, typ = self.initialize_transform_pair(
                                    (k, l, m), sdims, pdims
                                    )
                                with self.subTest(type=typ, dir="forward"):
                                    nptest.assert_almost_equal(
                                        numpy.asarray(p.to_spectral()),
                                        numpy.asarray(s)
                                        )
                                if typ == 'trunc':
                                    continue
                                with self.subTest(type=typ, dir="backward"):
                                    nptest.assert_almost_equal(
                                        numpy.asarray(s.to_physical()),
                                        numpy.asarray(p)
                                        )

class TestSymmetries(tests.TestCase):
    r"""Test that spectral transforms have the correct symmetries.

    Hermitian symmetry imposes certain contraints on spectral arrays
    (see :ref:`Three-dimensional transforms`).  This test confirms that,
    for a random array, these symmetries occur.
    """
    def test_z_zero(self):
        r"""Test Hermitian symmetry when z=0"""
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                s = random_spectral_array(SpectralGrid(sdims, pdims))
                for k in range(1, sdims[0]//2):
                    for l in range(1, sdims[1]//2):
                        with self.subTest(k=k, l=l):
                            with self.rank_zero(s.grid.comm):
                                self.assertAlmostEqual(
                                    s.get_mode([k, l, 0]),
                                    s.get_mode([-k, -l, 0]).conjugate()
                                    )

    def test_z_max(self):
        r"""Test Hermitian symmetry when z=Nz/2"""
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                if sdims[2] % 2 == 0:
                    s = random_spectral_array(SpectralGrid(sdims, pdims))
                    for k in range(1, sdims[0]//2):
                        for l in range(1, sdims[1]//2):
                            with self.subTest(k=k, l=l):
                                with self.rank_zero(s.grid.comm):
                                    self.assertAlmostEqual(
                                        s.get_mode([k, l, 0]),
                                        s.get_mode([-k, -l, 0]).conjugate()
                                        )

    def test_real_corners(self):
        r"""Test for real values in corners"""
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                krange = [0, -sdims[0]//2] \
                    if sdims[0] == pdims[0] and sdims[0] % 2 == 0 \
                    else [0]
                lrange = [0, -sdims[1]//2] \
                    if sdims[1] == pdims[1] and sdims[1] % 2 == 0 \
                    else [0]
                mrange = [0, sdims[2]//2] \
                    if sdims[2] == pdims[2] and sdims[2] % 2 == 0 \
                    else [0]
                s = random_spectral_array(SpectralGrid(sdims, pdims))
                for k in krange:
                    for l in lrange:
                        for m in mrange:
                            with self.subTest(k=k, l=l, m=m):
                                with self.rank_zero(s.grid.comm):
                                    self.assertAlmostEqual(
                                        s.get_mode([k, l, m]).imag,
                                        0
                                        )

class TestProperties(tests.TestCase):
    """Test that various mathematical properties of the FFT obeyed.
    """
    def test_round_trip1(self):
        """Transforming to spectral and back to physical returns the original value.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                p = random_physical_array(SpectralGrid(sdims, pdims))
                # Filter out unsupported spectral modes
                p = p.to_spectral().to_physical()
                nptest.assert_allclose(
                    numpy.asarray(p.to_spectral().to_physical()),
                    numpy.asarray(p)
                    )

    def test_round_trip2(self):
        """Transforming to physical and back to spectral returns the original value.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                s = random_spectral_array(SpectralGrid(sdims, pdims))
                nptest.assert_allclose(
                    numpy.asarray(s.to_physical().to_spectral()),
                    numpy.asarray(s)
                    )

    def test_linear1(self):
        """Physical-to-spectral transforms are linear.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                grid = SpectralGrid(sdims, pdims)
                p1 = random_physical_array(grid)
                p2 = random_physical_array(grid)
                a = grid.comm.bcast(numpy.random.rand(), root=0)
                nptest.assert_allclose(
                    numpy.asarray((p1+a*p2).to_spectral()),
                    numpy.asarray(p1.to_spectral()+a*p2.to_spectral())
                    )

    def test_linear2(self):
        """Spectral-to-physical transforms are linear.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                grid = SpectralGrid(sdims, pdims)
                s1 = random_spectral_array(grid)
                s2 = random_spectral_array(grid)
                a = grid.comm.bcast(numpy.random.rand(), root=0)
                nptest.assert_allclose(
                    numpy.asarray((s1+a*s2).to_physical()),
                    numpy.asarray(s1.to_physical()+a*s2.to_physical())
                    )

    def test_norm(self):
        """Spectral norm should match physical space norm.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                s = random_spectral_array(SpectralGrid(sdims, pdims))
                with self.rank_zero(s.grid.comm):
                    self.assertAlmostEqual(
                        s.norm(),
                        s.to_physical().norm()
                        )

    def test_norm2(self):
        """Physical norm should match spectral space norm.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                p = random_physical_array(SpectralGrid(sdims, pdims))
                # Filter out unsupported spectral modes
                p = p.to_spectral().to_physical()
                with self.rank_zero(p.grid.comm):
                    self.assertAlmostEqual(
                        p.norm(),
                        p.to_spectral().norm()
                        )

    def test_norm3(self):
        """Check magnitude (scaling) of physical norm"""
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                p = random_physical_array(SpectralGrid(sdims, pdims))
                p[...] = numpy.cos(p.grid.x[0])
                with self.rank_zero(p.grid.comm):
                    self.assertAlmostEqual(p.norm(), 0.5)

    def test_vector_to_spectral(self):
        """Vectors transform to spectral elementwise
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                p = random_physical_array(
                    SpectralGrid(sdims, pdims),
                    shape=(3,)
                    )
                s = p.to_spectral()
                with self.subTest("check shape"):
                    self.assertEqual(
                        p.shape[:-3],
                        s.shape[:-3]
                        )
                for i in range(3):
                    with self.subTest(vector_element=i):
                        nptest.assert_almost_equal(
                            numpy.asarray(p[i].to_spectral()),
                            numpy.asarray(s[i])
                            )

    def test_vector_to_physical(self):
        """Vectors transform to physical elementwise
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                s = random_spectral_array(
                    SpectralGrid(sdims, pdims),
                    shape=(3,)
                    )
                p = s.to_physical()
                with self.subTest("check shape"):
                    self.assertEqual(
                        s.shape[:-3],
                        p.shape[:-3]
                        )
                for i in range(3):
                    with self.subTest(vector_element=i):
                        nptest.assert_almost_equal(
                            numpy.asarray(s[i].to_physical()),
                            numpy.asarray(p[i])
                            )

@unittest.skipIf(
    MPI.COMM_WORLD.size != 1,
    "Test may fail if array subsizes don't match (see documentation)"
    )
class TestMPI4PyFFT(tests.TestCase):
    """Test that our transforms return the same results as mpi4py-fft.

    Compare the results of PsDNS transforms to those provided by the
    mpi4py-fft library, with ``aliasing_stragegy=mpi4py`` for the
    :class:`~psdns.bases.SpectralGrid`.  Note that, since the two
    codes may decompse domains differently, this test may fail if run
    on multiple MPI ranks.
    """
    def test_mpi4py_fft_forward(self):
        """Physical-to-spectral transforms match mpi4py-fft.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                fft = PFFT(
                    MPI.COMM_WORLD,
                    sdims,
                    padding=numpy.asarray(pdims)/numpy.asarray(sdims),
                    axes=(0, 1, 2),
                    dtype=float,
                    grid=(-1,)
                    )
                p = random_physical_array(
                    SpectralGrid(sdims, pdims, aliasing_strategy='mpi4py')
                    )
                u = newDistArray(fft, False)
                u[...] = p
                u_hat = fft.forward(u, normalize=True)
                s = p.to_spectral()
                numpy.set_printoptions(linewidth=120, precision=2)
                nptest.assert_allclose(
                    u_hat,
                    numpy.asarray(s),
                    )

    def test_mpi4py_fft_backward(self):
        """Spectral-to-physical transforms match mpi4py-fft.
        """
        for sdims, pdims in domains:
            with self.subTest(sdims=sdims, pdims=pdims):
                fft = PFFT(
                    MPI.COMM_WORLD,
                    sdims,
                    padding=numpy.asarray(pdims)/numpy.asarray(sdims),
                    axes=(0, 1, 2),
                    dtype=float,
                    grid=(-1,)
                    )
                s = random_spectral_array(
                    SpectralGrid(sdims, pdims, aliasing_strategy='mpi4py')
                    )
                u_hat = newDistArray(fft, True)
                u_hat[...] = s
                u = fft.backward(u_hat, normalize=False)
                nptest.assert_allclose(
                    u,
                    numpy.asarray(s.to_physical())
                    )


# -*- coding: utf-8 -*-
# Copyright (c) 2013 - 2019, Gorka Zamora-López <gorka@Zamora-Lopez.xyz>
#
# Released under the Apache License, Version 2.0 (the "License");
# you may not use this software except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0

"""
Graph Analysis Library
======================

A package for the analysis of graphs and complex networks in Python.
Compatible with Python 2.7 and 3.X

GAlib treats networks as adjacency matrices, represented as 2D NumPy arrays,
thus taking advantage of the faster operations over pure Pythen data types.
The core of the library consists of three modules:

metrics
    Basic graph metrics (degrees, clustering, graph distance, etc)
models
    Generation of synthetic networks and randomization.
tools
    Miscelaneous helper functions.

The rest of modules provide additional functionalities:

metrics_numba
    Uses package Numba to accelerate calculation of some metrics.
models_numba
    Uses package Numba to accelerate generation of some graph models.
extra.py
    Additional measures and functionalities related to network analysis.

USING pyGAlib
^^^^^^^^^^^^^^
The library is organised in the following modules:

- *metrics.py*: Common graph metrics (degrees, clustering, graph distance, etc)
- *models.py*: Generation of synthetic networks and randomization.
- *tools.py*: Miscelaneous helper functions.
- *metrics_numba.py*: Uses the Numba package to accelerate calculation of some
metrics.
- *models_numba.py*: Uses the Numba package to accelerate generation of some
graph models.
- *extra.py*: Additional measures and functionalities related to network analysis.

Getting started
***************
Since pyGAlib depends on NumPy, it is recommended to import NumPy first.
Although this is not necessary for loading pyGAlib, NumPy functionalities and
array manipulation will be often needed. Try importing pyGAlib: ::

    >>> import numpy as np
    >>> import galib

..note::
    Importing galib imports also all functions in module *metrics.py*
    into its namespace. The rest of modules are imported separately. Therefore,
    if the import is relative those functions can be called as, e.g.,

::

    >>> import galib
    >>> ...
    >>> deg = galib.Degree(net)
    >>> C, Cnodes = galib.Clustering(net)

See that we did not have to call ``galib.metrics.Degree(net)``. In the case of
an absolute import (using an asterisk ``*``) all functions in *metrics.py* are
imported to the base namespace:  ::

    >>> from galib import *
    >>> ...
    >>> deg = Degree(net)
    >>> C, Cnodes = Clustering(net)

Example
*******
Let's generate a random graph following the Erdos-Renyi model, G(N,p), with
*N = 100* nodes and link probability *p = 0.1*:  ::

    >>> import galib
    >>> import galib.models
    >>> N = 100; p = 0.1
    >>> net = galib.models.ErdosRenyiGraph(N, p, directed=False)

Here, *net* is the adjacency matrix of the random graph represented as a
2-dimensional NumPy array. Let's calculate some basic properties.  ::

    >>> galib.Density(net)
    0.09838383838383838

As expected, the density of an Erdos-Renyi random graph is close to the
*p = 0.1* value given. We now calculate the degree of every node:  ::

    >>> deg = galib.Degree(net)
    >>> deg
    array([10,  7, 10, 10, 11,  7,  5, 11, 13, 12, 14, 13,  8, 10,  9,  8,  7,
       10, 11,  9, 11, 11,  8, 10,  5,  9, 13, 10, 13, 12, 12, 11, 11,  7,
       13, 11,  7, 10, 10,  6, 12, 10,  6, 10,  7,  6,  9, 10,  9,  9,  7,
        9,  8, 13, 10,  9,  7,  7, 11,  8, 13,  6,  7, 12, 14,  6,  5, 11,
        5, 12, 14, 14, 13,  8,  7, 12,  4, 19,  9, 13,  7, 10, 15, 15,  4,
        9,  7, 12,  7,  8, 12,  4, 11, 12,  6, 13,  6, 12, 16, 12])

The degree is returned as a numpy array of rank 1, of integer type. The function
``Clustering()`` returns both the global clustering coefficient and the local
clustering of every node:  ::

    >>> C, Cnodes = galib.Clustering(net)
    >>> C
    0.096051227321238
    >>> Cnodes
    array([0.13333333, 0.04761905, 0.04444444, 0.08888889, 0.10909091,
       0.19047619, 0.3       , 0.12727273, 0.08974359, 0.06060606,
      	... ... ...
      	... ... ...
       0.04545455, 0.        , 0.10909091, 0.13636364, 0.        ,
       0.07692308, 0.13333333, 0.09090909, 0.08333333, 0.10606061])

We compute the pair-wise graph distance between all the nodes using the
Floyd-Warshall algorithm, which is returned as a matrix ``dij`` (numpy array of
rank 2):  ::

	>>> dij = galib.FloydWarshall(net)
	>>> avlen = (dij.sum() - dij.trace()) / (N*(N-1))
	>>> avlen
	2.248080808080808

.. note::
    For calculating the distance matrix of larger networks please use
    the version of the function located in the module *metrics_numba.py*. Here,
    the function ``FloydWarshall_Numba()`` works the same but makes use of the
    Numba library to significantly speed the calculation.

Most network generators and graph metrics in pyGAlib work with directed graphs
as well. Check for the optional parameter ``directed``. Following the example
above, we generate a directed Erdos-Renyi graph and calculate its input and
output degrees for every node:  ::

    >>> net = galib.models.ErdosRenyiGraph(N, p, directed=True)
    >>> galib.Density(net)
    0.10272727272727272
    >>> indeg, outdeg = galib.Degree(net, directed=True)
    >>> indeg
    array([17,  7,  9,  8, 11, 10,  9,  8, 13, 13,  5,  9, 13,  9, 10, 10, 13,
       10,  9,  9,  7, 11, 13, 10,  4, 15, 11, 11, 10,  6,  6,  8,  8,  8,
       11,  8,  4, 12,  8, 13, 13, 14, 12,  5,  6,  5, 16, 12,  5, 10,  9,
       13,  8,  9,  7,  8, 13, 14,  9, 18,  7, 11,  5,  4, 12,  8,  8, 10,
        7,  9, 15, 12, 14,  9, 15, 11, 13, 12, 15, 10, 11, 11, 15,  7, 10,
       13,  7, 14,  9, 16, 11, 11,  6, 18,  7,  4, 14, 12, 12, 10])
    >>> outdeg
    array([ 9, 10,  7,  9, 12,  9, 19,  9, 11, 16, 11, 12, 11, 15, 11,  6,  9,
        8, 11, 12,  9, 13,  9,  8, 11,  6,  7, 11, 11, 12, 10,  8, 11, 12,
       10, 12, 13,  8, 18, 11,  8, 13, 10,  8, 10, 10, 11,  8, 11, 11, 11,
       10, 11, 10,  9, 12,  6, 10,  7, 10, 10, 11, 15, 12, 11,  7, 10,  8,
        5, 11,  7, 11, 13,  8,  5,  6, 13, 11, 10, 13,  7,  6, 13, 11,  8,
        8, 10,  6, 10,  9, 12, 15, 11,  9, 15, 11,  7,  8, 11, 10])

Data I/O
********
Since GAlib is based on NumPy arrays, saving and reading of adjacency matrices,
as well as any other output of GAlib functions, can be performed using the usual
data I/O functionalities of NumPy. See for example the documentation for
functions: ``loadtxt()``, ``savetxt()``, ``load()``, ``save()`` and ``savez()``.
The *tools.py* module in pyGAlib provides also some data conversion
functionalities.

How to find further documentation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
While working in an interactive session, after importing a module, the built-in
``help()`` function will show further details:  ::

    >>> help(modulename)

The help for galib (``help(galib)``) shows the general summary of the package
and a list of all the modules in the library. The help for each module,
e.g., ``help(galib.metrics)`` or ``help(galib.models)`` will display module
specific information and a list of all the functions in the module. For further
details regarding each function, type:  ::

    >>> help(galib.modulename.functionname)

For IPython and Jupyter notebook users the help command is replaced by a
question mark after the module's or function's name, e.g.:  ::

    >>> modulename?
    >>> functionname?

For questions, bug reports, etc, please write to <galib@Zamora-Lopez.xyz>, or
open an issue in GitHub.

License
-------

Copyright (c) 2013 - 2019, Gorka Zamora-López <gorka@Zamora-Lopez.xyz>

Released under the Apache License, Version 2.0 (the "License");
you may not use this software except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

or see the LICENSE.txt file.

.. note::
    Please, use the logos provided in the Branding/ folder whenever possible.

"""
from __future__ import absolute_import

from . import metrics
from .metrics import*

__author__ = "Gorka Zamora-Lopez"
__email__ = "galib@Zamora-Lopez.xyz"
__copyright__ = "Copyright 2013-2020"
__license__ = "Apache License version 2.0"
__version__ = "1.1.2"
__update__="15/06/2020"

#


import numpy as np
import pandas as pd
import collections

def test_create_dataframe(dataFrame,columnNames,dataTypes,nRows = 10):

    ##STEP - 1

    #Obtain actual column names from dataFrame
    actualColNames = list(dataFrame.columns)

    if(len(columnNames) == len(actualColNames)) : #Check for no. of columns match
        if(collections.Counter(columnNames) != collections.Counter(actualColNames)): #Check if column names match irrespective of order
            print("Column name mismatch")
            return False
    else:
        print("Column number mismatch")
        return False

    ## STEP-2

    #initialize step flag value to true
    flag = True

    #For each actual column name
    for i in actualColNames:
        flag = flag & (dataFrame[i].dtype == dataTypes[columnNames.index(i)]) #Check for that column name index data type
                                                                                #And update step flag

    if(flag==False):
        print("Data type mismatch")
        return False

    ## STEP-3

    if(nRows <= dataFrame.shape[0]): #check that number of rows are at least nRows
        flag = True
    else:
        print("Minimum number of rows criterion not met")
        flag = False

    #return flag value
    return flag


"""
Getting Started
===============

**Author**: Yi-Hsiang Lai (seanlatias@github)

In this tutorial, we demonstrate the basic usage of HeteroCL.

Import HeteroCL
---------------
We usually use ``hcl`` as the acronym of HeteroCL.
"""

import heterocl as hcl

##############################################################################
# Initialize the Environment
# --------------------------
# We need to initialize the environment for each HeteroCL application. We can
# do this by calling the API ``hcl.init()``. We can also set the default data
# type for every computation via this API. The default data type is **32-bit**
# integers.
#
# .. note::
#
#    For more information on the data types, please see
#    :ref:`sphx_glr_tutorials_tutorial_05_dtype.py`.

hcl.init()

##############################################################################
# Algorithm Definition
# --------------------
# After we initialize, we define the algorithm by using a Python function
# definition, where the arguments are the input tensors. The function can
# optionally return tensors as outputs. In this example, the two inputs are a
# scalar `a` and a tensor `A`, and the output is also a tensor `B`. The main
# difference between a scalar and a tensor is that *a scalar cannot be updated*.
#
# Within the algorithm definition, we use HeteroCL APIs to describe the
# operations. In this example, we use a tensor-based declarative-style
# operation ``hcl.compute``. We also show the equivalent  Python code.
#
# .. note::
#
#    For more information on the APIs, please see
#    :ref:`sphx_glr_tutorials_tutorial_03_api.py`

def simple_compute(a, A):

    B = hcl.compute(A.shape, lambda x, y: A[x, y] + a, "B")
    """
    The above API is equivalent to the following Python code.

    for x in range(0, 10):
        for y in range(0, 10):
            B[x, y] = A[x, y] + a
    """

    return B

##############################################################################
# Inputs/Outputs Definition
# -------------------------
# One of the advantages of such *modularized algorithm definition* is that we
# can reuse the defined function with different input settings. We use
# ``hcl.placeholder`` to set the inputs, where we specify the shape, name,
# and data type. The shape must be specified and should be in the form of a
# **tuple**. If it is empty (i.e., `()`), the returned object is a *scalar*.
# Otherwise, the returned object is a *tensor*. The rest two fields are
# optional. In this example, we define a scalar input `a` and a
# two-dimensional tensor input `A`.
#
# .. note::
#
#    For more information on the interfaces, please see
#    :obj:`heterocl.placeholder`

a = hcl.placeholder((), "a")
A = hcl.placeholder((10, 10), "A")

##############################################################################
# Apply Hardware Customization
# ----------------------------
# Usually, our next step is apply various hardware customization techniques to
# the application. In this tutorial, we skip this step which will be discussed
# in the later tutorials. However, we still need to build a default schedule
# by using ``hcl.create_schedule`` whose inputs are a list of inputs and
# the Python function that defines the algorithm.

s = hcl.create_schedule([a, A], simple_compute)

##############################################################################
# Inspect the Intermediate Representation (IR)
# --------------------------------------------
# A HeteroCL program will be lowered to an IR before backend code generation.
# HeteroCL provides an API for users to inspect the lowered IR. This could be
# helpful for debugging.

print(hcl.lower(s))

##############################################################################
# Create the Executable
# ---------------------
# The next step is to build the executable by using ``hcl.build``. You can
# define the target of the executable, where the default target is `llvm`.
# Namely, the executable will be run on CPU. The input for this API is the
# schedule we just created.

f = hcl.build(s)

##############################################################################
# Prepare the Inputs/Outputs for the Executable
# ---------------------------------------------
# To run the generated executable, we can feed it with Numpy arrays by using
# ``hcl.asarray``. This API transforms a Numpy array to a HeteroCL container
# that is used as inputs/outputs to the executable. In this tutorial, we
# randomly generate the values for our input tensor `A`. Note that since we
# return a new tensor at the end of our algorithm, we also need to prepare
# an input array for tensor `B`.

import numpy as np

hcl_a = 10
np_A = np.random.randint(100, size = A.shape)
hcl_A = hcl.asarray(np_A)
hcl_B = hcl.asarray(np.zeros(A.shape))

##############################################################################
# Run the Executable
# ------------------
# With the prepared inputs/outputs, we can finally feed them to our executable.

f(hcl_a, hcl_A, hcl_B)

##############################################################################
# View the Results
# ----------------
# To view the results, we can transform the HeteroCL tensors back to Numpy
# arrays by using ``asnumpy()``.

np_A = hcl_A.asnumpy()
np_B = hcl_B.asnumpy()

print(hcl_a)
print(np_A)
print(np_B)

##############################################################################
# Let's run some test

for i in range(10):
    for j in range(10):
        assert np_B[i][j] == np_A[i][j] + 10


import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    while True:
      city = input("\nWould you like to see data for Chicago, New York City, or Washington?\n")
      city = city.lower()
      if city not in ('new york city', 'chicago', 'washington'):
        print("Please choose Chicago, New York City, or Washington ")
        continue
      else:
        break

    # TO DO: get user input for month (all, january, february, ... , june)
    while True:
      month = input("\nWould you like to filter by month? Choose January, February, March, April, May, June or type 'all'\n")
      month = month.lower()
      if month not in ('january', 'february', 'march', 'april', 'may', 'june', 'all'):
        print("Please choose January, February, March, April, May, June or type 'all'.")
        continue
      else:
        break

    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    while True:
      day = input("\nWould you like to filter by day of week? Choose Sunday, Monday, Tuesday, Wednesday, Thursday, Friday, Saturday or type 'all'.\n")
      day = day.lower()
      if day not in ('sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'all'):
        print("Please choose Sunday, Monday, Tuesday, Wednesday, Thursday, Friday, Saturday or type 'all' ")
        continue
      else:
        break

    print('-'*40)
    return city, month, day

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """
    # load data file into a dataframe
    df = pd.read_csv(CITY_DATA[city])

    # convert the Start Time column to datetime
    df['Start Time'] = pd.to_datetime(df['Start Time'])

    # extract month and day of week from Start Time to create new columns
    df['month'] = df['Start Time'].dt.month
    df['day_of_week'] = df['Start Time'].dt.weekday_name

    # filter by month if applicable
    if month != 'all':
        # use the index of the months list to get the corresponding int
        months = ['january', 'february', 'march', 'april', 'may', 'june']
        month = months.index(month) + 1

        # filter by month to create the new dataframe
        df = df[df['month'] == month]

    # filter by day of week if applicable
    if day != 'all':
        # filter by day of week to create the new dataframe
        df = df[df['day_of_week'] == day.title()]

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month
    common_month = df['month'].mode()[0]
    print('The most common month:', common_month)

    # TO DO: display the most common day of week
    common_day = df['day_of_week'].mode()[0]
    print('The most common day:', common_day)

    # TO DO: display the most common start hour
    df['hour'] = df['Start Time'].dt.hour
    common_hour = df['hour'].mode()[0]
    print('The most common hour:', common_hour)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # TO DO: display most commonly used start station
    common_start_station = df['Start Station'].mode()[0]
    print('Most commonly used start station:', common_start_station)

    # TO DO: display most commonly used end station
    common_end_station = df['End Station'].mode()[0]
    print('Most commonly used end station:', common_end_station)

    # TO DO: display most frequent combination of start station and end station trip
    df['start end station'] = df['Start Station'] + ' and ' + df['End Station']
    common_start_end_station = df['start end station'].mode()[0]
    print('Most frequent combination of start station and end station trip :', common_start_end_station)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # TO DO: display total travel time
    total_travel_time = df['Trip Duration'].sum()
    print('Total travel time in seconds : ', total_travel_time)

    # TO DO: display mean travel time
    mean_travel_time = df['Trip Duration'].mean()
    print('Mean of travel time in seconds : ', mean_travel_time)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df):
    """Displays statistics on bikeshare users."""

    print('Calculating User Stats...\n')
    start_time = time.time()

    # TO DO: Display counts of user types
    n_user_types = df['User Type'].value_counts()

    print('Counts of user types:\n', n_user_types)

    # TO DO: Display counts of gender
    if 'Gender' in df.columns:
        n_gender = df['Gender'].value_counts()
        print('Counts of gender:\n', n_gender)
    else:
        print('There is no information for gender')

    # TO DO: Display earliest, most recent, and most common year of birth
    if 'Birth Year' in df.columns:
        earliest_year = int(df['Birth Year'].min())
        most_recent_year = int(df['Birth Year'].max())
        most_common_year = int(df['Birth Year'].mode())
        print('The earliest year of birth: ', earliest_year)
        print('The most recent year of birth: ', most_recent_year)
        print('The most common year of birth: ', most_common_year)
    else:
        print('There is no information for year of birth')

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def display_data(df):
    index=0
    user_input=input('would you like to display five rows of raw data? Please type Yes or No.\n').lower()
    while user_input in ['yes','y','yep','yea'] and index+5 < df.shape[0]:
        print(df.iloc[index:index+5])
        index += 5
        user_input = input('would you like to display five more rows of raw data? Please type Yes or No.\n').lower()

def main():
    while True:
      city, month, day = get_filters()
      df = load_data(city, month, day)

      time_stats(df)
      station_stats(df)
      trip_duration_stats(df)
      user_stats(df)
      display_data(df)

      restart = input('\nWould you like to restart? Enter yes or no.\n')
      if restart.lower() != 'yes':
          break

if __name__ == "__main__":
	main()


from __future__ import division, print_function
import sys, os, re
import numpy as np
import matplotlib.pyplot as plt

import pandas as pd # import data tables from NIST and store with pandas

"""
Generate isothermal data from NIST Thermophysical fluid database using Python 3

Copyright (c) 2019 - 2020 Jordan K. Pommerenck

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

usage = '''arguments: dp pmax T FLUID
  pressures are in bar, T in Kelvin

This program downloads data from the (NIST) Thermophysical properties of fluid
system database. This data can then be read using python with pandas or the
users program of choice.

Typical usage:
    python3 isothermal-save-gas-csv.py 0.1 2000 298 xenon
    # The above only needs to be done once
    python3 isothermal-ads-gas-plot.py xenon 298 5.8 200
'''

# --- Command line arguments --- #
if len(sys.argv) != 5:
    print(usage)
    exit(1)
pInc = float(sys.argv[1])
pMax = float(sys.argv[2])
Temperature = float(sys.argv[3])
fluid = sys.argv[4]

basename = '%s-%g' % (fluid, Temperature)
csvname = basename + '.csv'

# The nist_ids dictionary holds the CAS registry identifiers for a few
# interesting species from available species (NIST).
nist_ids = {
    'water':            'C7732185',
    'H20':              'C7732185',
    'nitrogen':         'C7727379',
    'N2':               'C7727379',
    'hydrogen':         'C1333740',
    'H2':               'C1333740',
    'parahydrogen':     'B5000001',
    'deuterium':        '7782390',
    'D2':               '7782390',
    'oxygen':           '7782447',
    'O2':               '7782447',
    'flourine':         'C7782414',
    'F':                'C7782414',
    'carbon-monoxide':  'C630080',
    'CO':               'C630080',
    'carbon-dioxide':   'C124389',
    'CO2':              'C124389',
    'methanol':         'C67561',
    'CH3OH':            'C67561',
    'CH4O':             'C67561',
    'methane':          'C74828',
    'CH4':              'C74828',
    'ethane':           'C74840',
    'C2H6':             'C74840',
    'ethene':           'C74851',
    'propane':          'C74986',
    'C3H8':             'C74986',
    'propene':          'C115071',
    'propyne':          'C74997',
    'cyclopropane':     'C75194',
    'butane':           'C106978',
    'isobutane':        'C75285',
    'pentane':          'C109660',
    'heptane':          'C142825',
    'octane':           'C111659',
    'nonane':           'C111842',
    'decane':           'C124185',
    'dodecane':         'C112403',
    'helium':           'C7440597',
    'neon':             'C7440019',
    'argon':            'C7440371',
    'krypton':          'C7439909',
    'xenon':            'C7440633',
    'ammonia':          'C7664417',
    'benzene':          'C71432',
    'toluene':          'C108883',
    'sulfur-dioxide':   'C7446095',
    'hydrogen-sulfide': 'C7783064',
}

# Check the fluid against the keys in the dictionary. If the input key is not
# found then the user is instructed to add the key to the dictionary.
if (nist_ids.get(fluid)):
    my_id = nist_ids[fluid]
    print(f'{fluid} has a CAS registry of {my_id}.')
else:
    raise KeyError(f"{fluid} is not a valid fluid. You must add name and CAS registry to nist_ids!")

open('data/' + csvname, 'w').close() # empty the file I am creating

num_rows = 500

inc_by = [2, 2, 2, 10/8.0]
inc_counter = 0

p = pInc

while p < pMax:
    print('pressure =', p, 'pressure increment =', pInc)
    p_final = min(pMax, p+pInc*(num_rows-1))
    if p == pInc:
        p_final = min(pMax, p+pInc*(num_rows-2))
    html_web = "https://webbook.nist.gov/cgi/fluid.cgi?Action=Load&ID=%s&" % my_id
    html_arg = "Type=IsoTherm&Digits=12&PLow=%g&PHigh=%g&PInc=%g&T=%g" % (p,
                                                                          p_final,
                                                                          pInc,
                                                                          Temperature)
    html_units = "&RefState=DEF&TUnit=K&PUnit=bar&DUnit=mol%2Fl&HUnit=kJ%2Fmol&WUnit=m%2Fs&VisUnit=uPa*s&STUnit=N%2Fm"
    # print(html_arg)
    tables = pd.read_html(html_web + html_arg + html_units)[0]
    if p != pInc:
        tables = tables.iloc[1:]
    tables = tables.iloc[:,:7]
    saveData = pd.DataFrame(data = tables)
    saveData.to_csv('data/' + csvname, header=None, index=False, sep=' ', mode='a') # mode = append
    p = p_final + pInc

    pInc *= inc_by[inc_counter % len(inc_by)]
    inc_counter += 1
print('Saved data as', csvname)


"""
Tuning High Performance Convolution on NVIDIA GPUs
=========================================================================
**Author**: `Lianmin Zheng <https://https://github.com/merrymercy>`_

This is an advanced tutorial for writing high performance tunable template for
NVIDIA GPU. By running auto-tuner on this template, we can outperform the
vendor provided library CuDNN in many cases.
"""

######################################################################
# Install dependencies
# --------------------
# To use autotvm package in tvm, we need to install some extra dependencies.
# (change "3" to "2" if you use python2):
#
# .. code-block:: bash
#
#   pip3 install --user psutil xgboost tornado
#
# To make tvm run faster in tuning, it is recommended to use cython
# as FFI of tvm. In the root directory of tvm, execute
#
# .. code-block:: bash
#
#   pip3 install --user cython
#   sudo make cython3
#
# Now return to python code. Import packages.

import logging
import sys
import numpy as np

import tvm
from tvm import topi
from tvm.topi import cuda

from tvm import autotvm

######################################################################
# Step 1:  Define the search space
# --------------------------------
# There are plenty of useful schedule primitives in tvm. You can also find
# some tutorials that describe them in more details, such as
# (1). :ref:`opt-conv-gpu`
# (2). `Optimizing DepthwiseConv on NVIDIA GPU <https://tvm.ai/2017/08/22/Optimize-Deep-Learning-GPU-Operators-with-TVM-A-Depthwise-Convolution-Example.html>`_
#
# However, their implementations are manually tuned for some special input
# shapes. In this section, we build a large enough space to cover
# the techniques used in these tutorials. Then we rely on the efficient auto-tuner
# to search through this space and pick some good configurations.
#
# If you are familiar with writing cuda schedule, you can find the following
# template is very general. Actually this template can be easily modified
# to tune other operators such as depthwise convolution and gemm.
# In order to fully understand this template, you should be familiar with
# the schedule primitives and auto tuning API. You can refer to the above
# tutorials and :doc:`autotvm tutorial <tune_simple_template>`
#
# It is worth noting that the search space for a conv2d operator
# can be very large (at the level of 10^9 for some input shapes)
#

# to run this, first:
# export PATH=/usr/local/cuda-10.1/nvvm/libdevice:$PATH

@autotvm.template("conv2d_nhwc_tensorcore_test")
def conv2d_nhwc_tensorcore_test(N, H, W, CO, CI, KH, KW, stride, padding):
    data = tvm.te.placeholder((N, H, W, CI), name='data', dtype="float16")
    kernel = tvm.te.placeholder((KH, KW, CI, CO), name='kernel', dtype="float16")
    cfg = autotvm.get_config()

    conv = topi.cuda.nhwc_tensorcore_cuda(
        cfg, data, kernel, stride, padding, dilation=1, out_dtype='float16')
    s = tvm.te.create_schedule([conv.op])
    topi.cuda.schedule_nhwc_tensorcore_cuda(cfg, s, conv)

    return s, [data, kernel, conv]

######################################################################
# Step 2:  Search through the space
# ---------------------------------
# We pick the last layer on resnet as test case.
# Since our space is very large, :code:`XGBoostTuner` is most suitable
# for our case. Here we only do 20 trials for demonstration.
# In practice, making 1000 trials usually can find some good kernels
# for this template

# logging config (for printing tuning log to screen)
logging.getLogger('autotvm').setLevel(logging.DEBUG)
logging.getLogger('autotvm').addHandler(logging.StreamHandler(sys.stdout))

# the last layer in yolo
def run(name, N, H, W, CO, CI, KH, KW, stride, pad):
    N, H, W, CO, CI, KH, KW, strides, padding = N, H, W, CO, CI, KH, KW, (stride, stride), (pad, pad)
    task = autotvm.task.create("conv2d_nhwc_tensorcore_test",
                               args=(N, H, W, CO, CI, KH, KW, strides, padding),
                               target='cuda')
    print(task.config_space)
    logfile = "conv2d_" + name + ".log"

    # Use local gpu, measure 10 times for every config to reduce variance
    # The timeout of compiling a program is 10 seconds, the timeout for running is 4 seconds
    measure_option = autotvm.measure_option(
        builder=autotvm.LocalBuilder(),
        runner=autotvm.LocalRunner(repeat=3, min_repeat_ms=100, timeout=4)
    )

    # Begin tuning, log records to file `conv2d.log`
    # During tuning we will also try many invalid configs, so you are expected to
    # see many error reports. As long as you can see non-zero GFLOPS, it is okay.
    tuner = autotvm.tuner.XGBTuner(task)
    tuner.tune(n_trial=1000,
               measure_option=measure_option,
               callbacks=[autotvm.callback.log_to_file(logfile)])

    #########################################################################
    # Finally we can inspect the best config from log file, check correctness,
    # and measure running time.

    # inspect the best config
    dispatch_context = autotvm.apply_history_best(logfile)
    best_config = dispatch_context.query(task.target, task.workload)
    print("\nBest config:")
    print(best_config)

    # apply history best from log file
    with autotvm.apply_history_best(logfile):
        with tvm.target.create("cuda"):
            s, arg_bufs = conv2d_nhwc_tensorcore_test(N, H, W, CO, CI, KH, KW, strides, padding)
            func = tvm.build(s, arg_bufs)

    # check correctness
    a_np = np.random.uniform(size=(N, H, W, CI)).astype(np.float16)
    w_np = np.random.uniform(size=(KH, KW, CI, CO)).astype(np.float16)
    c_np = np.random.uniform(size=(N, (H + 2 * pad - KH) // stride + 1, (W + 2 * pad - KW) // stride + 1, CO)).astype(np.float16)
    # c_np = conv2d_nchw_python(a_np, w_np, strides, padding)

    ctx = tvm.gpu()
    a_tvm = tvm.nd.array(a_np, ctx=ctx)
    w_tvm = tvm.nd.array(w_np, ctx=ctx)
    c_tvm = tvm.nd.array(c_np, ctx=ctx)
    # c_tvm = tvm.nd.empty((N, CO, (H + 2 * pad - KH) // stride + 1, (W + 2 * pad - KW) // stride + 1), ctx=ctx)
    # func(a_tvm, w_tvm, c_tvm)

    # tvm.testing.assert_allclose(c_np, c_tvm.asnumpy(), rtol=1e-2)

    # Evaluate running time. Here we choose a large repeat number (400) to reduce the noise
    # and the overhead of kernel launch. You can also use nvprof to validate the result.
    evaluator = func.time_evaluator(func.entry_name, ctx, number=400, min_repeat_ms=500)
    cost = evaluator(a_tvm, w_tvm, c_tvm).mean * 1e3
    # print('Time cost of this operator: %f' % cost)
    # with open("autotvm_conv_nhwc.txt", "a") as f:
    #     f.write("name, {}\n".format(cost))
    return cost

res18_shapes_b1 = [
    # resnet-18
    (16, 3, 224, 224, 64, 3, 7, 7, 1, 2, 3, 1, 1),  # conv1  0
    (16, 64, 56, 56, 64, 64, 3, 3, 1, 1, 1, 1, 1),  # conv2   1
    (16, 64, 56, 56, 64, 64, 1, 1, 1, 1, 0, 1, 1),  # conv3   2
    (16, 64, 56, 56, 128, 64, 3, 3, 1, 2, 1, 1, 1),  # conv4   3
    (16, 64, 56, 56, 128, 64, 1, 1, 1, 2, 0, 1, 1),  # conv5   4
    (16, 128, 28, 28, 128, 128, 3, 3, 1, 1, 1, 1, 1),  # conv6   5
    (16, 128, 28, 28, 256, 128, 3, 3, 1, 2, 1, 1, 1),  # conv7   6
    (16, 128, 28, 28, 256, 128, 1, 1, 1, 2, 0, 1, 1),  # conv8   7
    (16, 256, 14, 14, 256, 256, 3, 3, 1, 1, 1, 1, 1),  # conv9   8
    (16, 256, 14, 14, 512, 256, 3, 3, 1, 2, 1, 1, 1),  # conv10  9
    (16, 256, 14, 14, 512, 256, 1, 1, 1, 2, 0, 1, 1),  # conv11  10
    (16, 512, 7, 7, 512, 512, 3, 3, 1, 1, 1, 1, 1),  # conv12  11
    # (256, 512, 7, 7, 512, 512, 1, 1, 1, 1, 0, 1, 1)
]

if __name__ == "__main__":
    costs = []
    for i, args in enumerate(res18_shapes_b1):
        name = "resnet-18-layer-" + str(i+1)
        N, CI, H, W, CO, _, KW, KH, _, stride, pad, _, _ = args
        N = (N + 15) // 16 * 16
        CI = (CI + 15) // 16 * 16
        CO = (CO + 15) // 16 * 16
        try:
            cost = run(name, N, H, W, CO, CI, KH, KW, stride, pad)
        except Exception as e:
            print(e, flush=True)
            cost = float("inf")
        costs.append(cost)
    print("The costs:")
    for cost in costs:
        print(cost)


import json
import os

import joblib
import numpy as np
import scipy.sparse
import xgboost as xgb

def init():
    global model
    # AZUREML_MODEL_DIR is an environment variable created during deployment.
    # It's the path to the model folder (./azureml-models/$MODEL_NAME/$VERSION).
    # For multiple models, it points to the folder containing all deployed models (./azureml-models).
    model_path = os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'xgboost_model.pkl')
    model = joblib.load(model_path)

def run(raw_data):
    data = np.array(json.loads(raw_data)['data'])
    csr_data = scipy.sparse.csr_matrix(data)
    test_data = xgb.DMatrix(csr_data)

    # Make prediction.
    y_hat = model.predict(test_data)
    map_2_rating = {0: "Negative", 1: "Neutral", 2:"Positive"}
    y_hat_list = [map_2_rating[j] for j in y_hat.tolist()]

    # Perform some logging
    print(f"there was an incomding request {data}")
    print(f"Response to request {y_hat_list}")

    return y_hat_list


import pickle
import tensorflow as tf
import librosa
import librosa.display
import matplotlib.pyplot as plt
import numpy as np

emotions = {
    '0': 'angry',
    '1': 'disgust',
    '2': 'fear',
    '3': 'happy',
    '4': 'neutral',
    '5': 'sad'
}

file = "./Audios/Dataset/1092_Help_FEA_XX.wav"

data, sampling_rate = librosa.load(file)
X = []
mfcc_feature = np.mean(librosa.feature.mfcc(y=data, sr=sampling_rate, n_mfcc=40).T, axis=0)
X.append(mfcc_feature)
MFCCs = np.array(X)

model = tf.keras.models.load_model('./models')
p = model.predict(MFCCs, verbose=0)
yhat_classes = np.argmax(p, 1)
output = emotions.get(str(yhat_classes[0]))
if output == 'fear':
    print('Fear')
else:
    print(output)

Source Code/Speech Emotion Dection/Model.py


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.2.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

#

#

#

#
Scientific Python

#
A super quick crash course

#

#

#

# Seismo-Live: http://seismo-live.org
#
# ##### Authors:
# * Lion Krischer ([@krischer](https://github.com/krischer))
#
# ---

# This notebook is a very quick introduction to Python and in particular its scientific ecosystem in case you have never seen it before. It furthermore grants a possibility to get to know the [IPython/Jupyter notebook](http://www.nature.com/news/interactive-notebooks-sharing-the-code-1.16261). [See here for the official documentation](http://nbviewer.jupyter.org/github/jupyter/notebook/blob/master/docs/source/examples/Notebook/Notebook%20Basics.ipynb) of the Jupyter notebook - a ton more information can be found online.
#
#
# A lot of motivational writing on *Why Python?* is out there so we will not repeat it here and just condense it to a single sentence: **Python is a good and easy to learn, open-source, general purpose programming language that happens to be very good for many scientific tasks (due to its vast scientific ecosystem).**
#
#
# #### Quick Reference on How to Use This Notebook
#
#
#
#
# * `Shift + Enter`: Execute cell and jump to the next cell
# * `Ctrl/Cmd + Enter`: Execute cell and don't jump to the next cell
#
#
# #### Disclaimer
#
# The tutorials are employing Jupyter notebooks but these are only one way of using Python. Writing scripts to text files and executing them with the Python interpreter of course also works:
#
# ```bash
# $ python do_something.py
# ```
#
# Another alternative is interactive usage on the command line:
#
# ```bash
# $ ipython
# ```
#
# ## Notebook Setup
#
# First things first: In many notebooks you will find a cell similar to the following one. **Always execute it!** They do a couple of things:
# * Make plots appear in the browser (otherwise a window pops up)
# * Printing things works like this:
#
# ```python
# print("Hello")
# ```
#
# This essentially makes the notebooks work under Python 2 and Python 3.
#
# * Plots look quite a bit nicer (this is optional).
#

# +
# Plots now appear in the notebook.
# %matplotlib inline

import matplotlib.pyplot as plt
plt.style.use('ggplot')                            # Matplotlib style sheet - nicer plots!
plt.rcParams['figure.figsize'] = 12, 8             # Slightly bigger plots by default
# -

# ---
#
# ## Useful Links
#
# Here is collection of resources regarding the scientific Python ecosystem. They cover a number of different packages and topics; way more than we will manage today.
#
# If you have any question regarding some specific Python functionality you can consult the official [Python documenation](http://docs.python.org/).
#
# Furthermore a large number of Python tutorials, introductions, and books are available online. Here are some examples for those interested in learning more.
#
# * [Learn Python The Hard Way](http://learnpythonthehardway.org/book/)
# * [Dive Into Python](http://www.diveintopython.net/)
# * [The Official Python Tutorial](http://docs.python.org/2/tutorial/index.html)
# * [Think Python Book](http://www.greenteapress.com/thinkpython/thinkpython.html)
#
# Some people might be used to Matlab - this helps:
#
# * [NumPy for Matlab Users Introdution](http://wiki.scipy.org/NumPy_for_Matlab_Users)
# * [NumPy for Matlab Users Cheatsheet](http://mathesaurus.sourceforge.net/matlab-numpy.html)
#
#
# Additionally there is an abundance of resources introducing and teaching parts of the scientific Python ecosystem.
#
# * [NumPy Tutorial](http://wiki.scipy.org/Tentative_NumPy_Tutorial)
# * [Probabilistic Programming and Bayesian Methods for Hackers](http://camdavidsonpilon.github.io/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/): Great ebook introducing Bayesian methods from an understanding-first point of view with the examples done in Python.
# * [Python Scientific Lecture Notes](http://scipy-lectures.github.io/): Introduces the basics of scientific Python with lots of examples.
# * [Python for Signal Processing](http://python-for-signal-processing.blogspot.de/): Free blog which is the basis of a proper book written on the subject.
# * [Another NumPy Tutorial](http://www.loria.fr/~rougier/teaching/numpy/numpy.html), [Matplotlib Tutorial](http://www.loria.fr/~rougier/teaching/matplotlib/matplotlib.html)
#
# You might eventually have a need to create some custom plots. The quickest way to success is usually to start from some example that is somewhat similar to what you want to achieve and just modify it. These websites are good starting points:
#
# * [Matplotlib Gallery](http://matplotlib.org/gallery.html)
# * [ObsPy Gallery](http://docs.obspy.org/gallery.html)
# * [Basemap Gallery](http://matplotlib.org/basemap/users/examples.html)
#
#
# ---

# ## Core Python Crash Course
#
# This course is fairly non-interactive and serves to get you up to speed with Python assuming you have practical programming experience with at least one other language. Nonetheless please change things and play around an your own - it is the only way to really learn it!
#
# The first part will introduce you to the core Python language. This tutorial uses Python 3 but almost all things can be transferred to Python 2. If possible choose Python 3 for your own work!
#
#
# ### 1. Numbers
#
# Python is dynamically typed and assigning something to a variable will give it that type.

# +
# Three basic types of numbers
a = 1             # Integers
b = 2.0           # Floating Point Numbers
c = 3.0 + 4j      # Complex Numbers, note the use of j for the complex part

# Arithmetics work as expected.
# Upcasting from int -> float -> complex
d = a + b         # (int + float = float)
print(d)

e = c ** 2        # c to the second power, performs a complex multiplication
print(e)
# -

# ### 2. Strings

# Just enclose something in single or double quotes and it will become a string. On Python 3 it defaults to unicode strings, e.g. non Latin alphabets and other symbols.

# +
# You can use single or double quotes to create strings.
location = "New York"

# Concatenate strings with plus.
where_am_i = 'I am in ' + location

# Print things with the print() function.
print(location, 1, 2)
print(where_am_i)

# Strings have a lot of attached methods for common manipulations.
print(location.lower())

# Access single items with square bracket. Negative indices are from the back.
print(location[0], location[-1])

# Strings can also be sliced.
print(location[4:])
# -

# #### Exercise
#
# Save your name in all lower-case letters to a variable, and print a capitalized version of it. Protip: [Google for "How to capitalize a string in python"](http://www.google.com/search?q=how+to+capitalize+a+string+in+python). This works for almost any programming problem - someone will have had the same issue before!

# + {"tags": ["exercise"]}

# + {"tags": ["solution"]}
name = "lion"
print(name.capitalize())
# -

# ### 3. Lists

# Python has two main collection types: List and dictionaries. The former is just an ordered collection of objects and is introduced here.

# +
# List use square brackets and are simple ordered collections of things.
everything = [a, b, c, 1, 2, 3, "hello"]

# Access elements with the same slicing/indexing notation as strings.
# Note that Python indices are zero based!
print(everything[0])
print(everything[:3])
print(everything[2:-2])

# Negative indices are counted from the back of the list.
print(everything[-3:])

# Append things with the append method.
everything.append("you")
print(everything)
# -

# ### 4. Dictionaries
#
# The other main collection type in Python are dictionaries. They are similiar to associative arrays or (hash) maps in other languages. Each entry is a key-value pair.

# +
# Dictionaries have named fields and no inherent order. As is
# the case with lists, they can contain anything.
information = {
    "name": "Hans",
    "surname": "Mustermann",
    "age": 78,
    "kids": [1, 2, 3]
}

# Acccess items by using the key in square brackets.
print(information["kids"])

# Add new things by just assigning to a key.
print(information)
information["music"] = "jazz"
print(information)

# Delete things by using the del operator
del information["age"]
print(information)

# -

# ### 5. Functions
#
# The key to conquer a big problem is to divide it into many smaller ones and tackle them one by one. This is usually achieved by using functions.

# +
# Functions are defined using the def keyword.
def do_stuff(a, b):
    return a * b

# And called with the arguments in round brackets.
print(do_stuff(2, 3))

# Python function also can have optional arguments.
def do_more_stuff(a, b, power=1):
    return (a * b) ** power

print(do_more_stuff(2, 3))
print(do_more_stuff(2, 3, power=3))

# For more complex function it is oftentimes a good idea to
#explicitly name the arguments. This is easier to read and less error-prone.
print(do_more_stuff(a=2, b=3, power=3))
# -

# ### 6. Imports
#
# To use functions and objects not part of the default namespace, you have import them. You will have to do this a lot so it is necessary to learn how to do it.

# +
# Import anything, and use it with the dot accessor.
import math

a = math.cos(4 * math.pi)

# You can also selectively import things.
from math import pi

b = 3 * pi

# And even rename them if you don't like their name.
from math import cos as cosine
c = cosine(b)
# -

# How to know what is available?
#
# 1. Read the [documentation](https://docs.python.org/3/library/math.html)
# 2. Interactively query the module

print(dir(math))

# Typing the dot and the TAB will kick off tab-completion.

# math.

# In the IPython framework you can also use a question mark to view the documentation of modules and functions.

# math.cos?

# ### 7. Control Flow
#
# Loops and conditionals are needed for any non-trivial task. Please note that **whitespace matters in Python**. Everything that is indented at the same level is part of the same block. By far the most common loops in Python are for-each loops as shown in the following. While loops also exist but are rarely used.

# +
temp = ["a", "b", "c"]

# The typical Python loop is a for-each loop, e.g.
for item in temp:
    # Everything with the same indentation is part of the loop.
    new_item = item + " " + item
    print(new_item)

print("No more part of the loop.")
# -

# Useful to know is the range() function.
for i in range(5):
    print(i)

# The second crucial control flow structure are if/else conditional and they work the same as in any other language.

# +
# If/else works as expected.
age = 77

if age >= 0 and age < 10:
    print("Younger than ten.")
elif age >= 10:
    print("Older than ten.")
else:
    print("Wait what?")

# +
# List comprehensions are a nice way to write compact loops.
# Make sure you understand this as it is very common in Python.

a = list(range(10))
print(a)
b = [i for i in a if not i % 2]
print(b)

# Equivalant loop for b.
b = []
for i in a:
    if not i % 2:
        b.append(i)
print(b)

# -

# ### 8. Error Messages
#
# You will eventually run into some error messages. Learn to read them! The last line is often the one that matters - reading upwards traces the error back in time and shows what calls led to it. If stuck: just google the error message!

# +
def do_something(a, b):
    print(a + b + something_else)

# do_something(1, 2)
# -

# ## The Scientific Python Ecosystem
#
# The [SciPy Stack](https://www.scipy.org/stackspec.html) forms the basis for essentially all applications of scientific Python. Here we will quickly introduce the three core libraries:
#
# * `NumPy`
# * `SciPy`
# * `Matplotlib`
#
# The SciPy stack furthermore contains `pandas` (library for data analysis on tabular and time series data) and `sympy` (package for symbolic math), both very powerful packages, but we will omit them in this tutorial.

# ### 9. NumPy
#
# Large parts of the scientific Python ecosystem use NumPy, an array computation package offering N-dimensional, typed arrays and useful functions for linear algebra, Fourier transforms, random numbers, and other basic scientific tasks.

# +
import numpy as np

# Create a large array with with 1 million samples.
x = np.linspace(start=0, stop=100, num=1E6, dtype=np.float64)

# Most operations work per-element.
y = x ** 2

# Uses C and Fortran under the hood for speed.
print(y.sum())

# FFT and inverse
x = np.random.random(100)
large_X = np.fft.fft(x)
x = np.fft.ifft(large_X)
# -

# ### 10. SciPy
#
# `SciPy`, in contrast to `NumPy` which only offers basic numerical routines, contains a lot of additional functionality needed for scientific work. Examples are solvers for basic differential equations, numeric integration and optimization, spare matrices, interpolation routines, signal processing methods, and a lot of other things.

# +
from scipy.interpolate import interp1d

x = np.linspace(0, 10, num=11, endpoint=True)
y = np.cos(-x ** 2 / 9.0)

# Cubic spline interpolation to new points.
f2 = interp1d(x, y, kind='cubic')(np.linspace(0, 10, num=101, endpoint=True))
# -

# ### 11. Matplotlib
#
# Plotting is done using `Matplotlib`, a package for greating high-quality static plots. It has an interface that mimics Matlab which many people are familiar with.

# +
import matplotlib.pyplot as plt

plt.plot(np.sin(np.linspace(0, 2 * np.pi, 2000)), color="green",
         label="Some Curve")
plt.legend()
plt.ylim(-1.1, 1.1)
plt.show()
# -

# ## Exercises
#
# #### Functions, NumPy, and Matplotlib
#
# A. Write a function that takes a NumPy array `x` and `a`, `b`, and `c` and returns
#
# $$
# f(x) = a x^2 + b x + c
# $$
#
# B. Plot the result of that function with matplotlib.

# + {"tags": ["exercise"]}

# + {"tags": ["solution"]}
import matplotlib.pyplot as plt
import numpy as np

def simple_poly(x, a, b, c):
    return a * x ** 2 + b * x + c

plt.plot(simple_poly(np.linspace(-5, 5), 10, 2, 2))
plt.show()
# -

# #### 99 Bottles of Beer
#
# *(stolen from http://www.ling.gu.se/~lager/python_exercises.html)*
#
#
# "99 Bottles of Beer" is a traditional song in the United States and Canada. It is popular to sing on long trips, as it has a very repetitive format which is easy to memorize, and can take a long time to sing. The song's simple lyrics are as follows:
#
# ```
# 99 bottles of beer on the wall, 99 bottles of beer.
# Take one down, pass it around, 98 bottles of beer on the wall.
# ```
#
# The same verse is repeated, each time with one fewer bottle. The song is completed when the singer or singers reach zero.
#
# Your task here is write a Python program capable of generating all the verses of the song.
#

# + {"tags": ["exercise"]}

# + {"tags": ["solution"]}
print("99 bottles of beer on the wall, 99 bottles of beer.")
for i in range(98, -1, -1):
    print("Take one down, pass it around, %i bottles of beer on the wall." % i)
# -

# #### Ceasar Cipher
#
# *(stolen from http://www.ling.gu.se/~lager/python_exercises.html)*
#
# In cryptography, a Caesar cipher is a very simple encryption techniques in which each letter in the plain text is replaced by a letter some fixed number of positions down the alphabet. For example, with a shift of 3, A would be replaced by D, B would become E, and so on. The method is named after Julius Caesar, who used it to communicate with his generals. ROT-13 ("rotate by 13 places") is a widely used example of a Caesar cipher where the shift is 13. In Python, the key for ROT-13 may be represented by means of the following dictionary:
#
# ```python
# key = {'a':'n', 'b':'o', 'c':'p', 'd':'q', 'e':'r', 'f':'s', 'g':'t', 'h':'u',
#        'i':'v', 'j':'w', 'k':'x', 'l':'y', 'm':'z', 'n':'a', 'o':'b', 'p':'c',
#        'q':'d', 'r':'e', 's':'f', 't':'g', 'u':'h', 'v':'i', 'w':'j', 'x':'k',
#        'y':'l', 'z':'m', 'A':'N', 'B':'O', 'C':'P', 'D':'Q', 'E':'R', 'F':'S',
#        'G':'T', 'H':'U', 'I':'V', 'J':'W', 'K':'X', 'L':'Y', 'M':'Z', 'N':'A',
#        'O':'B', 'P':'C', 'Q':'D', 'R':'E', 'S':'F', 'T':'G', 'U':'H', 'V':'I',
#        'W':'J', 'X':'K', 'Y':'L', 'Z':'M'}
# ```
#
# Your task in this exercise is to implement an decoder of ROT-13. Once you're done, you will be able to read the following secret message:
#
# ```
# Pnrfne pvcure? V zhpu cersre Pnrfne fnynq!
# ```
#
# **BONUS:** Write an encoder!

# + {"tags": ["exercise"]}

# + {"tags": ["solution"]}
sentence = "Pnrfne pvcure? V zhpu cersre Pnrfne fnynq!"

key = {'a':'n', 'b':'o', 'c':'p', 'd':'q', 'e':'r', 'f':'s', 'g':'t', 'h':'u',
       'i':'v', 'j':'w', 'k':'x', 'l':'y', 'm':'z', 'n':'a', 'o':'b', 'p':'c',
       'q':'d', 'r':'e', 's':'f', 't':'g', 'u':'h', 'v':'i', 'w':'j', 'x':'k',
       'y':'l', 'z':'m', 'A':'N', 'B':'O', 'C':'P', 'D':'Q', 'E':'R', 'F':'S',
       'G':'T', 'H':'U', 'I':'V', 'J':'W', 'K':'X', 'L':'Y', 'M':'Z', 'N':'A',
       'O':'B', 'P':'C', 'Q':'D', 'R':'E', 'S':'F', 'T':'G', 'U':'H', 'V':'I',
       'W':'J', 'X':'K', 'Y':'L', 'Z':'M'}

result = ""
for letter in sentence:
    if letter not in key:
        result += letter
    else:
        result += key[letter]
print(result)


#!/usr/bin/env python3
"""
generate snippets for VSCode
"""
import os
import json
DIR = os.path.dirname(__file__)
snippets = {}
snippets_for_global = {}

def push(prefix, desc, body=None, for_global=False):
    to_generate_desc = False
    if not body:
        body = desc
        to_generate_desc = True
    body = body.strip()
    lines = body.splitlines()

    if to_generate_desc:
        if lines[0] == '"""':
            desc = lines[1]
        else:
            desc = lines[0]

    if desc in snippets:
        print(f"{desc} already exists")
        desc = desc + "'"

    snip = dict(
        scope="python",
        prefix=prefix,
        body=lines,
        description=desc
    )
    if for_global:
        snippets_for_global[desc] = snip
    else:
        snippets[desc] = snip

def make_template(code, args):
    args = args.split()
    for i, arg in enumerate(args):
        placeholder = "${%d:%s}" % (i + 1, arg)
        code = code.replace(arg, placeholder)
    return code

push("readlist", "read list of integers", """
list(map(int, input().split()))
""")
push("readtuple", "tuple(map(int, input().split()))")

push("readints", "map(int, input().split())")

push("readanint", "read an integer", "int(input())")

push("readstr",  "input().strip()")

push("readstrascii", "input().strip().decode('ascii')")

push("readquery", """
N, Q = map(int, input().split())
# Q = int(input())
QS = []
for _q in range(Q):
    QS.append(tuple(map(int, input().split())))
""")

push("readtuples", make_template("""
XS = []
for _i in range(N):
    XS.append(tuple(map(int, input().split())))
""", "XS N"))

push("readedgecost", "read edges with cost", make_template("""
from collections import defaultdict
INF = 9223372036854775807
edges = defaultdict(lambda: defaultdict(lambda: INF))
for _i in range(NUM_EDGES):
    frm, to, cost = map(int, input().split())
    edges[frm-1][to-1] = cost  # -1 for 1-origin vertexes
    edges[to-1][frm-1] = cost  # if bidirectional
    # edges[frm-1][to-1] = min(edges[frm-1][to-1], cost)  # for multiple edges
""", "NUM_EDGES -1"))

push("readedges", "read costless edges", make_template("""
from collections import defaultdict
edges = defaultdict(list)
for _i in range(NUM_EDGES):
    frm, to = map(int, input().split())
    edges[frm-1].append(to-1)  # -1 for 1-origin vertexes
    edges[to-1].append(frm-1)  # if bidirectional
""", "NUM_EDGES -1"))

push("readbigint", "[x - ord('0') for x in input().strip()]")
push("profile", "define @profile if not exist", """
try:
    profile
except:
    def profile(f): return f
""", for_global=True)

push("perf", "use perf_counter", """
start_time = perf_counter()
$CLIPBOARD
debug(f"$1: {(perf_counter() - start_time):.2f}")
""", for_global=True)

push("impperf", "perf_counter", "from time import perf_counter", for_global=True)

push("test", "define testcode", '''
T${1:} = """
$2
"""
TEST_T$1 = """
>>> as_input(T$1)
>>> main()
${3:result}
"""
''', for_global=True)

push("dpold", "debug print", """
debug("$1", $1)
""", for_global=True)

push("dp", "debug print", """
debug($1, msg="$2:$1")
""", for_global=True)

push("bp", "conditional breakpoint", """
if ${1:True}:
    import pdb
    pdb.set_trace()
""", for_global=True)

push("cache", """
from functools import lru_cache
@lru_cache(maxsize=None)
""", for_global=True)

push("unpack", make_template("""
x = pair >> 32
y = pair - (x << 32)
""", "pair 32 x y"))

push("pack", make_template("""
pair = (x << 32) + y
""", "pair 32 x y"))

# import
push("impdef", "from collections import defaultdict", for_global=True)
push("impcou", "from collections import Counter", for_global=True)
push("impdeq", "from collections import deque", for_global=True)
push("impheap", "from heapq import heappush, heappop", for_global=True)
push("impnp", "import numpy as np", for_global=True)

push("deftest", """
def _test():
    import doctest
    doctest.testmod()
    g = globals()
    for k in sorted(g):
        if k.startswith("TEST_"):
            print(k)
            doctest.run_docstring_examples(g[k], g, name=k)
""", for_global=True)

push("defdebug", """
def debug(*x, msg=""):
    import sys
    print(msg, *x, file=sys.stderr)
""", for_global=True)

push("ifmain", """
if __name__ == "__main__":
    import sys
    input = sys.stdin.buffer.readline
    read = sys.stdin.buffer.read
    if sys.argv[-1] == "-t":
        _test()
        sys.exit()
    main()
""", for_global=True)

push("yesno", """
if $1:
    print("Yes")
else:
    print("No")
""")

push("as_input", """
def as_input(s):
    "use in test, use given string as input file"
    import io
    g = globals()
    f = io.StringIO(s.strip())

    g["input"] = lambda: bytes(f.readline(), "ascii")
    g["read"] = lambda: bytes(f.read(), "ascii")
""")

push("MOD1", "MOD = 1_000_000_007")
push("MOD9", "MOD = 998_244_353")
push("INF", "INF = 9223372036854775807")

push("for_subset", make_template("""
for subset in range(2 ** N):
    for i in range(N):
        if subset & (1 << i):
            pass
    pass
""", "subset N i"))

push("use_mincostflow", """
global mcf
mcf = MinCostFlow(numVertex)
# construct graph
mcf.add_edge(frm, to, capacity, cost)
# debug
print(list(mcf.edges()))
# flow
cap, cost = mcf.flow(start, goal)
""")

EOL = "# --- end of library ---"
push("eol", "end of library", EOL)

def read_file(filename):
    data = open(os.path.join(DIR, filename)).read()
    if EOL in data:
        data = data.split(EOL)[0]
    data = data.strip() + "\n"
    return f"# included from {filename}\n{data}\n# end of {filename}"

push("main", read_file("snippets/main.py"))
push("numbamain", read_file("snippets/numbamain.py"))
push("def_debug_indent", read_file("snippets/debug_indent.py"))

# register libs/*.py
for filename in os.listdir(os.path.join(DIR, "libs")):
    if filename.endswith(".py"):
        prefix = filename.replace(".py", "")
        push(prefix, read_file(os.path.join("libs", filename)))

def main():
    path = os.path.join(DIR, ".vscode/snippet.code-snippets")
    json.dump(snippets, open(path, "w"), indent=2)
    path = "/Users/nishio/Library/Application Support/Code/User/snippets/python.code-snippets"
    json.dump(snippets_for_global, open(path, "w"), indent=2)

if __name__ == "__main__":
    main()