# Licensed under a 3-clause BSD style license - see LICENSE.rst
import os
from copy import deepcopy
from itertools import chain, count
from math import floor
from pathlib import Path

import numba
import numpy as np

__all__ = ["ACAImage", "centroid_fm", "AcaPsfLibrary", "EIGHT_LABELS"]

EIGHT_LABELS = np.array(
    [
        ["A1", "B1", "C1", "D1", "E1", "F1", "G1", "H1"],
        ["I1", "J1", "K1", "L1", "M1", "N1", "O1", "P1"],
        ["A2", "B2", "C2", "D2", "E2", "F2", "G2", "H2"],
        ["I2", "J2", "K2", "L2", "M2", "N2", "O2", "P2"],
        ["A3", "B3", "C3", "D3", "E3", "F3", "G3", "H3"],
        ["I3", "J3", "K3", "L3", "M3", "N3", "O3", "P3"],
        ["A4", "B4", "C4", "D4", "E4", "F4", "G4", "H4"],
        ["I4", "J4", "K4", "L4", "M4", "N4", "O4", "P4"],
    ]
)
"""Constant for labeling ACA image pixels using the EQ-278 spec format.
Pixel A1 has the lowest values of row and column; pixel H1 has the lowest
row and highest col; pixel I4 has the highest row and lowest column."""

def _operator_factory(operator, inplace=False):
    """
    Generate data model methods like __add__(self, other) and
    __iadd__(self, other).  These always operate in the coordinate
    system of the left and right operands.  If both are in ACA
    coordinates then any non-overlapping pixels are ignored.
    """
    # Define the operator and the in-place version (which might be the
    # same if op is already in-place)
    op = getattr(np.ndarray, "__{}__".format(operator))
    inplace_op = op if inplace else getattr(np.ndarray, "__i{}__".format(operator))

    def _operator(self, other):
        if isinstance(other, ACAImage) and (other._aca_coords or self._aca_coords):
            # If inplace then work on the original self, else use a copy
            out = self if inplace else self.copy()

            sz_r0, sz_c0 = self.shape
            sz_r1, sz_c1 = other.shape

            # If images overlap do this process, else return unmodified ``out``.
            if all(
                diff > 0
                for diff in [
                    self.row0 + sz_r0 - other.row0,
                    self.col0 + sz_c0 - other.col0,
                    other.row0 + sz_r1 - self.row0,
                    other.col0 + sz_c1 - self.col0,
                ]
            ):
                dr = other.row0 - self.row0
                dc = other.col0 - self.col0

                r_min, r_max = -min(0, dr), min(sz_r1, sz_r0 - dr)
                c_min, c_max = -min(0, dc), min(sz_c1, sz_c0 - dc)

                row0 = max(self.row0, other.row0)
                col0 = max(self.col0, other.col0)
                sz_r = r_max - r_min
                sz_c = c_max - c_min
                section = ACAImage(shape=(sz_r, sz_c), row0=row0, col0=col0)

                # Always use the inplace operator, but remember that ``out`` is a copy of
                # self for inplace=False (thus mimicking the non-inplace version).
                inplace_op(
                    out[section], other.view(np.ndarray)[r_min:r_max, c_min:c_max]
                )

        else:
            out = op(self, other)  # returns self for inplace ops

        return out

    return _operator

class ACAImage(np.ndarray):
    """
    ACAImage is an ndarray subclass that supports functionality for the Chandra
    ACA. Most importantly it allows image indexing and slicing in absolute
    "aca" coordinates, where the image lower left coordinate is specified
    by object ``row0`` and ``col0`` attributes.

    It also provides a ``meta`` dict that can be used to store additional useful
    information.  Any keys which are all upper-case will be exposed as object
    attributes, e.g. ``img.BGDAVG`` <=> ``img.meta['BGDAVG']``.  The ``row0``
    attribute  is a proxy for ``img.meta['IMGROW0']``, and likewise for ``col0``.

    When initializing an ``ACAImage``, additional ``*args`` and ``**kwargs`` are
    used to try initializing via ``np.array(*args, **kwargs)``.  If this fails
    then ``np.zeros(*args, **kwargs)`` is tried.  In this way one can either
    initialize from array data or create a new array of zeros.

    Examples::

      >>> import numpy as np
      >>> from chandra_aca.aca_image import ACAImage
      >>> dat = np.random.uniform(size=(1024, 1024))
      >>> a = ACAImage(dat, row0=-512, col0=-512)
      >>> a = ACAImage([[1,2], [3,4]], meta={'BGDAVG': 5.2})
      >>> a = ACAImage(shape=(1024, 1024), row0=-512, col0=-512)

    :param row0: row coordinate of lower left image pixel (int, default=0)
    :param col0: col coordinate of lower left image pixel (int, default=0)
    :param meta: dict of object attributes
    :param ``*args``: additional args passed to np.array() or np.zeros()
    :param ``**kwargs``: additional kwargs passed to np.array() or np.zeros()
    """

    @property
    def aca(self):
        """
        Return a light copy (same data) of self but with the _aca_coords
        attribute switched on so that indexing is absolute.
        """
        obj = self.view(self.__class__)
        obj.meta = self.meta
        obj._aca_coords = True
        return obj

    def __new__(cls, *args, **kwargs):
        meta = kwargs.pop("meta", {})

        # Set default row0 and col0 to 0 (if not already in meta), and
        # then override with like-named kwargs.  row0 attribute => meta['IMGROW0']
        for ax in ("row0", "col0"):
            imgax = "IMG" + ax.upper()
            meta.setdefault(imgax, 0)
            if ax in kwargs:
                meta[imgax] = np.int64(kwargs.pop(ax))

        try:
            arr = np.array(*args, **kwargs)
        except Exception:
            arr = np.zeros(*args, **kwargs)

        # Input array is an already formed ndarray instance
        # We first cast to be our class type
        obj = arr.view(cls)

        if obj.ndim != 2:
            raise ValueError("{} must be 2-d".format(cls.__name__))

        # add the new attribute to the created instance
        obj.meta = meta
        obj._aca_coords = False

        # Finally, we must return the newly created object:
        return obj

    def __array_finalize__(self, obj):
        # see InfoArray.__array_finalize__ for comments
        if obj is None:
            return

        self.meta = deepcopy(getattr(obj, "meta", {}))
        self._aca_coords = getattr(obj, "_aca_coords", False)

    __add__ = _operator_factory("add")
    __sub__ = _operator_factory("sub")
    __mul__ = _operator_factory("mul")
    __truediv__ = _operator_factory("truediv")
    __floordiv__ = _operator_factory("floordiv")
    __mod__ = _operator_factory("mod")
    __pow__ = _operator_factory("pow")

    __iadd__ = _operator_factory("iadd", inplace=True)
    __isub__ = _operator_factory("isub", inplace=True)
    __imul__ = _operator_factory("imul", inplace=True)
    __itruediv__ = _operator_factory("itruediv", inplace=True)
    __ifloordiv__ = _operator_factory("ifloordiv", inplace=True)
    __imod__ = _operator_factory("imod", inplace=True)
    __ipow__ = _operator_factory("ipow", inplace=True)

    def _adjust_item(self, item):
        """
        This is the money method that does all the work of manipulating
        an item and subsequent row0/col0 when accessing and slicing.
        """
        # Allow slicing via an existing ACAImage object
        aca_coords = self._aca_coords
        if isinstance(item, ACAImage):
            item = (
                slice(item.row0, item.row0 + item.shape[0]),
                slice(item.col0, item.col0 + item.shape[1]),
            )
            aca_coords = True

        out_rc = [None, None]  # New [row0, col0]

        if isinstance(item, (int, np.integer)):
            item = (item,)

        if isinstance(item, tuple):
            if aca_coords:
                # Interpret input `item` indices as being expressed in absolute
                # terms and subtract row0/col0 as needed.
                item = list(item)
                for i, it, rc0 in zip(count(), item, (self.row0, self.col0)):
                    if isinstance(it, slice):
                        start = None if it.start is None else it.start - rc0
                        stop = None if it.stop is None else it.stop - rc0
                        item[i] = slice(start, stop, it.step)
                    else:
                        item[i] = it - rc0
                item = tuple(item)

            # Compute new row0, col0 (stored in out_rc) based on input item
            for i, it, rc0 in zip(count(), item, (self.row0, self.col0)):
                if isinstance(it, slice):
                    if it.start is not None:
                        out_rc[i] = rc0 + it.start
                else:
                    out_rc[i] = rc0 + it

        return item, out_rc[0], out_rc[1]

    def __getitem__(self, item):
        item, row0, col0 = self._adjust_item(item)

        out = super(ACAImage, self).__getitem__(item)

        if isinstance(out, ACAImage):
            if row0 is not None:
                out.row0 = row0
            if col0 is not None:
                out.col0 = col0
            out._aca_coords = False

        return out

    def __setitem__(self, item, value):
        item, row0, col0 = self._adjust_item(item)

        aca_coords = self._aca_coords
        try:
            self._aca_coords = False
            super(ACAImage, self).__setitem__(item, value)
        finally:
            self._aca_coords = aca_coords

    def __repr__(self):
        # Make an integerized version for viewing more nicely
        outarr = np.asarray(np.round(self)).astype(int)
        out = "<{} row0={} col0={}\n{}>".format(
            self.__class__.__name__, self.row0, self.col0, outarr.__repr__()
        )
        return out

    def __getattr__(self, attr):
        if attr.isupper():
            try:
                return self.meta[attr]
            except KeyError:
                pass

        return super(ACAImage, self).__getattribute__(attr)

    def __setattr__(self, attr, value):
        if attr.isupper():
            self.meta[attr] = value
        else:
            super(ACAImage, self).__setattr__(attr, value)

    def centroid_fm(self, bgd=None, pix_zero_loc="center", norm_clip=None):
        """
        First moment centroid of ``self`` using 6x6 mousebitten image for input
        6x6 or 8x8 images.

        Note that the returned ``norm`` is the sum of the background-subtracted 6x6
        mousebitten image, not the entire image.

        :param bgd: background to subtract, scalar or NxN ndarray (float)
        :param pix_zero_loc: row/col coords are integral at 'edge' or 'center'
        :param norm_clip: clip image norm at this min value (default is None and
                          implies Exception for non-positive norm)

        :returns: row, col, norm float
        """
        row, col, norm = centroid_fm(
            self, bgd=bgd, pix_zero_loc=pix_zero_loc, norm_clip=norm_clip
        )
        if self._aca_coords:
            row += self.row0
            col += self.col0

        return row, col, norm

    def __dir__(self):
        return sorted(super().__dir__() + list(self.meta))

    @property
    def row0(self):
        return self.meta["IMGROW0"]

    @row0.setter
    def row0(self, value):
        self.meta["IMGROW0"] = np.int64(value)

    @property
    def col0(self):
        return self.meta["IMGCOL0"]

    @col0.setter
    def col0(self, value):
        self.meta["IMGCOL0"] = np.int64(value)

    @classmethod
    def _read_flicker_cdfs(cls):
        """Read flickering pixel model cumulative distribution functions
        and associated metadata.  Set up class variables accordingly.

        The flicker_cdf file here was created using:
        /proj/sot/ska/www/ASPECT/ipynb/chandra_aca/flickering-pixel-model.ipynb

        """
        from astropy.io import fits

        filename = Path(__file__).parent / "data" / "flicker_cdf.fits.gz"
        with fits.open(filename) as hdus:
            hdu = hdus[0]
            hdr = hdu.header

            # Get the main data, which is an n_cdf * n_cdf_x array.  Each row corresponds
            # to the CDF for a particular bin range in e-/sec, e.g. 200 to 300 e-/sec.
            # CDF will go from 0.0 to 1.0
            cls.flicker_cdfs = hdu.data.astype(np.float64)

            # CDF_x is the x-value of the distribution, namely the log-amplitude change
            # in pixel value due to a flicker event.
            cls.flicker_cdf_x = np.linspace(
                hdr["cdf_x0"], hdr["cdf_x1"], hdr["n_cdf_x"]
            )

            # CDF bin range (e-/sec) for each for in flicker_cdfs.
            cdf_bins = []
            for ii in range(hdr["n_bin"]):
                cdf_bins.append(hdr[f"cdf_bin{ii}"])
            cls.flicker_cdf_bins = np.array(cdf_bins)

    def flicker_init(self, flicker_mean_time=10000, flicker_scale=1.0, seed=None):
        """Initialize instance variables to allow for flickering pixel updates.

        The ``flicker_scale`` can be interpreted as follows: if the pixel
        was going to flicker by a multiplicative factor of (1 + x), now
        make it flicker by (1 + x * flicker_scale).  This applies for flickers
        that increase the amplitude.  For flickers that make the value smaller,
        then it would be 1 / (1 + x) => 1 / (1 + x * flicker_scale).

        The flicker_cdf file here was created using:
        /proj/sot/ska/www/ASPECT/ipynb/chandra_aca/flickering-pixel-model.ipynb

        Examples and performance details at:
        /proj/sot/ska/www/ASPECT/ipynb/chandra_aca/flickering-implementation.ipynb

        The model was reviewed and approved at SS&AWG on 2019-05-22.

        :param flicker_mean_time: mean flickering time (sec, default=10000)
        :param flicker_scale: multiplicative factor beyond model default for
               flickering amplitude (default=1.0)
        :param seed: random seed for reproducibility (default=None => no seed)
        """
        if not hasattr(self, "flicker_cdf_bins"):
            self._read_flicker_cdfs()

        self.flicker_mean_time = flicker_mean_time
        self.flicker_scale = flicker_scale
        self.test_idx = 1 if seed == -1 else 0

        if seed is not None and seed != -1:
            np.random.seed(seed)
            _numba_random_seed(seed)

        # Make a flattened view of the image for easier update processing.
        # Also store the initial pixel values, since flicker updates are
        # relative to the initial value, not the current value (which would
        # end up random-walking).
        self.flicker_vals = self.view(np.ndarray).ravel()
        self.flicker_vals0 = self.flicker_vals.copy()
        self.flicker_n_vals = len(self.flicker_vals)

        # Make a bool ACAImage like self to allow convenient mask/unmask of
        # pixels to flicker.  This is used in annie.  Also make the corresponding
        # 1-d ravelled version.
        self.flicker_mask = ACAImage(
            np.ones(self.shape, dtype=bool), row0=self.row0, col0=self.col0
        )
        self.flicker_mask_vals = self.flicker_mask.view(np.ndarray).ravel()

        # Get the index to the CDFs which is appropriate for each pixel
        # based on its initial value.
        self.flicker_cdf_idxs = (
            np.searchsorted(self.flicker_cdf_bins, self.flicker_vals0) - 1
        )

        # Create an array of time (secs) until next flicker for each pixel
        if seed == -1:
            # Special case of testing, use a constant flicker_mean_time initially
            t_flicker = np.ones(shape=(self.flicker_n_vals,)) * flicker_mean_time
            phase = 1.0
        else:
            # This is drawing from an exponential distribution.  For the initial
            # time assume the flickering is randomly phased within that interval.
            phase = np.random.uniform(0.0, 1.0, size=self.flicker_n_vals)
            rand_unifs = np.random.uniform(0.0, 1.0, size=self.flicker_n_vals)
            t_flicker = -np.log(1.0 - rand_unifs) * flicker_mean_time

        self.flicker_times = t_flicker * phase

    def flicker_update(self, dt, use_numba=True):
        """Propagate the image forward by ``dt`` seconds and update any pixels
        that have flickered during that interval.

        This has the option to use one of two implementations.  The default is
        to use the numba-based version which is about 6 times faster.  The
        vectorized version is left in for reference.

        :param dt: time (secs) to propagate image
        :param use_numba: use the numba version of updating (default=True)
        """
        if not hasattr(self, "flicker_times"):
            self.flicker_init()

        if use_numba:
            _flicker_update_numba(
                dt,
                len(self.flicker_vals),
                self.test_idx,
                self.flicker_vals0,
                self.flicker_vals,
                self.flicker_mask_vals,
                self.flicker_times,
                self.flicker_cdf_idxs,
                self.flicker_cdf_x,
                self.flicker_cdfs,
                self.flicker_scale,
                self.flicker_mean_time,
            )
            if self.test_idx > 0:
                self.test_idx += 1
        else:
            self._flicker_update_vectorized(dt)

    def _flicker_update_vectorized(self, dt):
        self.flicker_times[self.flicker_mask_vals] -= dt

        # When flicker_times < 0 that means a flicker occurs
        ok = (self.flicker_times < 0) & (self.flicker_cdf_idxs >= 0)
        idxs = np.where(ok)[0]

        # Random uniform used for (1) distribution of flickering amplitude
        # via the CDFs and (2) distribution of time to next flicker.
        rand_ampls = np.random.uniform(0.0, 1.0, size=len(idxs))
        rand_times = np.random.uniform(0.0, 1.0, size=len(idxs))

        for idx, rand_time, rand_ampl in zip(idxs, rand_times, rand_ampls):
            # Determine the new value after flickering and set in array view.
            # First get the right CDF from the list of CDFs based on the pixel value.
            cdf_idx = self.flicker_cdf_idxs[idx]
            y = np.interp(
                fp=self.flicker_cdf_x, xp=self.flicker_cdfs[cdf_idx], x=rand_ampl
            )

            if self.flicker_scale != 1.0:
                # Express the multiplicative change as (1 + x) and change
                # it to be (1 + x * scale).  This makes sense for positive y,
                # so use abs(y) and then flip the sign back at the end.  For
                # negative y this is the same as doing this trick expressing the
                # change as 1 / (1 + x).
                dy = (10 ** np.abs(y) - 1.0) * self.flicker_scale + 1.0
                y = np.log10(dy) * np.sign(y)

            val = self.flicker_vals0[idx] * 10**y
            self.flicker_vals[idx] = val

            # Get the new time before next flicker
            t_flicker = -np.log(1.0 - rand_time) * self.flicker_mean_time
            self.flicker_times[idx] = t_flicker

@numba.jit(nopython=True)
def _numba_random_seed(seed):
    np.random.seed(seed)

@numba.jit(nopython=True)
def _flicker_update_numba(
    dt,
    nvals,
    test_idx,
    flicker_vals0,
    flicker_vals,
    flicker_mask_vals,
    flicker_times,
    flicker_cdf_idxs,
    flicker_cdf_x,
    flicker_cdfs,
    flicker_scale,
    flicker_mean_time,
):
    """
    Propagate the image forward by ``dt`` seconds and update any pixels
    that have flickered during that interval.
    """
    for ii in range(nvals):  # nvals
        # cdf_idx is -1 for pixels with dark current in range that does not flicker.
        # Don't flicker those ones or pixels that are masked out.
        cdf_idx = flicker_cdf_idxs[ii]
        if cdf_idx < 0 or not flicker_mask_vals[ii]:
            continue

        flicker_times[ii] -= dt

        if flicker_times[ii] > 0:
            continue

        if test_idx > 0:
            # Deterministic and reproducible but bouncy sequence that is reproducible in C
            # (which has a different random number generator).
            rand_ampl = np.abs(np.sin(float(ii + test_idx)))
            rand_time = np.abs(np.cos(float(ii + test_idx)))
        else:
            # Random uniform used for (1) distribution of flickering amplitude
            # via the CDFs and (2) distribution of time to next flicker.
            rand_ampl = np.random.uniform(0.0, 1.0)
            rand_time = np.random.uniform(0.0, 1.0)

        # Determine the new value after flickering and set in array view.
        # First get the right CDF from the list of CDFs based on the pixel value.
        y = np_interp(yin=flicker_cdf_x, xin=flicker_cdfs[cdf_idx], xout=rand_ampl)

        if flicker_scale != 1.0:
            # Express the multiplicative change as (1 + x) and change
            # it to be (1 + x * scale).  This makes sense for positive y,
            # so use abs(y) and then flip the sign back at the end.  For
            # negative y this is the same as doing this trick expressing the
            # change as 1 / (1 + x).
            dy = (10 ** np.abs(y) - 1.0) * flicker_scale + 1.0
            y = np.log10(dy) * np.sign(y)

        val = 10 ** (np.log10(flicker_vals0[ii]) + y)
        flicker_vals[ii] = val

        # Get the new time before next flicker
        t_flicker = -np.log(1.0 - rand_time) * flicker_mean_time
        flicker_times[ii] = t_flicker

@numba.jit(nopython=True)
def np_interp(yin, xin, xout):
    idx = np.searchsorted(xin, xout)

    if idx == 0:
        return yin[0]

    if idx == len(xin):
        return yin[-1]

    x0 = xin[idx - 1]
    x1 = xin[idx]
    y0 = yin[idx - 1]
    y1 = yin[idx]
    yout = (xout - x0) / (x1 - x0) * (y1 - y0) + y0

    return yout

def _prep_6x6(img, bgd=None):
    """
    Subtract background and in case of 8x8 image
    cut and return the 6x6 inner section.
    """
    if isinstance(bgd, np.ndarray):
        bgd = bgd.view(np.ndarray)

    # Subtract background and/or ensure a copy of img is made since downstream
    # code may modify the image.
    if bgd is not None:
        img = img - bgd
    else:
        img = img.copy()

    if img.shape == (8, 8):
        img = img[1:7, 1:7]

    return img

def centroid_fm(img, bgd=None, pix_zero_loc="center", norm_clip=None):
    """
    First moment centroid of ``img``.

    Return FM centroid in coords where lower left pixel of image has value
    (0.0, 0.0) at the center (for pix_zero_loc='center') or the lower-left edge
    (for pix_zero_loc='edge').

    :param img: NxN ndarray
    :param bgd: background to subtract, float of NXN ndarray
    :param pix_zero_loc: row/col coords are integral at 'edge' or 'center'
    :param norm_clip: clip image norm at this min value (default is None and
                      implies Exception for non-positive norm)

    :returns: row, col, norm float
    """
    # Cast to an ndarray (without copying)
    img = img.view(np.ndarray)

    sz_r, sz_c = img.shape
    if sz_r != sz_c:
        raise ValueError("input img must be square")

    rw, cw = np.mgrid[1:7, 1:7] if sz_r == 8 else np.mgrid[0:sz_r, 0:sz_r]

    if sz_r in (6, 8):
        img = _prep_6x6(img, bgd)
        img[[0, 0, 5, 5], [0, 5, 0, 5]] = 0

    norm = np.sum(img)
    if norm_clip is not None:
        norm = norm.clip(norm_clip, None)
    else:
        if norm <= 0:
            raise ValueError("non-positive image norm {}".format(norm))

    row = np.sum(rw * img) / norm
    col = np.sum(cw * img) / norm

    if pix_zero_loc == "edge":
        # Transform row/col values from 'center' convention (as returned
        # by centroiding) to the 'edge' convention requested by user.
        row = row + 0.5
        col = col + 0.5
    elif pix_zero_loc != "center":
        raise ValueError("pix_zero_loc can be only 'edge' or 'center'")

    return row, col, norm

class AcaPsfLibrary(object):
    """
    Access the ACA PSF library, whch is a library of 8x8 images providing the integrated
    (pixelated) ACA PSF over a grid of subpixel locations.

    Example::

      >>> from chandra_aca.aca_image import AcaPsfLibrary
      >>> apl = AcaPsfLibrary()  # Reads in PSF library data file
      >>> img = apl.get_psf_image(row=-10.456, col=250.123, norm=100000)
      >>> img
      <ACAImage row0=-14 col0=247
      array([[   39,    54,    56,    52,    37,    33,    30,    21],
             [   79,   144,   260,   252,   156,    86,    67,    36],
             [  162,   544,  2474,  5269,  2012,   443,   163,    57],
             [  255,  1420, 10083, 12688, 11273,  1627,   302,    78],
             [  186,  1423,  8926,  8480, 12292,  2142,   231,    64],
             [   80,   344,  1384,  6509,  4187,   665,   111,    43],
             [   40,    78,   241,   828,   616,   188,    65,    29],
             [   24,    39,    86,   157,   139,    69,    48,    32]])>

    :param filename: file name of ACA PSF library (default=built-in file)
    :returns: AcaPsfLibrary object
    """

    def __init__(self, filename=None):
        from astropy.table import Table  # Table is a somewhat-heavy import

        psfs = {}

        if filename is None:
            filename = os.path.join(
                os.path.dirname(__file__), "data", "aca_psf_lib.dat"
            )
        dat = Table.read(filename, format="ascii.basic", guess=False)
        self.dat = dat

        # Sub-pixel grid spacing in pixels.  This assumes the sub-pixels are
        # all the same size and square, which is indeed the case.
        self.drc = dat["row_bin_right_edge"][0] - dat["row_bin_left_edge"][0]

        for row in dat:
            ii = row["row_bin_idx"]
            jj = row["col_bin_idx"]
            psf = np.array([row[label] for label in chain(*EIGHT_LABELS)]).reshape(8, 8)
            psfs[ii, jj] = psf

        self.psfs = psfs

    def get_psf_image(
        self,
        row,
        col,
        norm=1.0,
        pix_zero_loc="center",
        interpolation="bilinear",
        aca_image=True,
    ):
        """
        Get interpolated ACA PSF image that corresponds to pixel location
        ``row``, ``col``.

        :param row: (float) row value of PSF centroid
        :param col: (float) col value of PSF centroid
        :param norm: (float) summed intensity of PSF image
        :param pix_zero_loc: row/col coords are integral at 'edge' or 'center'
        :param interpolation: 'nearest' | 'bilinear' (default)
        :param aca_image: return ACAImage if True, else return ndarray

        :returns: ACAImage if (aca_image is True) else (ndarray image, row0, col0)
        """
        drc = self.drc

        if pix_zero_loc == "center":
            # Transform to 'edge' coordinates (pixel lower-left corner at 0.0, 0.0)
            row = row + 0.5
            col = col + 0.5
        elif pix_zero_loc != "edge":
            raise ValueError("pix_zero_loc can be only 'edge' or 'center'")

        # 8x8 image row0, col0
        round_row = round(row)
        round_col = round(col)
        row0 = int(round_row) - 4
        col0 = int(round_col) - 4

        # Subpixel position in range (-0.5, 0.5)
        r = row - round_row
        c = col - round_col

        # Floating point index into PSF library in range (0.0, 10.0)
        # (assuming 10x10 grid of PSFs covering central pixel
        ix = (r + 0.5) / drc - 0.5
        iy = (c + 0.5) / drc - 0.5

        if interpolation == "nearest":
            # Int index into PSF library
            ii = int(round(ix))
            jj = int(round(iy))
            psf = self.psfs[ii, jj].copy()

        elif interpolation == "bilinear":
            # Int index into PSF library
            ii = int(floor(ix))
            jj = int(floor(iy))

            # Following wikipedia notation (Unit Square section of
            # https://en.wikipedia.org/wiki/Bilinear_interpolation)

            # Float index within subpixel bin in range (0, 1)
            x = ix - ii
            y = iy - jj

            # Finally the bilinear interpolation of the PSF images.
            f = self.psfs
            b0 = (1 - x) * (1 - y)
            b1 = x * (1 - y)
            b2 = (1 - x) * y
            b3 = x * y
            P0 = f[ii, jj]
            P1 = f[ii + 1, jj]
            P2 = f[ii, jj + 1]
            P3 = f[ii + 1, jj + 1]
            psf = P0 * b0 + P1 * b1 + P2 * b2 + P3 * b3

        else:
            raise ValueError("interpolation must be 'nearest' or 'bilinear'")

        if norm != 1.0:
            psf *= norm

        out = ACAImage(psf, row0=row0, col0=col0) if aca_image else (psf, row0, col0)
        return out


from flask import Flask

app = Flask(__name__)

@app.route('/')
def home():
    return "Hello World"

@app.route('/python')
def python():
    html = """
        <html>
                {python_h1}
        </html>
    """
    python_h1 ="<h1>{title}</h1>".format(title="Python Programming Language")

    return html.format(python_h1=python_h1)


"""List billing information of orders."""
# :license: MIT, see LICENSE for more details.

import SoftLayer
from SoftLayer.CLI import environment
from SoftLayer.CLI import formatting
from SoftLayer import utils

import click

@click.command()
@click.option('--start_date', '-f', help='cost incurred from this start date')
@click.option('--end_date', '-e',
              help='cost incurred before this end date'
                   ' (default is current time stamp)')
@environment.pass_env
def cli(env, start_date, end_date):
    """List billing information for orders."""
    billing = SoftLayer.BillingManager(env.client)
    from_date = start_date
    to_date = end_date
    table = formatting.Table(['Order ID', 'Resource Name', 'Resource Type',
                              'cost', 'create_date'])
    resources = billing.list_resources(from_date, to_date)

    for resource in resources:
        resource = utils.NestedDict(resource)
        table.add_row([
            resource['id'],
            resource['hostName'],
            resource['resourceType'],
            resource['cost'],
            resource['createDate']
        ])

    return table


import cv2
import socket
import pickle
import base64

HOST = '192.168.1.27'
PORT = 9999

socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.bind((HOST,PORT))
socket.listen()
conn,addr = socket.accept()
print(conn)

while True:
    try:
        datas = conn.recv(1000000)
        datas = base64.b64decode(datas)
        frame = pickle.loads(datas)
        frame = cv2.imdecode(frame, cv2.IMREAD_COLOR)
        cv2.imshow("deneme", frame)
        cv2.waitKey(1)
    except:
        continue


import matplotlib.pyplot as plt
import numpy as np
import matplotlib.ticker as ticker
import math
import time
import random
import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Encoder

MAX_LENGTH = 100

class EncoderRNN(nn.Module):
  def __init__(self, input_size, hidden_size):
    super(EncoderRNN, self).__init__()
    self.hidden_size = hidden_size
    # 行が各単語ベクトル、列が埋め込みの次元である行列を生成
    # Embedding(扱う単語の数, 隱れ層のサイズ(埋め込みの次元))
    self.embedding = nn.Embedding(input_size, hidden_size)
    self.gru = nn.GRU(hidden_size, hidden_size)

  def forward(self, input, hidden):
    embedded = self.embedding(input).view(1, 1, -1)
    # 1 x 1 x n 型にベクトルサイズを変える
    # n の値は自動的に設定される
    output = embedded
    output, hidden = self.gru(output, hidden)
    return output, hidden
    # output が各系列のGRUの隱れ層ベクトル

  def initHidden(self):
    return torch.zeros(1, 1, self.hidden_size, device=device)

# Decoder

class DecoderRNN(nn.Module):
  def __init__(self, hidden_size, output_size):
    super(DecoderRNN, self).__init__()
    self.hidden_size = hidden_size

    self.embedding = nn.Embedding(output_size, hidden_size)
    self.gru = nn.GRU(hidden_size, hidden_size)
    self.out = nn.Linear(hidden_size, output_size)
    self.softmax = nn.LogSoftmax(dim=1)

  def forward(self, input, hidden):
    output = self.embedding(input).view(1, 1, -1)
    output = F.relu(output)
    output, hidden = self.gru(output, hidden)
    output = self.softmax(self.out(output[0]))
    return output, hidden

  def initHidden(self):
    return torch.zeros(1, 1, self.hidden_size, device=device)

class AttnDecoderRNN(nn.Module):
  def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
    super(AttnDecoderRNN, self).__init__()
    self.hidden_size = hidden_size
    self.output_size = output_size
    self.dropout_p = dropout_p
    self.max_length = max_length

    self.embedding = nn.Embedding(self.output_size, self.hidden_size)
    self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
    self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
    # 線形結合を計算
    # hidden_size * 2
    # → 各系列のGRUの隠れ層とAttention層で計算したコンテキストベクトルをtorch.catでつなぎ合わせることで長さが２倍になる
    self.dropout = nn.Dropout(self.dropout_p)
    # 過学習の回避
    self.gru = nn.GRU(self.hidden_size, self.hidden_size)
    self.out = nn.Linear(self.hidden_size, self.output_size)

  def forward(self, input, hidden, encoder_outputs):
    embedded = self.embedding(input).view(1, 1, -1)
    embedded = self.dropout(embedded)

    attn_weights = F.softmax(
        self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
    # 列方向を確率変換したいから dim = 1
    attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                             encoder_outputs.unsqueeze(0))
    # bmm でバッチも考慮してまとめて行列計算
    # ここでバッチが考慮されるから unsqueeze(0) が必要

    output = torch.cat((embedded[0], attn_applied[0]), 1)
    output = self.attn_combine(output).unsqueeze(0)

    output = F.relu(output)
    output, hidden = self.gru(output, hidden)

    output = F.log_softmax(self.out(output[0]), dim=1)
    return output, hidden, attn_weights

  def initHidden(self):
    # コンテキストベクトルをまとめるための入れ物を用意する
    return torch.zeros(1, 1, self.hidden_size, device=device)

# train 関数

teacher_forcing_ratio = 0.5

def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):
  encoder_hidden = encoder.initHidden()

  # 勾配の初期化
  encoder_optimizer.zero_grad()
  decoder_optimizer.zero_grad()

  # データをテンソルに変換する
  input_length = input_tensor.size(0)
  target_length = target_tensor.size(0)

  encoder_outputs = torch.zeros(
      max_length, encoder.hidden_size, device=device)

  loss = 0

  for ei in range(input_length):
    # for ei in range(5):
    encoder_output, encoder_hidden = encoder(
        input_tensor[ei], encoder_hidden)
    encoder_outputs[ei] = encoder_output[0, 0]

  decoder_input = torch.tensor([[SOS_token]], device=device)

  decoder_hidden = encoder_hidden

  use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

  if use_teacher_forcing:
    # 教師強制を使用する場合
    # Teacher forcing: Feed the target as the next input
    # 教師強制 : 次の入力としてターゲットを送る
    for di in range(target_length):
      decoder_output, decoder_hidden, decoder_attention = decoder(
          decoder_input, decoder_hidden, encoder_outputs)
      loss += criterion(decoder_output, target_tensor[di])
      decoder_input = target_tensor[di]
  else:
    # Without teacher forcing: use its own predictions as the next input
    # 教師強制の使用をしない : 次の入力として独自の予測値を使用する
    for di in range(target_length):
      decoder_output, decoder_hidden, decoder_attention = decoder(
          decoder_input, decoder_hidden, encoder_outputs)
      topv, topi = decoder_output.topk(1)
      decoder_input = topi.squeeze().detach()  # detach from history as input

      loss += criterion(decoder_output, target_tensor[di])
      if decoder_input.item() == EOS_token:
          break

  # 誤差逆伝播
  loss.backward()

  # パラメータの更新
  encoder_optimizer.step()
  decoder_optimizer.step()

  return loss.item() / target_length

# Language

SOS_token = 0
EOS_token = 1

# デフォルト字句解析器

def defaultTokenizer(s):
  return s.split(' ')

class Lang(object):
  def __init__(self, name, tokenizer=defaultTokenizer):
    self.name = name
    self.tokenizer = tokenizer
    self.word2index = {}
    self.word2count = {}
    self.index2word = {0: "SOS", 1: "EOS"}
    self.n_words = 2

  def addSentence(self, sentence):
    tokenizer = self.tokenizer
    for word in tokenizer(sentence):
      self.addWord(word)

  def addWord(self, word):
    if word not in self.word2index:
      self.word2index[word] = self.n_words
      self.word2count[word] = 1
      self.index2word[self.n_words] = word
      self.n_words += 1
    else:
      self.word2count[word] += 1

  def indexesFromSentence(self, sentence):
    return [self.word2index[word] for word in self.tokenizer(sentence)]

  def tensorFromSentence(self, sentence):
    indexes = self.indexesFromSentence(sentence)
    indexes.append(EOS_token)
    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)

def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))

#plt.switch_backend('agg')

def showPlot(points):
    plt.figure()
    fig, ax = plt.subplots()
    loc = ticker.MultipleLocator(base=0.2)
    # this locator puts ticks at regular intervals
    # loc は定期的に ticker を配置する
    ax.yaxis.set_major_locator(loc)
    plt.plot(points)
    plt.savefig("plot.png")
    # content フォルダ下に保存される
    plt.show()

class Seq2SeqModel(object):

  def fit(self, pairs, inputTokenizer=defaultTokenizer, outputTokenizer=defaultTokenizer, hidden_size=256, n_iters=20000, print_every=2500):
    self.input_lang = Lang('Source', inputTokenizer)
    self.output_lang = Lang('Target', outputTokenizer)
    print(f'Read {len(pairs)} sentence pairs')
    for pair in pairs:
      if len(pair) >= 2:
        self.input_lang.addSentence(pair[0])
        self.output_lang.addSentence(pair[1])
    print("Counted words:")
    print(self.input_lang.name, self.input_lang.n_words)
    print(self.output_lang.name, self.output_lang.n_words)

    self.encoder = EncoderRNN(self.input_lang.n_words, hidden_size).to(device)
    self.decoder = AttnDecoderRNN(
        hidden_size, self.output_lang.n_words, dropout_p=0.1).to(device)
    self.trainIters(pairs, n_iters, print_every)

  def trainIters(self, pairs, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):
    encoder = self.encoder
    decoder = self.decoder
    start = time.time()
    plot_losses = []
    print_loss_total = 0
    plot_loss_total = 0

    # 最適化
    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)

    training_pairs = [self.tensorsFromPair(
        random.choice(pairs)) for i in range(n_iters)]

    # 損失関数
    criterion = nn.NLLLoss()

    for iter in range(1, n_iters + 1):
      training_pair = training_pairs[iter - 1]
      input_tensor = training_pair[0]
      target_tensor = training_pair[1]

      loss = train(input_tensor, target_tensor, encoder,
                   decoder, encoder_optimizer, decoder_optimizer, criterion)
      print_loss_total += loss
      plot_loss_total += loss

      if iter % print_every == 0:
          print_loss_avg = print_loss_total / print_every
          print_loss_total = 0
          print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
                                       iter, iter / n_iters * 100, print_loss_avg))

      if iter % plot_every == 0:
          plot_loss_avg = plot_loss_total / plot_every
          plot_losses.append(plot_loss_avg)
          plot_loss_total = 0

    showPlot(plot_losses)

  def tensorsFromPair(self, pair):
    input_tensor = self.input_lang.tensorFromSentence(pair[0])
    target_tensor = self.output_lang.tensorFromSentence(pair[1])
    return (input_tensor, target_tensor)

  def predict(self, sentence, max_length=MAX_LENGTH):
    encoder = self.encoder
    decoder = self.decoder
    with torch.no_grad():
      input_tensor = self.input_lang.tensorFromSentence(sentence)
      input_length = input_tensor.size()[0]
      encoder_hidden = encoder.initHidden()

      encoder_outputs = torch.zeros(
          max_length, encoder.hidden_size, device=device)
      # output_length =  encoder_outputs.size()[0]

      for ei in range(input_length):
        # for ei in range(5):
        encoder_output, encoder_hidden = encoder(
            input_tensor[ei], encoder_hidden)
        encoder_outputs[ei] += encoder_output[0, 0]

      decoder_input = torch.tensor([[SOS_token]], device=device)
      # SOS

      decoder_hidden = encoder_hidden

      decoded_words = []
      decoder_attentions = torch.zeros(max_length, max_length)

      for di in range(max_length):
        decoder_output, decoder_hidden, decoder_attention = decoder(
            decoder_input, decoder_hidden, encoder_outputs)
        decoder_attentions[di] = decoder_attention.data
        topv, topi = decoder_output.data.topk(1)
        if topi.item() == EOS_token:
            decoded_words.append('<EOS>')
            break
        else:
            decoded_words.append(self.output_lang.index2word[topi.item()])

        decoder_input = topi.squeeze().detach()

      return decoded_words, decoder_attentions[:di + 1]


# coding=utf-8
import tensorflow as tf
from data import *
# from model import Model
from model import Model
from model_musk_ver_1 import Model_musk
from my_metrics import *
from tensorflow.python import debug as tf_debug
import numpy as np
# import jieba
import xlrd,xlwt

input_steps = 50
embedding_size = 4819
#embedding_size = 64
hidden_size = 150
n_layers = 2
batch_size = 1

'''
vocab_size = 780
slot_size = 21
intent_size = 19
'''

# after embedding
vocab_size = 42452
slot_size = 12
intent_size = 5
epoch_num = 1

model_path = './model/ijia/'

def transform(inputs_strs):
    utterance_temp = inputs_strs.split(' ')

    #print("len utterance_temp: ", len(utterance_temp))

    slot_format = ""
    for i in range(0, (len(utterance_temp)-1)):
        slot_format = slot_format + "O "

    iob_format = inputs_strs + "\t" + slot_format + "<UNK>"
    # print("iob_format: ") + str(iob_format).decode('string_escape')
    #print ('iob_format: '+ iob_format)

    return iob_format

def get_model():
    model = Model_musk(input_steps, embedding_size, hidden_size, vocab_size, slot_size,
                 intent_size, epoch_num, batch_size, n_layers)
    model.build()
    return model

def infer():
    model = get_model()
    sess = tf.Session()

    sess.run(tf.global_variables_initializer())
    #JerryB +++ 建立 saver 物件
    saver = tf.train.Saver(tf.global_variables())

    ckpt = tf.train.get_checkpoint_state(model_path)
    if ckpt is not None:
        print(ckpt.model_checkpoint_path)
        saver.restore(sess, ckpt.model_checkpoint_path)
    else:
        print("没找到模型")
    #JerryB ---

    test_data = []
    action = False

    #train_data = open("dataset/ijia_dataset.iob", "r").readlines()
    #test_data = open("dataset/ijia_primiry130.iob", "r").readlines()
    train_data = open( "dataset/redwine_training_data_musk.iob", "r" ).readlines()
    test_data = open( "dataset/redwine_training_data_musk.iob", "r" ).readlines()

    #print("JerryB train_data[0]: ", train_data[0])
    #print("JerryB train_data[1]: ", train_data[1])
    #print("JerryB train_data[2]: ", train_data[2])
    train_data_ed = data_pipeline(train_data)
    test_data_ed = data_pipeline(test_data)
    # print("test_data_ed[0]: ") + str(test_data_ed[0]).decode('string_escape')
    word2index, index2word, slot2index, index2slot, intent2index, index2intent = \
        get_info_from_training_data(train_data_ed)
    #print("JerryB word2index: ", word2index)
    #print("JerryB index2word: ", index2word)
    #print("JerryB slot2index: ", slot2index)
    #print("JerryB index2slot: ", index2slot)
    #print("JerryB intent2index: ", intent2index)
    #print("JerryB index2intent: ", index2intent)
    #index_train = to_index(train_data_ed, word2index, slot2index, intent2index)
    #index_test = to_index(test_data_ed, word2index, slot2index, intent2index)
    # print ('len(index_test):' + str(len(index_test)))
    #print("JerryB index_train[0]: ", index_train[0])
    #print("JerryB index_train[1]: ", index_train[1])
    #print("JerryB index_train[2]: ", index_train[2])
    #print("JerryB index_test[0]: ", index_test[0])

    #indexxxx = np.concatenate((index_train,index_test))

    pred_slots = []
    slot_accs = []
    intent_accs = []

    # prepare workbook
    fail_workbook = xlwt.Workbook()
    wt_sheet = fail_workbook.add_sheet('fails')
    wt_style = xlwt.XFStyle()
    wt_style.alignment.wrap = 1
    total_fail_item_count = 1

    # write header
    wt_sheet.write(0, 0, "ID")
    wt_sheet.write(0, 1, "Utterance")
    wt_sheet.write(0, 2, "Correct Intent")
    wt_sheet.write(0, 3, "Correct Entities")
    wt_sheet.write(0, 4, "Err Intent")
    wt_sheet.write(0, 5, "Err Entities")

    intnet_err_count = 0
    slot_err_count = 0

    # Musk_embedding +++
    word_list = pd.read_csv( "word_list.csv" )
    word_dic = {}
    num = 0
    # a = word_list.iloc[:,3]
    for i in word_list.iloc[:, 4]:
        word_dic[i.decode( "utf-8" )] = num
        num += 1
    for i in range(0,10):
        word_dic[str(i)] = num
        num += 1
    word_dic[u"non"] = num

    index_train = to_index_musk_ver_1( train_data_ed, word2index, slot2index, intent2index )
    index_test = to_index_musk_ver_1( test_data_ed, word2index, slot2index, intent2index )
    # Musk_embedding ---

    count = 0
    #embed_npy_musk = np.load( 'embedding_4809.npy' )
    #embeddings = embed_npy_musk.astype( np.float32 )

    for j, batch in enumerate(getBatchUT(batch_size, index_test)):
        # print (batch)
        # print (type(j))

        decoder_prediction, intent, intent_accuracy = model.step(sess, "test", batch, word_dic)
        #prediction_confidence = tf.Variable.eval(prediction_confidence)
        decoder_prediction = np.transpose(decoder_prediction, [1, 0])
        count += 1

        index = 0   # since batch size is 1
        sen_len = batch[index][1]
        input_sentence = " ".join(batch[index][0][:sen_len])
        #input_sentence  = str(index_seq2word(batch[index][0], index2word)[:sen_len]).decode('string_escape')
        slot_truth      = str(index_seq2slot(batch[index][2], index2slot)[:sen_len]).decode('string_escape')
        slot_prediction = str(index_seq2slot(decoder_prediction[index], index2slot)[:sen_len]).decode('string_escape')
        intent_truth    = str(index2intent[batch[index][3]]).decode('string_escape')
        intent_prediction = str(index2intent[intent[index]]).decode('string_escape')
        print ("Input Sentence        : ") + input_sentence
        print ("Slot Truth            : ") + slot_truth
        print ("Slot Prediction       : ") + slot_prediction
        print ("Intent Truth          : ") + intent_truth
        print ("Intent Prediction     : ") + intent_prediction
        # print("Input Sentence        : ", index_seq2word(batch[index][0], index2word)[:sen_len])
        #print("Slot Truth            : ", index_seq2slot(batch[index][2], index2slot)[:sen_len])
        #print("Slot Prediction       : ", index_seq2slot(decoder_prediction[index], index2slot)[:sen_len])
        #print("Intent Truth          : ", index2intent[batch[index][3]])
        #print("Intent Prediction     : ", index2intent[intent[index]])
        slot_pred_length = list(np.shape(decoder_prediction))[1]
        pred_padded = np.lib.pad(decoder_prediction, ((0, 0), (0, input_steps-slot_pred_length)),
                                 mode="constant", constant_values=0)
        pred_slots.append(pred_padded)
        # print("slot_pred_length: ", slot_pred_length)
        true_slot = np.array((list(zip(*batch))[2]))
        true_length = np.array((list(zip(*batch))[1]))
        true_slot = true_slot[:, :slot_pred_length]
        # print(np.shape(true_slot), np.shape(decoder_prediction))
        # print(true_slot, decoder_prediction)
        slot_acc = accuracy_score(true_slot, decoder_prediction, true_length)
        intent_acc = accuracy_score(list(zip(*batch))[3], intent)
        print("slot accuracy: {}, intent accuracy: {}".format(slot_acc, intent_acc))

        if intent_acc < 1.0:
            wt_sheet.write(total_fail_item_count, 0, str(j+1))   # ID
            wt_sheet.write(total_fail_item_count, 1, input_sentence.decode('utf-8'))   # segmented sentence
            wt_sheet.write(total_fail_item_count, 2, intent_truth.decode('utf-8'), wt_style)   # right intent
            wt_sheet.write(total_fail_item_count, 4, intent_prediction.decode('utf-8') + "(" +  str(round(intent_accuracy, 3)) + ")", wt_style)   # wrong intent
            total_fail_item_count +=1
            intnet_err_count += 1

        if slot_acc < 1.0:
            wt_sheet.write(total_fail_item_count, 0, str(j+1))
            wt_sheet.write(total_fail_item_count, 1, input_sentence.decode('utf-8'))   # sentence
            wt_sheet.write(total_fail_item_count, 3, slot_truth.decode('utf-8'), wt_style)   # right entity
            wt_sheet.write(total_fail_item_count, 5, slot_prediction.decode('utf-8'), wt_style)   # wrong entity
            total_fail_item_count +=1
            slot_err_count += 1

        '''
        if intent_accuracy <= 1:
            wt_sheet.write( total_fail_item_count, 0, str( j + 1 ) )  # ID
            wt_sheet.write( total_fail_item_count, 1, input_sentence.decode( 'utf-8' ) )  # segmented sentence
            #wt_sheet.write( total_fail_item_count, 2, intent_truth.decode( 'utf-8' ), wt_style )  # right intent
            wt_sheet.write( total_fail_item_count, 4,
                            intent_prediction.decode( 'utf-8' ) + "(" + str( round( intent_accuracy, 3 ) ) + ")",
                            wt_style )  # wrong intent
            total_fail_item_count += 1
            intnet_err_count += 1
        '''

        slot_accs.append(slot_acc)
        intent_accs.append(intent_acc)
        fail_workbook.save('UT_result.xls')
        print ('processed sentence #: ' + str(count))
    pred_slots_a = np.vstack(pred_slots)
    print("pred_slots_a: ", pred_slots_a.shape)
    true_slots_a = np.array(list(zip(*index_test))[2])[:pred_slots_a.shape[0]]
    print("true_slots_a: ", true_slots_a.shape)
    print("Intent accuracy for epoch 0: {}".format(np.average(intent_accs)))
    print("Slot accuracy for epoch 0: {}".format(np.average(slot_accs)))
    print("Slot F1 score for epoch 0: {}".format(f1_for_sequence_batch(true_slots_a, pred_slots_a)))

    print('Intent error count: {}'.format(intnet_err_count))
    print('Slot error count: {}'.format(slot_err_count))

if __name__ == '__main__':
    # jieba dict
    # jieba_dict = "./jieba-zh_TW/dict.txt"
    # corpus_dict = "./jieba-zh_TW/corpus_jieba.txt"
    # jieba.set_dictionary(jieba_dict)
    # jieba.load_userdict(corpus_dict)
    infer()


# -*- coding: utf-8 -*-
"""
Created on Sun Jun 17 22:40:05 2018

@author: tzhou
"""

import numpy as np
import cv2

def main():
    canvas = np.zeros((400, 600, 3), dtype=np.uint8) +255
    cv2.circle(canvas, (200,300), 75, (0,0,255), 5)
    cv2.imshow('Basic drawing with CV', canvas)
    #cv2.waitKey()

if __name__ == '__main__':
    main()


import csv
import numpy as np
from scipy import spatial
from sklearn import svm
from sklearn import metrics
vectorLength = 100

tags = {'PDT': 'PT', '.': '.', 'VB': 'VB', ':': ':', '#': '#', 'VBN': 'VN', 'RBR': 'RR', 'PRP$': 'PR', 'DT': 'DT', 'VBZ': 'VZ', 'CC': 'CC', 'TO': 'TO', 'LS': 'LS', 'SYM': 'SM', 'RBS': 'RS', 'JJ': 'JJ', 'EX': 'EX', 'WP': 'WP', 'POS': 'PS', 'WDT': 'WT', 'VBP': 'VP', 'WRB': 'WB', 'PRP': 'PP', 'JJR': 'JR', 'VBD': 'VD', 'NNPS': 'NQ', 'RB': 'RB', '-LRB-': 'L$', 'RP': 'RP', 'JJS': 'JS', 'CD': 'CD', '-RRB-': 'R$', 'NNP': 'NP', '$': '$', 'WP$': 'WP', 'FW': 'FW', 'VBG': 'VG', "''": "''", ',': ',', 'NN': 'NN', 'UH': 'UH', 'NNS': 'NS', 'MD': 'MD', '``': '``', 'IN': 'IN'}
inverseTags = dict()
for key, item in tags.items():
	inverseTags[item] = key
#missing = open("missingMSRLemmaUntaggedUnhyphenWords.txt",'w')
#missingSet = set()
with open('../vectors/lemmaTagVectors.vec', 'r') as vectorFile:
        vectors = dict()
        vectorFile.readline()
        for line in vectorFile:
                splitLine = line.split()
                start = len(splitLine) - vectorLength
                #for i in range(start, len(splitLine)):
                #        splitLine[i] = float(splitLine[i])
                vectors[" ".join(splitLine[0:start])] = np.array(splitLine[start:], dtype = float)

with open('missing/missingLEMMATAGVectors.txt', 'r') as missingFile:
	for line in missingFile:
		splitLine = line.split()
		start = len(splitLine) - vectorLength
		if " ".join(splitLine[0:start]) not in vectors:
			vectors[" ".join(splitLine[0:start])] = np.array(splitLine[start:], dtype = float)
		else:
			print("Weird!",splitLine)

#with open('../vectors/tagWordVectors.vec','r') as tagFile:
#	tagVectors = dict()
#	tagFile.readline()
#	for line in tagFile:
#		splitLine = line.split()
#		tagVectors[splitLine[0]] = np.array(splitLine[1:], dtype=float)

def getVector(sentence, vectors):
    resultVector = np.zeros(vectorLength)
    tagVector = np.zeros(10)
    splitLine = sentence.split()
    for word in splitLine:
        if word in vectors:
             resultVector = resultVector + vectors[word]
        else:
             print("Woah", word, splitLine)
    return resultVector
#        try:
#            if word[-2] == '_':
#                tempWord = word[:-2]
#                if tempWord in vectors:
#                    resultVector = resultVector + vectors[tempWord]
#                    tagVector = tagVector + tagVectors[inverseTags[word[-1]]]
#            elif word[-3] == '_':
#                tempWord = word[:-3]
#                if tempWord in vectors:
#                    resultVector = resultVector + vectors[tempWord]
#                    tagVector = tagVector + tagVectors[inverseTags[word[-2:]]]
#        except IndexError:
#            pass
#else:
#                print("Cray", word, splitLine)
#        else:
#            print("Woah", word, splitLine)
#    return resultVector
#        else:
#            tempWord = word
#            index = tempWord.find('-')
#            while index != -1:
#                current = tempWord[:index]
#                if current in vectors:
#                    resultVector = resultVector + vectors[current]
               #elif current != "" and current not in missingSet:
                #    print("Problem1", current)
                #    missing.write(current + '\n')
                #    missingSet.add(current)
#                tempWord = tempWord[index+1:]
#                index = tempWord.find('-')
#            if tempWord in vectors:
#                resultVector = resultVector + vectors[tempWord]
            #elif tempWord != "" and tempWord not in missingSet:
            #    print("Problem2", tempWord)
            #    missing.write(tempWord + '\n')
            #    missingSet.add(tempWord)
                #raise(ValueError)"""
#    return resultVector
#    return np.concatenate((resultVector,tagVector))

from sklearn import linear_model
from sklearn.metrics import mean_squared_error
from scipy.stats import spearmanr

with open('lemmaTagPPDBTrain.txt', 'r', encoding='utf8') as trainFile:
    distances = []
    truths = []
    fileSize = int(trainFile.readline())

    for i in range(fileSize):
        data1 = getVector(trainFile.readline()[:-1], vectors)
        data2 = getVector(trainFile.readline()[:-1], vectors)
        distances.append([spatial.distance.cosine(data1,data2)])
        truths.append(float(trainFile.readline()))

#    cutoff = len(distances)*8//10
#    best = 0

    model = linear_model.LinearRegression()
    model.fit(distances, truths)
    #for i in range(-4, 5):
    #    print(i)
    #    model = svm.SVC(kernel = 'linear',C=10**i)
    #    model.fit(distances[:cutoff], truths[:cutoff])

    #y_pred = model.predict(distances)

     #   f1 = metrics.f1_score(truths[cutoff:],y_pred[cutoff:])
     #   print(f1)
   #     if f1 > best:
   #         best = f1
   #         cvalue = i
#    model = svm.SVC(kernel='linear', C=10**cvalue)
#    model.fit(distances,truths)

#    outcome = [truths[i]==y_pred[i] for i in range(len(truths))]
    print("training error: ", mean_squared_error(model.predict(distances), truths))
    print("Training Spearman's: ", spearmanr(model.predict(distances),truths))
#    print("Training F1 score: ", metrics.f1_score(truths,y_pred))
#    print(cvalue)

with open('lemmaTagPPDBTest.txt', 'r', encoding='utf8') as testFile:
    distances = []
    truths = []
    fileSize = int(testFile.readline())
    for i in range(fileSize):
        data1 = getVector(testFile.readline()[:-1], vectors)
        data2 = getVector(testFile.readline()[:-1], vectors)
        distances.append([spatial.distance.cosine(data1,data2)])
        truths.append(float(testFile.readline()))

    y_pred = model.predict(distances)
    print("Test error: ", mean_squared_error(y_pred,truths))
    print("Test spearman's: ", spearmanr(y_pred, truths))
	#missing.close()


from flask import (
   Flask,
   render_template,
   request,
   redirect,
   url_for,
   flash,
   jsonify,
   abort,
   g
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy import create_engine
from flask_httpauth import HTTPBasicAuth
from models import Base, Category, Item, User
from werkzeug.security import generate_password_hash, check_password_hash
from flask import session as login_session
import random
import string
from oauth2client.client import flow_from_clientsecrets
from oauth2client.client import FlowExchangeError
import httplib2
import json
from flask import make_response
import requests

app = Flask(__name__)
auth = HTTPBasicAuth()

CLIENT_ID = json.loads(
    open('client_secrets.json', 'r').read())['web']['client_id']
APPLICATION_NAME = "Item Catalog"

# Connect to Database and create database session
engine = create_engine('sqlite:///itemCatalogwithusers.db',
    connect_args={'check_same_thread': False})

Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()

# Create anti-forgery state token
@app.route('/login')
def showLogin():
    state = ''.join(random.choice(string.ascii_uppercase + string.digits)
                    for x in range(32))
    login_session['state'] = state
    #return "The current session state is %s" % login_session['state']
    return render_template('login.html', STATE=state)

@app.route('/gconnect', methods=['POST'])
def gconnect():
    # Validate state token
    if request.args.get('state') != login_session['state']:
        response = make_response(json.dumps('Invalid state parameter.'), 401)
        response.headers['Content-Type'] = 'application/json'
        return response
    # Obtain authorization code
    code = request.data

    try:
        # Upgrade the authorization code into a credentials object
        oauth_flow = flow_from_clientsecrets('client_secrets.json', scope='')
        oauth_flow.redirect_uri = 'postmessage'
        credentials = oauth_flow.step2_exchange(code)
    except FlowExchangeError:
        response = make_response(
            json.dumps('Failed to upgrade the authorization code.'), 401)
        response.headers['Content-Type'] = 'application/json'
        return response

    # Check that the access token is valid.
    access_token = credentials.access_token
    url = ('https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=%s'
           % access_token)
    h = httplib2.Http()
    result = json.loads(h.request(url, 'GET')[1].decode('utf-8'))
    # If there was an error in the access token info, abort.
    if result.get('error') is not None:
        response = make_response(json.dumps(result.get('error')), 500)
        response.headers['Content-Type'] = 'application/json'
        return response

    # Verify that the access token is used for the intended user.
    gplus_id = credentials.id_token['sub']
    if result['user_id'] != gplus_id:
        response = make_response(
            json.dumps("Token's user ID doesn't match given user ID."), 401)
        response.headers['Content-Type'] = 'application/json'
        return response

    # Verify that the access token is valid for this app.
    if result['issued_to'] != CLIENT_ID:
        response = make_response(
            json.dumps("Token's client ID does not match app's."), 401)
        print ("Token's client ID does not match app's.")
        response.headers['Content-Type'] = 'application/json'
        return response

    stored_access_token = login_session.get('access_token')
    stored_gplus_id = login_session.get('gplus_id')
    if stored_access_token is not None and gplus_id == stored_gplus_id:
        response = make_response(json.dumps('Current user is already'
            'connected.'),200)
        response.headers['Content-Type'] = 'application/json'
        return response

    # Store the access token in the session for later use.
    login_session['access_token'] = credentials.access_token
    login_session['gplus_id'] = gplus_id

    # Get user info
    userinfo_url = "https://www.googleapis.com/oauth2/v1/userinfo"
    params = {'access_token': credentials.access_token, 'alt': 'json'}
    answer = requests.get(userinfo_url, params=params)

    data = answer.json()

    login_session['username'] = data['name']
    login_session['picture'] = data['picture']
    login_session['email'] = data['email']

    # See if a user exists, if it doesn't make a new one
    user_id = getUserID(login_session['email'])
    if not user_id:
        user_id = createUser(login_session)
    login_session['user_id'] = user_id

    output = ''
    output += '
Welcome, '
    output += login_session['username']
    output += '!
'
    output += ' '
    flash("you are now logged in as %s" % login_session['username'])
    print ("done!")
    return output

    # DISCONNECT - Revoke a current user's token and reset their login_session

@app.route('/gdisconnect')
def gdisconnect():
    access_token = login_session.get('access_token')
    if access_token is None:
        print ('Access Token is None')
        response = make_response(json.dumps('Current user not connected.'), 401)
        response.headers['Content-Type'] = 'application/json'
        return response
    print ('In gdisconnect access token is %s', access_token)
    print ('User name is: ')
    print (login_session['username'])
    url = 'https://accounts.google.com/o/oauth2/revoke?token=%s'\
        % login_session['access_token']
    h = httplib2.Http()
    result = h.request(url, 'GET')[0]
    print ('result is ')
    print (result)
    if result['status'] == '200':
        del login_session['access_token']
        del login_session['gplus_id']
        del login_session['username']
        del login_session['email']
        del login_session['picture']
        response = make_response(json.dumps('Successfully disconnected.'), 200)
        response.headers['Content-Type'] = 'application/json'
        return response
    else:
        response = make_response(json.dumps('Failed to revoke token'
            'for given user.', 400))
        response.headers['Content-Type'] = 'application/json'
        return response

# User Helper Functions

def createUser(login_session):
    newUser = User(name=login_session['username'], email=login_session[
                   'email'], picture=login_session['picture'])
    session.add(newUser)
    session.commit()
    user = session.query(User).filter_by(email=login_session['email']).first()
    return user.id

def getUserInfo(user_id):
    user = session.query(User).filter_by(id=user_id).first()
    return user

def getUserID(email):
    try:
        user = session.query(User).filter_by(email=email).one()
        return user.id
    except:
        return None

@app.route('/users/<int:id>')
def get_user(id):
    user = session.query(User).filter_by(id=id).one()
    if not user:
        abort(400)
    return jsonify({'username': user.name})

# Provide a JSON endpoint
@app.route('/catalog.json')
def categoryItemJSON():
    categories = session.query(Category).all()
    final_list = []
    final_dict = {}
    for c in categories:
        items = session.query(Item).filter_by(
            cat_id = c.id).all()
        lst = jsonify(Items=[i.serialize for i in items])
        cat = jsonify(c.serialize)
        res = [cat, lst]

        final_dict[cat]=lst
    #return result
    #final_dict['Category'] = final_list
    return lst

# Show all Categories and Latest Items
@app.route('/')
@app.route('/catalog/')
def showCatalog():
    categories = session.query(Category).order_by(Category.name.asc())
    items = session.query(Item).order_by(Item.title.asc())
    if 'username' not in login_session:
        return render_template('catalog.html',categories=categories,
            items=items)
    else:
        return render_template('catalog_loggedIn.html',categories=categories,
            items=items)

# Show Category Items
@app.route('/catalog/<string:category>/items')
def categoryItems(category):
    categories = session.query(Category).all()
    selectedCategory = session.query(Category).filter_by(name = category).first()
    items = session.query(Item).filter_by(
        category = selectedCategory).all()
    nItems = len((items))
    userId=selectedCategory.user_id
    creator = getUserInfo(userId)
    if 'username' not in login_session or creator.id!=login_session['user_id']:
        return render_template('categoryitems.html',
            categories=categories, category=selectedCategory,
            items=items, nItems=nItems)
    else:
        return render_template('categoryitems_loggedIn.html',
            categories=categories, category=selectedCategory,
            items=items, nItems=nItems, creator=creator)

# Show Item Information
@app.route('/catalog/<string:category>/<string:item>/')
def itemInformation(category,item):
    selectedCategory = session.query(Category).filter_by(
        name = category).first()
    selectedItem = session.query(Item).filter_by(title = item).first()
    creator = getUserInfo(selectedCategory.user_id)
    if 'username' not in login_session or creator.id!=login_session['user_id']:
        return render_template('categoryiteminfo.html',
            category = selectedCategory, item = selectedItem)
    else:
        return render_template('categoryiteminfo_loggedIn.html',
            category = selectedCategory, item = selectedItem, creator=creator)

# Create a new category item
@app.route('/catalog/<string:category>/add', methods=['GET', 'POST'])
def newCategoryItem(category):
    selectedCategory = session.query(Category).filter_by(
            name = category).one()
    if 'username' not in login_session:
        return redirect('/login')
    if selectedCategory.user_id != login_session['user_id']:
        return ""
    if request.method == 'POST':
        newItem = Item(user_id=login_session['user_id'],title = request.form['title'], description = request.form['description'], cat_id=selectedCategory.id)
        session.add(newItem)
        session.commit()
        return redirect(url_for('categoryItems', category=selectedCategory.name))
    else:
        return render_template('newcategoryitem.html',category=selectedCategory)

# Edit a new category item
@app.route('/catalog/<string:category>/<string:item>/edit',
    methods = ['GET', 'POST'])
def categoryItemEdit(category, item):
    selectedCategory = session.query(Category).filter_by(
        name = category).first()
    editedItem = session.query(Item).filter_by(
        title = item).first()
    if 'username' not in login_session:
        return redirect('/login')
    if selectedCategory.user_id != login_session['user_id']:
        return ""
    if request.method == 'POST':
        if request.form['title']:
            editedItem.title = request.form['title']
        if request.form['description']:
            editedItem.description = request.form['description']
        session.add(editedItem)
        session.commit()
        return redirect(url_for('itemInformation',
            category=selectedCategory.name ,
            item = editedItem.title))
    else:
        return render_template('editcategoryitem.html',
            category=selectedCategory ,
            item = editedItem)

# Delete a category item
@app.route('/catalog/<string:category>/<string:item>/delete',
    methods = ['GET', 'POST'])
def categoryItemDelete(category, item):
    selectedCategory = session.query(Category).filter_by(
        name = category).first()
    itemToDelete = session.query(Item).filter_by(
        title = item).first()
    if 'username' not in login_session:
        return redirect('/login')
    if selectedCategory.user_id != login_session['user_id']:
        return ""
    if request.method == 'POST':
        session.delete(itemToDelete)
        session.commit()
        return redirect(url_for('categoryItems',
            category=selectedCategory.name))
    else:
        return render_template('deletecategoryitem.html',
            category=selectedCategory ,
            item = itemToDelete)

if __name__ == '__main__':
     app.secret_key = 'super_secret_key'
     app.debug = True
     app.run(host='0.0.0.0', port=8000)


import logging
import multiprocessing
import threading
import os
import numpy as np
import tensorflow as tf

import Utils.utils
from Environment.environment import UnoEnvironment
from Environment.state import State
from Players.A3CPlayer import A3CPlayer
from Players.AgentPlaceholder import AgentPlaceholder
from Players.randomplayer import RandomPlayer
from Agents.ListMemory import ListMemory
from Agents.ActorCriticModel import ActorCriticModel

logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('matplotlib.font_manager').disabled = True

# Prirejeno po zgledu: https://blog.tensorflow.org/2018/07/deep-reinforcement-learning-keras-eager-execution.html
class A3CWorker(threading.Thread):
    GLOBAL_MAX_EPISODES = 1000
    global_model_update_frequency = 20
    global_best_episode_score = None
    global_total_episodes_across_all_workers = 0
    global_semaphore = threading.Lock()
    global_statistic = []

    def __init__(self, master_model, optimizer, worker_id, folder, beta, gamma, opponent_model_path=None):
        super(A3CWorker, self).__init__()
        self.master_model = master_model
        self.optimizer = optimizer
        self.worker_id = worker_id
        self.env = UnoEnvironment(False)
        self.state_size = State.STATE_SIZE
        self.action_size = State.ACTION_SIZE
        self.gamma = gamma  # discounting factor
        self.beta = beta
        self.worker_model = ActorCriticModel(self.action_size)
        self.memory = ListMemory()
        self.folder = folder
        self.episode_loss = 0
        self.episode_steps = 0
        self.episode_reward = 0
        self.episode_discounted_reward = 0
        self.steps = 0
        self.save = True
        self.agent_placeholder = AgentPlaceholder()
        if opponent_model_path is not None:
            self.opponent = A3CPlayer(opponent_model_path)
        else:
            self.opponent = RandomPlayer()

    def run(self):
        while A3CWorker.global_total_episodes_across_all_workers < A3CWorker.GLOBAL_MAX_EPISODES:
            A3CWorker.global_total_episodes_across_all_workers += 1
            logger.info("Starting episode " + str(A3CWorker.global_total_episodes_across_all_workers) + "/" +
                        str(A3CWorker.GLOBAL_MAX_EPISODES) + " with worker " + str(self.worker_id))

            done = False
            self.episode_steps = 0
            self.episode_loss = 0
            self.episode_reward = 0
            self.episode_discounted_reward = 0
            self.memory.empty()
            state = self.env.reset(self.agent_placeholder, self.opponent)

            while not done:
                action = self.get_action()
                next_state, reward, done = self.env.step_with_opp_step(action)
                self.episode_reward += reward
                self.memory.store(state, action, reward, next_state, done)
                state = next_state

                self.steps += 1
                self.episode_steps += 1

                if self.steps % A3CWorker.global_model_update_frequency == 0 or done:
                    self.sync_gradients()

                if done:
                    A3CWorker.global_statistic.append((self.episode_reward, self.episode_loss / self.episode_steps))
                    if A3CWorker.global_best_episode_score is None or \
                            self.episode_reward > A3CWorker.global_best_episode_score:
                        self.update_master_model()

    def get_action(self):
        # You are only interested in the legal policy logits
        policy_logits, values = self.worker_model(tf.convert_to_tensor(np.random.random((1, self.state_size)),
                                                                       dtype=tf.float32))
        legal_actions = self.env.get_legal_actions()
        legal_policy_logits = np.take(policy_logits, legal_actions)
        stochastic_legal_action_probabilities = tf.nn.softmax(legal_policy_logits)
        stochastic_legal_action_probabilities = stochastic_legal_action_probabilities.numpy()
        stochastic_legal_action_probabilities /= stochastic_legal_action_probabilities.sum()
        stochastic_policy_driven_legal_action = np.random.choice(legal_actions,
                                                                 p=stochastic_legal_action_probabilities)
        return stochastic_policy_driven_legal_action

    def update_master_model(self):
        A3CWorker.global_best_episode_score = self.episode_reward
        if self.save:
            with A3CWorker.global_semaphore:
                logger.info("Saving best model "
                            "- worker: " + str(self.worker_id) +
                            " episode: " + str(A3CWorker.global_total_episodes_across_all_workers) +
                            " reward: " + str(self.episode_reward) +
                            " discounted reward:" + str(self.episode_discounted_reward) +
                            " loss: " + str(self.episode_loss.numpy()))
                self.master_model.save_weights(os.path.join(self.folder, 'model_' + Utils.utils.get_timestamp() +'.h5'))

    def sync_gradients(self):
        with tf.GradientTape() as tape:
            total_loss = self.compute_loss()
        self.episode_loss += total_loss
        grads = tape.gradient(total_loss, self.worker_model.trainable_weights)
        self.optimizer.apply_gradients(zip(grads, self.master_model.trainable_weights))

        self.worker_model.set_weights(self.master_model.get_weights())
        self.memory.empty()

    def compute_loss(self):
        if self.memory.dones[-1]:
            reward_sum = 0.
        else:
            reward_sum = self.worker_model(tf.convert_to_tensor(self.memory.next_states[-1][None, :], dtype=tf.float32))[-1].numpy()[0]
        discounted_rewards = []
        for reward in self.memory.rewards[::-1]:
            reward_sum = reward + self.gamma * reward_sum
            discounted_rewards.append(reward_sum)
        discounted_rewards.reverse()

        self.episode_discounted_reward = discounted_rewards[0]

        logits, values = self.worker_model(tf.convert_to_tensor(np.vstack(self.memory.states), dtype=tf.float32))

        advantage = tf.convert_to_tensor(np.array(discounted_rewards)[:, None], dtype=tf.float32) - values # Baseline
        value_loss = advantage ** 2

        actions_one_hot = tf.one_hot(self.memory.actions, self.action_size, dtype=tf.float32)

        policy = tf.nn.softmax(logits)
        entropy = tf.reduce_sum(policy * tf.math.log(policy + self.beta), axis=1)

        # Entropija prepreč prehitro konvergiranje k suboptimalnem determinističnmem pravilniku
        policy_loss = tf.nn.softmax_cross_entropy_with_logits(labels=actions_one_hot, logits=logits)
        policy_loss *= tf.stop_gradient(advantage)
        policy_loss -= 0.01 * entropy
        total_loss = tf.reduce_mean((0.5 * value_loss + policy_loss))
        return total_loss


import uuid
import datetime
from common.database import Database
from models.post import Post

class AllPosts(object):
    def __init__(self, authorName, author, title, description, author_id, _id=None):
        self.authorName = authorName
        self.author = author
        self.author_id = author_id
        self.title = title
        self.description = description
        self._id = uuid.uuid4().hex if _id is None else _id

    def json(self):
        return {'authorName': self.authorName,
                'author': self.author,
                'author_id': self.author_id,
                'title': self.title,
                'description': self.description,
                '_id': self._id}

    def save_to_mongo(self):
        Database.insert(collection='posts', data=self.json())

    def new_post(self, title, content, date=datetime.datetime.utcnow()):
        post = Post(post_id=self._id,
                    post_title=title,
                    post_content=content,
                    authorName=self.authorName,
                    author=self.author,
                    created_time=date)
        post.save_to_mongo()

    def get_posts(self):
        return Post.from_post(self._id)

    @classmethod
    def from_mongo(cls, id):
        post = Database.find_one(collection='posts', query={'_id': id})
        return cls(**post)

    @classmethod
    def get_post_from_author_id(cls, author_id):
        posts = Database.find(collection='posts', query={'author_id': author_id})
        return [cls(**post) for post in posts]


from csv import DictReader
from pathlib import Path
from datetime import datetime
import re
from collections import defaultdict
from fnmatch import fnmatch

from ingest_validation_tools.validation_utils import (
    get_metadata_tsv_errors,
    get_data_dir_errors
)

def _get_directory_type_from_path(path):
    return re.match(r'(.*)-metadata\.tsv$', Path(path).name)[1]

def _get_tsv_rows(path):
    with open(path, encoding='latin-1') as f:
        rows = list(DictReader(f, dialect='excel-tab'))
    return rows

class Submission:
    def __init__(self, directory_path=None, override_tsv_paths={},
                 optional_fields=[], add_notes=True,
                 dataset_ignore_globs=[], submission_ignore_globs=[]):
        self.directory_path = directory_path
        self.optional_fields = optional_fields
        self.dataset_ignore_globs = dataset_ignore_globs
        self.submission_ignore_globs = submission_ignore_globs
        unsorted_effective_tsv_paths = (
            override_tsv_paths if override_tsv_paths
            else {
                _get_directory_type_from_path(path): path
                for path in directory_path.glob('*-metadata.tsv')
            }
        )
        self.effective_tsv_paths = {
            k: unsorted_effective_tsv_paths[k]
            for k in sorted(unsorted_effective_tsv_paths.keys())
        }
        self.add_notes = add_notes

    def get_errors(self):
        # This creates a deeply nested dict.
        # Keys are present only if there is actually an error to report.
        errors = {}
        tsv_errors = self._get_tsv_errors()
        reference_errors = self._get_reference_errors()
        if tsv_errors:
            errors['Metadata TSV Errors'] = tsv_errors
        if reference_errors:
            errors['Reference Errors'] = reference_errors
        if errors and self.add_notes:
            errors['Notes'] = {
                'Time': datetime.now(),
                'Directory': str(self.directory_path),
                'Effective TSVs': {
                    type: str(path) for type, path
                    in self.effective_tsv_paths.items()
                }
            }
        return errors

    def _get_tsv_errors(self):
        errors = {}
        types_paths = self.effective_tsv_paths.items()
        if not types_paths:
            errors['Missing'] = 'There are no effective TSVs.'
        for type, path in types_paths:
            single_tsv_internal_errors = \
                self._get_single_tsv_internal_errors(type, path)
            single_tsv_external_errors = \
                self._get_single_tsv_external_errors(type, path)
            single_tsv_errors = {}
            if single_tsv_internal_errors:
                single_tsv_errors['Internal'] = single_tsv_internal_errors
            if single_tsv_external_errors:
                single_tsv_errors['External'] = single_tsv_external_errors
            if single_tsv_errors:
                errors[f'{path} (as {type})'] = single_tsv_errors
        return errors

    def _get_single_tsv_internal_errors(self, type, path):
        return get_metadata_tsv_errors(
            type=type, metadata_path=path,
            optional_fields=self.optional_fields)

    def _get_single_tsv_external_errors(self, type, path):
        errors = {}
        rows = _get_tsv_rows(path)
        if not rows:
            errors['Warning'] = f'File has no data rows.'
        if self.directory_path:
            for i, row in enumerate(rows):
                full_data_path = self.directory_path / row['data_path']
                data_dir_errors = self._get_data_dir_errors(
                    type, full_data_path)
                if data_dir_errors:
                    errors[f'{path.name} (row {i+2})'] = data_dir_errors
        return errors

    def _get_data_dir_errors(self, type, path):
        return get_data_dir_errors(
            type, path, dataset_ignore_globs=self.dataset_ignore_globs)

    def _get_reference_errors(self):
        errors = {}
        no_ref_errors = self._get_no_ref_errors()
        multi_ref_errors = self._get_multi_ref_errors()
        if no_ref_errors:
            errors['No References'] = no_ref_errors
        if multi_ref_errors:
            errors['Multiple References'] = multi_ref_errors
        return errors

    def _get_no_ref_errors(self):
        if not self.directory_path:
            return {}
        referenced_data_paths = set(self._get_data_references().keys())
        non_metadata_paths = {
            path.name for path in self.directory_path.iterdir()
            if not path.name.endswith('-metadata.tsv')
            and not any([
                fnmatch(path.name, glob)
                for glob in self.submission_ignore_globs
            ])
        }
        unreferenced_paths = non_metadata_paths - referenced_data_paths
        return [str(path) for path in unreferenced_paths]

    def _get_multi_ref_errors(self):
        errors = {}
        data_references = self._get_data_references()
        for path, references in data_references.items():
            if len(references) > 1:
                errors[path] = references
        return errors

    def _get_data_references(self):
        # TODO: Move this to __init__
        data_references = defaultdict(list)
        for tsv_path in self.effective_tsv_paths.values():
            for i, row in enumerate(_get_tsv_rows(tsv_path)):
                if 'data_path' in row:
                    reference = f'{tsv_path} (row {i+2})'
                    data_references[row['data_path']].append(reference)
        return data_references


#!/bin/python

import pandas as pd
import datetime as dt
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import seaborn as sn
import time
import pytz
import os
import shutil

import config
from bettor import Bettor, Team

bet_file_postfix = config.BET_FILE_POSTFIX
schedule_file = config.SCHEDULE_FILE
SEASON = config.SEASON
START_DATE = config.START_DATE
tz = pytz.timezone('EST')
today = dt.datetime.now(tz)
STR_TODAY = '%i/%i/%i' % (today.month, today.day, today.year)

def todays_games(person_list):
    team_bettor_dict = {team: person_obj.name for person_obj in person_list for team in person_obj.team_objs.keys()}
    team_bettor_dict['Blazers'] = team_bettor_dict['Trail Blazers']
    team_bettor_dict['76ers'] = team_bettor_dict['Sixers']
    # Upload the current schedule and convert date to timestamp
    nba_sched = pd.read_csv(schedule_file, index_col=0)
    nba_sched.index = pd.to_datetime(nba_sched.index)

    # Get today's date
    today = dt.datetime.strptime(STR_TODAY, '%m/%d/%Y').isoformat()

    # Grabs todays games
    nba_sched_today = nba_sched.loc[nba_sched.index == today]
    nba_sched_today['HomeShort'] = nba_sched_today['Home/Neutral'].str.split().str[-1]
    nba_sched_today['VisitorShort'] = nba_sched_today['Visitor/Neutral'].str.split().str[-1]

    games = ''
    for index, row in nba_sched_today.iterrows():
        ht = row['HomeShort']
        at = row['VisitorShort']

        games += html_table_add_row([ht + ' ('+team_bettor_dict[ht].title()+')',at + ' ('+team_bettor_dict[at].title()+')'])

    return games

def determine_current_pick_order(person_list):
    """
    Assign a new pick number for every team based on current win percentage.
    """
    win_pct = pd.DataFrame(columns=['win pct'])
    for person_obj in person_list:

        for team in person_obj.teams:

            team_obj = person_obj.team_objs[team]
            s = pd.Series(name=team_obj.name, data={'win pct': float(team_obj.wins())/float(team_obj.wins()+team_obj.losses())})
            win_pct = win_pct.append(s)

    win_pct.sort_values('win pct', inplace=True, ascending=False)
    win_pct['new pickno'] = range(1, 31)

    for person_obj in person_list:

        for team in person_obj.teams:

            person_obj.team_objs[team].new_pickno = win_pct.loc[team, 'new pickno']

    return

def plot_graph_all(person_list):
    """Creates bettor 30-day and all season Wins-Losses graphs and creates
       graphs that displays the Wins-Losses for a bettor's set of teams"""

    # Create a DF with every calendar day with the summed Wins-Losses for each bettor
    summed_diff_calendar_list = [reduce((lambda x, y: x+y),
        [person_obj.team_objs[team].diff_calendar() for team in person_obj.teams])
        for person_obj in person_list ]

     # Plot Wins-Losses as a function of time for last 30 days
    sn.set(color_codes=True)
    fig = plt.figure(figsize=(6.5, 6.5))
    ax = fig.add_subplot(111)
    for i in range(len(person_list)):
        summed_diff_calendar_list[i][today - dt.timedelta(days=30):today].plot()
    ax.legend(pick_order, loc='lower left')
    ax.set_ylabel('Wins - Losses', fontsize=12)
    fig.savefig('../images/win_percent_recent.png', bbox_inches='tight')
    plt.close(fig)

     # Plot Wins-Losses as a function of time for season so far
    sn.set(color_codes=True)
    fig = plt.figure(figsize=(6.5, 3))
    ax = fig.add_subplot(111)
    for i in range(len(person_list)):
        summed_diff_calendar_list[i].plot()
    ax.legend([person_obj.name for person_obj in person_list], loc='lower left')
    ax.set_ylabel('Wins - Losses', fontsize=12)
    fig.savefig('../images/win_percent_all.png', bbox_inches='tight')
    plt.close(fig)

    # Plot Wins-Losses for each bettor by team
    sn.set_palette(sn.color_palette("hls", 10))
    for order, person_obj in enumerate(person_list):
        # DF with every calendar day for each of the Bettor's teams
        team_diff_calendar_list = [person_obj.team_objs[team].diff_calendar() for team in person_obj.teams]
        fig = plt.figure(figsize=(10, 6.5))
        ax = fig.add_subplot(111)
        for iteam in person_obj.teams:
            person_obj.team_objs[iteam].diff_calendar().plot(label=iteam, ax=ax)
        ax.set_ylabel('Wins - Losses', fontsize=12)
        ax.legend()
        fig.savefig('../images/'+str(order+1)+'_teams.png', bbox_inches='tight')
        plt.close(fig)

    return

def make_html(person_list):
    """Populate main HTML page that summarizes bettors status and tonight's matchups"""
    with open('../template.html', 'r') as ftemp:
        temp = ftemp.read()

    # Person with the most wins
    current_winner = max([(person_obj.all_wins(), person_obj.name) for person_obj in person_list])[1]

    shutil.copyfile('../images/current_winner_'+current_winner+'.jpg', '../images/current_winner.jpg')

    # Use html_string to dump information into HTML page
    html_string = [current_winner, STR_TODAY, current_winner.title()]

    # Every bettor's quick stats
    wins, losses, remaining, draft_value = [],[],[],[]
    for person_obj in person_list:
        html_string += [person_obj.name.title()]
        wins.append(person_obj.all_wins())
        losses.append(person_obj.all_losses())
        remaining.append(person_obj.all_remaining()-person_obj.dup_remaining_games())
        draft_value.append(person_obj.all_draft_value())

    html_string += wins + losses + remaining + draft_value

    # Order of teams that were picked by a bettor
    # -> Bettor 1 pick 1, Bettor 2 pick 1, Bettor 3 pick 1, Bettor 1 pick 2, etc
    team_pickorder = [person_obj.name.title() for person_obj in person_list] + [person_obj.teams[i]+' ('+str(person_obj.team_objs[person_obj.teams[i]].new_pickno)+')' for i in range(10) for person_obj in person_list]

    html_string += team_pickorder

    # Today's games (by team and bettor)
    html_string += [todays_games(person_list)]
    updated = temp % tuple(html_string)

    with open('../bet.html', 'w') as findex:
        findex.write(updated)

    return

def make_bettor_html(person_list):
    """Make individual bettor page information that breaks down a bettor's teams"""
    with open('../template_bettor.html', 'r') as ftemp:
        temp = ftemp.read()
    html_string = [person_obj.name.title() for person_obj in person_list]
    for person_obj in person_list:

        team_list = ''
        # Summary of team information for a bettor
        for teamname in person_obj.teams:
            team_obj = person_obj.team_objs[teamname]
            team_list += html_table_add_row([teamname,
                                             team_obj.wins(),
                                             team_obj.losses(),
                                             team_obj.remaining(),
                                             team_obj.last10(),
                                             '{0:.2f}'.format(team_obj.expected_wins()),
                                             '{0:.2f}'.format(team_obj.draft_value())], align="center")

        html_string += [person_obj.name.title(), person_obj.all_wins(), person_obj.all_losses(),
                        person_obj.all_remaining()-person_obj.dup_remaining_games(),team_list]

    updated = temp % tuple(html_string)

    with open('../bettors.html', 'w') as findex:
        findex.write(updated)

    return

def html_table_add_row(element_list, align="left"):
    """Simple html table row"""
    connector = '    	'
    table_str = '
'
    table_str += connector.join(str(e) for e in element_list)
    table_str += '
'
    return table_str

if __name__ == '__main__':

    # Read in bettor pick order
    pick_order = np.loadtxt('pickorder.txt', dtype=str, delimiter='\t')

    # Create list of Bettor objects and initialize them
    person_list = []
    for order, bettor in enumerate(pick_order):
        bettor = bettor.lower()
        bettor_teams = np.loadtxt(bettor + bet_file_postfix, dtype=str, delimiter='\t')

        bettor_obj = Bettor(bettor, order, bettor_teams)

        person_list += [bettor_obj]

    # Add new pick number to each team
    determine_current_pick_order(person_list)

    # Create graphs and html pages
    plot_graph_all(person_list)
    make_html(person_list)
    make_bettor_html(person_list)


# partners/urls.py
from django.conf.urls.defaults import *
from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns('',
    url(r'^(?P<blog_id>\d+)$', 'partners.views.blogd', name='blogd'),
    url(r'^$', 'partners.views.partners', name='partners'),
)


from werkzeug.exceptions import NotFound
from flask import Blueprint, render_template, abort

products_app = Blueprint('products_app', __name__)

PRODUCTS = {
    1: 'Phone',
    2: 'Tablet',
    3: 'Laptop',
}

@products_app.route('/', endpoint='products')
def products_list():
    return render_template(
        'products/index.html',
        products=PRODUCTS.items(),
    )

@products_app.route('/<int:id>/', endpoint='product')
def product_info(id):
    try:
        product_name = PRODUCTS[id]
    except KeyError:
        raise NotFound(f'There is no product #{id}')
        # abort(404)
    return f'Product #{id}: {product_name}'


# Victor Ferreira Ferrari, RA 187890
# MC920 - Introduction to Image Digital Processing
# Project 5 - Principal Component Analysis
# Last modified: 21/11/2019

from cv2 import imwrite, imread
from cv2 import imshow, waitKey, destroyAllWindows, IMREAD_COLOR
from os.path import join, basename, getsize
import matplotlib.pyplot as plt
import numpy as np
import argparse

def main():
    parser = argparse.ArgumentParser(description='Uses Principal Component Analysis (PCA) for compression.')
    parser.add_argument('file', help='Name of the file containing the image (PNG)')
    parser.add_argument('comp', help='Amount of components to split.', type=int)
    parser.add_argument('--plot', help='Plot component variance graph.', action='store_true')
    parser.add_argument('--folder', help='Folder to save compressed image (defaults to Outputs/).', default='Outputs')
    args = parser.parse_args()

    # Read color image
    img = imread(args.file, IMREAD_COLOR)

    # Checks if plot mode.
    if args.plot:
        plot_components(img, args.comp)
        name = 'comp_graph_' + basename(args.file)
        plt.savefig(join(args.folder,name))
        plt.show()
    else:
        # Compress image
        out = pca_compression(img, args.comp)

        # Show/Save result and original image
        base = basename(args.file)
        name = str(args.comp) + 'comp_' + base
        show_image(out.astype(np.uint8), name='Compressed')
        save_image(out, name, args.folder)
        save_image(img, base, args.folder)

        # Evaluate result
        metrics = eval_compression(img.astype(np.float64), out, join(args.folder, base), join(args.folder, name))
        print("Compression factor:", metrics['rho'])
        print("Compression RMSE:", metrics['rmse'])

# Compressing an image via principal component analysis.
# Calculated using SVD.
def pca_compression(img, n_comp):
    k = n_comp

    # Divide in 3 channels (BGR) and apply SVD to each one.
    USVb = np.linalg.svd(img[:,:,0], full_matrices=False)
    USVg = np.linalg.svd(img[:,:,1], full_matrices=False)
    USVr = np.linalg.svd(img[:,:,2], full_matrices=False)

    U = np.dstack((USVb[0],USVg[0],USVr[0]))
    S = np.dstack((USVb[1],USVg[1],USVr[1]))
    V = np.dstack((USVb[2],USVg[2],USVr[2]))

    # Consider k components and combine channels.
    out = np.zeros(img.shape)

    Ug = U[:,:k,:]
    Sg = S[:k,:k,:]
    Vg = V[:k,:,:]

    for i in range(img.shape[2]):
        out[:,:,i] = (Ug[:,:,i]*Sg[:,:,i]) @ Vg[:,:,i]

    return out

# Evaluates compression by RMSE and compression factor.
def eval_compression(f, g, f_path, g_path, factor=True):
    rho = getsize(g_path)/getsize(f_path) if factor else 0

    diff = (f-g)**2
    rmse = np.sqrt(diff.sum().sum()/(f.shape[0] * f.shape[1]))

    return {'rho':rho, 'rmse':rmse}

# Plots component graph to verify optimal amount of components to compress.
def plot_components(img, amount):
    rmse = np.zeros(amount)
    for i in range(1,amount+1):
        out = pca_compression(img, i)
        met = eval_compression(img, out, '', '', factor=False)
        rmse[i-1] = met['rmse']

    plt.plot(range(1,amount+1), rmse, 'bo')
    plt.xlabel('Component Amount')
    plt.ylabel('RMSE')
    plt.title('Compression error per component amount')

# Self-explanatory
def show_image(img, name='Image'):
    imshow(name, img)
    waitKey(0)
    destroyAllWindows()

def save_image(img, name, folder):
    imwrite(join(folder, name), img)

if __name__ == "__main__":
    main()


# https://github.com/opencv/opencv/tree/master/data/haarcascades

import cv2, time

face_cascade = cv2.CascadeClassifier('haarcascade_frontalcatface.xml')
smile_cascade  =cv2.CascadeClassifier('haarcascade_smile.xml')

video = cv2.VideoCapture(0)

while True:
	check, frame=video.read()
	gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
	face = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5)
	for x,y,w,h in face:
		img = cv2.rectangle(frame,(x,y), (x+w,y+h),(0,0,255),3)   # (x+w,y+h) w- widht h- height  3- widht of rectangle
		#                                      colorB  G  R

		#this is detect smile
		smile = smile_cascade.detectMultiScale(gray, scaleFactor=1.8, minNeighbors=20)
		for x,y,w,h in smile:
			img = cv2.rectangle(frame,(x,y), (x+w,y+h),(255,0,0),3)

	cv2.imshow('gotcha', frame)
	key = cv2.waitKey(1)

	if key == ord('q'):
		break

video.release()
cv2.destroyAllWindows


# 3
# 简单搜索实现的进化

titles = "本周的金枪鱼之夜，" \
         "我们邀请了计算机系高性能所的陈康副教授给大家讲授关于分布式系统的基础知识，" \
         "以及著名的分布式一致性算法 Paxos 协议。"

title_list = titles.split('，')
title_list = set(title_list)

import jieba

def word_match(word, title):
    ifmatch = True
    keyword_ = ' '.join(jieba.cut(word))
    for word in keyword_.split():
        if word != ' ' and word not in title:
            ifmatch = False
    return ifmatch

query = '分布式 and (Paxos or Raft)'  # 分词
query_new_parts = []
for part in list(jieba.cut(query)):
    if part == '(' or part == ')':
        query_new_parts.append(part)
    elif part in ('AND', 'and', 'OR', 'or', 'NOT', 'not', ' '):
        query_new_parts.append(part.lower())
    else:
        query_new_parts.append("word_match('{}', title)".format(part))

query_new = ''.join(query_new_parts)

for title in title_list:
    if eval(query_new):
        for part in list(jieba.cut(query)):
            if part not in ('AND', 'and', 'OR', 'or', 'NOT', 'not', '(', ')', ' '):
                print(title, part)