"""
Home-Monitor:
    AI system for the detection of anomalous and possibly harmful events for people.

    Written by Gabriel Rojas - 2019
    Copyright (c) 2019 G0 S.A.S.
    See LICENSE file for details

Class information:
    Class to detect a posible person in infarcting.
"""

import sys
import numpy as np
from os.path import dirname, normpath
import json
import math
from time import time, sleep

from cv2 import cv2
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.models import load_model

# Including Home Monitor Paths to do visible the modules
sys.path.insert(0, './Tools/')
sys.path.insert(0, './Core/')

import Misc
from EventRecognizer import EventRecognizer
from DataPool import LogTypes, SourceTypes, Messages, Data

class SkeletonInfarctRecognizer(EventRecognizer):
    """ Class to detect a posible person in infarcting. """

    simulationStep = 0

    def preLoad(self):
        """ Implement me! :: Do anything necessary for processing """
        #pass
        import os
        os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

    def loadModel(self):
        """ Loads model """
        ModelPath = normpath(self.ME_PATH + "/" + self.ME_CONFIG['MODEL'])
        self.log(Messages.controller_loading_model.format(ModelPath), LogTypes.DEBUG)
        self.MODEL = load_model(ModelPath)
        self.MODEL._make_predict_function()
        if self.ME_STANDALONE:
            self.MODEL.summary()
        ModelPath = normpath(self.ME_PATH + "/" + self.ME_CONFIG['MODEL_FACE'])
        self.log(Messages.controller_loading_model.format(ModelPath), LogTypes.DEBUG)
        self.MODEL_FACE = load_model(ModelPath)
        self.MODEL_FACE._make_predict_function()
        if self.ME_STANDALONE:
            self.MODEL_FACE.summary()

    def loaded(self):
        """ Implement me! :: Just after load the model """
        # TODO: This method is called just after load model
        pass

    def predict(self, data:Data):
        """ Exec prediction to recognize an activity """
        rgbFilter = Data()
        rgbFilter.id = None
        rgbFilter.package  = data.package
        rgbFilter.source_name = 'CamController'
        rgbFilter.source_type = SourceTypes.CONTROLLER
        rgbData = self.receive(dataFilter=rgbFilter, limit=1, lastTime=0)

        if len(rgbData) < 2:
            return []
        rgbData = rgbData[1]

        dataReturn = []

        per = data.data
        if per[0].any() == None or per[1].any() == None or per[2].any() == None or per[5].any() == None:
            return []

        skeletonResult = self.predict_skeleton(person=per)

        img = self.extract_face(frame=rgbData.data, person=per)
        faceResult = self.predict_face(frame=img)

        array = self.predict_full(skeletonResult, faceResult)
        result = array[0]
        answer = np.argmax(result)

        if result[answer] < 0.65:
            return []
        #print(result[answer], skeletonResult, faceResult)

        if answer > 0:
            dataInf = Data()
            dataInf.source_item = self.CLASSES[answer]
            dataInf.data = { 'class':self.CLASSES[answer], 'acc':result[answer] }
            dataReturn.append(dataInf)

        return dataReturn

    def showData(self, dataPredicted:Data, dataSource:Data):
        """ To show data if this module start standalone """
        self.log('Class detected:' + dataPredicted.data['class'] + \
            ' with acc:' + str(dataPredicted.data['acc']), LogTypes.INFO)

        #cv2.imshow(dataSource.source_name + '-' + dataSource.source_item, dataSource.data)
        #if cv2.waitKey(1) & 0xFF == ord('q'):
        #    self.stop()
        #    cv2.destroyAllWindows()

        #with open("M:/tmp/HM-SimulatingData/SkeletonInfarctRecognizer_OutPut.txt",'a+') as file:
        #    file.write('\n' + dataPredicted.toString(False, True))

    def simulateData(self, dataFilter:Data, limit:int=-1, lastTime:float=-1):
        """ Implement me! :: Allows to simulate data if this module start standalone """

        if self.simulationStep == 0:
            self.file = open(self.SimulatingPath, 'r').readlines()
            self.file_length = len(self.file)

        dataReturn = []

        if dataFilter.package != '':
            for target_list in self.file:
                if len(target_list) < 10:
                    continue
                dataSimulated = Data()
                dataSimulated = dataSimulated.parse(target_list, False, True)
                if dataSimulated.package == dataFilter.package and dataSimulated.source_name == dataFilter.source_name:
                    dataReturn.append(dataSimulated)
                    dataReturn.insert(0, {'queryTime':time()})
                    return dataReturn

        if self.simulationStep < self.file_length:
            if len(self.file[self.simulationStep]) < 10:
                dataReturn.insert(0, {'queryTime':time()})
                self.simulationStep += 1
                return dataReturn

            dataSimulated = Data()
            dataSimulated = dataSimulated.parse(self.file[self.simulationStep], False, True)

            if dataSimulated.source_name != dataFilter.source_name:
                dataReturn.insert(0, {'queryTime':time()})
                self.simulationStep += 1
                return dataReturn

            self.simulationStep += 1
            dataReturn.append(dataSimulated)
            dataReturn.insert(0, {'queryTime':time()})
        else:
            self.simulationStep = 0
            dataReturn = self.simulateData(dataFilter)

        return dataReturn

    # =========== Auxiliar methods =========== #

    def extract_face(self, frame, person):
        """ Cut and return image of face base in neck and nose points """
        Nose = person[0]
        Neck = person[1]
        RShoulder = person[2]
        LShoulder = person[5]
        Radio1 = math.sqrt(math.pow(Neck[1] - Nose[1], 2) + math.pow(Neck[0] - Nose[0], 2))
        Radio2 = math.sqrt(math.pow(Neck[1] - RShoulder[1], 2) + math.pow(Neck[0] - RShoulder[0], 2))
        Radio3 = math.sqrt(math.pow(Neck[1] - LShoulder[1], 2) + math.pow(Neck[0] - LShoulder[0], 2))
        Radio = max(Radio1, Radio2, Radio3)
        Radio = int(Radio)
        x = max(Nose[0] - Radio, 0)
        y = max(Nose[1] - Radio, 0)

        Face = frame[y:Nose[1] + Radio, x:Nose[0] + Radio]
        Face = cv2.resize(Face, (64, 64))
        return Face

    def furthest_shouder(self, person):
        """ Returns the distance to the shoulder furthest from the neck """
        neck = person[1]
        if person[2][0] is not None:
            length_RShoulder = math.pow(person[2][0] - neck[0], 2) + math.pow(person[2][1] - neck[1], 2)
        else:
            length_RShoulder = 0
        if person[5][0] is not None:
            length_LShoulder = math.pow(person[5][0] - neck[0], 2) + math.pow(person[5][1] - neck[1], 2)
        else:
            length_LShoulder = 0

        if length_RShoulder > length_LShoulder:
            Shoulder = 2 # Right
            H = math.sqrt(length_RShoulder)
        else:
            Shoulder = 5 # Left
            H = math.sqrt(length_LShoulder)

        CosA = (person[Shoulder][0] - neck[0]) / H
        SinA = (neck[1] - person[Shoulder][1]) / H

        return H, CosA, SinA

    def complete_body(self, person):

        nose = person[0]
        neck = person[1]
        h = math.sqrt(math.pow(neck[0] - nose[0],2) + math.pow(neck[1] - nose[1], 2))
        h = int(h)

        if person[2][0] is None: # if not RShoulder mirror LShoulder
            auxX = neck[0] - person[5][0]
            auxY = person[5][1]
            person[2] = (neck[0] + auxX, auxY)

        if person[5][0] is None: # if not LShoulder mirror RShoulder
            auxX = neck[0] - person[2][0]
            auxY = person[2][1]
            person[5] = (neck[0] + auxX, auxY)

        #if person[8][0] is None: # No RHip calculate it
        #    person[8] = (person[2][0], neck[1] + int(2 * h))

        #if person[9][0] is None: # No LHip calculate it
        #    person[9] = (person[5][0], neck[1] + int(2 * h))

        if person[3][0] is None: # Not RElbow
            if not person[6][0] is None: # LElbow
                auxX = neck[0] - person[6][0]
                auxY = person[6][1]
                person[3] = (neck[0] + auxX, auxY)
            else:
                person[3] = (person[2][0], person[2][1] + 2*h)

        if person[6][0] is None: # Not LElbow
            if not person[3][0] is None: # RElbow
                auxX = neck[0] - person[3][0]
                auxY = person[3][1]
                person[6] = (neck[0] + auxX, auxY)
            else:
                person[6] = (person[5][0], person[5][1] + 2*h)

        if person[4][0] is None: # Not RWrist
            if not person[7][0] is None: # LWrist
                auxX = neck[0] - person[7][0]
                auxY = person[7][1]
                person[4] = (neck[0] + auxX, auxY)
            else:
                person[4] = (person[3][0], person[3][1] + 2*h)

        if person[7][0] is None: # Not LWrist
            if not person[4][0] is None: # RWrist
                auxX = neck[0] - person[4][0]
                auxY = person[4][1]
                person[7] = (neck[0] + auxX, auxY)
            else:
                person[7] = (person[6][0], person[6][1] + 2*h)

        return person

    def rotate_point(self,cX, cY, cosA, sinA, X, Y):
        X1 = X - cX
        Y1 = cY - Y
        X2 = (X1 * -cosA) - (Y1 * sinA)
        Y2 = (X1 * sinA) + (Y1 * -cosA)
        resX = (-1 if cosA > 0.0 else 1) * X2 + cX
        resY = (1 if cosA > 0.0 else -1) * Y2 + cY
        return int(resX), int(resY)

    def rotate_person(self, person):
        """ Rotate joints arround neck """
        neck = person[1]
        _, CosA, SinA = self.furthest_shouder(person)

        for p in range(len(person)):
            point = person[p]
            if not point[0] is None:
                x, y = self.rotate_point(neck[0], neck[1], CosA, SinA, point[0], point[1])
                #x, y = point[0], point[1]
                person[p] = (x, y)

        return person

    def norm_person(self, person):
        """ Normalize the distances between joints """
        H, _, _ = self.furthest_shouder(person)
        neck = person[1]
        nPerson = []
        for p in range(len(person)):
            point = person[p]
            if point[0] is None:
                x = None
                y = None
            else:
                x = (point[0] - neck[0]) / H
                y = (neck[1] - point[1]) / H
            nPerson.append((x, y))

        return nPerson

    def predict_skeleton(self, person):
        """ Make skeleton prediction """
        self.rotate_person(person)
        self.complete_body(person)
        nPerson = self.norm_person(person)
        nPerson = np.expand_dims(nPerson, axis=0)
        return self.MODEL.predict(nPerson[:,0:8])

    def predict_face(self, frame):
        """ Make face prediction """
        #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame = np.expand_dims(frame, axis=0)
        return self.MODEL_FACE.predict(frame)

    def predict_full(self, skeletonResult, faceResult):
        """ Combine skeleton and face results """
        result = skeletonResult + faceResult
        result /= 2
        return result

# =========== Start standalone =========== #
if __name__ == "__main__":
    comp = SkeletonInfarctRecognizer()
    comp.setLoggingSettings(LogTypes.DEBUG)
    comp.init_standalone(path=dirname(__file__))
    sleep(600)


import ast
import argparse
import numpy as np

from keras.preprocessing import sequence
from sklearn.preprocessing import LabelEncoder
from keras.models import load_model
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

def load_data(data_path, max_len=200):
    data = []
    l = []
    ids = []
    i = 0
    l_encoder = LabelEncoder()
    with open(data_path, 'rb') as inf:
        for line in inf:
            gzip_fields = line.decode('utf-8').split('\t')
            gzip_id = gzip_fields[0]
            gzip_label = gzip_fields[1]
            elmo_embd_str = gzip_fields[4].strip()
            elmo_embd_list = ast.literal_eval(elmo_embd_str)
            elmo_embd_array = np.array(elmo_embd_list)
            padded_seq = sequence.pad_sequences([elmo_embd_array], maxlen=max_len, dtype='float32')[0]
            data.append(padded_seq)
            l.append(gzip_label)
            ids.append(gzip_id)
            i += 1
    label = l_encoder.fit_transform(l)
    return np.array(data), np.array(label), np.array(ids)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--inputTSV", help="Location of input data tsv")
    parser.add_argument("--savedModel", help="Location of saved model file")
    args, _ = parser.parse_known_args()

    maxLen = 200
    data, target, docId = load_data(args.inputTSV, max_len=maxLen)
    model = load_model(args.savedModel)
    pred = model.predict(data)
    pred[pred > 0.5] = 1
    pred[pred <= 0.5] = 0
    print("Accuracy: {}".format(accuracy_score(target, pred)))
    print("F1 score: {}".format(f1_score(target, pred, average="macro")))
    print("Precision: {}".format(precision_score(target, pred, average="macro")))
    print("Recall: {}".format(recall_score(target, pred, average="macro")))


#!/usr/bin/env python

##############################################
# The MIT License (MIT)
# Copyright (c) 2016 Kevin Walchko
# see LICENSE for full details
##############################################

from __future__ import print_function
from __future__ import division
import numpy as np
from math import cos, sin, sqrt, pi
# from math import radians as d2r
# from Servo import Servo
# import time

def rot_z_tuple(t, c):
	"""
	t - theta [radians]
	c - [x,y,z]
	return - (x,y,z) tuple rotated about z-axis
	"""
	ans = (
		c[0]*cos(t)-c[1]*sin(t),
		c[0]*sin(t)+c[1]*cos(t),
		c[2]
	)

	return ans

# make a static method in Gait? Nothing else uses it
def rot_z(t, c):
	"""
	t - theta [radians]
	c - [x,y,z]
	return - [x,y,z] numpy array rotated about z-axis
	"""
	ans = np.array([
		c[0]*cos(t)-c[1]*sin(t),
		c[0]*sin(t)+c[1]*cos(t),
		c[2]
	])
	# ans = np.array(rot_z_tuple(t, c))

	return ans

class Gait(object):
	"""
	Base class for gaits. Gait only plan all foot locations for 1 complete cycle
	of the gait.
	"""
	# these are the offsets of each leg
	legOffset = [0, 6, 3, 9]
	# frame rotations for each leg
	cmrot = [pi/4, -pi/4, -3*pi/4, 3*pi/4]
	frame = [-pi/4, pi/4, 3*pi/4, -3*pi/4]  # this seem to work better ... wtf?
	moveFoot = None
	rest = None
	scale = 50.0

	def __init__(self, rest):
		# the resting or idle position/orientation of a leg
		self.rest = rest
		# self.moveFoot = mf
		# self.write = w

	def command(self, cmd):
		"""
		func is the quadruped move foot function for a specific leg
		"""
		x, y, rz = cmd
		d = sqrt(x**2+y**2)

		# commands should be unit length, oneCyle scales it
		if 1.0 < d:
			x /= d
			y /= d

		# handle no movement command ... do else where?
		if d < 0.1 and abs(rz) < 0.1:
			x = y = 0.0
			rz = 0.0

			return None

		# if abs(rz) < 0.001:
		# 	rz = 0.0

		return self.oneCycle(x, y, rz)

	def oneCycle(x, y, rz):
		print('*** wrong function! ***')
		return None

class DiscreteRippleGait(Gait):
	"""
	Discrete 12 step gait
	"""
	steps = 0

	def __init__(self, h, r):
		Gait.__init__(self, r)
		self.phi = [9/9, 6/9, 3/9, 0/9, 1/9, 2/9, 3/9, 4/9, 5/9, 6/9, 7/9, 8/9]  # foot pos in gait sequence
		maxl = h  # lifting higher gives me errors
		minl = maxl/2
		self.z = [minl, maxl, minl, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]  # leg height
		self.steps = len(self.phi)

	def eachLeg(self, index, cmd):
		"""
		interpolates the foot position of each leg
		cmd:
			linear (mm)
			angle (rads)
		"""
		rest = self.rest
		i = index
		phi = self.phi[i]
		xx, yy, rzz = cmd

		# rotational commands -----------------------------------------------
		angle = rzz/2-rzz*phi
		rest_rot = rot_z(-angle, rest)

		# create new move command
		move = np.array([
			xx/2 - phi*xx,
			yy/2 - phi*yy,
			self.z[i]
		])

		# new foot position: newpos = rot + move ----------------------------
		newpos = move + rest_rot
		# print('New  [](x,y,z): {:.2f}\t{:.2f}\t{:.2f}'.format(newpos[0], newpos[1], newpos[2]))
		return newpos

	def oneCycle(self, x, y, rz):
		scale = self.scale
		cmd = (scale*x, scale*y, rz)
		ret = []  # 4 leg foot positions for the entire 12 count cycle is returned

		for i in range(0, self.steps):  # iteration, there are 12 steps in gait cycle
			footPos = []
			for legNum in [0, 2, 1, 3]:  # order them diagonally
				# rcmd = self.calcRotatedOffset(cmd, legNum)
				rcmd = rot_z_tuple(self.frame[legNum], cmd)
				index = (i + self.legOffset[legNum]) % 12
				pos = self.eachLeg(index, rcmd)  # move each leg appropriately
				# print('Foot[{}]: {:.2f} {:.2f} {:.2f}'.format(legNum, *(pos)))
				# if legNum == 0: print('New  [{}](x,y,z): {:.2f}\t{:.2f}\t{:.2f}'.format(i, pos[0], pos[1], pos[2]))
				footPos.append([index, legNum, pos])  # all in leg frame
			# print('footPos', footPos)

			# corr = Correction()
			# c = corr.calcCorrection(footPos)
			# feet = corr.rotateFeetCorrected(footPos, c)

			ret.append(footPos)  # 4 feet at index i: [index, legNum, footposition]

# <<<<<<< HEAD:quadruped/Gait.py
		return ret
# =======
# 			feet = footPos
# 			# print('----------------------------')
# 			for foot in feet:
# 				legNum = foot[1]
# 				ft = foot[2]
# 				self.moveFoot(legNum, ft)
# 				# print('Foot[{}]: {:.2f} {:.2f} {:.2f}'.format(legNum, *ft))
#
# 			# Servo.bulkWrite(Servo.ser)
# 			# Servo.syncWrite(Servo.ser)
# 			self.write()
# 			time.sleep(0.1)
# >>>>>>> master:quadruped/quadruped/Gait.py

# class ContinousRippleGait(Gait):
# 	alpha = 1.0
#
# 	def __init__(self, h, r):
# 		Gait.__init__(self)
# 		self.height = h
# 		self.rest = r
#
# 	@staticmethod
# 	def phi(x):
# 		"""
# 		The phase
# 		"""
# 		phi = 0.0
# 		if x <= 3.0:
# 			phi = 1/3*(3.0-x)
# 		else:
# 			phi = 1/9*(x-3)
# 		return phi
#
# 	def z(self, x):
# 		"""
# 		Leg height
#
# 		duty cycle:
# 			0-3: leg lifted
# 			3-12: leg on ground
# 			duty = (12-3)/12 = 0.75 = 75% a walking gait
# 		"""
# 		height = self.height
# 		z = 0.0
# 		if x <= 1:
# 			z = height/1.0*x
# 		elif x <= 2.0:
# 			z = height
# 		elif x <= 3.0:
# 			z = -height/1.0*(x-2.0)+height
# 		return z
#
# 	def eachLeg(self, index, cmd):
# 		"""
# 		interpolates the foot position of each leg
# 		"""
# 		rest = self.rest
# 		i = (index*self.alpha) % 12
# 		phi = self.phi(i)
# 		z = self.z(i)
#
# 		# rotational commands -----------------------------------------------
# 		angle = cmd['angle']/2-cmd['angle']*phi
# 		rest_rot = rot_z(-angle, rest)
# 		# rest_rot[2] = 0  # let linear handle z height
#
# 		# linear commands ----------------------------------------------------
# 		linear = cmd['linear']
# 		xx = linear[0]
# 		yy = linear[1]
#
# 		# create new move command
# 		move = np.array([
# 			xx/2 - phi*xx,
# 			yy/2 - phi*yy,
# 			z
# 		])
#
# 		# new foot position: newpos = rot + move ----------------------------
# 		newpos = move + rest_rot
# 		return newpos


# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""Functions for performing aperture photometry on 2-D arrays."""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import math
import abc
import numpy as np
import warnings
import astropy.units as u
from astropy.io import fits
from astropy.table import Table
from astropy.coordinates import SkyCoord
from astropy.extern import six
from astropy.utils.misc import InheritDocstrings
from astropy.utils.exceptions import AstropyUserWarning
from .aperture_funcs import do_circular_photometry, do_elliptical_photometry, \
                            do_annulus_photometry

__all__ = ["Aperture",
           "CircularAperture", "CircularAnnulus",
           "EllipticalAperture", "EllipticalAnnulus",
           "RectangularAperture",
           "aperture_photometry"]

def _make_annulus_path(patch_inner, patch_outer):
    import matplotlib.path as mpath
    verts_inner = patch_inner.get_verts()
    verts_outer = patch_outer.get_verts()
    codes_inner = (np.ones(len(verts_inner), dtype=mpath.Path.code_type) *
                   mpath.Path.LINETO)
    codes_inner[0] = mpath.Path.MOVETO
    codes_outer = (np.ones(len(verts_outer), dtype=mpath.Path.code_type) *
                   mpath.Path.LINETO)
    codes_outer[0] = mpath.Path.MOVETO
    codes = np.concatenate((codes_inner, codes_outer))
    verts = np.concatenate((verts_inner, verts_outer[::-1]))
    return mpath.Path(verts, codes)

class _ABCMetaAndInheritDocstrings(InheritDocstrings, abc.ABCMeta):
    pass

@six.add_metaclass(_ABCMetaAndInheritDocstrings)
class Aperture(object):
    """
    Abstract base class for an arbitrary 2-d aperture.

    Derived classes should contain whatever internal data is needed to
    define the aperture, and provide methods `do_photometry` and `extent`
    (and optionally, ``area``).
    """

    @abc.abstractmethod
    def extent(self):
        """
        Extent of apertures. In the case when part of an aperture's extent
        falls out of the actual data region, the
        `~photutils.Aperture.get_phot_extents` method redefines the extent
        which has data coverage.

        Returns
        -------
        x_min, x_max, y_min, y_max : lists of floats
            Extent of the apertures.
        """

    @abc.abstractmethod
    def plot(self, ax=None, fill=False, **kwargs):
        """
        Plot the aperture(s) on a matplotlib Axes instance.

        Parameters
        ----------
        ax : `matplotlib.axes.Axes` instance, optional
            If `None`, then the current ``Axes`` instance is used.

        fill : bool, optional
            Set whether to fill the aperture patch.  The default is
            `False`.

        kwargs
            Any keyword arguments accepted by `matplotlib.patches.Patch`.
        """

    @abc.abstractmethod
    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='exact', subpixels=5):
        """Sum flux within aperture(s).

        Parameters
        ----------
        data : array_like
            The 2-d array on which to perform photometry.
        error : array_like, optional
            Error in each pixel, interpreted as Gaussian 1-sigma uncertainty.
            ``error`` has to have the same shape as ``data``.
        gain : array_like, optional
            Ratio of counts (e.g., electrons or photons) to units of the
            data (e.g., ADU), for the purpose of calculating Poisson error from
            the object itself. ``gain`` has to have the same shape as ``data``.
        pixelwise_error : bool, optional
            For error and/or gain arrays. If `True`, assume error and/or gain
            vary significantly within an aperture: sum contribution from each
            pixel. If `False`, assume error and gain do not vary significantly
            within an aperture.
        method : str, optional
            Method to threat masked pixels. For the currently supported methods
            see the documentation of `aperture_photometry`.
        subpixels : int, optional
            For the ``'subpixel'`` method, resample pixels by this factor (in
            each dimension). That is, each pixel is divided into
            ``subpixels ** 2`` subpixels.

        Returns
        -------
        flux : `~astropy.units.Quantity`
            Sum of the values withint the aperture(s). Unit is kept to be the
            unit of the input ``data``.

        fluxvar :  `~astropy.units.Quantity`
            Corresponting uncertainity in the ``flux`` values. Returned only
            if input ``error`` is not `None`.
        """

    def get_phot_extents(self, data):
        """
        Get the photometry extents and check if the apertures is fully out
        of data.

        Parameters
        ----------
        data : array_like
            The 2-d array on which to perform photometry.

        Returns
        -------
        extents : dict
            The ``extents`` dictionary contains 3 elements:

            * ``'ood_filter'``
                A boolean array with `True` elements where the aperture is
                falling out of the data region.
            * ``'pixel_extent'``
                x_min, x_max, y_min, y_max : Refined extent of apertures with
                data coverage.
            * ``'phot_extent'``
                x_pmin, x_pmax, y_pmin, y_pmax: Extent centered to the 0, 0
                positions as required by the `~photutils.geometry` functions.
        """

        extents = self.extent()

        # Check if an aperture is fully out of data
        ood_filter = np.logical_or(extents[:, 0] >= data.shape[1],
                                   extents[:, 1] <= 0)
        np.logical_or(ood_filter, extents[:, 2] >= data.shape[0],
                      out=ood_filter)
        np.logical_or(ood_filter, extents[:, 3] <= 0, out=ood_filter)

        # TODO check whether it makes sense to have negative pixel
        # coordinate, one could imagine a stackes image where the reference
        # was a bit offset from some of the images? Or in those cases just
        # give Skycoord to the Aperture and it should deal with the
        # conversion for the actual case?
        x_min = np.maximum(extents[:, 0], 0)
        x_max = np.minimum(extents[:, 1], data.shape[1])
        y_min = np.maximum(extents[:, 2], 0)
        y_max = np.minimum(extents[:, 3], data.shape[0])

        x_pmin = x_min - self.positions[:, 0] - 0.5
        x_pmax = x_max - self.positions[:, 0] - 0.5
        y_pmin = y_min - self.positions[:, 1] - 0.5
        y_pmax = y_max - self.positions[:, 1] - 0.5

        # TODO: check whether any pixel is nan in data[y_min[i]:y_max[i],
        # x_min[i]:x_max[i])), if yes return something valid rather than nan

        return {'ood_filter': ood_filter,
                'pixel_extent': [x_min, x_max, y_min, y_max],
                'phot_extent': [x_pmin, x_pmax, y_pmin, y_pmax]}

class CircularAperture(Aperture):
    """
    Circular aperture(s).

    Parameters
    ----------
    positions : tuple, or list, or array
        Center coordinates of the apertures as list or array of (x, y)
        pixelcoordinates.
    r : float
        The radius of the aperture.

    Raises
    ------
    ValueError : `~.exceptions.ValueError`
        If the radius is negative.
    """

    def __init__(self, positions, r):
        try:
            self.r = float(r)
        except TypeError:
            raise TypeError('r must be numeric, received {0}'.format(type(r)))

        if r < 0:
            raise ValueError('r must be non-negative')

        if isinstance(positions, u.Quantity):
            positions = positions.value
        if isinstance(positions, (list, tuple, np.ndarray)):
            self.positions = np.atleast_2d(positions)
        else:
            raise TypeError("List or array of (x,y) pixel coordinates is "
                            "expected got '{0}'.".format(positions))

        if self.positions.ndim > 2:
            raise ValueError('{0}-d position array not supported. Only 2-d '
                             'arrays supported.'.format(self.positions.ndim))

    def extent(self):
        extents = []
        for x, y in self.positions:
            extents.append((int(x - self.r + 0.5), int(x + self.r + 1.5),
                            int(y - self.r + 0.5), int(y + self.r + 1.5)))

        return np.array(extents)

    def area(self):
        """
        Returns
        -------
        area : float
            Area of aperture.
        """
        return math.pi * self.r ** 2

    def plot(self, ax=None, fill=False, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.patches as mpatches
        kwargs['fill'] = fill
        if ax is None:
            ax = plt.gca()
        for position in self.positions:
            patch = mpatches.Circle(position, self.r, **kwargs)
            ax.add_patch(patch)

    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='exact', subpixels=5):
        extents = super(CircularAperture, self).get_phot_extents(data)

        if method not in ('center', 'subpixel', 'exact'):
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        flux = do_circular_photometry(data, self.positions, extents,
                                      self.r, error=error, gain=gain,
                                      pixelwise_error=pixelwise_error,
                                      method=method,
                                      subpixels=subpixels)
        return flux

class CircularAnnulus(Aperture):
    """
    Circular annulus aperture.

    Parameters
    ----------
    positions : tuple, or list, or array
        Center coordinates of the apertures as list or array of (x, y)
        pixelcoordinates.
    r_in : float
        The inner radius of the annulus.
    r_out : float
        The outer radius of the annulus.

    Raises
    ------
    ValueError : `~.exceptions.ValueError`
        If inner radius (``r_in``) is greater than outer radius (``r_out``).
    ValueError : `~.exceptions.ValueError`
        If inner radius is negative.
    """

    def __init__(self, positions, r_in, r_out):
        try:
            self.r_in = r_in
            self.r_out = r_out
        except TypeError:
            raise TypeError("'r_in' and 'r_out' must be numeric, received {0} "
                            "and {1}".format((type(r_in), type(r_out))))

        if not (r_out > r_in):
            raise ValueError('r_out must be greater than r_in')
        if r_in < 0:
            raise ValueError('r_in must be non-negative')

        if isinstance(positions, u.Quantity):
            positions = positions.value
        if isinstance(positions, (list, tuple, np.ndarray)):
            self.positions = np.atleast_2d(positions)
        else:
            raise TypeError("List or array of (x,y) pixel coordinates is "
                            "expected got '{0}'.".format(positions))

        if self.positions.ndim > 2:
            raise ValueError('{0}-d position array not supported. Only 2-d '
                             'arrays supported.'.format(self.positions.ndim))

    def extent(self):
        extents = []
        centers = []
        for x, y in self.positions:
            extents.append((int(x - self.r_out + 0.5),
                            int(x + self.r_out + 1.5),
                            int(y - self.r_out + 0.5),
                            int(y + self.r_out + 1.5)))
            centers.append((x, x, y, y))

        self._centers = np.array(centers)
        return np.array(extents)

    def area(self):
        """
        Returns
        -------
        area : float
            Area of aperture.
        """
        return math.pi * (self.r_out ** 2 - self.r_in ** 2)

    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='exact', subpixels=5):
        extents = super(CircularAnnulus, self).get_phot_extents(data)

        if method not in ('center', 'subpixel', 'exact'):
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        flux = do_annulus_photometry(data, self.positions, 'circular', extents,
                                     (self.r_in, ), (self.r_out, ),
                                     error=error, gain=gain,
                                     pixelwise_error=pixelwise_error,
                                     method=method,
                                     subpixels=subpixels)

        return flux

    def plot(self, ax=None, fill=False, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.patches as mpatches
        kwargs['fill'] = fill
        if ax is None:
            ax = plt.gca()
        resolution = 20
        for position in self.positions:
            patch_inner = mpatches.CirclePolygon(position, self.r_in,
                                                 resolution=resolution)
            patch_outer = mpatches.CirclePolygon(position, self.r_out,
                                                 resolution=resolution)
            path = _make_annulus_path(patch_inner, patch_outer)
            patch = mpatches.PathPatch(path, **kwargs)
            ax.add_patch(patch)

class EllipticalAperture(Aperture):
    """
    An elliptical aperture.

    Parameters
    ----------
    positions : tuple, or list, or array
        Center coordinates of the apertures as list or array of (x, y)
        pixelcoordinates.
    a : float
        The semimajor axis.
    b : float
        The semiminor axis.
    theta : float
        The position angle of the semimajor axis in radians
        (counterclockwise).

    Raises
    ------
    ValueError : `~.exceptions.ValueError`
        If either axis (``a`` or ``b``) is negative.

    """

    def __init__(self, positions, a, b, theta):
        try:
            self.a = float(a)
            self.b = float(b)
            self.theta = float(theta)
        except TypeError:
            raise TypeError("'a' and 'b' and 'theta' must be numeric, received"
                            "{0} and {1} and {2}."
                            .format((type(a), type(b), type(theta))))

        if a < 0 or b < 0:
            raise ValueError("'a' and 'b' must be non-negative.")

        if isinstance(positions, u.Quantity):
            positions = positions.value
        if isinstance(positions, (list, tuple, np.ndarray)):
            self.positions = np.atleast_2d(positions)
        else:
            raise TypeError("List or array of (x,y) pixel coordinates is "
                            "expected got '{0}'.".format(positions))

        if self.positions.ndim > 2:
            raise ValueError('{0}-d position array not supported. Only 2-d '
                             'arrays supported.'.format(self.positions.ndim))

    def extent(self):
        r = max(self.a, self.b)
        extents = []
        centers = []
        for x, y in self.positions:
            extents.append((int(x - r + 0.5), int(x + r + 1.5),
                            int(y - r + 0.5), int(y + r + 1.5)))
            centers.append((x, x, y, y))

        self._centers = np.array(centers)
        return np.array(extents)

    def area(self):
        """
        Returns
        -------
        area : float
            Area of aperture.
        """
        return math.pi * self.a * self.b

    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='exact', subpixels=5):
        extents = super(EllipticalAperture, self).get_phot_extents(data)

        if method not in ('center', 'subpixel', 'exact'):
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        flux = do_elliptical_photometry(data, self.positions, extents,
                                        self.a, self.b, self.theta,
                                        error=error, gain=gain,
                                        pixelwise_error=pixelwise_error,
                                        method=method,
                                        subpixels=subpixels)
        return flux

    def plot(self, ax=None, fill=False, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.patches as mpatches
        kwargs['fill'] = fill
        if ax is None:
            ax = plt.gca()
        theta_deg = self.theta * 180. / np.pi
        for position in self.positions:
            patch = mpatches.Ellipse(position, self.a, self.b, theta_deg,
                                     **kwargs)
            ax.add_patch(patch)

class EllipticalAnnulus(Aperture):
    """
    An elliptical annulus aperture.

    Parameters
    ----------
    positions : tuple, or list, or array
        Center coordinates of the apertures as list or array of (x, y)
        pixelcoordinates.
    a_in : float
        The inner semimajor axis.
    a_out : float
        The outer semimajor axis.
    b_out : float
        The outer semiminor axis. (The inner semiminor axis is determined
        by scaling by a_in/a_out.)
    theta : float
        The position angle of the semimajor axis in radians.
        (counterclockwise).

    Raises
    ------
    ValueError : `~.exceptions.ValueError`
        If inner semimajor axis (``a_in``) is greater than outer semimajor
        axis (``a_out``).
    ValueError : `~.exceptions.ValueError`
        If either the inner semimajor axis (``a_in``) or the outer semiminor
        axis (``b_out``) is negative.
    """

    def __init__(self, positions, a_in, a_out, b_out, theta):
        try:
            self.a_in = float(a_in)
            self.a_out = float(a_out)
            self.b_out = float(b_out)
            self.theta = float(theta)
        except TypeError:
            raise TypeError("'a_in' and 'a_out' and 'b_out' and 'theta' must "
                            "be numeric, received {0} and {1} and {2} and {3}."
                            .format((type(a_in), type(a_out),
                                     type(b_out), type(theta))))

        if not (a_out > a_in):
            raise ValueError("'a_out' must be greater than 'a_in'")
        if a_in < 0 or b_out < 0:
            raise ValueError("'a_in' and 'b_out' must be non-negative")

        self.b_in = a_in * b_out / a_out

        if isinstance(positions, u.Quantity):
            positions = positions.value
        if isinstance(positions, (list, tuple, np.ndarray)):
            self.positions = np.atleast_2d(positions)
        else:
            raise TypeError("List or array of (x,y) pixel coordinates is "
                            "expected got '{0}'.".format(positions))

        if self.positions.ndim > 2:
            raise ValueError('{0}-d position array not supported. Only 2-d '
                             'arrays supported.'.format(self.positions.ndim))

    def extent(self):
        r = max(self.a_out, self.b_out)
        extents = []
        centers = []
        for x, y in self.positions:
            extents.append((int(x - r + 0.5), int(x + r + 1.5),
                            int(y - r + 0.5), int(y + r + 1.5)))
            centers.append((x, x, y, y))

        self._centers = np.array(centers)
        return np.array(extents)

    def area(self):
        """
        Returns
        -------
        area : float
            Area of aperture.
        """
        return math.pi * (self.a_out * self.b_out - self.a_in * self.b_in)

    def plot(self, ax=None, fill=False, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.patches as mpatches
        kwargs['fill'] = fill
        if ax is None:
            ax = plt.gca()
        theta_deg = self.theta * 180. / np.pi
        for position in self.positions:
            patch_inner = mpatches.Ellipse(position, self.a_in, self.b_in,
                                           theta_deg, **kwargs)
            patch_outer = mpatches.Ellipse(position, self.a_out, self.b_out,
                                           theta_deg, **kwargs)
            path = _make_annulus_path(patch_inner, patch_outer)
            patch = mpatches.PathPatch(path, **kwargs)
            ax.add_patch(patch)

    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='exact', subpixels=5):
        extents = super(EllipticalAnnulus, self).get_phot_extents(data)

        if method not in ('center', 'subpixel', 'exact'):
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        flux = do_annulus_photometry(data, self.positions, 'elliptical',
                                     extents,
                                     (self.a_in, self.b_in, self.theta),
                                     (self.a_out, self.b_out, self.theta),
                                     error=error, gain=gain,
                                     pixelwise_error=pixelwise_error,
                                     method=method, subpixels=subpixels)
        return flux

class RectangularAperture(Aperture):
    """
    A rectangular aperture.

    Parameters
    ----------
    positions : tuple, or list, or array
        Center coordinates of the apertures as list or array of (x, y)
        pixelcoordinates.
    w : float
        The full width of the aperture (at theta = 0, this is the "x" axis).
    h : float
        The full height of the aperture (at theta = 0, this is the "y" axis).
    theta : float
        The position angle of the semimajor axis in radians
        (counterclockwise).

    Raises
    ------
    ValueError : `~.exceptions.ValueError`
        If either width (``w``) or height (``h``) is negative.
    """

    def __init__(self, positions, w, h, theta):
        try:
            self.w = float(w)
            self.h = float(h)
            self.theta = float(theta)
        except TypeError:
            raise TypeError("'w' and 'h' and 'theta' must "
                            "be numeric, received {0} and {1} and {2}."
                            .format((type(w), type(h), type(theta))))
        if w < 0 or h < 0:
            raise ValueError("'w' and 'h' must be nonnegative.")

        if isinstance(positions, u.Quantity):
            positions = positions.value
        if isinstance(positions, (list, tuple, np.ndarray)):
            self.positions = np.atleast_2d(positions)
        else:
            raise TypeError("List or array of (x,y) pixel coordinates is "
                            "expected got '{0}'.".format(positions))

        if self.positions.ndim > 2:
            raise ValueError('{0}-d position array not supported. Only 2-d '
                             'arrays supported.'.format(self.positions.ndim))

    def extent(self):
        r = max(self.h, self.w) * 2 ** -0.5
        # this is an overestimate by up to sqrt(2) unless theta = 45 deg
        extents = []
        centers = []
        for x, y in self.positions:
            extents.append((int(x - r + 0.5), int(x + r + 1.5),
                            int(y - r + 0.5), int(y + r + 1.5)))
            centers.append((x, x, y, y))

        self._centers = np.array(centers)
        return np.array(extents)

    def area(self):
        """
        Returns
        -------
        area : float
            Area of aperture.
        """
        return self.w * self.h

    def plot(self, ax=None, fill=False, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.patches as mpatches
        kwargs['fill'] = fill
        if ax is None:
            ax = plt.gca()
        hw = self.w / 2.
        hh = self.h / 2.
        sint = math.sin(self.theta)
        cost = math.cos(self.theta)
        dx = (hh * sint) - (hw * cost)
        dy = -(hh * cost) - (hw * sint)
        positions = self.positions + np.array([dx, dy])
        theta_deg = self.theta * 180. / np.pi
        for position in positions:
            patch = mpatches.Rectangle(position, self.w, self.h, theta_deg,
                                       **kwargs)
            ax.add_patch(patch)

    def do_photometry(self, data, error=None, gain=None, pixelwise_error=True,
                      method='subpixel', subpixels=5):

        extents = super(RectangularAperture, self).get_phot_extents(data)

        if method not in ('center', 'subpixel', 'exact'):
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        ood_filter = extents['ood_filter']
        x_min, x_max, y_min, y_max = extents['pixel_extent']
        x_pmin, x_pmax, y_pmin, y_pmax = extents['phot_extent']

        flux = u.Quantity(np.zeros(len(self.positions), dtype=np.float),
                          unit=data.unit)

        # Check for invalid aperture
        if self.w == 0 or self.h == 0:
            return (flux, )

        # TODO: flag these objects
        if np.sum(ood_filter):
            flux[ood_filter] = np.nan
            warnings.warn("The aperture at position {0} does not have any "
                          "overlap with the data"
                          .format(self.positions[ood_filter]),
                          AstropyUserWarning)
            if np.sum(ood_filter) == len(self.positions):
                return (flux, )

        if error is not None:
            fluxvar = u.Quantity(np.zeros(len(self.positions), dtype=np.float),
                                 unit=error.unit ** 2)

        if method in ('center', 'subpixel'):
            if method == 'center': subpixels = 1
            if method == 'subpixel': from imageutils import downsample

            for i in range(len(flux)):
                x_size = ((x_pmax[i] - x_pmin[i]) /
                          (data[:, x_min[i]:x_max[i]].shape[1] * subpixels))
                y_size = ((y_pmax[i] - y_pmin[i]) /
                          (data[y_min[i]:y_max[i], :].shape[0] * subpixels))

                x_centers = np.arange(x_pmin[i] + x_size / 2.,
                                      x_pmax[i], x_size)
                y_centers = np.arange(y_pmin[i] + y_size / 2.,
                                      y_pmax[i], y_size)

                xx, yy = np.meshgrid(x_centers, y_centers)

                newx = (xx * math.cos(self.theta) +
                        yy * math.sin(self.theta))
                newy = (yy * math.cos(self.theta) -
                        xx * math.sin(self.theta))

                halfw = self.w / 2
                halfh = self.h / 2
                in_aper = (((-halfw < newx) & (newx < halfw) &
                            (-halfh < newy) & (newy < halfh)).astype(float)
                           / subpixels ** 2)

                if method == 'center':
                    if not np.isnan(flux[i]):
                        flux[i] = np.sum(data[y_min[i]:y_max[i],
                                              x_min[i]:x_max[i]] * in_aper)
                        if error is not None:
                            if pixelwise_error:
                                subvariance = error[y_min[i]:y_max[i],
                                                    x_min[i]:x_max[i]] ** 2
                                if gain is not None:
                                    subvariance += (data[y_min[i]:y_max[i],
                                                         x_min[i]:x_max[i]] /
                                                    gain[y_min[i]:y_max[i],
                                                         x_min[i]:x_max[i]])
                                # Make sure variance is > 0
                                fluxvar[i] = max(np.sum(subvariance * in_aper), 0)
                            else:
                                local_error = error[int((y_min[i] + y_max[i]) / 2 + 0.5),
                                                    int((x_min[i] + x_max[i]) / 2 + 0.5)]
                                fluxvar[i] = max(local_error ** 2 * np.sum(in_aper), 0)
                                if gain is not None:
                                    local_gain = gain[int((y_min[i] + y_max[i]) / 2 + 0.5),
                                                      int((x_min[i] + x_max[i]) / 2 + 0.5)]
                                    fluxvar[i] += flux[i] / local_gain
                else:
                    if not np.isnan(flux[i]):
                        if error is None:
                            flux[i] = np.sum(data[y_min[i]:y_max[i],
                                                  x_min[i]:x_max[i]] *
                                             downsample(in_aper, subpixels))
                        else:
                            fraction = downsample(in_aper, subpixels)
                            flux[i] = np.sum(data[y_min[i]:y_max[i],
                                                  x_min[i]:x_max[i]] * fraction)

                            if pixelwise_error:
                                subvariance = error[y_min[i]:y_max[i],
                                                    x_min[i]:x_max[i]] ** 2
                                if gain is not None:
                                    subvariance += (data[y_min[i]:y_max[i],
                                                         x_min[i]:x_max[i]] /
                                                    gain[y_min[i]:y_max[i],
                                                         x_min[i]:x_max[i]])
                                # Make sure variance is > 0
                                fluxvar[i] = max(np.sum(subvariance * fraction), 0)
                            else:
                                local_error = error[int((y_min[i] + y_max[i]) / 2 + 0.5),
                                                    int((x_min[i] + x_max[i]) / 2 + 0.5)]
                                fluxvar[i] = max(local_error ** 2 * np.sum(fraction), 0)
                                if gain is not None:
                                    local_gain = gain[int((y_min[i] + y_max[i]) / 2 + 0.5),
                                                      int((x_min[i] + x_max[i]) / 2 + 0.5)]
                                    fluxvar[i] += flux[i] / local_gain

        elif method == 'exact':
            raise NotImplementedError("'exact' method not yet supported for "
                                      "RectangularAperture")
        else:
            raise ValueError('{0} method not supported for aperture class '
                             '{1}'.format(method, self.__class__.__name__))

        if error is None:
            return (flux, )
        else:
            return (flux, np.sqrt(fluxvar))

def aperture_photometry(data, positions, apertures, unit=None, wcs=None,
                        error=None, gain=None, mask=None, method='exact',
                        subpixels=5, pixelcoord=True, pixelwise_error=True,
                        mask_method='skip'):
    """
    Sum flux within an aperture at the given position(s).

    Parameters
    ----------
    data : array_like, `~astropy.io.fits.ImageHDU`, `~astropy.io.fits.HDUList`
        The 2-d array on which to perform photometry. Units are used during
        the photometry, either provided along with the data array, or stored
        in the header keyword ``'BUNIT'``.
    positions : list, tuple, nd.array or `~astropy.coordinates.SkyCoord`
        Positions of the aperture centers, either in pixel or sky
        coordinates. If positions is `~astropy.coordinates.SkyCoord` or
        ``pixelcoord`` is `False` a wcs transformation is also needed to
        convert the input positions to pixel positions. If ``positions`` are
        sky positions but not an `~astropy.coordinates.SkyCoord` object, it
        need to be in the same celestial frame as the wcs transformation.
    apertures : tuple
        First element of the tuple is the mode, the currently supported ones
        are: ``'circular'``, ``'elliptical'``, ``'circular_annulus'``,
        ``'elliptical_annulus'``, ``'rectangular'``. The remaining (1 to 4)
        elements are the parameters for the given mode. Check the
        documentation of the relevant ``Aperture`` classes for more
        information.
    unit : `~astropy.units.UnitBase` instance, str
        An object that represents the unit associated with ``data``.  Must
        be an `~astropy.units.UnitBase` object or a string parseable by the
        :mod:`~astropy.units` package. An error is raised if ``data``
        already has a different unit.
    wcs : `~astropy.wcs.WCS`, optional
        Use this as the wcs transformation when either ``pixelcoord`` is
        `False` or ``positions`` is `~astropy.coordinates.SkyCoord`. It
        overrides any wcs transformation passed along with ``data`` either
        in the header or in an attribute.
    error : float or array_like, optional
        Error in each pixel, interpreted as Gaussian 1-sigma uncertainty.
    gain : float or array_like, optional
        Ratio of counts (e.g., electrons or photons) to units of the data
        (e.g., ADU), for the purpose of calculating Poisson error from the
        object itself. If ``gain`` is `None` (default), ``error`` is assumed to
        include all uncertainty in each pixel. If ``gain`` is given, ``error``
        is assumed to be the "background error" only (not accounting for
        Poisson error in the flux in the apertures).
    mask : array_like (bool), optional
        Mask to apply to the data.
    method : str, optional
        Method to use for determining overlap between the aperture and pixels.
        Options include ['center', 'subpixel', 'exact'], but not all options
        are available for all types of apertures. More precise methods will
        generally be slower.

        * ``'center'``
            A pixel is considered to be entirely in or out of the aperture
            depending on whether its center is in or out of the aperture.
        * ``'subpixel'``
            A pixel is divided into subpixels and the center of each
            subpixel is tested (as above). With ``subpixels`` set to 1, this
            method is equivalent to 'center'. Note that for subpixel
            sampling, the input array is only resampled once for each
            object.
        * ``'exact'`` (default)
            The exact overlap between the aperture and each pixel is
            calculated.
    subpixels : int, optional
        For the ``'subpixel'`` method, resample pixels by this factor (in
        each dimension). That is, each pixel is divided into
        ``subpixels ** 2`` subpixels.
    pixelcoord : bool, optional
        If `True` (default), assume ``positions`` are pixel positions. If
        `False`, assume the input positions are sky coordinates and uses the
        wcs transformation (provided either via ``wcs`` or along with
        ``data``) to convert them to pixel positions.
    pixelwise_error : bool, optional
        For error and/or gain arrays. If `True`, assume error and/or gain
        vary significantly within an aperture: sum contribution from each
        pixel. If `False`, assume error and gain do not vary significantly
        within an aperture. Use the single value of error and/or gain at
        the center of each aperture as the value for the entire aperture.
        Default is `True`.
    mask_method : str, optional
        Method to treat masked pixels. Currently supported methods:

        * ``'skip'``
            Leave out the masked pixels from all calculations.
        * ``'interpolation'``
            The value of the masked pixels are replaced by the mean value of
            the neighbouring non-masked pixels.

    Returns
    -------
    phot_table : `~astropy.table.Table`
        A table of the photometry with the following columns:

        * ``'aperture_sum'``: Sum of the values within the aperture.
        * ``'aperture_sum_err'``: Corresponding uncertainty in
          ``'aperture_sum'`` values.  Returned only if input ``error`` is not
          `None`.
        * ``'pixel_center'``: pixel coordinate pairs of the center of the
          apertures. Unit is pixel.
        * ``'input_center'``: input coordinate pairs as they were given in the
          ``positions`` parameter.

        The metadata of the table stores the version numbers of both astropy
        and photutils, as well as the calling arguments.
    aux_dict : dict
        Auxilary dictionary storing all the auxilary information
        available. The element are the following:

        * ``'apertures'``
            The `~photutils.Aperture` object containing the apertures to use
            during the photometry.

    """
    dataunit = None
    datamask = None
    wcs_transformation = wcs

    if isinstance(data, (fits.PrimaryHDU, fits.ImageHDU)):
        header = data.header
        data = data.data

        if 'BUNIT' in header:
            dataunit = header['BUNIT']

        # TODO check how a mask can be stored in the header, it seems like
        # full pixel masks are not supported by the FITS standard, look for
        # real life examples (e.g. header value stores the fits number of
        # fits extension where the pixelmask is stored?)
        if 'MASK' in header:
            datamask = header.mask

    elif isinstance(data, fits.HDUList):
        # TODO: do it in a 2d array, and thus get the light curves as a
        # side-product? Although it's not usual to store time series as
        # HDUList

        for i in range(len(data)):
            if data[i].data is not None:
                warnings.warn("Input data is a HDUList object, photometry is "
                              "only run for the {0}. HDU."
                              .format(i), AstropyUserWarning)
                return aperture_photometry(data[i], positions, apertures, unit,
                                           wcs, error, gain, mask, method,
                                           subpixels, pixelcoord,
                                           pixelwise_error, mask_method)

    # this is basically for NDData inputs and alike
    elif hasattr(data, 'data') and not isinstance(data, np.ndarray):
        if data.wcs is not None and wcs_transformation is None:
            wcs_transformation = data.wcs
        datamask = data.mask

    if hasattr(data, 'unit'):
        dataunit = data.unit

    if unit is not None and dataunit is not None:
        if unit != dataunit:
            raise u.UnitsError('Unit of input data ({0}) and unit given by '
                               'unit argument ({1}) are not identical.'.
                               format(dataunit, unit))
        data = u.Quantity(data, unit=dataunit, copy=False)
    elif unit is None:
        if dataunit is not None:
            data = u.Quantity(data, unit=dataunit, copy=False)
        else:
            data = u.Quantity(data, copy=False)
    else:
        data = u.Quantity(data, unit=unit, copy=False)

    if datamask is None:
        data.mask = datamask

    # Check input array type and dimension.
    if np.iscomplexobj(data):
        raise TypeError('Complex type not supported')
    if data.ndim != 2:
        raise ValueError('{0}-d array not supported. '
                         'Only 2-d arrays supported.'.format(data.ndim))

    # Deal with the mask if it exist
    if mask is not None or datamask is not None:
        if mask is None:
            mask = datamask
        else:
            mask = np.asarray(mask)
            if np.iscomplexobj(mask):
                raise TypeError('Complex type not supported')
            if mask.ndim != 2:
                raise ValueError('{0}-d array not supported. '
                                 'Only 2-d arrays supported.'
                                 .format(mask.ndim))
            if mask.shape != data.shape:
                raise ValueError('Shapes of mask array and data array '
                                 'must match')

            if datamask is not None:
                mask *= datamask

        if mask_method == 'skip':
            data *= ~mask

        if mask_method == 'interpolation':
            for i, j in zip(*np.nonzero(mask)):
                y0, y1 = max(i - 1, 0), min(i + 2, data.shape[0])
                x0, x1 = max(j - 1, 0), min(j + 2, data.shape[1])
                data[i, j] = np.mean(data[y0:y1, x0:x1][~mask[y0:y1, x0:x1]])

    # Check whether we really need to calculate pixelwise errors, even if
    # requested. (If neither error nor gain is an array, we don't need to.)
    if ((error is None) or
        (np.isscalar(error) and gain is None) or
        (np.isscalar(error) and np.isscalar(gain))):
        pixelwise_error = False

    # Check error shape.
    if error is not None:
        if isinstance(error, u.Quantity):
            if np.isscalar(error.value):
                error = u.Quantity(np.broadcast_arrays(error, data),
                                   unit=error.unit)[0]
        elif np.isscalar(error):
            error = u.Quantity(np.broadcast_arrays(error, data),
                               unit=data.unit)[0]
        else:
            error = u.Quantity(error, unit=data.unit, copy=False)

        if error.shape != data.shape:
            raise ValueError('shapes of error array and data array must'
                             ' match')

    # Check gain shape.
    if gain is not None:
        # Gain doesn't do anything without error set, so raise an exception.
        # (TODO: instead, should we just set gain = None and ignore it?)
        if error is None:
            raise ValueError('gain requires error')

        if isinstance(gain, u.Quantity):
            if np.isscalar(gain.value):
                gain = u.Quantity(np.broadcast_arrays(gain, data),
                                  unit=gain.unit)[0]

        elif np.isscalar(gain):
            gain = np.broadcast_arrays(gain, data)[0]
        if gain.shape != data.shape:
            raise ValueError('shapes of gain array and data array must match')

    # Check that 'subpixels' is an int and is 1 or greater.
    if method == 'subpixel':
        subpixels = int(subpixels)
        if subpixels < 1:
            raise ValueError('subpixels: an integer greater than 0 is '
                             'required')

    if not pixelcoord or isinstance(positions, SkyCoord):

        from astropy.wcs import wcs

        try:
            from astropy.wcs.utils import wcs_to_celestial_frame
        except ImportError:  # Astropy < 1.0
            from .extern.wcs_utils import wcs_to_celestial_frame

        if wcs_transformation is None:
            wcs_transformation = wcs.WCS(header)

        # TODO this should be simplified once wcs_world2pix() supports
        # SkyCoord objects as input
        if isinstance(positions, SkyCoord):
            # Check which frame the wcs uses
            framename = wcs_to_celestial_frame(wcs_transformation).name
            frame = getattr(positions, framename)
            component_names = list(frame.representation_component_names.keys())[0:2]
            if len(positions.shape) > 0:
                positions_repr = u.Quantity(zip(getattr(frame,
                                                        component_names[0]).deg,
                                                getattr(frame,
                                                        component_names[1]).deg),
                                            unit=u.deg)
            else:
                positions_repr = (u.Quantity((getattr(frame,
                                                      component_names[0]).deg,
                                              getattr(frame,
                                                      component_names[1]).deg),
                                             unit=u.deg), )
        elif not isinstance(positions, u.Quantity):
            # TODO figure out the unit of the input positions for this case
            # TODO revise this once wcs_world2pix() accepts more input formats
            if len(positions) > 1 and not isinstance(positions, tuple):
                positions_repr = u.Quantity(positions, copy=False)
            else:
                positions_repr = (u.Quantity(positions, copy=False), )
        pixelpositions = u.Quantity(wcs_transformation.wcs_world2pix
                                    (positions_repr, 0), unit=u.pixel,
                                    copy=False)
    else:
        positions = u.Quantity(positions, unit=u.pixel, copy=False)
        pixelpositions = positions

    if apertures[0] == 'circular':
        ap = CircularAperture(pixelpositions.value, apertures[1])
    elif apertures[0] == 'circular_annulus':
        ap = CircularAnnulus(pixelpositions.value, *apertures[1:3])
    elif apertures[0] == 'elliptical':
        ap = EllipticalAperture(pixelpositions.value, *apertures[1:4])
    elif apertures[0] == 'elliptical_annulus':
        ap = EllipticalAnnulus(pixelpositions.value, *apertures[1:5])
    elif apertures[0] == 'rectangular':
        if method == 'exact':
            warnings.warn("'exact' method is not implemented, defaults to "
                          "'subpixel' instead", AstropyUserWarning)
            method = 'subpixel'
        ap = RectangularAperture(pixelpositions.value, *apertures[1:4])

    # Prepare version return data
    from astropy import __version__
    astropy_version = __version__

    from photutils import __version__
    photutils_version = __version__

    photometry_result = ap.do_photometry(data, method=method,
                                         subpixels=subpixels,
                                         error=error, gain=gain,
                                         pixelwise_error=pixelwise_error)
    if error is None:
        phot_col_names = ('aperture_sum', )
    else:
        phot_col_names = ('aperture_sum', 'aperture_sum_err')

    # Note: if wcs_transformation is None, 'pixel_center' will be the same
    # as 'input_center'

    # check whether single or multiple positions
    if len(pixelpositions) > 1 and pixelpositions[0].size >= 2:
        coord_columns = (pixelpositions, positions)
    else:
        coord_columns = ((pixelpositions,), (positions,))

    coord_col_names = ('pixel_center', 'input_center')

    return (Table(data=(photometry_result + coord_columns),
                  names=(phot_col_names + coord_col_names),
                  meta={'name': 'Aperture photometry results',
                        'version': 'astropy: {0}, photutils: {1}'
                        .format(astropy_version, photutils_version),
                        'calling_args': ('method={0}, subpixels={1}, '
                                         'error={2}, gain={3}, '
                                         'pixelwise_error={4}')
                        .format(method, subpixels, error is not None,
                                gain is not None, pixelwise_error)}),
            {'apertures': ap})

import numpy,import astropy,from astropy


import pyaudio
import numpy as np
import librosa

pyau = pyaudio.PyAudio()
CHUNK = 1024 * 2
RATE = 44100

stream = pyau.open(
    format = pyaudio.paInt16,
    channels = 1,
    rate = RATE,
    frames_per_buffer = CHUNK,
    input = True,
    output = True)

while stream.is_active():
    # input audio
    au_in = stream.read(CHUNK)
    # convert wave
    au_in_wave = np.frombuffer(au_in, dtype = "int16") / float(2**15)
    # process
    pitch, temp = librosa.beat.beat_track(y = au_in_wave, sr = RATE)
    print(pitch)
    # output audio
    au_out = stream.write(au_in)

stream.stop_stream()
stream.close()
pyau.terminate()

print("stop")


import numpy as np
import matplotlib.pyplot as plt
from collections import deque
import pyaudio
from scipy import signal
from my_midgets.midget import Midget

plt.ion()

class BarkMidget(Midget):
    FORMAT = pyaudio.paInt16
    CHANNELS = 2
    RATE = 1000
    CHUNK = RATE//5
    FRAME_SIZE = 5
    DOWNSAMPLE_FACTOR = 4

    def __init__(self):
        super().__init__()
        self.pyaudio = pyaudio.PyAudio()
        self.stream = self.pyaudio.open(format=self.FORMAT,
                                        channels=self.CHANNELS,
                                        rate=self.RATE,
                                        input=True,
                                        frames_per_buffer=self.CHUNK)

        self.frames = deque(maxlen=(self.RATE * self.FRAME_SIZE))

    def listen(self):
        num_peaks = 0
        while True:
            raw_data = self.stream.read(self.CHUNK)
            data = np.fromstring(raw_data, dtype=np.int16)
            data = signal.decimate(data, self.DOWNSAMPLE_FACTOR)
            num_peaks += len(signal.find_peaks(data, threshold=20000))
            self.frames.extend(list(data))

    def run(self):
        self.listen()


"""
Copyright 2020 The OneFlow Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import oneflow as flow
import oneflow.typing as tp
from tsn_model import restsn
import argparse
import time
import mmcv
import os.path as osp
import os
import sys
import tempfile
import numpy as np
from collections import Counter
import warnings
warnings.filterwarnings("ignore", category=UserWarning)
from video_dataset import *
import shutil

def parse_args():
    parser = argparse.ArgumentParser(description='Test an action recognizer')
    parser.add_argument('--launcher',
        choices=['none', 'pytorch', 'mpi', 'slurm'],
        default='pytorch',
        help='job launcher')
    parser.add_argument('--out', help='output result file', default='default.pkl')
    parser.add_argument('--num_classes', help='number of class', type=int, default=400)
    parser.add_argument('--use_softmax', action='store_true',
                        help='whether to use softmax score')
    parser.add_argument("--train_num_segments", type=int, default=3, required=False)
    parser.add_argument("--epoch", type=int, default=100, required=False)

    # for oneflow
    parser.add_argument("--gpu_num_per_node", type=int, default=1, required=False)
    parser.add_argument("--lr", type=int, default=0.1, required=False)
    parser.add_argument("--use_tensorrt", dest="use_tensorrt", action="store_true", default=False, required=False, help="inference with tensorrt")
    parser.add_argument("--model_load_dir", type=str, default='/home/zjlab/liuxy/mmaction/modelzoo/model_pcb', required=False, help="model load directory")
    parser.add_argument("--log_dir", type=str, default="./output", required=False, help="log info save directory")
    parser.add_argument("--image_height", type=int, default=224, required=False)
    parser.add_argument("--image_width", type=int, default=224, required=False)
    parser.add_argument("--train_batch_size", type=int, default=8, required=False)
    parser.add_argument("--out_dir", type=str, default="output/save_model/", required=False)

    args = parser.parse_args()
    return args

# train config
args = parse_args()
func_config = flow.function_config()
func_config.default_data_type(flow.float)
flow.config.gpu_device_num(args.gpu_num_per_node)

if args.use_tensorrt:
        func_config.use_tensorrt(True)

@flow.global_function('train',func_config)
def train_tsn(image:tp.Numpy.Placeholder((args.train_batch_size*args.train_num_segments,3,224,224)),
              label:tp.Numpy.Placeholder((args.train_batch_size,400))):
    features = restsn(image, args.train_batch_size, trainable=True)

    loss = flow.nn.softmax_cross_entropy_with_logits(label, features, name="loss_liu")
    # loss = flow.nn.sparse_softmax_cross_entropy_with_logits(label, features, name="loss_liu")
    # set learning rate as 0.1
    lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [0.00001])
    # Set SGD optimizer, weight_decay factor is 0.00001
    gradient_clip = flow.optimizer.grad_clipping.by_global_norm(40.0)
    flow.optimizer.AdamW(lr_scheduler,
            weight_decay=0.0001,grad_clipping=gradient_clip).minimize(loss)
    # flow.optimizer.Adam(lr_scheduler,
    #     grad_clipping=gradient_clip).minimize(loss)
    return loss

@flow.global_function('predict')
def val_tsn(image:tp.Numpy.Placeholder((250,3,224,224))):
    output= restsn(image, 1, trainable=False)
    # loss = flow.nn.softmax_cross_entropy_with_logits(label, output, name="loss_liu")
    return output
check_point = flow.train.CheckPoint()
check_point.load(args.model_load_dir)
class TSNTrain(object):
    def __init__(self):
        train_mode = True
    def train(self, imgs, labels):
        img_group = np.ascontiguousarray(imgs)
        bs = img_group.shape[0]
        img_group = img_group.reshape(
            (-1, 3) + img_group.shape[3:])
        num_seg = img_group.shape[0] // bs
        one_hot_labels = (np.arange(args.num_classes)==labels[:,None]).astype(np.float32)
        loss = train_tsn(img_group, one_hot_labels).get()

        return loss.numpy()

def multi_train():
    global args
    count = 0
    # train VideoDataset config
    ann_file = "data/kinetics400/kinetics400_train_list_videos.txt"
    img_prefix = "/data/liuxy/videos_train_big"
    img_norm_cfg = {'mean': [123.675, 116.28, 103.53], 'std': [58.395, 57.12, 57.375], 'to_rgb': True}
    anno_open = open(ann_file, 'r')
    anno_len = len(anno_open.readlines())
    anno_open.close()
    oneflow_dataset = VideoDataset(ann_file,
        img_prefix,
        img_norm_cfg,
        num_segments=args.train_num_segments,
        new_length=1,
        new_step=1,
        random_shift=True,
        modality='RGB',
        image_tmpl='img_{:05d}.jpg',
        img_scale=256,
        input_size=224,
        div_255=False,
        flip_ratio=0.5,
        resize_keep_ratio=True,
        oversample=None,
        random_crop=False,
        more_fix_crop=False,
        multiscale_crop=True,
        scales=[1, 0.875, 0.75, 0.66],
        max_distort=1,
        test_mode=False)

    # val VideoDataset config
    ann_file_val = "data/kinetics400/kinetics400_val_list_videos.txt"
    img_prefix_val = "/data/liuxy/videos_val_mp4"
    anno_open_val = open(ann_file_val, 'r')
    anno_len_val = len(anno_open_val.readlines())
    anno_open_val.close()
    oneflow_dataset_val = VideoDataset(ann_file_val, img_prefix_val, img_norm_cfg)

    flow.env.grpc_use_no_signal()
    flow.env.log_dir(args.log_dir)

    obj = TSNTrain()
    val_prec = 0

    for j in range(args.epoch):
        wrong_count = 0
        iter_num = anno_len//args.train_batch_size

        for i in range(iter_num):
            img_group = []
            labels = []
            for k in range(args.train_batch_size):
                try:
                    img, label = oneflow_dataset[args.train_batch_size*i+k]
                    img_group.append(img)
                    labels.append(label)
                except:
                    print(args.train_batch_size*i+k)
                    break
            img_group = np.array(img_group)
            labels = np.array(labels)
            loss = obj.train(img_group, labels)

            if i % 20 == 0:
                print("Epoch: {}\t iter: [{}/{}]\t loss: {}\t".format(j, i, iter_num, loss.mean()))
        # val after each epoch
        for i in range(anno_len_val):
            img_group, label = oneflow_dataset_val[i]
            array = np.ascontiguousarray(img_group)
            feature = val_tsn(array).get()
            flow_result = np.argmax(feature.numpy().flatten())
            if label!=flow_result:
                wrong_count = wrong_count +1
            count = count + 1
        final_precision = float(anno_len_val - wrong_count)/anno_len_val
        print("val precision is: {}".format(final_precision))

        if final_precision > val_prec:
            val_prec = final_precision
            shutil.rmtree(args.out_dir)
            os.mkdir(args.out_dir)
            check_point.save(args.out_dir)
            print("saving model...")

def main():
    global args
    args = parse_args()

    if args.out is not None and not args.out.endswith(('.pkl', '.pickle')):
        raise ValueError('The output file must be a pkl file.')

    multi_train()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Process bounding boxes on multiple images analyzed using gigatest_folder.py.
Saves resulting data in gigafolder_bb_reults.pkl and images in processed_images
folder.

Part of gigadetector repo:
https://github.com/EricThomson/gigadetector
"""
import os
import sys
import cv2
import numpy as np
import joblib
import logging
logging.basicConfig(level = logging.DEBUG)

base_path = os.path.expanduser("~") + r"/gigadetector/"
sys.path.append(base_path + r'/gigadetector/')
import utils

#%% set parameters
max_num_objects =100# will winnow down to this
conf_threshold = 0.9  #filter out boxes below this
nms_threshold = 0.5    # with overlap equal to/greater than this, include it for nms
overlap_threshold = 90 #overlap filter (for those nms doesn't get...overlap to the rescue)
area_threshold = 12_000
debug = 0  #display images (not really useful for full folder)
save = 2  #0 do not, 1: save bboxes etc, 2: also save images with bboxes drawn on them
bb_color = (255, 255, 255)

#%% set paths and initialize vars
od_filename = r'gigafolder_od_results.pkl'
bb_filename = r'gigafolder_bb_results.pkl'
analysis_path = base_path + r'data/processed/'
bb_results_filepath = analysis_path + bb_filename
od_results_filepath = analysis_path + od_filename

# load results from gigatest_folder run
od_data_all = joblib.load(od_results_filepath)
num_images = len(od_data_all)
print(f"There is data from {num_images} images")

processed_images_path = analysis_path + 'processed_images/'
if os.path.isdir(processed_images_path):
    pass
else:
    os.mkdir(processed_images_path)

all_bboxes = []
all_scores = []
all_areas = []
all_filepaths = []

#%% pull bbs
for current_ind, current_data in enumerate(od_data_all):
    print(f"{current_ind+1}/{num_images}")

    image_path = current_data['fname']
    image_name = os.path.splitext(os.path.basename(image_path))[0]
    image_save_path = processed_images_path + image_name + r'_processed.png'

    print(f"\tWorking on {image_name}")
    bboxes = current_data['bboxes']
    scores = current_data['scores']
    rois = current_data['rois']
    num_rois = len(rois)

    #% Filter out empty boxes yielding *_initial estimates
    bboxes_initial, scores_initial = utils.bb_filter_initial(bboxes, scores)
    num_initial = len(scores_initial)
    bboxes_initial_nms = utils.bbox_std2ocv(bboxes_initial)

    #% Filter using non-max suppression yielding bboxes_nms_filtered
    box_inds_nms = list(cv2.dnn.NMSBoxes(bboxes_initial_nms, scores_initial, conf_threshold, nms_threshold).reshape(-1,))
    bboxes_nms_filtered = [bboxes_initial[ind] for ind in box_inds_nms]
    scores_nms_filtered = [scores_initial[ind] for ind in box_inds_nms]
    num_nms_filtered = len(scores_nms_filtered)

    #% Get bbox areas (will save these for later so do for all initial boxes)
    areas_nms_filtered = []
    for bbox in bboxes_nms_filtered:
        areas_nms_filtered.append(utils.bb_area(np.asarray(bbox)))

    #%  Filter out the little guys
    bboxes_area_filtered = bboxes_nms_filtered
    scores_area_filtered = scores_nms_filtered
    areas_area_filtered = areas_nms_filtered
    for ind, area in enumerate(areas_nms_filtered):
        if area < area_threshold:
            bboxes_area_filtered = np.delete(bboxes_area_filtered, ind, axis = 0)
            scores_area_filtered = np.delete(scores_area_filtered, ind)
            areas_area_filtered = np.delete(bboxes_area_filtered, ind)
    num_area_filtered = len(scores_area_filtered)

    #% Percent overlap filter
    overlaps = utils.pairwise_overlap(bboxes_area_filtered)
    inds_to_remove = utils.overlap_suppression(overlaps, overlap_threshold)
    bboxes_overlap_filtered = bboxes_area_filtered
    scores_overlap_filtered = scores_area_filtered
    areas_overlap_filtered = areas_area_filtered
    if inds_to_remove:
        bboxes_overlap_filtered = np.delete(bboxes_overlap_filtered, inds_to_remove, axis = 0)
        scores_overlap_filtered = np.delete(scores_overlap_filtered, inds_to_remove)
        areas_overlap_filtered = np.delete(areas_overlap_filtered, inds_to_remove)

    #That's all she wrote: add these go the list of boxes, scores, areas
    all_bboxes.append(bboxes_overlap_filtered)
    all_scores.append(scores_overlap_filtered)
    all_areas.append(areas_overlap_filtered)
    all_filepaths.append(image_path)

    #% If image save ==2, draw final bbox_overlap filtered boxes into the image and save that shit
    #% save image/data
    if save == 2:
        image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
        image = utils.put_bboxes_scores(image,
                         bboxes_overlap_filtered,
                         scores_overlap_filtered,
                         bb_color = (255, 255, 255),
                         line_width = 10,
                         text_thickness = 3)

        save_fname = processed_images_path + image_name + '_bbs.png'
        print(f"\tSaving to {save_fname}")
        cv2.imwrite(save_fname, image, [cv2.IMWRITE_PNG_COMPRESSION, 2])

        # To see it -- this is really for debugging not testing in full loop (yet)
        if debug:
            utils.cv_imshow(image, name = "filtered", xy = (2300, 50), shape = (1000, 1000))

#%%
if save:
    full_bb_data = {'all_bboxes': all_bboxes,
                 'all_scores': all_scores,
                 'all_areas': all_areas,
                 'all_filepaths': all_filepaths}
    with open(bb_results_filepath, 'wb') as fp:
        joblib.dump(full_bb_data, fp)
        print(f"DONE!!!\nBox/score/area data/path saved to\n{bb_results_filepath}")


import numpy as np
from PIL import ImageGrab, Image
import cv2
import json

while True:
    coords = json.loads(open('../coords.json').read())

    fullscreen_coords = coords['fullscreen']
    full_img = np.array(ImageGrab.grab(bbox=(fullscreen_coords['x-off'],fullscreen_coords['y-off'],fullscreen_coords['width'],fullscreen_coords['height'])))
    cv2.imshow('FullImageCanvas', full_img)

    question_game_image = np.array(ImageGrab.grab(bbox=(fullscreen_coords['x-off'],fullscreen_coords['y-off']+5,fullscreen_coords['width'],420)))
    cv2.imshow('InGameCanvas', question_game_image)

    question_loading_image = np.array(ImageGrab.grab(bbox=(fullscreen_coords['x-off'],fullscreen_coords['y-off']+200,fullscreen_coords['width'],645)))
    cv2.imshow('LoadingCanvas', question_loading_image)

    '''
    #VIEW BUTTONS UI
    top_left = coords['answer_top_left']
    top_right = coords['answer_top_right']
    bottom_right = coords['answer_bottom_right']
    bottom_left = coords['answer_bottom_left']

    fiftyfifty_left = coords['answer_5050_left']
    fiftyfifty_right = coords['answer_5050_right']

    top_left_btn = np.array(ImageGrab.grab(bbox=(top_left['x-off'],top_left['y-off'],top_left['width'],top_left['height'])))
    top_right_btn = np.array(ImageGrab.grab(bbox=(top_right['x-off'],top_right['y-off'],top_right['width'],top_right['height'])))
    bottom_left_btn = np.array(ImageGrab.grab(bbox=(bottom_left['x-off'],bottom_left['y-off'],bottom_left['width'],bottom_left['height'])))
    bottom_right_btn = np.array(ImageGrab.grab(bbox=(bottom_right['x-off'],bottom_right['y-off'],bottom_right['width'],bottom_right['height'])))

    fiftyfifty_left_btn = np.array(ImageGrab.grab(bbox=(fiftyfifty_left['x-off'],fiftyfifty_left['y-off'],fiftyfifty_left['width'],fiftyfifty_left['height'])))

    fiftyfifty_right_btn = np.array(ImageGrab.grab(bbox=(fiftyfifty_right['x-off'],fiftyfifty_right['y-off'],fiftyfifty_right['width'],fiftyfifty_right['height'])))

    #cv2.imshow('top_left', top_left_btn)
    #cv2.imshow('top_right', top_right_btn)
    #cv2.imshow('bottom_right', bottom_right_btn)
    #cv2.imshow('bottom_left', bottom_left_btn)

    cv2.imshow('bottom_right',  fiftyfifty_left_btn)
    cv2.imshow('bottom_left',  fiftyfifty_right_btn)
    '''

    if cv2.waitKey(25) & 0xFF == ord('q'):
            cv2.destroyAllWindows()
            break


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
websocket wrap shared library
"""
import argparse
import time
import websockets
import numpy as np
from PIL import Image

class WebsocketWrap:
    """
    websocket sending class
    """
    tiffsend = True
    duration = 0.166
    pixel_dwell_time = 1
    count = 4
    frequency = 60
    savefile = "/tmp/work/websocket.bin"
    input_file = ""

    def __init__(self):
        parser = argparse.ArgumentParser(
            description="tiff file websocket sender")
        parser.add_argument("--port", type=int, required=True)
        parser.add_argument("--savefile", type=str, default=self.savefile)
        parser.add_argument("--input-file", type=str, default=self.input_file)
        parser.add_argument("--count", type=int, default=self.count)
        parser.add_argument("--frequency",
                            type=int,
                            default=self.frequency,
                            help="video rate: 60")
        parser.add_argument("--pixel-dwell-time",
                            type=float,
                            default=self.pixel_dwell_time,
                            help="unit us")
        parsed = parser.parse_args()
        print(repr(parsed))
        self.port = parsed.port
        self.savefile = parsed.savefile
        self.input_file = parsed.input_file
        self.count = parsed.count
        self.frequency = parsed.frequency
        self.pixel_dwell_time = parsed.pixel_dwell_time

    def start_server(self):
        """
        return server object
        """
        return websockets.serve(self.send_img_data, "localhost", self.port)

    def y_coord(self, size, websocket_i) -> list:
        """
        return y_coord of the websocket packet whose index is websocket_i

        Parameters
        ----------
        size: tuple
            (width, height)
        websocket_i : int
            websocket index

        Returns
        -------
        y_coords : list
            [y_begin1, y_end1, y_begin2, y_end2, z_current_index]
        """
        (width, height) = size

        # pixel num per message
        pixel_num = int((1000000 / self.pixel_dwell_time) / self.frequency)
        y_begin_accm = websocket_i * pixel_num // width
        y_end_accm = (websocket_i + 1) * pixel_num // width
        y_begin_mod = y_begin_accm % height
        y_end_mod = y_end_accm % height
        z_current_index = 0
        z_previous_index = -1
        if websocket_i > 0:
            z_current_index = websocket_i * pixel_num // (width * height)
            z_previous_index = (websocket_i - 1) * pixel_num // (width *
                                                                 height)
        new_z = 0
        if z_previous_index < z_current_index:
            new_z = 1

        if pixel_num >= width * height or (height == 1 and new_z == 1):
            return [0, height, 0, 0, z_current_index]
        if y_begin_mod > y_end_mod:
            return [y_begin_mod, height, 0, y_end_mod, z_current_index]
        return [y_begin_mod, y_end_mod, -1, -1, z_current_index]

    # pylint: disable=too-many-locals
    async def send_img_data(self, websocket, _):
        """
        main function send image data
        """
        tiff_img = Image.open(self.input_file)

        tiff_img.seek(0)
        (width, height) = tiff_img.size
        n_frames = tiff_img.n_frames
        recvtxt = await websocket.recv()
        print(recvtxt)
        # while True:
        f_i = -1
        ui16arr = np.array([])
        for w_i in range(self.count):
            time_start = time.perf_counter()
            y_coord = self.y_coord((width, height), w_i)
            imgbytes = bytes(np.array(y_coord[:4], dtype=np.uint16))
            imgbytes += bytes(np.array(y_coord[4:5], dtype=np.uint32))
            print(y_coord)
            f_i = y_coord[4] % n_frames
            tiff_img.seek(f_i)
            ui16arr = np.asarray(tiff_img, dtype=np.uint16)
            imgbytes += bytes(ui16arr[y_coord[0]:y_coord[1], :])
            if y_coord[2] != -1:
                f_i = (f_i + 1) % n_frames
                tiff_img.seek(f_i)
                ui16arr2 = np.asarray(tiff_img, dtype=np.uint16)
                imgbytes2 = bytes(ui16arr2[y_coord[2]:y_coord[3], :])
                print(ui16arr2[y_coord[2]])
                imgbytes += imgbytes2
            print(len(imgbytes))
            await websocket.send(imgbytes)
            time_end = time.perf_counter()
            print(time_end - time_start)
            time.sleep(max(self.duration - time_end + time_start, 0))
        imgbytes = bytes(np.asarray([0, 0, -1], dtype=np.int32))
        await websocket.send(imgbytes)

    async def recv_img_data(self):
        """
        main function receive image data
        """
        uri = f"ws://localhost:{self.port}"
        async with websockets.connect(uri) as websocket:
            await websocket.send("Hello World")
            with open(f"{self.savefile}", "wb") as f:
                for i in range(self.count):
                    print(i)
                    data = await websocket.recv()
                    f.write(data)