from FaceRecognition_imagepreprocessing import image_cropping, face_recording, take_image
from FaceRecognition_eigenfaces_core import FaceRecognitionEigenfaces

import cv2 as cv
import os
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
import pickle

class EigenfaceRecognition:

    face_data = None

    def __init__(self, filepath = None, data = None):
        """Either load the data from filepath or get the data directly."""
        if filepath:
            self.face_data = pickle.load(open(filepath, "rb"))
        if data:
            self.face_data = data

        self.knn_classifier = KNeighborsClassifier(n_neighbors=5)
        self.knn_classifier.fit(self.face_data.face_weights, self.face_data.labels)
        self.newfacedir = os.path.join(self.face_data.datadir, 'new_faces_notindetected')

    def recognize_face(self, **kwargs):
        """Check if face can be assigned to some person in the database with some degree of confidence
        (which is distance of class1 * 1.6 > class2 as we search for the closest class to face).
        The class is chosen based on return from the 'sum_class_distances' function.
        Returns:
            - Find id of the best suited class.
            - Return True if 'class confidence' is fulfilled and False if not."""

        #Enable inGui plotting
        gui = kwargs.get('gui', False)
        image = take_image(gui=gui)
        image_representation = self.face_data.transfer_image(image.flatten())

        # fit again in case new data appears
        self.knn_classifier.fit(self.face_data.face_weights, self.face_data.labels)
        probabilities = self.knn_classifier.predict_proba(image_representation.reshape(1, -1))
        prob_person = np.argsort(probabilities)[0][-3:]
        prob_val = probabilities[0][prob_person]

        isnew = 0
        dist, ids = self.knn_classifier.kneighbors(X=image_representation.reshape(1, -1),
                                                   n_neighbors=5, return_distance=True)
        person_ids = self.face_data.labels[ids]
        candidates_n = len([1 for x in prob_val if x > 0])

        class_distances = sum_class_distances(dist, person_ids)
        class_distances = sorted(class_distances, key=lambda x: x[1])

        # If one candidate
        if candidates_n == 1:
            face_found_id = prob_person[-1]
        # If more than one candidate
        else:
            # If too many candidates =  not confident
            if candidates_n >= 4:
                isnew = 1

            # Check for 'confidence'
            # If there is more than one candidate
            elif candidates_n != 1:
                # If first class not smaller than 0.7 * second class
                print(class_distances)
                if class_distances[1][1][0]*0.8 < class_distances[0][1][0]:
                    isnew = 1

            # if one class more probable pick this one
            if prob_val[-1] > prob_val[-2]:
                face_found_id = prob_person[-1]
                # If some class was chosen but its images were not the closest
                # (majority rule decided, but not confident pick)
                if prob_person[-1] != person_ids[0][0]:
                    isnew=1
            # If equally probable choose based on distace
            elif prob_val[-1] == prob_val[-2]:
                if class_distances[-1][1][0] < class_distances[-2][1][0]:
                    face_found_id = prob_person[-1]
                else:
                    face_found_id = prob_person[-2]

        #Find closest face for representation
        if person_ids[0][0] == face_found_id:
            closest_face = np.reshape(self.face_data.image_matrix_raw.T[ids[0][0]],
                                      (self.face_data.image_shape, self.face_data.image_shape,))
            closest_face_id = ids[0][0]
        else:
            closest_face = np.reshape(self.face_data.image_matrix_raw.T[ids[0][1]],
                                      (self.face_data.image_shape, self.face_data.image_shape,))
            closest_face_id = ids[0][1]

        if isnew:
            print("Face found without confidence, label = {}".format(face_found_id))
            return False, face_found_id, image, closest_face, closest_face_id

        if not isnew:
            print("Face found with confidence, label = {}".format(face_found_id))
            return True, face_found_id, image, closest_face, closest_face_id

    def add_person(self, **kwargs):
        gui = kwargs.get('gui', False)
        if gui:
            data = face_recording(gui)
        else:
            data = face_recording()
        data = np.array(data)

        label = max(self.face_data.labels) + 1
        count = len(data)
        data = data.reshape(count, self.face_data.image_shape**2)

        for image in data:
            self.add_image2database(image, label)
        print("Added face with label = {}.".format(label))

    def add_image2database(self, image, label):
        """Input is a flattened image."""

        # Increase image count to keep consistency
        self.face_data.image_count += 1
        # Label the image
        self.face_data.labels = np.append(self.face_data.labels, label)

        # Create new 'mean' image
        self.face_data.image_matrix_raw = np.vstack((self.face_data.image_matrix_raw.T, image)).T
        self.mean_img = np.sum(self.face_data.image_matrix_raw, axis=1) / self.face_data.image_count
        self.mean_img = self.mean_img.reshape(self.face_data.image_shape, self.face_data.image_shape)

        # Substract mean from raw images
        self.face_data.image_matrix_flat = np.array(
            [x - self.mean_img.flatten() for x in self.face_data.image_matrix_raw.transpose()]).transpose()

        # Add face weights for the given image
        self.face_data.face_weights = np.matmul(self.face_data.image_matrix_flat.transpose(),
                                                self.face_data.eigenfaces_flat.transpose())

def sum_class_distances(distances, class_labels):
    """Sums distances to face from respective classes and calculates the average."""
    un_val = np.unique(class_labels)
    arr = []
    for i in un_val:
        sum = 0
        count = 0
        arr_pos = 0
        for elem in class_labels[0]:
            if elem==i:
                sum +=distances[0][arr_pos]
                count+=1
            arr_pos+=1
        arr.append(np.vstack((i, sum/count)))

    return arr

if __name__ == "__main__":
    fr = FaceRecognitionEigenfaces()
    fr.get_images()
    fr.get_eigenfaces()
    fr.save_to_file()
    efr = EigenfaceRecognition(data=fr)
    # efr.add_person()
    # confidence, person_id, im_searched, im_found, im_found_id =
    #  efr.recognize_face(image_path='1.pgm-s5_newface.pgm')


import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

def annotate_heatmap(cls, im, data=None, valfmt="{x:.2f}",
                     textcolors=["black", "white"],
                     threshold=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A list or array of two color specifications.  The first is used for
        values below a threshold, the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max()) / 2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

fig, ax = plt.subplots()
im, cbar = heatmap(z_np, equips, dates, ax=ax,
                   cmap="YlGn", cbarlabel="harvest [t/year]")
# texts = annotate_heatmap(im, valfmt="{x:.1f} t")

fig.tight_layout()
fig.savefig("../tmp/tst.png")
plt.show()


import numpy as np
import cv2
import cv

cap = cv2.VideoCapture(0)

while(True):
  ret, frame = cap.read()
  gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
  cv2.imshow('frame', gray)

#  res = cv2.resize(gray, None, fx=2, fy=2, interpolation = cv2.INTER_CUBIC)
  res = cv2.resize(gray, (92,112), interpolation = cv2.INTER_CUBIC)
  cv2.imshow('resize', res)

  if cv2.waitKey(1) & 0xFF == ord('q'):
    break

cap.release()
cv2.destroyAllWindows()


from .fhirbase import fhirbase

class Condition(fhirbase):
    """
    A clinical condition, problem, diagnosis, or other event, situation,
    issue, or clinical concept that has risen to a level of concern.

    Args:
        resourceType: This is a Condition resource
        identifier: This records identifiers associated with this condition
            that are defined by business processes and/or used to refer to it when
            a direct URL reference to the resource itself is not appropriate (e.g.
            in CDA documents, or in written / printed documentation).
        clinicalStatus: The clinical status of the condition.
        verificationStatus: The verification status to support the clinical
            status of the condition.
        category: A category assigned to the condition.
        severity: A subjective assessment of the severity of the condition as
            evaluated by the clinician.
        code: Identification of the condition, problem or diagnosis.
        bodySite: The anatomical location where this condition manifests
            itself.
        subject: Indicates the patient or group who the condition record is
            associated with.
        context: Encounter during which the condition was first asserted.
        onsetDateTime: Estimated or actual date or date-time  the condition
            began, in the opinion of the clinician.
        onsetAge: Estimated or actual date or date-time  the condition began,
            in the opinion of the clinician.
        onsetPeriod: Estimated or actual date or date-time  the condition
            began, in the opinion of the clinician.
        onsetRange: Estimated or actual date or date-time  the condition
            began, in the opinion of the clinician.
        onsetString: Estimated or actual date or date-time  the condition
            began, in the opinion of the clinician.
        abatementDateTime: The date or estimated date that the condition
            resolved or went into remission. This is called "abatement" because of
            the many overloaded connotations associated with "remission" or
            "resolution" - Conditions are never really resolved, but they can
            abate.
        abatementAge: The date or estimated date that the condition resolved
            or went into remission. This is called "abatement" because of the many
            overloaded connotations associated with "remission" or "resolution" -
            Conditions are never really resolved, but they can abate.
        abatementBoolean: The date or estimated date that the condition
            resolved or went into remission. This is called "abatement" because of
            the many overloaded connotations associated with "remission" or
            "resolution" - Conditions are never really resolved, but they can
            abate.
        abatementPeriod: The date or estimated date that the condition
            resolved or went into remission. This is called "abatement" because of
            the many overloaded connotations associated with "remission" or
            "resolution" - Conditions are never really resolved, but they can
            abate.
        abatementRange: The date or estimated date that the condition resolved
            or went into remission. This is called "abatement" because of the many
            overloaded connotations associated with "remission" or "resolution" -
            Conditions are never really resolved, but they can abate.
        abatementString: The date or estimated date that the condition
            resolved or went into remission. This is called "abatement" because of
            the many overloaded connotations associated with "remission" or
            "resolution" - Conditions are never really resolved, but they can
            abate.
        assertedDate: The date on which the existance of the Condition was
            first asserted or acknowledged.
        asserter: Individual who is making the condition statement.
        stage: Clinical stage or grade of a condition. May include formal
            severity assessments.
        evidence: Supporting Evidence / manifestations that are the basis on
            which this condition is suspected or confirmed.
        note: Additional information about the Condition. This is a general
            notes/comments entry  for description of the Condition, its diagnosis
            and prognosis.
    """

    __name__ = 'Condition'

    def __init__(self, dict_values=None):
        self.resourceType = 'Condition'
        # type: str
        # possible values: Condition

        self.clinicalStatus = None
        # type: str

        self.verificationStatus = None
        # type: str
        # possible values: provisional, differential, confirmed,
        # refuted, entered-in-error, unknown

        self.category = None
        # type: list
        # reference to CodeableConcept

        self.severity = None
        # reference to CodeableConcept

        self.code = None
        # reference to CodeableConcept

        self.bodySite = None
        # type: list
        # reference to CodeableConcept

        self.subject = None
        # reference to Reference: identifier

        self.context = None
        # reference to Reference: identifier

        self.onsetDateTime = None
        # type: str

        self.onsetAge = None
        # reference to Age

        self.onsetPeriod = None
        # reference to Period

        self.onsetRange = None
        # reference to Range

        self.onsetString = None
        # type: str

        self.abatementDateTime = None
        # type: str

        self.abatementAge = None
        # reference to Age

        self.abatementBoolean = None
        # type: bool

        self.abatementPeriod = None
        # reference to Period

        self.abatementRange = None
        # reference to Range

        self.abatementString = None
        # type: str

        self.assertedDate = None
        # type: str

        self.asserter = None
        # reference to Reference: identifier

        self.stage = None
        # reference to Condition_Stage

        self.evidence = None
        # type: list
        # reference to Condition_Evidence

        self.note = None
        # type: list
        # reference to Annotation

        self.identifier = None
        # type: list
        # reference to Identifier

        if dict_values:
            self.set_attributes(dict_values)
            self.assert_type()

    def assert_type(self):

        if self.verificationStatus is not None:
            for value in self.verificationStatus:
                if value is not None and value.lower() not in [
                    'provisional', 'differential', 'confirmed', 'refuted',
                        'entered-in-error', 'unknown']:
                    raise ValueError('"{}" does not match possible values: {}'.format(
                        value, 'provisional, differential, confirmed, refuted, entered-in-error,'
                        'unknown'))

    def get_relationships(self):

        return [
            {'parent_entity': 'Range',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'onsetRange'},

            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'category'},

            {'parent_entity': 'Reference',
             'parent_variable': 'identifier',
             'child_entity': 'Condition',
             'child_variable': 'subject'},

            {'parent_entity': 'Condition_Evidence',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'evidence'},

            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'code'},

            {'parent_entity': 'Identifier',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'identifier'},

            {'parent_entity': 'Period',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'onsetPeriod'},

            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'bodySite'},

            {'parent_entity': 'Reference',
             'parent_variable': 'identifier',
             'child_entity': 'Condition',
             'child_variable': 'asserter'},

            {'parent_entity': 'Condition_Stage',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'stage'},

            {'parent_entity': 'Age',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'onsetAge'},

            {'parent_entity': 'Period',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'abatementPeriod'},

            {'parent_entity': 'Range',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'abatementRange'},

            {'parent_entity': 'Annotation',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'note'},

            {'parent_entity': 'Age',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'abatementAge'},

            {'parent_entity': 'Reference',
             'parent_variable': 'identifier',
             'child_entity': 'Condition',
             'child_variable': 'context'},

            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition',
             'child_variable': 'severity'},
        ]

class Condition_Stage(fhirbase):
    """
    A clinical condition, problem, diagnosis, or other event, situation,
    issue, or clinical concept that has risen to a level of concern.

    Args:
        summary: A simple summary of the stage such as "Stage 3". The
            determination of the stage is disease-specific.
        assessment: Reference to a formal record of the evidence on which the
            staging assessment is based.
    """

    __name__ = 'Condition_Stage'

    def __init__(self, dict_values=None):
        self.summary = None
        # reference to CodeableConcept

        self.assessment = None
        # type: list
        # reference to Reference: identifier

        self.object_id = None
        # unique identifier for object class

        if dict_values:
            self.set_attributes(dict_values)

    def get_relationships(self):

        return [
            {'parent_entity': 'Reference',
             'parent_variable': 'identifier',
             'child_entity': 'Condition_Stage',
             'child_variable': 'assessment'},

            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition_Stage',
             'child_variable': 'summary'},
        ]

class Condition_Evidence(fhirbase):
    """
    A clinical condition, problem, diagnosis, or other event, situation,
    issue, or clinical concept that has risen to a level of concern.

    Args:
        code: A manifestation or symptom that led to the recording of this
            condition.
        detail: Links to other relevant information, including pathology
            reports.
    """

    __name__ = 'Condition_Evidence'

    def __init__(self, dict_values=None):
        self.code = None
        # type: list
        # reference to CodeableConcept

        self.detail = None
        # type: list
        # reference to Reference: identifier

        self.object_id = None
        # unique identifier for object class

        if dict_values:
            self.set_attributes(dict_values)

    def get_relationships(self):

        return [
            {'parent_entity': 'CodeableConcept',
             'parent_variable': 'object_id',
             'child_entity': 'Condition_Evidence',
             'child_variable': 'code'},

            {'parent_entity': 'Reference',
             'parent_variable': 'identifier',
             'child_entity': 'Condition_Evidence',
             'child_variable': 'detail'},
        ]


import tensorflow as tf
import time
import numpy as np

import helpers.helper_funcs as helpers
import helpers.cifar_models as models

def run_combinations(simple_model, complex_model, x_data, y_data):
    '''
    Attempt all confidence values in 0:0.1:1
    Store accuracy and time for each confidence value
    '''

    y_test_np = y_data.numpy()
    count_dicts = []
    coeffs = np.zeros((2, 10))

    s_model_probs = simple_model.predict(x_data)
    s_model_preds = np.argmax(s_model_probs, axis=1)
    unique, counts = np.unique(s_model_preds, return_counts=True)
    count_dicts.append(dict(zip(unique, counts)))

    c_model_probs = complex_model.predict(x_data)
    c_model_preds = np.argmax(c_model_probs, axis=1)
    unique, counts = np.unique(c_model_preds, return_counts=True)
    count_dicts.append(dict(zip(unique, counts)))

    for j in range(10):
        # Compute the number of times where the prediction matches the test output for the simple model
        class_count = len(np.where((s_model_preds == j) & (y_test_np == j))[0])
        coeffs[0][j] = class_count / count_dicts[0][j]
    for j in range(10):
        # Compute the number of times where the prediction matches the test output for the complex model
        class_count = len(np.where((c_model_preds == j) & (y_test_np == j))[0])
        coeffs[1][j] = class_count / count_dicts[1][j]

    conf_values = np.arange(0, 0.9, 0.1)
    conf_values = np.append(conf_values, np.arange(0.81, 0.91, 0.01))

    accuracies = []
    times = []
    simplePercents = []
    for conf_value in conf_values:
        accuracy, time, simplePercent = run_combined(simple_model, complex_model, x_data, y_data, conf_value, coeffs)
        accuracies.append(accuracy)
        times.append(time)
        simplePercents.append(simplePercent)

    return accuracies, times, simplePercents

def run_combined(simple_model, complex_model, inputs, labels, conf_value, coeffs):
    '''
    Runs both models on the input data with the input confidence value
    '''
    before = time.time()
    # -----------------------------------
    # All Simple
    simple_probs = simple_model.predict(inputs)
    simple_highest_probs = np.amax(simple_probs, axis=1)
    simple_preds = np.argmax(simple_probs, axis=1)

    for i in range(inputs.shape[0]):
        pred = simple_preds[i]
        simple_highest_probs[i] = simple_highest_probs[i] * coeffs[0][pred]

    # complex_probs = simple_model.predict(inputs)
    # complex_highest_probs = np.amax(simple_probs, axis=1)
    # complex_preds = np.argmax(simple_probs, axis=1)

    # for i in range(inputs.shape[0]):
    #     pred = complex_preds[i]
    #     complex_highest_probs[i] = complex_highest_probs[i] * coeffs[1][pred]

    # combined_preds = np.arange(inputs.shape[0])
    # simple_count = 0

    # for i in range(inputs.shape[0]):
    #     if simple_highest_probs[i] > complex_highest_probs[i]:
    #         combined_preds[i] = simple_preds[i]
    #         simple_count += 1
    #     else:
    #         combined_preds[i] = complex_preds[i]

    # -----------------------------------
    # Complex predictions: get inputs of complex predictions
    indices = [i for i in range(inputs.shape[0])]
    complex_indices = np.where(simple_highest_probs < conf_value, indices, None)
    complex_indices = complex_indices[complex_indices != np.array(None)] # remove None values
    complex_indices = np.asarray(complex_indices, dtype=np.int64)

    complex_inputs = np.take(inputs, complex_indices, axis=0)
    if complex_inputs.shape[0] == 0:
        complex_preds = []
    else:
        complex_preds = np.argmax(complex_model.predict(complex_inputs), axis=1)
    # -----------------------------------
    # Select simple
    simple_indices = np.where(simple_highest_probs >= conf_value, indices, None)
    simple_indices = simple_indices[simple_indices != np.array(None)] # remove None values
    simple_indices = np.asarray(simple_indices, dtype=np.int64)

    # ------------------------------------
    # Reorganize preds
    combined_preds = np.arange(inputs.shape[0])

    np.put(combined_preds, indices, simple_preds)
    np.put(combined_preds, complex_indices, complex_preds)

    simplePercent = simple_indices.shape[0] / (simple_indices.shape[0] + complex_indices.shape[0])
    return tf.reduce_mean(tf.cast(tf.equal(combined_preds, labels), tf.float32)).numpy(), time.time() - before, simplePercent

def main():
    print('Loading data...')
    x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
    y_test = tf.squeeze(y_test)

    #train_and_save_models(x_train, y_train)

    print("Loading models...")
    # l0_model = tf.keras.models.load_model('models/l0_model')
    # l1_model = tf.keras.models.load_model('models/cifar/l1_model')
    # l2_model = tf.keras.models.load_model('models/cifar/l2_model')
    # l3_model = tf.keras.models.load_model('models/cifar/l3_model')
    # l4_model = tf.keras.models.load_model('models/cifar/l4_model')
    l7_model = tf.keras.models.load_model('models/cifar/l7_model')
    l8_model = tf.keras.models.load_model('models/cifar/l8_model')

    l7_times = []
    l8_times = []

    # for i in range(5):
    #     before_time = time.time()
    #     accuracy = l7_model.predict(x_test)
    #     #print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test, verbose=0)[1])
    #     l7_times.append(time.time() - before_time)

    #     before_time = time.time()
    #     accuracy = l8_model.predict(x_test)
    #     #print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test, verbose=0)[1])
    #     l8_times.append(time.time() - before_time)

    # print(l7_times)
    # print(l8_times)
    # print("L7 Time:", np.mean(l7_times[1:], axis=0))
    # print("L8 Time:", np.mean(l8_times[1:], axis=0))

    l7_l8_accuracies = []
    l7_l8_times = []
    l7_l8_percents = []

    for i in range(3):
        print("Run l7 l8... #" + str(i))
        accuracies, times, simplePercents = run_combinations(l7_model, l8_model, x_test, y_test)
        l7_l8_accuracies.append(accuracies)
        l7_l8_times.append(times)
        l7_l8_percents.append(simplePercents)

    print("L7 L8 Accuracies:", np.mean(l7_l8_accuracies, axis=0))
    print("L7 L8 Times:", np.mean(l7_l8_times, axis=0))
    print("L7 L8 Simple Percents", np.mean(l7_l8_percents, axis=0))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
from nlp.parser import fill_slots

def test_no_url():
    text = "Закрыл задачу"
    slots = fill_slots(text)
    assert slots['url'] is None
    assert slots['text'] == text

def test_only_url():
    text = "https://pm.is74.ru/issues/41918"
    slots = fill_slots(text)
    assert slots['text'] == ''

def test_wrong_url():
    text = "https://git.is74.ru/issues/41918101 Закрыл задачу"
    slots = fill_slots(text)
    assert slots['url'] is None
    assert slots['text'] == text

def test_multiple_lines():
    text = "http://pm.is74.ru/issues/41918#note-18 Построила прогноз,\n " \
           "незначительно улучшилось качество, у лин регрессии - 25с/ч,\n " \
           "у XGBoost'a - 33с/ч https://pm.is74.ru/issues/41918#note-17"

    slots = fill_slots(text)
    print(text, '\n')
    print(slots)
    assert slots[
        'text'] == "Построила прогноз,\n " \
        "незначительно улучшилось качество, у лин регрессии - 25с/ч,\n " \
        "у XGBoost'a - 33с/ч https://pm.is74.ru/issues/41918#note-17"
    assert slots['url'] == "http://pm.is74.ru/issues/41918#note-18"

def test_multiple_urls():
    text = "pm.is74.ru/issues/41918 Связал задачу с " \
           "pm.is74.ru/issues/101 Тест"
    slots = fill_slots(text)
    assert slots['url'] == "pm.is74.ru/issues/41918"
    assert slots['text'] == "Связал задачу с " \
        "pm.is74.ru/issues/101 Тест"

def test_empty_line():
    text = ''
    slots = fill_slots(text)
    assert slots['url'] is None
    assert slots['text'] == text


import math
class C45:

    def __init__(self, archivo_datos,archivo_descripcion):
        self.archivo_datos = archivo_datos
        self.archivo_descripcion = archivo_descripcion
        self.data = []
        self.clases = []
        self.numero_atributos = -1
        self.valores_de_atributos = {}
        self.atributos = []
        self.arbol = None

    def Obtener_datos(self):
        with open(self.archivo_descripcion, "r") as file:
            clases = file.readline()
            self.clases = [x.strip() for x in clases.split(",")]
            for line in file:
                [atributo, valores] = [x.strip() for x in line.split(":")]
                valores = [x.strip() for x in valores.split(",")]
                self.valores_de_atributos[atributo] = valores
        self.numero_atributos = len(self.valores_de_atributos.keys())
        self.atributos = list(self.valores_de_atributos.keys())
        with open(self.archivo_datos, "r") as file:
            for line in file:
                fila = [x.strip() for x in line.split(",")]
                if fila != [] or fila != [""]:
                    self.data.append(fila)

    def prepocesar_datos(self):
        for indice,fila in enumerate(self.data):
            for attr_indice in range(self.numero_atributos):
                if(not self.es_atributo_discreto(self.atributos[attr_indice])):
                    self.data[indice][attr_indice] = float(self.data[indice][attr_indice])

    def printarbol(self):
        self.mostrar_nodo(self.arbol)

    def mostrar_nodo(self, nodo, identacion=""):
        if not nodo.esHoja:
            if nodo.limite is None:
                for indice,hijo in enumerate(nodo.hijoren):
                    if hijo.esHoja:
                        print(identacion + nodo.etiqueta + " = " + nodo.valores_de_atributo[nodo.etiqueta][indice] + " : " + hijo.etiqueta)
                    else:
                        print(identacion + nodo.etiqueta + " = " + nodo.valores_de_atributo[nodo.etiqueta][indice] + " : ")
                        self.mostrar_nodo(hijo, identacion + "    ")
            else:
                hijo_izquierdo = nodo.hijoren[0]
                hijo_derecho = nodo.hijoren[1]
                if hijo_izquierdo.esHoja:
                    print(identacion + nodo.etiqueta + " <= " + str(nodo.limite) + " : " + hijo_izquierdo.etiqueta)
                else:
                    print(identacion + nodo.etiqueta + " <= " + str(nodo.limite)+" : ")
                    self.mostrar_nodo(hijo_izquierdo, identacion + "    ")

                if hijo_derecho.esHoja:
                    print(identacion + nodo.etiqueta + " > " + str(nodo.limite) + " : " + hijo_derecho.etiqueta)
                else:
                    print(identacion + nodo.etiqueta + " > " + str(nodo.limite) + " : ")
                    self.mostrar_nodo(hijo_derecho , identacion + "    ")

    def generar_arbol(self):
        self.arbol = self.generar_arbol_recursivamente(self.data, self.atributos)

    def generar_arbol_recursivamente(self, curData, curatributos):

        misma_clases = self.misma_clase(curData)

        if len(curData) == 0:
            #Fail
            return Nodo(True, "Fail", None , None )
        elif misma_clases is not False:
            return Nodo(True, misma_clases, None , None )
        elif len(curatributos) == 0:
            majClass = self.getMajClass(curData)
            return Nodo(True, majClass, None)
        else:
            (mejor,mejor_limite,separado_) = self.separar_atributos(curData, curatributos)
            atributos_restantes = curatributos[:]
            atributos_restantes.remove(mejor)
            nodo = Nodo(False, mejor, mejor_limite , self.valores_de_atributos)
            nodo.hijoren = [self.generar_arbol_recursivamente(subset, atributos_restantes) for subset in separado_]
            return nodo

    def getMajClass(self, curData):
        freq = [0]*len(self.clases)
        for fila in curData:
            indice = self.clases.index(fila[-1])
            freq[indice] += 1
        maxInd = freq.index(max(freq))
        return self.clases[maxInd]

    def misma_clase(self, data):
        for fila in data:
            if fila[-1] != data[0][-1]:
                return False
        return data[0][-1]

    def es_atributo_discreto(self, atributo):
        if atributo not in self.atributos:
            raise ValueError("atributo not listed")
        elif len(self.valores_de_atributos[atributo]) == 1 and self.valores_de_atributos[atributo][0] == "continuous":
            return False
        else:
            return True

    def separar_atributos(self, curData, curatributos):
        separado_ = []
        maxEnt = -1*float("inf")
        mejor_atributo = -1
        mejor_limite = None
        for atributo in curatributos:
            indice_de_atributo = self.atributos.index(atributo)
            if self.es_atributo_discreto(atributo):
                valores_para_atributoo = self.valores_de_atributos[atributo]
                sub_conjuntos = [[] for a in valores_para_atributoo]
                for fila in curData:
                    for indice in range(len(valores_para_atributoo)):
                        if fila[indice_de_atributo] == valores_para_atributoo[indice]:
                            sub_conjuntos[indice].append(fila)
                e = self.ganancia(curData, sub_conjuntos)
                if e > maxEnt:
                    maxEnt = e
                    separado_ = sub_conjuntos
                    mejor_atributo = atributo
                    mejor_limite = None
            else:
                curData.sort(key = lambda x: x[indice_de_atributo])
                for j in range(0, len(curData) - 1):
                    if curData[j][indice_de_atributo] != curData[j+1][indice_de_atributo]:
                        limite = (curData[j][indice_de_atributo] + curData[j+1][indice_de_atributo]) / 2
                        less = []
                        greater = []
                        for fila in curData:
                            if(fila[indice_de_atributo] > limite):
                                greater.append(fila)
                            else:
                                less.append(fila)
                        e = self.ganancia(curData, [less, greater])
                        if e >= maxEnt:
                            separado_ = [less, greater]
                            maxEnt = e
                            mejor_atributo = atributo
                            mejor_limite = limite
        return (mejor_atributo,mejor_limite,separado_)

    def ganancia(self,unionSet, sub_conjuntos):

        S = len(unionSet)
        impureza_antes_de_separar = self.entropia(unionSet)
        pesos = [len(subset)/S for subset in sub_conjuntos]
        impureza_depues_de_separar = 0
        for i in range(len(sub_conjuntos)):
            impureza_depues_de_separar += pesos[i]*self.entropia(sub_conjuntos[i])
        totalGain = impureza_antes_de_separar - impureza_depues_de_separar
        return totalGain

    def entropia(self, dataSet):
        S = len(dataSet)
        if S == 0:
            return 0
        num_clases = [0 for i in self.clases]
        for fila in dataSet:
            classindice = list(self.clases).index(fila[-1])
            num_clases[classindice] += 1
        num_clases = [x/S for x in num_clases]
        ent = 0
        for num in num_clases:
            ent += num*self.log(num)
        return ent*-1

    def log(self, x):
        if x == 0:
            return 0
        else:
            return math.log(x,2)

class Nodo:
    def __init__(self,esHoja, etiqueta, limite , valores_de_atributo ):
        self.etiqueta = etiqueta
        self.limite = limite
        self.esHoja = esHoja
        self.hijoren = []
        self.valores_de_atributo = valores_de_atributo


"""
Crawler for 7Eleven based on ATSSpider
scrapy crawl 7eleven -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="http://careers.7-eleven.com/careers/SalesAssociateJobs.html"
Sample URLS:
    http://careers.7-eleven.com/careers/SalesAssociateJobs_list.html
"""

from re import compile
from scrapy.http import Request
from scrapy.selector import Selector
from urllib import urlencode

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import Prefix, RemoveBadElements
from urlparse import urljoin

REF_REGEX = compile(r'storeID=(\d+)$')

class Eleven(ATSSpider):

    name = '7eleven'
    allowed_domains = ['7-eleven.com']

    def parse(self, response):
        sel = Selector(response)
        self.description = sel.xpath('//div[@id="content"]').extract()
        all_sales_jobs = sel.xpath(
            '//a[contains(text(), "All Other Areas")]/@href'
        ).extract()
        if all_sales_jobs:
            yield Request(
                url=urljoin(response.url, all_sales_jobs[0]),
                callback=self.parse_state_list
            )

    def parse_state_list(self, response):
        """
            Extract all the states from the search form.
            Made GET call request to the url and encode the parameters.
        """
        sel = Selector(response)

        states = filter(None, sel.xpath(
            '//div[@class="form-group"]/select[@name="state"]/option/@value'
        ).extract())
        for state in states:
            yield Request(
                callback=self.parse_job_callback(),
                url='%(domain)s?%(params)s' % {
                    'domain': response.url,
                    'params': urlencode({'state': state, 'city': '', 'zip': ''})
                },
            )

    def parse_job(self, response):
        """
            Extract all the required information.
            In this website, we can get only following fields.
            title - For all jobs its same, because seed url is for Sales Associate Jobs.
            location, zip_code, referencenumber, apply_url.
            When you hit apply_url, it leads to submit the applications.
        """
        sel = Selector(response)
        for tr in sel.xpath(
            '//table[@class="table search-results-jobs"]/tbody/tr'
        ):
            loader = BrightcorpItemLoader(selector=tr)
            url = tr.xpath('./td/a/@href').extract()
            loader.add_value(
                'description', self.description, RemoveBadElements(['a', 'iframe', ])
            )
            loader.add_xpath(
                'title', '//div[@id="content"]/h1[@class="text-green"]/text()'
            )
            address = tr.xpath('./td/text()').extract()
            loader.add_value('location', ', '.join(address[:-1]))
            loader.add_value('zip_code', address[-1])
            loader.add_value('apply_url', url)
            loader.add_value('url', url)
            loader.add_xpath(
                'referencenumber',
                './td/a/@href',
                Prefix('%s-' % self.name),
                re=REF_REGEX
            )

            yield loader.load_item()


""" game_stableford.py - Stableford Golf Game class."""
from .game import GolfGame
from .exceptions import GolfException

class StablefordGame(GolfGame):
    """The Stableford game."""

    short_description = "Stableford"
    dct_scoring = {
        "Modified": {-3: 8, -2: 5, -1: 2, 0: 0, 1: -1, 2: -3, "min": 8, "max": -3},
        "Classic": {-3: 8, -2: 5, -1: 2, 0: 0, 1: -1, 2: -2, "min": 8, "max": -2},
        "British": {-3: 4, -2: 4, -1: 3, 0: 2, 1: 1, 2: 0, "min": 4, "max": 0},
        "Spanish": {-3: 4, -2: 4, -1: 3, 0: 2, 1: 1, 2: 0, "min": 8, "max": 0},
    }

    def __init__(self, golf_round, scores, **kwargs):
        self.stableford_type = kwargs.get("stableford_type", "Classic")
        self.jokers = kwargs.get("jokers")
        super(StablefordGame, self).__init__(golf_round, scores, **kwargs)

    def validate(self):
        super(StablefordGame, self).validate()
        if self.stableford_type not in self.dct_scoring:
            raise GolfException(
                "stableford_type {} not supported.".format(self.stableford_type)
            )
        if self.stableford_type == "Spanish":
            if not self.jokers:
                raise GolfException("stableford_type Spanish needs jokers set.")
            if len(self.jokers) != len(self.scores):
                raise GolfException(
                    "stableford_type Spanish needs a joker set for each player."
                )
            for joker in self.jokers:
                if len(joker) != 2:
                    raise GolfException(
                        "stableford_type Spanish jokers must have 2 values."
                    )
                if joker[0] not in (1, 2, 3, 4, 5, 6, 7, 8, 9):
                    raise GolfException(
                        "stableford_type Spanish joker[0] must be in 1-9."
                    )
                if joker[1] not in (10, 11, 12, 13, 14, 15, 16, 17, 18):
                    raise GolfException(
                        "stableford_type Spanish joker[1] must be in 10-18."
                    )

    def start(self):
        self.dct_stableford = self.dct_scoring[self.stableford_type]
        # use full handicap for all players
        for n, pl in enumerate(self.scores):
            pl._score = [None for _ in range(len(self.golf_round.course.holes))]
            pl._bumps = self.golf_round.course.calcBumps(pl.course_handicap)
            pl.dct_points = self._init_dict()
            pl._jokers = self.jokers[n] if self.stableford_type == "Spanish" else None
            pl.dct_money = self._init_dict(score_type=float) if self._wager else None
        self.dctScorecard["header"] = "{0:*^98}".format(" Stableford ")
        self._thru = 0
        if self._wager:
            self._calc_money_from_points()

    def _calc_score(self, net_score):
        if net_score in self.dct_stableford:
            return self.dct_stableford[net_score]
        if net_score < 0:
            return self.dct_stableford["min"]
        else:
            return self.dct_stableford["max"]

    def addScore(self, index, lstGross):
        """add scores for a hole."""
        self._thru = index + 1
        for pl, gross in zip(self.scores, lstGross):
            # update net
            net_score = gross - pl._bumps[index]
            pl._score[index] = net_score
            pl.dct_points["holes"][index] = self._calc_score(
                net_score - self.golf_round.course.holes[index].par
            )
            if pl._jokers:
                if (index + 1) in pl._jokers:
                    pl.dct_points["holes"][index] *= 2
            self._update_totals(pl.dct_points)
        if self._wager and self._thru in (9, 18):
            self._calc_money_from_points()

    def getScorecard(self, **kwargs):
        """Scorecard with all players."""
        lstPlayers = []
        for n, sc in enumerate(self.scores):
            dct = {"player": sc.player}
            dct["in"] = sc.dct_points["in"]
            dct["out"] = sc.dct_points["out"]
            dct["total"] = sc.dct_points["total"]
            line = "{:<6}".format(sc.player.nick_name)
            for i, point in enumerate(sc.dct_points["holes"][:9]):
                s = "*" if sc._jokers and (i + 1) in sc._jokers else ""
                s += "" if point is None else "{:d}".format(point)
                line += " {:>3}".format(s)
            line += " {:>4d}".format(dct["out"])
            for i, point in enumerate(sc.dct_points["holes"][9:]):
                s = "*" if sc._jokers and (i + 10) in sc._jokers else ""
                s += "" if point is None else "{:d}".format(point)
                line += " {:>3}".format(s)
            line += " {:>4d} {:>4d}".format(dct["in"], dct["total"])
            dct["line"] = line
            lstPlayers.append(dct)
        self.dctScorecard["players"] = lstPlayers
        return self.dctScorecard

    def getLeaderboard(self, **kwargs):
        board = []
        sort_type = kwargs.get("sort_type", "points")
        if sort_type == "money" and self._wager:
            self.dctLeaderboard["hdr"] = "Pos Name  Money  Thru"
            scores = sorted(
                self.scores, key=lambda score: score.dct_money["total"], reverse=True
            )
            sort_by = "money"
        else:
            self.dctLeaderboard["hdr"] = "Pos Name  Points  Thru"
            scores = sorted(
                self.scores, key=lambda score: score.dct_points["total"], reverse=True
            )
            sort_by = "total"
        pos = 1
        prev_total = None
        for sc in scores:
            score_dct = {
                "player": sc.player,
                "total": sc.dct_points["total"],
                "money": sc.dct_money["total"] if self._wager else None,
            }
            if prev_total != None and score_dct[sort_by] != prev_total:
                pos += 1
            prev_total = score_dct[sort_by]
            score_dct["pos"] = pos

            for n, point in enumerate(sc.dct_points["holes"]):
                if point is None:
                    break
            else:
                n += 1
            score_dct["thru"] = n
            if sort_by == "money":
                money = (
                    "---"
                    if score_dct["money"] == 0.0
                    else "${:2g}".format(score_dct["money"])
                )
                score_dct["line"] = "{:<3} {:<6} {:>5} {:>4}".format(
                    score_dct["pos"],
                    score_dct["player"].nick_name,
                    money,
                    score_dct["thru"],
                )
            else:
                score_dct["line"] = "{:<3} {:<6} {:>5} {:>4}".format(
                    score_dct["pos"],
                    score_dct["player"].nick_name,
                    score_dct["total"],
                    score_dct["thru"],
                )
            board.append(score_dct)
        self.dctLeaderboard["leaderboard"] = board
        return self.dctLeaderboard

    def getStatus(self, **kwargs):
        for n, net in enumerate(self.scores[0]._score):
            if net is None:
                self.dctStatus["next_hole"] = n + 1
                self.dctStatus["par"] = self.golf_round.course.holes[n].par
                self.dctStatus["handicap"] = self.golf_round.course.holes[n].handicap
                bumps = []
                bump_line = []
                for sc in self.scores:
                    if sc._bumps[n] > 0:
                        dct = {"player": sc.player, "bumps": sc._bumps[n]}
                        bumps.append(dct)
                        bump_line.append(
                            "{}{}".format(
                                sc.player.nick_name,
                                "({})".format(dct["bumps"]) if dct["bumps"] > 1 else "",
                            )
                        )
                self.dctStatus["bumps"] = bumps
                self.dctStatus["line"] = "Hole {} Par {} Hdcp {}".format(
                    self.dctStatus["next_hole"],
                    self.dctStatus["par"],
                    self.dctStatus["handicap"],
                )
                if bumps:
                    self.dctStatus["line"] += " Bumps:{}".format(",".join(bump_line))
                break
        else:
            # round complete
            self.dctStatus["next_hole"] = None
            self.dctStatus["par"] = self.golf_round.course.total
            self.dctStatus["handicap"] = None
            self.dctStatus["line"] = "Round Complete"
        return self.dctStatus


import re
from bs4 import BeautifulSoup as bs

from .exceptions import FullTextNotAvailable

WHITESPACE_RE = re.compile(r"\s+")
ALPHA_RE = re.compile(r"[^\w ]")

clean_whitespace = lambda string: WHITESPACE_RE.sub(" ", string).strip()
clean_key = lambda string: WHITESPACE_RE.sub("_", ALPHA_RE.sub("", string).strip().lower())

class FullTextParser(object):
    def parse(self, html_text):
        # r"\n\s*
": avoiding possible intermediate tail whitespaces from r"\n\s*"
        # r"\n\s*": \s* is only for extra heading spaces in appft
        # r"
\s*": \s is only for extra heading spaces in patft
        text = re.sub(r"
\s*", "\n", re.sub(r"\n\s*", " ", re.sub(r"\n\s*
", "
", html_text)))
        sections = text.split("
")
        parsed_sections = [bs(section.replace("&nbsp", " "), "lxml") for section in sections]

        if "Full text is not available for this patent" in text:
            raise FullTextNotAvailable()
            # This is for very old patents. Fail gently and get what you can.
            #data = dict([c.text.strip() for c in r.find_all(("td", "th"))] for r in parsed_sections[1].find_all("tr"))
            #return {
            #    "publication_number": data['United States Patent'].replace(",", ""),
            #    "publication_date": data['Issue Date:'],
            #    **self.get_classifications(parsed_sections),
            #}

        return {
            "publication_number": self.get_publication_number(parsed_sections),
            "kind_code": self.get_kind_code(parsed_sections),
            "publication_date": self.get_publication_date(parsed_sections),
            "title": self.get_title(parsed_sections),
            "abstract": self.get_abstract(parsed_sections),
            "description": self.get_description(parsed_sections),
            "claims": self.get_claims(parsed_sections),
            "foreign_priority": self.get_foreign_priority_data(parsed_sections),
            "prior_publications": self.get_prior_publication_data(parsed_sections),
            "related_us_applications": self.get_related_patent_documents(parsed_sections),
            **self.get_classifications(parsed_sections),
            **self.get_bibliographic_data(parsed_sections),
            **self.get_references_agent_and_examiner(parsed_sections),
        }

    def get_title(self, parsed_sections):
        return clean_whitespace(parsed_sections[2].find("font").text)

    def get_abstract(self, parsed_sections):
        try:
            return clean_whitespace(parsed_sections[2].find("p").text)
        except AttributeError:
            return None

    def get_description(self, parsed_sections):
        section_number = [s.find("center", text="Description") is not None for s in parsed_sections].index(True) + 1
        return parsed_sections[section_number].text

    def get_claims(self, parsed_sections):
        section_number = [s.find("center", text="Claims") is not None for s in parsed_sections].index(True) + 1
        return parsed_sections[section_number].text

    def get_foreign_priority_data(self, parsed_sections):
        try:
            section_number = [s.find("center", text="Foreign Application Priority Data") is not None for s in parsed_sections].index(True)
        except ValueError:
            return None
        section = parsed_sections[section_number]
        table = section.find("b", text="Foreign Application Priority Data").find_next('table')

        FOREIGN_PRIORITY_RE = re.compile(r"(?P<date>[^[]*)\[(?P<country_code>[^]]*)\](?P<number>.*)")
        return [FOREIGN_PRIORITY_RE.match(t.text.strip()).groupdict()
            for t in table.find_all("tr")
            if t.text.strip()]

    def get_classifications(self, parsed_sections):
        cpc_re = re.compile(r"(?P<class>[A-Z]\d{2}[A-Z] \d{1,}/\d{2,}) \(?(?P<version>\d{8})?\)?")
        def parse_cpc(element):
            text = element.text.strip()
            try:
                return [cpc_re.search(c).groupdict() for c in text.split(";")]
            except AttributeError: # Design Patent
                return [{"class": text, "version": None},]

        def parse_us_class(element):
            strings = element.text.strip().strip(";").split('; ')
            return list({"class": c.split('/')[0], "subclass": c.split('/')[1]} for c in strings)

        def parse_field_of_search(element):
            groups = element.text.strip().strip(";").split(" ;")
            output = list()
            for group in groups:
                cl, *subcls = re.split("[,/]", group)
                output += [{"class":cl, "subclass": subcl} for subcl in subcls]
            return output
        section_number = [s.find("b", text="Current U.S. Class:") is not None for s in parsed_sections].index(True)
        rows = (parsed_sections[section_number]
                    .find("b", text="Current U.S. Class:")
                    .find_parent("table")
                    .find_all("tr")
        )

        parsers = {
            "current_cpc_class": parse_cpc,
            "current_international_class": parse_cpc,
            "current_us_class": parse_us_class,
            "class_at_publication": parse_us_class,
            "field_of_search": parse_field_of_search,
        }
        cells = [row.find_all("td") for row in rows]
        data = {clean_key(c[0].text.strip()[:-1]): c[1] for c in cells}
        if "international_class" in data:
            data['current_international_class'] = data["international_class"]
            del data['international_class']
        return {k: parsers[k](v) for k, v in data.items()}

    def get_bibliographic_data(self, parsed_sections):
        def parse_applicant(element):
            def html_table_to_records(table):
                data = [
                    [c.text.strip() for c in r.find_all(["td", "th"])]
                    for r in table.find_all("tr")
                ]
                headings = [clean_key(k) for k in data[0]]
                return [dict(zip(headings, r)) for r in data[1:]]
            return html_table_to_records(element.find_next("table"))

        def parse_inventors(element):
            inventor_pattern = re.compile(r"(?P<last_name>[^;]+); (?P<first_name>[^;]+);? \((?P<city>[^,]+), (?P<region>[^)]+)\)")
            inventors = element.text.strip().split(" ; ")
            return [inventor_pattern.search(i).groupdict() for i in inventors]

        def parse_assignee(element):
            assignee_pattern = re.compile(r"(?P<name>[^;]+) \((?P<city>[^,]+), (?P<region>[^)]+)\)")
            assignees = clean_whitespace(element.text.strip()).split(" ; ")
            try:
                return [assignee_pattern.search(a).groupdict() for a in assignees]
            except AttributeError:
                result = list()
                for assignee in assignees:
                    if len(assignee.rsplit(" ", 2)) < 3: # Where only a name is given
                        result.append({
                            "name": assignee
                        })
                    else:
                        name, city, region = assignee.rsplit(" ", 2)
                        result.append({
                            "name": name,
                            "city": city,
                            "region": region
                        })
                return result

        section_number = [s.find(("th", "td"), text="Inventors:") is not None for s in parsed_sections].index(True)
        section = parsed_sections[section_number]

        rows = [
            r
            for table in section.find_all("table")
            for r in table.find_all("tr")
        ]

        data = {
            r.find_all(("td", "th"))[0].text.strip()[:-1]: r.find_all(("td", "th"))[1] for r in rows
        }
        parsers = {
            "Applicant": parse_applicant,
            "Inventors": parse_inventors,
            "Assignee": parse_assignee
        }
        result = {clean_key(k): parsers.get(k, lambda x: x.text.strip())(v) for k, v in data.items()}
        return result

    def get_references_agent_and_examiner(self, parsed_sections):
        def get_us_references(ref_section):
            section = ref_section.find("center", text="U.S. Patent Documents")
            if section is None:
                return list()
            table = section.find_next("table")
            keys = ("publication_number", "date", "name")
            data = [[cell.text.strip() for cell in row.find_all(["td", "th"])]
                    for row in table.find_all("tr")]
            data = [row for row in data if any(row)]
            return list(dict(zip(keys, row)) for row in data if len(row) == len(keys))

        def get_foreign_references(ref_section):
            section = ref_section.find("center", text="Foreign Patent Documents")
            if section is None:
                return list()
            table = section.find_next("table")
            rows = [row for row in table.find_all("tr") if row.text.strip()]
            return [
                {
                    "publication_number": row.select_one("td:nth-child(2)").text.strip(),
                    "name": row.select_one("td:nth-child(4)").text.strip(),
                    "country_code": row.select_one("td:nth-child(6)").text.strip(),
                }
                for row in rows
            ]

        def get_npl_references(ref_section):
            section = ref_section.find("center", text="Other References")
            if section is None:
                return list()
            return [{"citation": cite} for cite in section.find_next("td").text.strip().split("\n")]
        try:
            section_number = ["References Cited" in s.text for s in parsed_sections].index(True) + 1
        except ValueError:
            return dict()
        section = parsed_sections[section_number]
        examiner = section.find("i", text="Primary Examiner:").next_sibling.strip()

        try:
            agent = section.find("i", text="Attorney, Agent or Firm:").find_next("coma").text.strip()
        except AttributeError:
            agent = None

        return {
            "examiner": examiner,
            "agent": agent,
            "us_references": get_us_references(section),
            "foreign_references": get_foreign_references(section),
            "npl_references": get_npl_references(section)
        }

    def get_publication_date(self, parsed_sections):
        return parsed_sections[1].find_all("td", align=re.compile("right", re.IGNORECASE))[-1].text.strip().replace(",", '')

    def get_publication_number(self, parsed_sections):
        return parsed_sections[1].find("td", align=re.compile("right", re.IGNORECASE)).text.strip().replace(",", '')

    def get_kind_code(self, parsed_sections):
        section = parsed_sections[1]
        headings = [i.text.strip() for i in section.find_all("td", align=re.compile("left", re.IGNORECASE))]
        if "Kind Code" in headings:
            return section.select("td[align='RIGHT']")[1].text.strip()

        has_publication_data = any(s.find("center", text="Prior Publication Data") is not None for s in parsed_sections)
        if has_publication_data:
            return "B2"
        else:
            return "B1"

    def get_prior_publication_data(self, parsed_sections):
        try:
            section_number = [s.find("center", text="Prior Publication Data") is not None for s in parsed_sections].index(True) + 1
        except ValueError:
            return list()
        section = parsed_sections[section_number]
        to_text = lambda lst: [i.text.strip() for i in lst]
        return [
            dict(zip(("publication_number", "publication_date"), to_text(r.find_all("td")[1:])))
            for r in section.find("table").find_all("tr")[2:-1]
            ]

    def get_related_patent_documents(self, parsed_sections):
        try:
            section_number = [s.find("center", text="Related U.S. Patent Documents") is not None for s in parsed_sections].index(True) + 1
        except ValueError:
            return list()
        section = parsed_sections[section_number]
        to_text = lambda lst: [i.text.strip() for i in lst]

        return [
            dict(zip(("appl_id", "filing_date", "patent_number"), to_text(r.find_all("td")[1:])))
            for r in section.find('table').find_all("tr")[2:-1]
            ]


#!/usr/bin/env python3

import gi

gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, GLib

gi.require_version('Vte', '2.91')
from gi.repository import Vte

import signal
import os

class Term:
	app = None
	win = None
	term = None

	def __init__ (self):
		pass

	def init (self):
		self.init_term()

	def init_term (self):
		## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes/Terminal.html#Vte.Terminal.new
		self.term = term = Vte.Terminal()

		## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes/Terminal.html#Vte.Terminal.spawn_sync
		ok, pid = term.spawn_sync(
			Vte.PtyFlags.DEFAULT,
			os.environ['HOME'],
			['/bin/bash'],
			[],
			GLib.SpawnFlags.DO_NOT_REAP_CHILD,
			None,
			None,
		)

		print('ok:', ok)
		print('pid:', pid)

		if (not ok):
			return False

		term.watch_child(pid)

		## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes/Terminal.html#Vte.Terminal.signals.child_exited
		term.connect('child-exited', self.on_child_exited);

		term.show_all()

	def on_child_exited(self, term, status):
		## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes/Terminal.html#Vte.Terminal.signals.child_exited
		print('')
		print('on_child_exited:')
		print('	term:', term)
		print('	status:', status)

		Gtk.main_quit()

class Win:
	app = None
	win = None
	term = None

	title = 'Demo Vte.Terminal'

	def __init__ (self):
		pass

	def init (self):
		self.init_win()

	def init_term (self):
		self.term = term = Term()
		term.app = self.app
		term.init()

	def init_win (self):
		self.win = win = Gtk.Window()

		win.set_title(self.title)

		## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/classes/Widget.html#Gtk.Widget.signals.delete_event
		#win.connect('delete-event', Gtk.main_quit)
		win.connect('delete-event', self.on_close_win)

		self.init_term()
		win.add(self.term.term)

		win.show_all()

	def on_close_win (self, win, evt):
		## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/functions.html#Gtk.main_quit
		Gtk.main_quit()

		return True

class Signal:
	def __init__ (self):
		pass

	def init (self):
		self.init_signal()

	def init_signal (self):
		## https://docs.python.org/3/library/signal.html
		## https://docs.python.org/2/library/signal.html
		signal.signal(signal.SIGINT, signal.SIG_DFL)

class App:
	def __init__ (self):
		pass

	def init (self):
		self.signal = signal = Signal()
		signal.init()

		self.win = win = Win()
		win.app = self
		win.init()

	def run (self):
		self.init()
		Gtk.main()

def main ():
	app = App()
	app.run()

if __name__ == '__main__':
	main()

## Tutorial
## http://python-gtk-3-tutorial.readthedocs.io/en/latest/introduction.html
## http://python-gtk-3-tutorial.readthedocs.io/en/latest/dialogs.html

## Gtk
## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0
## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/classes.html
## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/classes/Window.html
## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/classes/Window.html#Gtk.Window.resize
## https://lazka.github.io/pgi-docs/index.html#Gtk-3.0/classes/Window.html#Gtk.Window.resize_to_geometry

## Vte
## https://lazka.github.io/pgi-docs/index.html#Vte-2.91
## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes.html

## Vte.Terminal
## https://lazka.github.io/pgi-docs/index.html#Vte-2.91/classes/Terminal.html


import requests

from tests.sample_data import BRAZIL_USER_POLY

def test_hello():
    url = 'http://localhost:9000/v1/polyIntersect/hello'
    result = requests.get(url)
    print(result)

def test_brazil_biomes_local():
    # host = 'https://staging-api.globalforestwatch.org'
    # url = '{}/v1/polyIntersect/brazil-biomes?'.format(host)
    url = 'http://localhost:5700/api/v1/polyIntersect/brazil-biomes'
    run_request(url)

def test_brazil_biomes_control_tower():
    url = 'http://localhost:9000/v1/polyIntersect/brazil-biomes'
    run_request(url)

def test_brazil_biomes_control_tower_remote():
    url = 'http://staging-api.globalforestwatch.org/v1/polyIntersect/brazil-biomes'
    run_request(url)

def run_request(url):
    payload = {}
    payload['user_json'] = BRAZIL_USER_POLY
    payload['category'] = 'name'

    result = requests.post(url, json=payload,
                           allow_redirects=True)
    print(result)
    result_obj = result.json()
    print(result_obj)

if __name__ == '__main__':
    test_hello()
    test_brazil_biomes_local()
    test_brazil_biomes_control_tower()
    test_brazil_biomes_control_tower_remote()


from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement

from datetime import datetime, timedelta
from django import forms
from django.core.exceptions import ValidationError
from django.db.models import Q
from django.utils import formats, timezone
from django.utils.translation import ugettext_lazy as _, ungettext_lazy as ungettext
from json import dumps

from ..forms.widgets import CheckboxSelectMultipleBootstrap
from ..models import TimesheetEntry
from ..models.fields import DAY_OF_WEEK
from ..utils import comma_separated

from .fields import ReadonlyField
from .form import FormMixin

class TimesheetEntryAdminForm(forms.ModelForm):
    """
    Validation of ManyToManyField must be performed in form.clean()
    Always use TimesheetEntryAdminForm to change TimesheetEntry
    """

    class Meta:
        model   = TimesheetEntry
        fields  = ('date', 'start', 'end', 'entry_type', 'description')

    def __init__(self, *args, **kwargs):
        timesheet = kwargs.pop('timesheet', None) or kwargs['instance'].timesheet
        super(TimesheetEntryAdminForm, self).__init__(*args, **kwargs)
        self.instance.timesheet = timesheet

        # make date, start and end read only, if timesheet is already submitted
        if self.instance.id and self.instance.timesheet.submitted:
            self.readonly = True
            self.readonly_fields = [
                ReadonlyField(label=_('Date'),      value=self.instance.date),
                ReadonlyField(label=_('Start'),     value=self.instance.start),
                ReadonlyField(label=_('End'),       value=self.instance.end),
                ReadonlyField(label=_('Entry type'),value=self.instance.entry_type),
            ]
            self.fields = {'description': self.fields['description']}
        else:
            self.readonly = False

    def clean_date(self):
        if self.cleaned_data['date'] < self.instance.timesheet.period.start:
            raise ValidationError(_('The timesheet period {period} starts {start}').format(
                period  = self.instance.timesheet.period.name,
                start   = formats.date_format(self.instance.timesheet.period.start, "SHORT_DATE_FORMAT"),
            ))
        if self.cleaned_data['date'] > self.instance.timesheet.period.end:
            raise ValidationError(_('The timesheet period {period} ends {end}').format(
                period  = self.instance.timesheet.period.name,
                end   = formats.date_format(self.instance.timesheet.period.end, "SHORT_DATE_FORMAT"),
            ))
        return self.cleaned_data['date']

    def clean(self):
        # check overlaping entries
        if 'date' in self.cleaned_data and 'start' in self.cleaned_data and 'end' in self.cleaned_data:
            qs = self.instance.timesheet.timesheet_entries.filter(
                date        = self.cleaned_data['date'],
                start__lt   = self.cleaned_data['end'],
                end__gt     = self.cleaned_data['start'],
            )
            if self.instance.id:
                qs = qs.exclude(id=self.instance.id)
            if qs.exists():
                raise ValidationError(_('An overlaping entry has already been added in the timesheet.'))
        return self.cleaned_data

class TimesheetEntryForm(FormMixin, TimesheetEntryAdminForm):
    pass


import gym
# env = gym.make('Breakout-ram-v0')
env = gym.make('Adventure-ram-v0')
# env = gym.make('SpaceInvaders-ram-v0')

env.reset()
print(env.action_space)
print(env.get_action_meanings())

for _ in range(1000):
    env.render()
    env.step(env.action_space.sample()) # take a random action
env.close()


# python featureExtractor_test.py ../../hw4_data/TrimmedVideos/video/test ../../hw4_data/TrimmedVideos/label/gt_test.csv ./ test
import os
import numpy as np
import sys

import torch
import torchvision.transforms as transforms

from tqdm import tqdm
from model import Resnet50
from reader import readShortVideo
from reader import getVideoList
tqdm.pandas()

# data source
video_path = sys.argv[1] #TrimmedVideos/video/valid
test_label_path = sys.argv[2] #TrimmedVideos/label/gt_valid.csv
save_path = sys.argv[3] #TrimmedVideos/label/gt_valid.csv
mode = sys.argv[4] #test
dict = getVideoList(test_label_path)
if mode != "test": action_labels = (dict['Action_labels'])
video_names = (dict['Video_name'])
video_categorys = (dict['Video_category'])
total_num = len(video_names)

# loading videos
test_videos = []
test_labels = []
print("\nloading videos...")
with tqdm(total=total_num) as pbar:
    for i,(video_category, video_name) in enumerate(zip(video_categorys,video_names)):
            train_video = readShortVideo(video_path, video_category, video_name)
            test_videos.append(train_video)
            pbar.update(1)
if mode != "test":
    print("\nloading labels...")
    for i,action_label in enumerate(action_labels):
        test_labels.append(int(action_label))
print("test_videos_len:",len(test_videos))

# extracting features
cnn_feature_extractor = Resnet50().cuda() # to 2048 dims
transform=transforms.Compose([
                        transforms.ToPILImage(),
                        transforms.Pad((0,40),fill=0,padding_mode='constant'),
                        transforms.Resize(224),
                        transforms.ToTensor(),
                        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
                        ])
cnn_feature_extractor.eval()
train_features = []
with torch.no_grad():
    print("\nextracting videos feature...")
    with tqdm(total=total_num) as pbar:
        for train_video in test_videos:
            local_batch = []
            for frame in train_video:
                frame = transform(frame)
                local_batch.append(frame)
            local_batch = torch.stack(local_batch)
            feature = cnn_feature_extractor(local_batch.cuda())
            train_features.append(torch.mean(feature,0))
            pbar.update(1)
train_features = torch.stack(train_features)

# save feature torch as file
# load feature file as torch for test
torch.save(train_features, os.path.join(save_path,mode+'_features.pt'))
train_features = torch.load(os.path.join(save_path,mode+'_features.pt'))
print(mode+"_features:",len(train_features))

if mode != "test":
    test_labels = torch.LongTensor(test_labels)
    torch.save(test_labels, os.path.join(save_path,mode+'_vals.pt'))
    test_labels = torch.load(os.path.join(save_path,mode+'_vals.pt'))
    print(mode+"_labels:",test_labels.shape)


#!/usr/bin/env python

from __future__ import print_function
from builtins import range
from builtins import object

import os.path
import re
import time

import fitsio
import astropy.io.fits as pyfits

import opscore.protocols.keys as keys
import opscore.protocols.types as types

try:
    import actorcore.utility.fits as actorFits
    import hxActor.charis.winFiles as winFiles
    import hxActor.charis.subaru as subaru
    import hxActor.charis.scexao as scexao
    reload(winFiles)
    reload(subaru)
    reload(scexao)
except:
    pass

class CharisCmd(object):

    def __init__(self, actor):

        # This lets us access the rest of the actor.
        self.actor = actor
        self.logger = self.actor.logger

        if self.actor.instrument != 'CHARIS':
            self.logger.info('not CHARIS, skipping CharisCmd.py')
            self.vocab = []
            self.keys = keys.KeysDictionary('charis', (1, 1))
            return

        # Declare the commands we implement. When the actor is started
        # these are registered with the parser, which will call the
        # associated methods when matched. The callbacks will be
        # passed a single argument, the parsed and typed command.
        #
        self.vocab = [
            ('backend', '@(windows|unix)', self.setBackend),
            ('win', '@raw', self.winRaw),
            ('wingetconfig', '', self.winGetconfig),
            ('winflush', '', self.flushProgramInput),
            ('charisConfig', '', self.charisConfig),
        ]

        # Define typed command arguments for the above commands.
        self.keys = keys.KeysDictionary("hx", (1, 2),
                                        keys.Key("seqno", types.Int(), default=None,
                                                 help='If set, the assigned sequence number.'),
                                        keys.Key("nramp", types.Int(), default=1,
                                                 help='number of ramps to take.'),
                                        keys.Key("nreset", types.Int(), default=1,
                                                 help='number of resets to make.'),
                                        keys.Key("nread", types.Int(), default=2,
                                                 help='number of readss to take.'),
                                        keys.Key("ngroup", types.Int(), default=1,
                                                 help='number of groups.'),
                                        keys.Key("ndrop", types.Int(), default=0,
                                                 help='number of drops to waste.'),
                                        keys.Key("itime", types.Float(), default=None,
                                                 help='desired integration time'),
                                        keys.Key("exptype", types.String(), default=None,
                                                 help='What to put in IMAGETYP/DATA-TYP.'),
                                        keys.Key("objname", types.String(), default=None,
                                                 help='What to put in OBJECT.'),
                                        keys.Key("configName", types.String(), default=None,
                                                 help='configuration name'),
                                        keys.Key("voltageName", types.String(), default=None,
                                                 help='voltage name'),
                                        keys.Key("voltage", types.Float(), default=None,
                                                 help='voltage'),
                                        )

        self.backend = 'hxhal'
        self.rampConfig = None

        self.dataRoot = "/home/data/charis"
        self.dataPrefix = "CRSA"

        from hxActor.charis import seqPath
        self.fileGenerator = seqPath.NightFilenameGen(self.dataRoot,
                                                      filePrefix=self.dataPrefix)

    @property
    def controller(self):
        return self.actor.controllers.get(self.backend, None)

    @property
    def sam(self):
        ctrlr = self.actor.controllers.get(self.backend, None)
        return ctrlr.sam

    def bounce(self, cmd):
        self.controller.disconnect()

    def hxconfig(self, cmd):
        """Set the given hxhal configuration. """

        if self.backend != 'hxhal' or self.controller is None:
            cmd.fail('text="No hxhal controller"')
            return

        cmdKeys = cmd.cmd.keywords
        configName = cmdKeys['configName'].values[0]

        sam = self.sam

        sam.updateHxRgConfigParameters('h2rgConfig', configName)
        cmd.finish()

    def charisConfig(self, cmd):
        if self.backend != 'hxhal' or self.controller is None:
            cmd.fail('text="No hxhal controller"')
            return

        cmdKeys = cmd.cmd.keywords
        sam = self.sam

        cmd.inform('text="setting ASIC configuration...."')
        sam.updateHxRgConfigParameters('h2rgConfig', 'cold_feb_05')

        cmd.inform('text="setting voltages...."')
        sam.link.WriteAsicReg(0x602c,0x82c3)

        cmd.finish()

    def setBackend(self, cmd):
        """Select the backend to use.

        Really not sure how to do this. Doing it right is impossible
        (you might need to steal/hand off the /dev/QUSB device between
        W7 and Unix. Among other horrors.)

        I think all this should do is:
          1. declare (via .backend), which backend we want.
          2. try (once) to (dis/re)-connect
        """

        cmdKeys = cmd.cmd.keywords
        if 'unix' in cmdKeys:
            want = 'hxhal'
        elif 'windows' in cmdKeys:
            want = 'winjarvis'
        elif 'none' in cmdKeys:
            want = None

        runningBackend = self.backend
        self.backend = None

        self.backend = want

        cmd.finish()
        return

        if self.controller is not None:
            cmd.inform('text="disconnecting from backend: %s"' % (runningBackend))
            try:
                self.actor.detachController(runningBackend)
            except Exception as e:
                cmd.warn("text='failed to disconnect: %s'" % (e))

        worked = self.actor.attachController(want)
        if worked:
            self.backend = want
            cmd.finish()
        else:
            cmd.fail()

    def winRaw(self, cmd):
        """ Tunnel a rawCmd command to these Windows IDL program. """

        cmdKeys = cmd.cmd.keywords

        ctrl = self.controller

        rawCmd = cmdKeys['raw'].values[0]
        cmd.diag('text="sending raw: %s"' % (rawCmd))
        ret = ctrl.sendOneCommand(rawCmd, cmd=cmd)
        cmd.finish('text="raw: %s"' % (ret))

    def winGetconfig(self, cmd, doFinish=True):
        """ Fetch the results of these Windows IDL 'getconfiguration' command. """

        ctrl = self.controller

        ret = ctrl.sendOneCommand('getconfig', cmd=cmd)
        ret = ret.replace('nOutputs', ' nOutputs')
        ret = ret.replace('winXStop', ' winXStop')
        ret = ret.replace('winYStart', ' winYStart')
        parts = ret.split()

        self.rampConfig = dict()
        for p in parts:
            k,v = p.split('=')
            k = k.lower()
            if k in {'nresets','nreads','nramps','ndrops','ngroups'}:
                print("config %s ->(%s, %s)" % (p, k[:-1], v))
                self.rampConfig[k[:-1]] = int(v)

        kparts = '; '.join(['win_%s' % p for p in parts])
        cmd.inform(kparts)
        if doFinish:
            cmd.finish()
        return parts

    def _calcAcquireTimeout(self, expType='ramp', cmd=None):
        """ Return the best estimate of the actual expected time for our current rampConfig. """

        frameTime = self.sam.frameTime
        if expType == 'ramp':
            return frameTime * (self.rampConfig['nread'] +
                                self.rampConfig['nreset'] +
                                self.rampConfig['ndrop'])
        elif expType == 'single':
            return frameTime * (1 + self.rampConfig['nreset'])
        elif expType == 'CDS':
            return frameTime * (2 + self.rampConfig['nreset'])
        else:
            raise RuntimeError("unknown expType %s" % (expType))

    def flushProgramInput(self, cmd, doFinish=True):
        debris = ''
        while True:
            try:
                ret = self.controller.getOneChar(timeout=0.2)
                debris = debris + ret
            except RuntimeError:
                break
            except:
                raise

        if debris != '':
            cmd.warn('text="flushed stray input: %r"' % (debris))

        if doFinish:
            cmd.finish()

    def _consumeRead(self, path, cmd, header=None):
        #  /home/data/wincharis/H2RG-C17206-ASIC-104/UpTheRamp/20160712210126/H2RG_R01_M01_N01.fits
        dirName, fileName = os.path.split(path)
        cmd.diag('text="checking %s"' % (fileName))
        match = re.match('^H2RG_R0*(\d+)_M0*(\d+)_N0*(\d+)\.fits', fileName)
        if match is None:
            cmd.warn("failed to split up filename: %s" % (file))
            return
        rampN, groupN, readN = [int(m) for m in match.group(1,2,3)]
        cmd.diag('text="new read %d %d %d"' % (rampN, groupN, readN))
        if readN == 1:
            if header is not None:
                cmd.diag('text="getting header"')
                subaruHdr = header
            else:
                subaruHdr = pyfits.Header()
            cards = [dict(name='IDLPATH', value=dirName)]
            for c in subaruHdr.cards:
                cards.append(dict(name=c.keyword, value=c.value, comment=c.comment))
            phdu = fitsio.FITSHDR(cards)
            fitsio.write(self.outfile, None, header=phdu, clobber=True)
            cmd.diag('text="new file %s"' % (self.outfile))

        inData, inHdr = fitsio.read(path, header=True)
        stackFile = fitsio.FITS(self.outfile, mode='rw')
        stackFile.write(inData, header=inHdr)
        stackFile[-1].write_checksum()
        stackFile.close()
        cmd.inform('readN=%d,%d,%d,%s' % (rampN,groupN,readN,self.outfile))

    def getSubaruHeader(self, frameId, timeout=1.0,
                        fullHeader=True, exptype='TEST', cmd=None):

        itime = self.sam.frameTime
        if exptype.lower() == 'nohdr':
            return pyfits.Header()

        headerTask = subaru.FetchHeader(fullHeader=True, frameId=frameId, itime=itime, exptype=exptype)
        self.logger.debug('text="starting header task timeout=%s frameId=%s"' % (timeout, frameId))
        headerTask.start()
        headerQ = headerTask.q
        self.logger.info('text="header q: %s"' % (headerQ))

        try:
            hdrString = headerQ.get(True, timeout)
            if hdrString is None:
                self.logger.debug('text=".get header: %s"' % (hdrString))
            else:
                self.logger.debug('text=".get header: %s"' % (len(hdrString)))
        except Exception as e:
            self.logger.warn('text="failed to .getHeader header: %s"' % (e))
            cmd.warn('text="failed to .getHeader header: %s"' % (e))
            hdrString = ''
        finally:
            headerTask.terminate()
            time.sleep(0.1)

        hdr = pyfits.Header.fromstring(hdrString)

        return hdr

    def getCharisCards(self, cmd):
        charisModel = self.actor.models['charis'].keyVarDict
        cards = []

        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'grism', 'Y_GRISM', idx=0,
                                               comment='deprecated: grism position'))
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'grism', 'Y_PRISM', idx=0,
                                               comment='prism position'))
        # Was CHARIS.FILTER.NAME
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'filterSlot', 'Y_FLTNAM', idx=1,
                                               comment='current filter name'))
        # Was CHARIS.FILTER.SLOT
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'filterSlot', 'Y_FLTSLT', cnv=int, idx=0,
                                               comment='current filter slot'))
        # Was CHARIS.SHUTTER
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'shutter', 'Y_SHUTTR', idx=1,
                                               comment='shutter position'))
        # Was CHARIS.LASER.ENABLED
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'laserState', 'Y_LSRENB', cnv=bool, idx=0,
                                               comment='is laserState enabled'))
        # Was CHARIS.LASER.POWER
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'laserState', 'Y_LSRPWR', idx=2,
                                               comment='laser power, percent'))
        # Was CHARIS.LASER.ALARMS
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'laserState', 'Y_LSRALM', idx=3,
                                               comment='laser alarms'))
        # Was CHARIS.TEMPS.%d
        for i in range(10):
            cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                                   'temps', 'Y_TEMP%02d'%i, idx=i,
                                                   comment='temperature sensor %d'%i))

        # Add global Subaru aliases
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'filterSlot', 'FILTER01', idx=1,
                                               comment='current filter name'))
        cards.append(actorFits.makeCardFromKey(cmd, charisModel,
                                               'grism', 'DISPERSR', idx=0,
                                               comment='prism position'))
        return cards

    def getHeader(self, frameId, fullHeader=True,
                  exptype='TEST', objname='TEST',
                  timeout=1.0, cmd=None):
        try:
            hdr = self.getSubaruHeader(frameId, fullHeader=fullHeader,
                                       exptype=exptype,
                                       timeout=timeout, cmd=cmd)
        except Exception as e:
            self.logger.warn('text="failed to fetch Subaru header: %s"' % (e))
            cmd.warn('text="failed to fetch Subaru header: %s"' % (e))
            hdr = pyfits.Header()

        hdr.set('OBJECT', objname, before=1)
        hxCards = self.getHxCards(cmd)
        for c in hxCards:
            hdr.append(c)

        scexaoCards = self.getSCExAOCards(cmd)
        for c in scexaoCards:
            hdr.append(c)

        charisCards = self.getCharisCards(cmd)
        for c in charisCards:
            hdr.append(c)

        return hdr

    def getCharisHeader(self, seqno=None,
                        exptype='TEST',
                        fullHeader=True, cmd=None):
        return self.getHeader(seqno, exptype=exptype,
                              fullHeader=fullHeader, cmd=cmd)

    def getHxCards(self, cmd=None):
        # voltageList = self.controller.getAllBiasVoltages
        return []

    def getSCExAOCards(self, cmd=None):
        hdr = []
        try:
            hdr = scexao.fetchHeader()
        except:
            pass

        return hdr

    def _consumeRamps(self, nramp, ngroup, nreset, nread, ndrop, cmd, timeLimits=None):
        if timeLimits is None:
            timeLimits = (nreset*1.5+15,
                          nread*1.5+10)

        root = '/home/data/wincharis/H2RG-C17206-ASIC-104'
        cmd.debug('text="starting winFiles i=on %s with timeout=%s"' % (root, timeLimits[0]))
        fileAlerts = hxActor.winFiles.FileAlert(root, timeLimit=timeLimits[0])
        fileQ = fileAlerts.q
        fileAlerts.start()

        try:
            rampsDone = 0
            readsDone = 0
            cmd.inform('text="ramp %d/%d starting %d resets..."' % (rampsDone+1, nramp, nreset))
            self.outfile = self.fileGenerator.getNextFileset()[0]
            cmd.inform('text="new filename %s"' % (self.outfile))
            try:
                header = self.getHeader(self.fileGenerator.seqno, nread*self.readTime, cmd=cmd)
                cmd.debug('text="header process returned %s"' % (None if header is None else len(header)))
            except Exception as e:
                cmd.warn('text="failed to start header process: %s"' % (e))
                header = None

            while rampsDone < nramp:
                if readsDone == 0:
                    cmd.debug('text="waiting for filesys event...')
                event = fileQ.get(timeout=timeLimits[0])

                cmd.debug('text="filesys event: %s"' % (event))
                fileOrDir, action, path = event.split()

                if fileOrDir == 'file' and action == 'done':
                    cmd.debug('text="new read (%d/%d in ramp %d/%d): %s"' % (readsDone+1,nread,
                                                                             rampsDone+1,nramp,
                                                                             path))
                    readsDone += 1
                    self._consumeRead(path, cmd, header)

                if readsDone >= nread:
                    rampsDone += 1
                    readsDone = 0
                    cmd.inform('filename=%s' % (self.outfile))
                    self.outfile = None
                    if rampsDone < nramp:
                        cmd.inform('text="ramp %d/%d starting %d resets..."' % (rampsDone+1, nramp, nreset))
                        self.outfile = self.fileGenerator.getNextFileset()[0]
                        cmd.diag('text="new filename %s"' % (self.outfile))
                        try:
                            header = self.getHeader(self.fileGenerator.seqno, nread*self.readTime, cmd=cmd)
                        except Exception as e:
                            cmd.warn('text="failed to start header process: %s"' % (e))
                            header = None

        except Exception as e:
            cmd.warn('winfile readers failed with %s' % (e))
            fileAlerts.terminate()
        finally:
            fileAlerts.terminate()
            del fileAlerts

        self.outfile = None


import tkinter as tk
import tkinter.messagebox as messagebox
import tkinter.ttk as ttk
import os.path

cmpdWidth = 60
numCmpds = 0
frameHeight = 0
saveFile = 'saveData.txt'
maxNumCmpds = 7

mapW = {}
options = ()
combos = []
cmpds = []
cmpdUIs = {}

def saveData():
	with open(saveFile, 'w') as saved:
		saved.write(str(mapW))
		saved.close()

def calculate():
	global cmpds
	global totalMass
	mixMass = 0
	for cmpd in cmpds:
		mixMass += (mapW[cmpd[0].get()] * float(cmpd[1].get()))
	moles = float(totalMass.get()) / mixMass
	for cmpd in cmpds:
		cmpd[2].set(str(moles * mapW[cmpd[0].get()] * float(cmpd[1].get())))

def changeValues():
	global combos
	for combo in combos:
		combo['values'] = options

def reloadOptions():
	global options
	options = tuple(mapW.keys())
	print(options)

def cancelCmpd(window):
	window.destroy()

def finishNewCmpd(window, name, weight):
	global mapW
	window.destroy()
	try:
		mapW[name] = float(weight)
	except ValueError:
		messagebox.showerror('Invalid weight', 'Weight must be a positive number!')
	reloadOptions()
	changeValues()
	saveData()

def newCmpd():
	w = tk.Toplevel()
	w.resizable(0, 0)
	w.wm_title('New')
	name = tk.StringVar()
	cmpdName = tk.Entry(w, textvariable=name)
	cmpdName.grid(row=0, column=0, columnspan=2)
	weight = tk.StringVar()
	cmpdWeight = tk.Entry(w, textvariable=weight)
	cmpdWeight.grid(row=1, column=0, columnspan=2)
	tk.Button(w, text='Ok', command=lambda: finishNewCmpd(w, name.get(), weight.get())).grid(row=2, column=0)
	tk.Button(w, text='Cancel', command=lambda: cancelCmpd(w)).grid(row=2, column=1)

def addCmpd():
	global numCmpds
	global cmpdframe
	global options
	global cmpds
	if numCmpds < maxNumCmpds:
		numCmpds += 1
		cmpd = tk.StringVar()
		combo = ttk.Combobox(cmpdframe, width=10, values=options, state='readonly', textvariable=cmpd)
		combos.append(combo)
		combo.grid(row=numCmpds, column=0)
		ratio = tk.StringVar()
		ratioEntry = tk.Entry(cmpdframe, width=17, textvariable=ratio)
		ratioEntry.grid(row=numCmpds, column=1)
		result = tk.StringVar()
		resultEntry = tk.Entry(cmpdframe, width=17, textvariable=result)
		resultEntry.grid(row=numCmpds, column=2)
		cmpds.append((cmpd, ratio, result))
		removeButton = tk.Button(cmpdframe, text='X', fg='red', command=lambda: removeCmpd(removeButton))
		removeButton.grid(row=numCmpds, column=3)
		cmpdUIs[removeButton] = (combo, ratioEntry, resultEntry, (cmpd, ratio, result))
	else:
		messagebox.showerror('ERROR!', 'Cannot create any more compounds!')

def removeCmpd(removeButton):
	for widget in cmpdUIs[removeButton][:3]:
		widget.destroy()
	combos.remove(cmpdUIs[removeButton][0])
	cmpds.remove(cmpdUIs[removeButton][3])
	removeButton.destroy()

if os.path.isfile(saveFile):
	with open(saveFile, 'r') as saved:
		mapW = eval(saved.read())
		saved.close()
		reloadOptions()

root = tk.Tk()
root.resizable(0, 0)
root.wm_title('Ratios')

tk.Button(text='Add', command=addCmpd).grid(row=0, column=0)
tk.Button(text='New', command=newCmpd).grid(row=0, column=1)

c = tk.Canvas(width=450)
c.grid(row=1, column=0, columnspan=2)
cmpdframe = tk.Frame(c)
c.create_window(10, 10, window=cmpdframe, anchor='nw')

totalMass = tk.StringVar()
tk.Entry(textvariable=totalMass).grid(row=2, column=0)

tk.Button(command=calculate).grid(row=2, column=1)

tk.mainloop()


import cv2
from facenet_pytorch import MTCNN, InceptionResnetV1, fixed_image_standardization
import torch
from torchvision import transforms
import numpy as np
from PIL import Image
import time

frame_size = (640,480)
IMG_PATH = './data/test_images'
DATA_PATH = './data'

def trans(img):
    transform = transforms.Compose([
            transforms.ToTensor(),
            fixed_image_standardization
        ])
    return transform(img)

def load_faceslist():
    if device == 'cpu':
        embeds = torch.load(DATA_PATH+'/faceslistCPU.pth')
    else:
        embeds = torch.load(DATA_PATH+'/faceslist.pth')
    names = np.load(DATA_PATH+'/usernames.npy')
    return embeds, names

def inference(model, face, local_embeds, threshold = 3):
    #local: [n,512] voi n la so nguoi trong faceslist
    embeds = []
    # print(trans(face).unsqueeze(0).shape)
    embeds.append(model(trans(face).to(device).unsqueeze(0)))
    detect_embeds = torch.cat(embeds) #[1,512]
    # print(detect_embeds.shape)
                    #[1,512,1]                                      [1,512,n]
    norm_diff = detect_embeds.unsqueeze(-1) - torch.transpose(local_embeds, 0, 1).unsqueeze(0)
    # print(norm_diff)
    norm_score = torch.sum(torch.pow(norm_diff, 2), dim=1) #(1,n), moi cot la tong khoang cach euclide so vs embed moi

    min_dist, embed_idx = torch.min(norm_score, dim = 1)
    print(min_dist*power, names[embed_idx])
    # print(min_dist.shape)
    if min_dist*power > threshold:
        return -1, -1
    else:
        return embed_idx, min_dist.double()

def extract_face(box, img, margin=20):
    face_size = 160
    img_size = frame_size
    margin = [
        margin * (box[2] - box[0]) / (face_size - margin),
        margin * (box[3] - box[1]) / (face_size - margin),
    ] #tạo margin bao quanh box cũ
    box = [
        int(max(box[0] - margin[0] / 2, 0)),
        int(max(box[1] - margin[1] / 2, 0)),
        int(min(box[2] + margin[0] / 2, img_size[0])),
        int(min(box[3] + margin[1] / 2, img_size[1])),
    ]
    img = img[box[1]:box[3], box[0]:box[2]]
    face = cv2.resize(img,(face_size, face_size), interpolation=cv2.INTER_AREA)
    face = Image.fromarray(face)
    return face

if __name__ == "__main__":
    prev_frame_time = 0
    new_frame_time = 0
    power = pow(10, 6)
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print(device)

    model = InceptionResnetV1(
        classify=False,
        pretrained="casia-webface"
    ).to(device)
    model.eval()

    mtcnn = MTCNN(thresholds= [0.7, 0.7, 0.8] ,keep_all=True, device = device)

    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH,640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT,480)
    embeddings, names = load_faceslist()
    while cap.isOpened():
        isSuccess, frame = cap.read()
        if isSuccess:
            boxes, _ = mtcnn.detect(frame)
            if boxes is not None:
                for box in boxes:
                    bbox = list(map(int,box.tolist()))
                    face = extract_face(bbox, frame)
                    idx, score = inference(model, face, embeddings)
                    if idx != -1:
                        frame = cv2.rectangle(frame, (bbox[0],bbox[1]), (bbox[2],bbox[3]), (0,0,255), 6)
                        score = torch.Tensor.cpu(score[0]).detach().numpy()*power
                        frame = cv2.putText(frame, names[idx] + '_{:.2f}'.format(score), (bbox[0],bbox[1]), cv2.FONT_HERSHEY_DUPLEX, 2, (0,255,0), 2, cv2.LINE_8)
                    else:
                        frame = cv2.rectangle(frame, (bbox[0],bbox[1]), (bbox[2],bbox[3]), (0,0,255), 6)
                        frame = cv2.putText(frame,'Unknown', (bbox[0],bbox[1]), cv2.FONT_HERSHEY_DUPLEX, 2, (0,255,0), 2, cv2.LINE_8)

            new_frame_time = time.time()
            fps = 1/(new_frame_time-prev_frame_time)
            prev_frame_time = new_frame_time
            fps = str(int(fps))
            cv2.putText(frame, fps, (7, 70), cv2.FONT_HERSHEY_DUPLEX, 3, (100, 255, 0), 3, cv2.LINE_AA)

        cv2.imshow('Face Recognition', frame)
        if cv2.waitKey(1)&0xFF == 27:
            break

    cap.release()
    cv2.destroyAllWindows()
    #+": {:.2f}".format(score)