import os
import cv2
import argparse
import cvlib as cv
from cvlib.object_detection import draw_bbox

import numpy as np
import imutils

parser = argparse.ArgumentParser()

parser.add_argument("--input", type=str, dest="input")

args = parser.parse_args()

input_name = args.input
print(input_name)
frame = cv2.imread(input_name, cv2.IMREAD_COLOR)

bbox, label, conf = cv.detect_common_objects(frame, model='yolov4', enable_gpu=True)

# for i, cla in enumerate(label):
#     if cla == 'car' or cla == 'bus' or cla == 'truck' or cla == 'train':
#         blur_h = int((bbox[i][3]-bbox[i][1])*1/3)
#         if bbox[i][0] > 0 and bbox[i][1] > 0:
#             blur_area = frame[bbox[i][3]-blur_h:bbox[i][3]+int(blur_h/3),bbox[i][0]:bbox[i][2]]
#             blur_img = cv2.blur(blur_area, (7,7))
#             frame[bbox[i][3]-blur_h:bbox[i][3]+int(blur_h/3),bbox[i][0]:bbox[i][2]] = blur_img
# draw bounding box over detected objects
frame = draw_bbox(frame, bbox, label, conf, write_conf=True)

# display output
cv2.namedWindow('frame',cv2.WINDOW_NORMAL)
cv2.resizeWindow('frame',720,360)
cv2.imshow("frame", frame)
# cv2.imwrite("output/"+input_name[6:-4]+"_result.jpg",frame)
cv2.waitKey(0)

cv2.destroyAllWindows()


# --------------------------------------------------------
# DenseCap-Tensorflow
# Written by InnerPeace
# This file is adapted from Linjie's work
# --------------------------------------------------------
# Tensorflow Faster R-CNN
# Licensed under The MIT License [see LICENSE for details]
# Written by Xinlei Chen
# --------------------------------------------------------
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow.contrib.slim import losses
from tensorflow.contrib.slim import arg_scope
import tensorflow.contrib.rnn as rnn

import numpy as np
import pdb

from lib.layers.snippets import generate_anchors_pre
from lib.layers.proposal_layer import proposal_layer
from lib.layers.proposal_top_layer import proposal_top_layer
from lib.layers.anchor_target_layer import anchor_target_layer
from lib.layers.proposal_target_layer import proposal_target_layer
from lib.layers.proposal_target_single_class_layer import proposal_target_single_class_layer
from lib.layers.sentence_data_layer import sentence_data_layer
from lib.utils.visualization import draw_bounding_boxes, draw_densecap

from lib.config import cfg

class Network(object):
    def __init__(self):
        self._predictions = {}
        self._losses = {}
        self._anchor_targets = {}
        self._proposal_targets = {}
        self._layers = {}
        self._gt_image = None
        self._act_summaries = []
        self._score_summaries = {}
        self._train_summaries = []
        self._event_summaries = {}
        self._variables_to_fix = {}

        self._image = tf.placeholder(tf.float32, shape=[1, None, None, 3])
        self._im_info = tf.placeholder(tf.float32, shape=[3])
        # add global roi
        if cfg.CONTEXT_FUSION:
            self._global_roi = tf.placeholder(tf.float32, shape=[1, 5])
        self._gt_boxes = tf.placeholder(tf.float32, shape=[None, 5])
        # add 2 for: <SOS> and <EOS>
        self._gt_phrases = tf.placeholder(tf.int32, shape=[None, cfg.MAX_WORDS])

        self._anchor_scales = cfg.ANCHOR_SCALES
        self._num_scales = len(self._anchor_scales)

        self._anchor_ratios = cfg.ANCHOR_RATIOS
        self._num_ratios = len(self._anchor_ratios)

        self._num_anchors = self._num_scales * self._num_ratios

        if cfg.DEBUG_ALL:
            self._for_debug = {}
            self._tag = 'pre'
            self._mode = 'TRAIN'
            self._num_classes = 1

    def _add_gt_image(self):
        # add back mean
        image = self._image + cfg.PIXEL_MEANS
        # BGR to RGB (opencv uses BGR)
        resized = tf.image.resize_bilinear(image, tf.to_int32(self._im_info[:2] / self._im_info[2]))
        self._gt_image = tf.reverse(resized, axis=[-1])

    def _add_gt_image_summary(self):
        # use a customized visualization function to visualize the boxes
        if self._gt_image is None:
            self._add_gt_image()
        image = tf.py_func(draw_bounding_boxes,
                           [self._gt_image, self._gt_boxes, self._im_info, self._gt_phrases],
                           tf.float32, name="gt_boxes")

        return tf.summary.image('GROUND_TRUTH', image)

    def _add_image_summary(self):
        # add back mean
        image = self._image + cfg.PIXEL_MEANS
        # BGR to RGB (opencv uses BGR)
        resized = tf.image.resize_bilinear(image, tf.to_int32(self._im_info[:2] / self._im_info[2]))
        gt_image = tf.reverse(resized, axis=[-1])
        img_wcap = tf.py_func(draw_densecap, [gt_image, self._predictions['cls_prob'],
            self._predictions['rois'], self._im_info, self._predictions['cap_probs'],
            self._predictions['bbox_pred']], tf.float32, name='image_summary')

        return tf.summary.image('TEMP_OUT', img_wcap)

    def _add_act_summary(self, tensor):
        tf.summary.histogram('ACT/' + tensor.op.name + '/activations', tensor)
        tf.summary.scalar('ACT/' + tensor.op.name + '/zero_fraction',
                          tf.nn.zero_fraction(tensor))

    def _add_score_summary(self, key, tensor):
        tf.summary.histogram('SCORE/' + tensor.op.name + '/' + key + '/scores', tensor)

    def _add_train_summary(self, var):
        tf.summary.histogram('TRAIN/' + var.op.name, var)

    def _reshape_layer(self, bottom, num_dim, name):
        input_shape = tf.shape(bottom)
        with tf.variable_scope(name) as scope:
            # change the channel to the caffe format
            to_caffe = tf.transpose(bottom, [0, 3, 1, 2])
            # then force it to have channel 2
            reshaped = tf.reshape(to_caffe,
                                  tf.concat(axis=0, values=[[1, num_dim, -1], [input_shape[2]]]))
            # then swap the channel back
            to_tf = tf.transpose(reshaped, [0, 2, 3, 1])
            return to_tf

    def _softmax_layer(self, bottom, name):
        if name.startswith('rpn_cls_prob_reshape'):
            input_shape = tf.shape(bottom)
            bottom_reshaped = tf.reshape(bottom, [-1, input_shape[-1]])
            reshaped_score = tf.nn.softmax(bottom_reshaped, name=name)
            return tf.reshape(reshaped_score, input_shape)
        return tf.nn.softmax(bottom, name=name)

    def _proposal_top_layer(self, rpn_cls_prob, rpn_bbox_pred, name):
        with tf.variable_scope(name) as scope:
            rois, rpn_scores = tf.py_func(proposal_top_layer,
                                          [rpn_cls_prob, rpn_bbox_pred, self._im_info,
                                           self._feat_stride, self._anchors, self._num_anchors],
                                          [tf.float32, tf.float32], name="proposal_top")
            rois.set_shape([cfg.TEST.RPN_TOP_N, 5])
            rpn_scores.set_shape([cfg.TEST.RPN_TOP_N, 1])

        return rois, rpn_scores

    def _proposal_layer(self, rpn_cls_prob, rpn_bbox_pred, name):
        with tf.variable_scope(name) as scope:
            rois, rpn_scores = tf.py_func(proposal_layer,
                                          [rpn_cls_prob, rpn_bbox_pred, self._im_info, self._mode,
                                           self._feat_stride, self._anchors, self._num_anchors],
                                          [tf.float32, tf.float32], name="proposal")
            rois.set_shape([None, 5])
            rpn_scores.set_shape([None, 1])

            if cfg.DEBUG_ALL:
                self._for_debug['proposal_rois'] = rois
                self._for_debug['proposal_rpn_scores'] = rpn_scores

        return rois, rpn_scores

    # Only use it if you have roi_pooling op written in tf.image
    def _roi_pool_layer(self, bootom, rois, name):
        with tf.variable_scope(name) as scope:
            return tf.image.roi_pooling(bootom, rois,
                                        pooled_height=cfg.POOLING_SIZE,
                                        pooled_width=cfg.POOLING_SIZE,
                                        spatial_scale=1. / 16.)[0]

    def _crop_pool_layer(self, bottom, rois, name):
        with tf.variable_scope(name) as scope:
            batch_ids = tf.squeeze(tf.slice(rois, [0, 0], [-1, 1], name="batch_id"), [1])
            # Get the normalized coordinates of bounding boxes
            bottom_shape = tf.shape(bottom)
            height = (tf.to_float(bottom_shape[1]) - 1.) * np.float32(self._feat_stride[0])
            width = (tf.to_float(bottom_shape[2]) - 1.) * np.float32(self._feat_stride[0])
            x1 = tf.slice(rois, [0, 1], [-1, 1], name="x1") / width
            y1 = tf.slice(rois, [0, 2], [-1, 1], name="y1") / height
            x2 = tf.slice(rois, [0, 3], [-1, 1], name="x2") / width
            y2 = tf.slice(rois, [0, 4], [-1, 1], name="y2") / height
            # Won't be back-propagated to rois anyway, but to save time
            bboxes = tf.stop_gradient(tf.concat([y1, x1, y2, x2], axis=1))
            pre_pool_size = cfg.POOLING_SIZE * 2
            crops = tf.image.crop_and_resize(bottom, bboxes, tf.to_int32(batch_ids),
                                             [pre_pool_size, pre_pool_size],
                                             name="crops")

        # slim.max_pool2d has stride 2 in default
        return slim.max_pool2d(crops, [2, 2], padding='SAME')

    def _dropout_layer(self, bottom, name, ratio=0.5):
        return tf.nn.dropout(bottom, ratio, name=name)

    def _anchor_target_layer(self, rpn_cls_score, name):
        with tf.variable_scope(name) as scope:
            rpn_labels, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = tf.py_func(
                anchor_target_layer,
                [rpn_cls_score, self._gt_boxes, self._im_info, self._feat_stride, self._anchors, self._num_anchors],
                [tf.float32, tf.float32, tf.float32, tf.float32],
                name="anchor_target")

            rpn_labels.set_shape([1, 1, None, None])
            rpn_bbox_targets.set_shape([1, None, None, self._num_anchors * 4])
            rpn_bbox_inside_weights.set_shape([1, None, None, self._num_anchors * 4])
            rpn_bbox_outside_weights.set_shape([1, None, None, self._num_anchors * 4])

            rpn_labels = tf.to_int32(rpn_labels, name="to_int32")
            self._anchor_targets['rpn_labels'] = rpn_labels
            self._anchor_targets['rpn_bbox_targets'] = rpn_bbox_targets
            self._anchor_targets['rpn_bbox_inside_weights'] = rpn_bbox_inside_weights
            self._anchor_targets['rpn_bbox_outside_weights'] = rpn_bbox_outside_weights

            self._score_summaries.update(self._anchor_targets)

        if cfg.DEBUG_ALL:
            self._for_debug['rpn_labels'] = rpn_labels
            self._for_debug['rpn_bbox_targets'] = rpn_bbox_targets
            self._for_debug['rpn_bbox_inside_weights'] = rpn_bbox_inside_weights
            self._for_debug['rpn_bbox_outside_weights'] = rpn_bbox_outside_weights

        return rpn_labels

    # TODO: about to delete
    def _proposal_target_layer(self, rois, roi_scores, name):
        with tf.variable_scope(name) as scope:
            rois, roi_scores, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights = tf.py_func(
                proposal_target_layer,
                [rois, roi_scores, self._gt_boxes, self._num_classes],
                [tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32],
                name="proposal_target")

            rois.set_shape([cfg.TRAIN.BATCH_SIZE, 5])
            roi_scores.set_shape([cfg.TRAIN.BATCH_SIZE])
            labels.set_shape([cfg.TRAIN.BATCH_SIZE, 1])
            bbox_targets.set_shape([cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])
            bbox_inside_weights.set_shape([cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])
            bbox_outside_weights.set_shape([cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])

            self._proposal_targets['rois'] = rois
            self._proposal_targets['labels'] = tf.to_int32(labels, name="to_int32")
            self._proposal_targets['bbox_targets'] = bbox_targets
            self._proposal_targets['bbox_inside_weights'] = bbox_inside_weights
            self._proposal_targets['bbox_outside_weights'] = bbox_outside_weights

            self._score_summaries.update(self._proposal_targets)

            return rois, roi_scores

    # TODO: roi_scores are not needed. About to delete
    def _proposal_target_single_class_layer(self, rois, roi_scores, name):

        with tf.variable_scope(name) as scope:
            rois, roi_scores, labels, bbox_targets, \
                bbox_inside_weights, bbox_outside_weights, clss, phrases = tf.py_func(
                    proposal_target_single_class_layer,
                    [rois, roi_scores, self._gt_boxes, self._gt_phrases],
                    [tf.float32, tf.float32, tf.float32, tf.float32,
                     tf.float32, tf.float32, tf.int32, tf.int32],
                    name="proposal_target_single_class")

            rois.set_shape([None, 5])
            roi_scores.set_shape([None])
            labels.set_shape([None, 1])
            phrases.set_shape([None, cfg.MAX_WORDS])
            bbox_targets.set_shape([None, 4])
            bbox_inside_weights.set_shape([None, 4])
            bbox_outside_weights.set_shape([None, 4])
            clss.set_shape([None, 1])

            # phrases = tf.to_int32(phrases, name='to_int32')
            labels = tf.to_int32(labels, name="to_int32")
            # clss = tf.to_int32(clss, name="to_int32")

            self._proposal_targets['rois'] = rois
            self._proposal_targets['labels'] = labels
            self._proposal_targets['clss'] = clss
            self._proposal_targets['phrases'] = phrases
            self._proposal_targets['bbox_targets'] = bbox_targets
            self._proposal_targets['bbox_inside_weights'] = bbox_inside_weights
            self._proposal_targets['bbox_outside_weights'] = bbox_outside_weights

            self._score_summaries.update(self._proposal_targets)

            if cfg.DEBUG_ALL:
                self._for_debug['rois'] = rois
                self._for_debug['labels'] = tf.to_int32(labels, name="to_int32")
                self._for_debug['clss'] = tf.to_int32(clss, name="to_int32")
                self._for_debug['phrases'] = phrases
                self._for_debug['bbox_targets'] = bbox_targets
                self._for_debug['bbox_inside_weights'] = bbox_inside_weights
                self._for_debug['bbox_outside_weights'] = bbox_outside_weights

            return rois, labels, phrases

    def _sentence_data_layer(self, name,
                             time_steps=cfg.TIME_STEPS, mode=cfg.CONTEXT_MODE):

        num_regions = self._roi_labels.shape[0]
        with tf.variable_scope(name) as scope:
            input_sentence, target_sentence, cont_sentence, cont_bbox = \
                tf.py_func(sentence_data_layer,
                           [self._roi_labels, self._roi_phrases, time_steps, mode],
                           [tf.float32, tf.float32, tf.float32, tf.float32],
                           name='sentence_data')
            if cfg.CONTEXT_MODE == 'concat':
                input_sentence.set_shape([num_regions, cfg.TIME_STEPS - 1])
            elif cfg.CONTEXT_MODE == 'repeat':
                input_sentence.set_shape([num_regions, cfg.TIME_STEPS])

            target_sentence.set_shape([num_regions, cfg.TIME_STEPS])
            cont_sentence.set_shape([num_regions, cfg.TIME_STEPS])
            cont_bbox.set_shape([num_regions, cfg.TIME_STEPS])

            input_sentence = tf.to_int32(input_sentence, name='to_int32')
            target_sentence = tf.to_int32(target_sentence, name='to_int32')

            self._sentence_data = {}
            self._sentence_data['input_sentence'] = input_sentence
            self._sentence_data['target_sentence'] = target_sentence
            self._sentence_data['cont_sentence'] = cont_sentence
            self._sentence_data['cont_bbox'] = cont_bbox

            self._score_summaries.update(self._sentence_data)

            if cfg.DEBUG_ALL:
                self._for_debug['input_sentence'] = input_sentence
                self._for_debug['target_sentence'] = target_sentence
                self._for_debug['cont_sentence'] = cont_sentence
                self._for_debug['cont_bbox'] = cont_bbox

        return input_sentence

    def _embed_caption_layer(self, fc7, input_sentence, initializer, is_training):
        """
        comput image context feature and embed input sentence,
        do 'concat' or 'repeat' image feature
        """
        region_features = slim.fully_connected(fc7, cfg.EMBED_DIM,
                                              weights_initializer=initializer,
                                              trainable=is_training,
                                              activation_fn=None, scope='region_features')
        if cfg.CONTEXT_FUSION:
            # global_feature [1, cfg.EMBED_DIM(512)]
            global_feature, region_features = tf.split(region_features, [1, -1], axis=0)
            batch_size = tf.shape(region_features)[0]
            # global_feature_rep [batch_size(256), cfg.EMBED_DIM(512)]
            global_feature_rep = tf.tile(global_feature, [batch_size, 1])
            gfeat_lstm_cell = rnn.BasicLSTMCell(cfg.EMBED_DIM, forget_bias=1.0,
                state_is_tuple=True)
        else:
            batch_size = tf.shape(region_features)[0]

        with tf.variable_scope('seq_embedding'), tf.device("/cpu:0"):
            if cfg.INIT_BY_GLOVE and is_training:
                glove_path = cfg.DATA_DIR + "/glove.trimmed.{}.npz".format(cfg.GLOVE_DIM)
                glove = np.load(glove_path)['glove'].astype(np.float32)
                print("load pre-trained glove from {}, with shape: {}".format(glove_path,
                    glove.shape))
                if not cfg.KEEP_AS_GLOVE_DIM:
                    assert cfg.EMBED_DIM > cfg.GLOVE_DIM
                    g_mean = np.mean(glove, axis=1)
                    g_std = np.std(glove, axis=1)
                    expand_glove = np.random.normal(g_mean, g_std,
                         (cfg.EMBED_DIM, cfg.VOCAB_SIZE + 3))
                    expand_glove[:glove.shape[1], :] = glove.T
                    embed_initializer = tf.constant_initializer(expand_glove.T)
                    print("Expanding glove vectors to be compatibale with embedding dimension.")
                else:
                    embed_initializer = tf.constant_initializer(glove)
                    assert cfg.EMBED_DIM == cfg.GLOVE_DIM
                    print("Keep the dimension of glove({}) as embedding dimension".format(
                        cfg.GLOVE_DIM))
            else:
                embed_initializer = initializer
                print("Initialize embedding vectors with default initializer.")

            self._embedding = tf.get_variable("embedding",
                                              # 0,1,2 for pad sos eof respectively.
                                              [cfg.VOCAB_SIZE + 3, cfg.EMBED_DIM],
                                              initializer=embed_initializer,
                                              trainable=is_training,
                                              dtype=tf.float32
                                              )
            print("Shape of embedding is {}".format(self._embedding.shape))
            # independent decoder and encoder for word representation.
            # self._inverse_embed = tf.get_variable('inverse_embed',
                                                  # [cfg.EMBED_DIM, cfg.VOCAB_SIZE + 3],
                                                  # initializer=initializer)
            embed_input_sentence = tf.nn.embedding_lookup(self._embedding,
                                                          input_sentence)

        location_lstm_cell = rnn.BasicLSTMCell(cfg.EMBED_DIM, forget_bias=1.0, state_is_tuple=True)
        caption_lstm_cell = rnn.BasicLSTMCell(cfg.EMBED_DIM, forget_bias=1.0, state_is_tuple=True)

        # add dropout in rnn
        # cell = rnn.DropoutWrapper(caption_lstm_cell,
        #                           input_keep_prob=prob,
        #                           output_keep_prob=prob,)

        with tf.variable_scope("lstm") as lstm_scope:
            # Feed the image embeddings to set the intial LSTM state
            cap_zero_state = caption_lstm_cell.zero_state(
                batch_size=batch_size, dtype=tf.float32)
            loc_zero_state = location_lstm_cell.zero_state(
                batch_size=batch_size, dtype=tf.float32)
            with tf.variable_scope('cap_lstm'):
                _, cap_init_state = caption_lstm_cell(region_features, cap_zero_state)
            with tf.variable_scope('loc_lstm'):
                _, loc_init_state = location_lstm_cell(region_features, loc_zero_state)

            if cfg.CONTEXT_FUSION:
                gfeat_zero_state = gfeat_lstm_cell.zero_state(
                    batch_size=batch_size, dtype=tf.float32)
                with tf.variable_scope('gfeat_lstm'):
                    # NOTE: gfeat_init_state [batch_size(256), cfg.EMBED_DIM(512)]
                    _, gfeat_init_state = gfeat_lstm_cell(global_feature_rep,
                        gfeat_zero_state)

            # Allow the LSTM variable to be reused
            lstm_scope.reuse_variables()

            if self._mode == 'TRAIN':
                if cfg.CONTEXT_MODE == 'concat':
                    im_context = tf.expand_dims(region_features, axis=1)
                    im_concat_words = tf.concat([im_context, embed_input_sentence],
                        axis=1)
                    if cfg.CONTEXT_FUSION:
                        global_feature_rep = tf.expand_dims(global_feature_rep, axis=1)
                        global_concat_words = tf.concat([global_feature_rep,
                            embed_input_sentence], axis=1)
                else:
                    raise NotImplementedError

                # TODO(wu) need to check again about the sequence length
                sequence_length = self._sequence_length(input_sentence) + 1
                cap_outputs, cap_states = tf.nn.dynamic_rnn(caption_lstm_cell, im_concat_words,
                                                            sequence_length=sequence_length,
                                                            dtype=tf.float32,
                                                            scope='captoin_lstm')

                loc_outputs, loc_states = tf.nn.dynamic_rnn(location_lstm_cell, im_concat_words,
                                                            sequence_length=sequence_length,
                                                            dtype=tf.float32,
                                                            scope='location_lstm')
                if cfg.CONTEXT_FUSION:
                    gfeat_outputs, gfeat_states = tf.nn.dynamic_rnn(gfeat_lstm_cell,
                        global_concat_words,
                        sequence_length=sequence_length,
                        dtype=tf.float32,
                        scope='gfeat_lstm')
                    # for now, it only support mode "sum"
                    if cfg.CONTEXT_FUSION_MODE == "sum":
                        cap_outputs = gfeat_outputs + cap_outputs
                    else:
                        raise NotImplementedError

                # OUT OF MEMORY ON GPU
                # inv_embedding = tf.tile(tf.expand_dims(tf.transpose(self._embedding),
                #                                        [0]),
                #                         [tf.shape(caption_outputs)[0], 1, 1])
                # predict_caption = tf.matmul(caption_outputs, inv_embedding)

                # cap_logits = tf.matmul(tf.reshape(cap_outputs, [-1, cfg.EMBED_DIM]),
                                       # tf.transpose(self._embedding))
                # cap_logits = tf.reshape(predict_cap_reshape,
                #                         [-1, cfg.TIME_STEPS, cfg.VOCAB_SIZE + 3])

                # problematic to always slice output of the last slice
                # loc_out_slice = tf.slice(loc_outputs, [0, cfg.TIME_STEPS - 1, 0], [-1, 1, -1])
                # loc_out_slice = tf.squeeze(loc_out_slice, [1])
                # BETTER SOLUTION
                cont_bbox = tf.expand_dims(self._sentence_data['cont_bbox'], axis=2)
                loc_out_slice = tf.reduce_sum(loc_outputs * cont_bbox, axis=1)

            elif self._mode == 'TEST':
                # In inference or test mode, use concatenated states for convenient feeding
                tf.concat(values=cap_init_state, axis=1, name='cap_init_state')
                tf.concat(values=loc_init_state, axis=1, name='loc_init_state')

                # placeholder for feeding a batch of concatnated states.
                cap_state_feed = tf.placeholder(dtype=tf.float32,
                                                shape=[None, sum(caption_lstm_cell.state_size)],
                                                name='cap_state_feed')
                loc_state_feed = tf.placeholder(dtype=tf.float32,
                                                shape=[None, sum(location_lstm_cell.state_size)],
                                                name='loc_state_feed')
                cap_state_tuple = tf.split(value=cap_state_feed, num_or_size_splits=2, axis=1)
                loc_state_tuple = tf.split(value=loc_state_feed, num_or_size_splits=2, axis=1)

                # Run a single LSTM step
                seq_embedding = tf.squeeze(embed_input_sentence, axis=[1])
                cap_outputs, cap_state_tuple = caption_lstm_cell(
                    inputs=seq_embedding,
                    state=cap_state_tuple
                )
                loc_outputs, loc_state_tuple = location_lstm_cell(
                    inputs=seq_embedding,
                    state=loc_state_tuple
                )

                # Concatenate the resulting state
                tf.concat(values=cap_state_tuple, axis=1, name='cap_state')
                tf.concat(values=loc_state_tuple, axis=1, name='loc_state')
                loc_out_slice = loc_outputs
                # NOTE CONTEXT FUSION
                if cfg.CONTEXT_FUSION:
                    tf.concat(values=gfeat_init_state, axis=1, name='gfeat_init_state')
                    gfeat_state_feed = tf.placeholder(dtype=tf.float32,
                        shape=[None, sum(gfeat_lstm_cell.state_size)],
                        name='gfeat_state_feed')
                    gfeat_state_tuple = tf.split(value=gfeat_state_feed, num_or_size_splits=2,
                        axis=1)
                    gfeat_outputs, gfeat_state_tuple = gfeat_lstm_cell(
                        inputs=seq_embedding,
                        state=gfeat_state_tuple)
                    tf.concat(values=gfeat_state_tuple, axis=1, name='gfeat_state')
                    if cfg.CONTEXT_FUSION_MODE == "sum":
                        cap_outputs += gfeat_outputs

            else:
                raise NotImplementedError

            # caption logits
            cap_logits = tf.matmul(tf.reshape(cap_outputs, [-1, cfg.EMBED_DIM]),
                tf.transpose(self._embedding), name='cap_logits')
            # cap_logits = tf.matmul(tf.reshape(cap_outputs, [-1, cfg.EMBED_DIM]),
                # self._inverse_embed, name='cap_logits')
            cap_probs = tf.nn.softmax(cap_logits, name='cap_probs')

        bbox_pred = slim.fully_connected(loc_out_slice, 4,
                                         weights_initializer=initializer,
                                         trainable=is_training,
                                         activation_fn=None, scope='bbox_pred')

        self._predictions['bbox_pred'] = bbox_pred
        self._predictions['cap_probs'] = cap_probs
        self._predictions['predict_caption'] = cap_logits

        if cfg.DEBUG_ALL:
            self._for_debug['embedding'] = self._embedding
            self._for_debug['embed_input_sentence'] = embed_input_sentence
            self._for_debug['fc8'] = region_features
            # self._for_debug['im_concat_words'] = im_concat_words
            self._for_debug['captoin_outputs'] = cap_outputs
            self._for_debug['loc_outputs'] = loc_outputs
            self._for_debug['loc_out_slice'] = loc_out_slice
            self._for_debug['bbox_pred'] = bbox_pred
            self._for_debug['predict_caption'] = cap_logits

    def _sequence_length(self, input_sentence):
        return tf.reduce_sum(tf.cast(tf.cast(input_sentence, tf.bool), tf.int32), axis=1)

    def _anchor_component(self):
        with tf.variable_scope('ANCHOR_' + self._tag) as scope:
            # just to get the shape right
            height = tf.to_int32(tf.ceil(self._im_info[0] / np.float32(self._feat_stride[0])))
            width = tf.to_int32(tf.ceil(self._im_info[1] / np.float32(self._feat_stride[0])))
            anchors, anchor_length = tf.py_func(generate_anchors_pre,
                                                [height, width,
                                                 self._feat_stride, self._anchor_scales, self._anchor_ratios],
                                                [tf.float32, tf.int32], name="generate_anchors")
            anchors.set_shape([None, 4])
            anchor_length.set_shape([])
            self._anchors = anchors
            self._anchor_length = anchor_length

        if cfg.DEBUG_ALL:
            self._for_debug['anchors'] = anchors

    def _build_network(self, is_training=True):
        # select initializers
        if cfg.TRAIN.WEIGHT_INITIALIZER == 'truncated':
            initializer = tf.truncated_normal_initializer(mean=0.0, stddev=0.01)
            # initializer_bbox = tf.truncated_normal_initializer(mean=0.0, stddev=0.001)
        elif cfg.TRAIN.WEIGHT_INITIALIZER == 'normal':
            initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)
            # initializer_bbox = tf.random_normal_initializer(mean=0.0, stddev=0.001)
        else:
            initializer = tf.contrib.layers.xavier_initializer()
            # initializer_bbox = tf.contrib.layers.xavier_initializer()

        net_conv = self._image_to_head(is_training)
        with tf.variable_scope(self._scope + '/Extraction'):
            # build the anchors for the image
            self._anchor_component()
            # region proposal network
            rois = self._region_proposal(net_conv, is_training, initializer)
            # region of interest pooling
            if cfg.POOLING_MODE == 'crop':
                pool5 = self._crop_pool_layer(net_conv, rois, "pool5")
            else:
                raise NotImplementedError

            if self._mode == 'TRAIN':
                # sentence data layer
                input_sentence = self._sentence_data_layer('sentence_data')
            elif self._mode == 'TEST':
                input_feed = tf.placeholder(dtype=tf.int32,
                                            shape=[None],
                                            name='input_feed')
                input_sentence = tf.expand_dims(input_feed, 1)
            else:
                raise NotImplementedError

        fc7 = self._head_to_tail(pool5, is_training)
        with tf.variable_scope(self._scope + '/Prediction'):
            # add context fusion
            if cfg.CONTEXT_FUSION:
                # global feature after "head_to_tail" is dumped
                _, fc7_1 = tf.split(fc7, [1, -1], axis=0)
            else:
                fc7_1 = fc7
            # region classification
            cls_prob = self._region_classification(fc7_1, is_training,
                                                   initializer)
            self._embed_caption_layer(fc7, input_sentence, initializer, is_training)

        self._score_summaries.update(self._predictions)

        if cfg.DEBUG_ALL:
            self._for_debug['pool5'] = pool5
            self._for_debug['cls_prob'] = cls_prob

        return rois, cls_prob

    def _smooth_l1_loss(self, bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights, sigma=1.0, dim=[1]):
        sigma_2 = sigma ** 2
        box_diff = bbox_pred - bbox_targets
        in_box_diff = bbox_inside_weights * box_diff
        abs_in_box_diff = tf.abs(in_box_diff)
        smoothL1_sign = tf.stop_gradient(tf.to_float(tf.less(abs_in_box_diff, 1. / sigma_2)))
        in_loss_box = tf.pow(in_box_diff, 2) * (sigma_2 / 2.) * smoothL1_sign \
            + (abs_in_box_diff - (0.5 / sigma_2)) * (1. - smoothL1_sign)
        out_loss_box = bbox_outside_weights * in_loss_box
        loss_box = tf.reduce_mean(tf.reduce_sum(
            out_loss_box,
            axis=dim
        ))
        return loss_box

    def _add_losses(self, sigma_rpn=3.0):
        with tf.variable_scope('LOSS_' + self._tag) as scope:
            # RPN, class loss
            rpn_cls_score = tf.reshape(self._predictions['rpn_cls_score_reshape'], [-1, 2])
            rpn_label = tf.reshape(self._anchor_targets['rpn_labels'], [-1])
            rpn_select = tf.where(tf.not_equal(rpn_label, -1))
            rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2])
            rpn_label = tf.reshape(tf.gather(rpn_label, rpn_select), [-1])
            rpn_cross_entropy = tf.reduce_mean(
                tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score,
                                                               labels=rpn_label))

            # RPN, bbox loss
            rpn_bbox_pred = self._predictions['rpn_bbox_pred']
            rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets']
            rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights']
            rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights']
            rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
                                                rpn_bbox_outside_weights,
                                                sigma=sigma_rpn, dim=[1, 2, 3])

            # class loss
            cls_score = self._predictions["cls_score"]
            label = tf.reshape(self._proposal_targets["clss"], [-1])
            clss_cross_entropy = tf.reduce_mean(
                tf.nn.sparse_softmax_cross_entropy_with_logits(logits=cls_score, labels=label))

            # bbox loss
            bbox_pred = self._predictions['bbox_pred']
            bbox_targets = self._proposal_targets['bbox_targets']
            # bbox_inside_weights = self._proposal_targets['bbox_inside_weights']
            # bbox_outside_weights = self._proposal_targets['bbox_outside_weights']
            loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets,
                                            1., 1.)

            # caption loss
            # shape [None*12, 10003]
            target_sentence = self._sentence_data['target_sentence']
            predict_caption = self._predictions['predict_caption']
            target_sentence = tf.reshape(target_sentence, [-1])
            cap_mask = tf.reshape(tf.to_float(tf.cast(target_sentence, tf.bool)), [-1])
            captoin_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=predict_caption,
                                                                                labels=target_sentence)
            caption_loss = tf.div(tf.reduce_sum(tf.multiply(captoin_cross_entropy, cap_mask)),
                                  tf.reduce_sum(cap_mask), name='caption_loss')

            self._losses['clss_cross_entropy'] = clss_cross_entropy
            self._losses['loss_box'] = loss_box
            self._losses['rpn_cross_entropy'] = rpn_cross_entropy
            self._losses['rpn_loss_box'] = rpn_loss_box
            self._losses['caption_loss'] = caption_loss

            loss = cfg.LOSS.CAP_W * caption_loss \
                + cfg.LOSS.CLS_W * clss_cross_entropy \
                + cfg.LOSS.BBOX_W * loss_box \
                + cfg.LOSS.RPN_CLS_W * rpn_cross_entropy \
                + cfg.LOSS.RPN_BBOX_W * rpn_loss_box
            regularization_loss = tf.add_n(tf.losses.get_regularization_losses(), 'regu')
            self._losses['total_loss'] = loss + regularization_loss

            self._event_summaries.update(self._losses)

            if cfg.DEBUG_ALL:
                self._for_debug['loss'] = loss
                self._for_debug['total_loss'] = loss

        return loss

    def _region_proposal(self, net_conv, is_training, initializer):
        rpn = slim.conv2d(net_conv, cfg.RPN_CHANNELS, [3, 3], trainable=is_training,
                          weights_initializer=initializer,
                          scope="rpn_conv/3x3")
        self._act_summaries.append(rpn)
        rpn_cls_score = slim.conv2d(rpn, self._num_anchors * 2, [1, 1], trainable=is_training,
                                    weights_initializer=initializer,
                                    padding='VALID', activation_fn=None, scope='rpn_cls_score')
        # change it so that the score has 2 as its channel size
        rpn_cls_score_reshape = self._reshape_layer(rpn_cls_score, 2, 'rpn_cls_score_reshape')
        rpn_cls_prob_reshape = self._softmax_layer(rpn_cls_score_reshape, "rpn_cls_prob_reshape")
        # rpn_cls_pred = tf.argmax(tf.reshape(rpn_cls_score_reshape, [-1, 2]), axis=1, name="rpn_cls_pred")
        rpn_cls_prob = self._reshape_layer(rpn_cls_prob_reshape, self._num_anchors * 2, "rpn_cls_prob")
        rpn_bbox_pred = slim.conv2d(rpn, self._num_anchors * 4, [1, 1], trainable=is_training,
                                    weights_initializer=initializer,
                                    padding='VALID', activation_fn=None, scope='rpn_bbox_pred')
        if is_training:
            rois, roi_scores = self._proposal_layer(rpn_cls_prob, rpn_bbox_pred, "rois")
            rpn_labels = self._anchor_target_layer(rpn_cls_score, "anchor")
            # Try to have a deterministic order for the computing graph, for reproducibility
            with tf.control_dependencies([rpn_labels]):
                # rois, _ = self._proposal_target_layer(rois, roi_scores, "rpn_rois")
                rois, labels, phrases = self._proposal_target_single_class_layer(rois, roi_scores, "rpn_rois")

                self._roi_labels = labels
                self._roi_phrases = phrases
        else:
            if cfg.TEST.MODE == 'nms':
                rois, _ = self._proposal_layer(rpn_cls_prob, rpn_bbox_pred, "rois")
            elif cfg.TEST.MODE == 'top':
                rois, _ = self._proposal_top_layer(rpn_cls_prob, rpn_bbox_pred, "rois")
            else:
                raise NotImplementedError

        self._predictions["rpn_cls_score"] = rpn_cls_score
        self._predictions["rpn_cls_score_reshape"] = rpn_cls_score_reshape
        self._predictions["rpn_cls_prob"] = rpn_cls_prob
        # self._predictions["rpn_cls_pred"] = rpn_cls_pred
        self._predictions["rpn_bbox_pred"] = rpn_bbox_pred
        self._predictions["rois"] = rois

        # NOTE: add context feature
        if cfg.CONTEXT_FUSION:
            rois = tf.concat((self._global_roi, rois), axis=0)
            print("Using context fusion, with shape of rois: {}".format(rois.shape))

        if cfg.DEBUG_ALL:
            self._for_debug['rpn'] = rpn
            self._for_debug['rpn_cls_score'] = rpn_cls_score
            self._for_debug['rpn_cls_prob'] = rpn_cls_prob
            self._for_debug['rpn_cls_prob_reshape'] = rpn_cls_prob_reshape
            self._for_debug['rpn_cls_score_reshape'] = rpn_cls_score_reshape
            self._for_debug['rpn_bbox_pred'] = rpn_bbox_pred
        return rois

    # TODO: clear stuff
    def _region_classification(self, fc7, is_training, initializer):
        # predict two class: fg or bg
        cls_score = slim.fully_connected(fc7, self._num_classes + 1,
                                         weights_initializer=initializer,
                                         trainable=is_training,
                                         activation_fn=None, scope='cls_score')
        cls_prob = self._softmax_layer(cls_score, "cls_prob")
        # cls_pred = tf.argmax(cls_score, axis=1, name="cls_pred")
        # bbox_pred = slim.fully_connected(fc7, self._num_classes * 4,
        #                                  weights_initializer=initializer_bbox,
        #                                  trainable=is_training,
        #                                  activation_fn=None, scope='bbox_pred')

        self._predictions["cls_score"] = cls_score
        # self._predictions["cls_pred"] = cls_pred
        self._predictions["cls_prob"] = cls_prob
        # self._predictions["bbox_pred"] = bbox_pred

        return cls_prob

    def _image_to_head(self, is_training, reuse=None):
        raise NotImplementedError

    def _head_to_tail(self, pool5, is_training, reuse=None):
        raise NotImplementedError

    def create_architecture(self, mode, num_classes=1, tag=None,
                            ):
        self._tag = tag

        self._num_classes = num_classes
        self._mode = mode

        training = mode == 'TRAIN'
        testing = mode == 'TEST'

        assert tag != None

        # handle most of the regularizers here
        weights_regularizer = tf.contrib.layers.l2_regularizer(cfg.TRAIN.WEIGHT_DECAY)
        if cfg.TRAIN.BIAS_DECAY:
            biases_regularizer = weights_regularizer
        else:
            biases_regularizer = tf.no_regularizer

        # list as many types of layers as possible, even if they are not used now
        with arg_scope([slim.conv2d, slim.conv2d_in_plane,
                        slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected],
                       weights_regularizer=weights_regularizer,
                       biases_regularizer=biases_regularizer,
                       biases_initializer=tf.constant_initializer(0.0)):
            rois, cls_prob = self._build_network(training)

        layers_to_output = {'rois': rois}

        for var in tf.trainable_variables():
            self._train_summaries.append(var)

        if testing:
            # stds = np.tile(np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (self._num_classes))
            stds = np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS)
            # means = np.tile(np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (self._num_classes))
            means = np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS)
            self._predictions["bbox_pred"] *= stds
            self._predictions["bbox_pred"] += means
        else:
            self._add_losses()
            layers_to_output.update(self._losses)

            val_summaries = []
            with tf.device("/cpu:0"):
                # val_summaries.append(self._add_gt_image_summary())
                val_summaries.append(self._add_image_summary())
                for key, var in self._event_summaries.items():
                    val_summaries.append(tf.summary.scalar(key, var))
                for key, var in self._score_summaries.items():
                    self._add_score_summary(key, var)
                for var in self._act_summaries:
                    self._add_act_summary(var)
                for var in self._train_summaries:
                    self._add_train_summary(var)

            self._summary_op = tf.summary.merge_all()
            self._summary_op_val = tf.summary.merge(val_summaries)

        layers_to_output.update(self._predictions)

        return layers_to_output

    def get_variables_to_restore(self, variables, var_keep_dic):
        raise NotImplementedError

    def fix_variables(self, sess, pretrained_model):
        raise NotImplementedError

    # Extract the head feature maps, for example for vgg16 it is conv5_3
    # only useful during testing mode
    def extract_head(self, sess, image):
        feed_dict = {self._image: image}
        feat = sess.run(self._layers["head"], feed_dict=feed_dict)
        return feat

    # only useful during testing mode
    def test_image(self, sess, image, im_info):
        feed_dict = {self._image: image,
                     self._im_info: im_info}

        cls_score, cls_prob, bbox_pred, rois = sess.run([self._predictions["cls_score"],
                                                         self._predictions['cls_prob'],
                                                         self._predictions['bbox_pred'],
                                                         self._predictions['rois']],
                                                        feed_dict=feed_dict)
        return cls_score, cls_prob, bbox_pred, rois

    def feed_image(self, sess, image, im_info):
        feed_dict = {self._image: image,
                     self._im_info: im_info}
        fetch_list = [
            '%s/Prediction/lstm/cap_init_state:0' % self._scope,
            '%s/Prediction/lstm/loc_init_state:0' % self._scope,
            self._predictions['cls_prob'],
            self._predictions['rois']]
        if cfg.CONTEXT_FUSION:
            feed_dict.update({self._global_roi: np.array([[0., 0., 0., im_info[1] - 1,
                        im_info[0] - 1]], dtype=np.float32)})
            fetch_list.append('%s/Prediction/lstm/gfeat_init_state:0' % self._scope)

        fetch = sess.run(fetch_list, feed_dict=feed_dict)

        return fetch

    def inference_step(self, sess, input_feed, cap_state_feed, loc_state_feed,
                        gfeat_state_feed=None):
        feed_dict = {'%s/Extraction/input_feed:0' % self._scope: input_feed,
                     '%s/Prediction/lstm/cap_state_feed:0' % self._scope: cap_state_feed,
                     '%s/Prediction/lstm/loc_state_feed:0' % self._scope: loc_state_feed}
        fetch_list = ['%s/Prediction/lstm/cap_probs:0' % self._scope,
                   self._predictions['bbox_pred'],
                   '%s/Prediction/lstm/cap_state:0' % self._scope,
                   '%s/Prediction/lstm/loc_state:0' % self._scope]
        if cfg.CONTEXT_FUSION:
            feed_dict.update({'%s/Prediction/lstm/gfeat_state_feed:0' % self._scope:
                              gfeat_state_feed})
            fetch_list.append('%s/Prediction/lstm/gfeat_state:0' % self._scope)
        fetch = sess.run(fetches=fetch_list, feed_dict=feed_dict)

        return fetch

    def get_summary(self, sess, blobs):
        feed_dict = self._feed_dict(blobs)

        summary = sess.run(self._summary_op_val, feed_dict=feed_dict)

        return summary

    def train_step(self, sess, blobs, train_op):
        feed_dict = self._feed_dict(blobs)

        rpn_loss_cls, rpn_loss_box, loss_cls, \
            loss_box, caption_loss, loss, \
            _ = sess.run([self._losses["rpn_cross_entropy"],
                          self._losses['rpn_loss_box'],
                          self._losses['clss_cross_entropy'],
                          self._losses['loss_box'],
                          self._losses['caption_loss'],
                          self._losses['total_loss'],
                          train_op],
                         feed_dict=feed_dict)
        return rpn_loss_cls, rpn_loss_box, loss_cls, \
            loss_box, caption_loss, loss

    def train_step_with_summary(self, sess, blobs, train_op):
        feed_dict = self._feed_dict(blobs)
        rpn_loss_cls, rpn_loss_box, loss_cls, \
            loss_box, caption_loss, loss, \
            summary, _ = sess.run([self._losses["rpn_cross_entropy"],
                                   self._losses['rpn_loss_box'],
                                   self._losses['clss_cross_entropy'],
                                   self._losses['loss_box'],
                                   self._losses['caption_loss'],
                                   self._losses['total_loss'],
                                   self._summary_op,
                                   train_op],
                                  feed_dict=feed_dict)

        return rpn_loss_cls, rpn_loss_box, loss_cls, \
            loss_box, caption_loss, loss, summary

    def train_step_no_return(self, sess, blobs, train_op):
        feed_dict = self._feed_dict(blobs)

        sess.run([train_op], feed_dict=feed_dict)

    def _feed_dict(self, blobs):
        feed_dict = {self._image: blobs['data'], self._im_info: blobs['im_info'],
                     self._gt_boxes: blobs['gt_boxes'],
                     self._gt_phrases: blobs['gt_phrases']}
        if cfg.CONTEXT_FUSION:
            feed_dict.update({self._global_roi:
                np.array([[0., 0., 0., blobs['im_info'][1] - 1,
                          blobs['im_info'][0] - 1]], dtype=np.float32)})

        return feed_dict


"""
python implementation of shadow3 OE object
"""
import numpy
from shadow4.compatibility.gfile import GFile

class OE(object):

    def __init__(self):

        self.FMIRR = 5
        self.F_TORUS = 0
        self.FCYL = 0
        self.F_EXT = 0
        self.FSTAT = 0
        self.F_SCREEN = 0
        self.F_PLATE = 0
        self.FSLIT = 0
        self.FWRITE = 0
        self.F_RIPPLE = 0
        self.F_MOVE = 0
        self.F_THICK = 0
        self.F_BRAGG_A = 0
        self.F_G_S = 0
        self.F_R_RAN = 0
        self.F_GRATING = 0
        self.F_MOSAIC = 0
        self.F_JOHANSSON = 0
        self.F_SIDE = 0
        self.F_CENTRAL = 0
        self.F_CONVEX = 0
        self.F_REFLEC = 0
        self.F_RUL_ABS = 0
        self.F_RULING = 0
        self.F_PW = 0
        self.F_PW_C = 0
        self.F_VIRTUAL = 0
        self.FSHAPE = 1
        self.FHIT_C = 0
        self.F_MONO = 0
        self.F_REFRAC = 0
        self.F_DEFAULT = 1
        self.F_REFL = 0
        self.F_HUNT = 1
        self.F_CRYSTAL = 0
        self.F_PHOT_CENT = 0
        self.F_ROUGHNESS = 0
        self.F_ANGLE = 0
        self.NPOINT = 5000
        self.NCOL = 18
        self.N_SCREEN = 0
        self.ISTAR1 = 0
        self.CIL_ANG = 0.00000000000000
        self.ELL_THE = 0.00000000000000
        self.N_PLATES = 0
        self.IG_SEED = 0
        self.MOSAIC_SEED = 1626261131
        self.ALPHA = 0.00000000000000
        self.SSOUR = 0.00000000000000
        self.THETA = 0.00000000000000
        self.SIMAG = 0.00000000000000
        self.RDSOUR = 0.00000000000000
        self.RTHETA = 0.00000000000000
        self.OFF_SOUX = 0.00000000000000
        self.OFF_SOUY = 0.00000000000000
        self.OFF_SOUZ = 0.00000000000000
        self.ALPHA_S = 0.00000000000000
        self.RLEN1 = 0.00000000000000
        self.RLEN2 = 0.00000000000000
        self.RMIRR = 0.00000000000000
        self.AXMAJ = 0.00000000000000
        self.AXMIN = 0.00000000000000
        self.CONE_A = 0.00000000000000
        self.R_MAJ = 0.00000000000000
        self.R_MIN = 0.00000000000000
        self.RWIDX1 = 0.00000000000000
        self.RWIDX2 = 0.00000000000000
        self.PARAM = 0.00000000000000
        self.HUNT_H = 0.00000000000000
        self.HUNT_L = 0.00000000000000
        self.BLAZE = 0.00000000000000
        self.RULING = 12000.0000000000
        self.ORDER = -1.00000000000000
        self.PHOT_CENT = 5.00000000000000
        self.X_ROT = 0.00000000000000
        self.D_SPACING = 0.00000000000000
        self.A_BRAGG = 0.00000000000000
        self.SPREAD_MOS = 0.00000000000000
        self.THICKNESS = 0.00000000000000
        self.R_JOHANSSON = 0.00000000000000
        self.Y_ROT = 0.00000000000000
        self.Z_ROT = 0.00000000000000
        self.OFFX = 0.00000000000000
        self.OFFY = 0.00000000000000
        self.OFFZ = 0.00000000000000
        self.SLLEN = 0.00000000000000
        self.SLWID = 0.00000000000000
        self.SLTILT = 0.00000000000000
        self.COD_LEN = 0.00000000000000
        self.COD_WID = 0.00000000000000
        self.X_SOUR = 0.00000000000000
        self.Y_SOUR = 0.00000000000000
        self.Z_SOUR = 0.00000000000000
        self.X_SOUR_ROT = 0.00000000000000
        self.Y_SOUR_ROT = 0.00000000000000
        self.Z_SOUR_ROT = 0.00000000000000
        self.R_LAMBDA = 5.00000000000000
        self.THETA_I = 0.00000000000000
        self.ALPHA_I = 0.00000000000000
        self.T_INCIDENCE = 88.0000000000000
        self.T_SOURCE = 10.0000000000000
        self.T_IMAGE = 20.0000000000000
        self.T_REFLECTION = 88.0000000000000
        self.FILE_SOURCE = "begin.dat"
        self.FILE_RIP = ""
        self.FILE_REFL = ""
        self.FILE_MIR = ""
        self.FILE_ROUGH = ""
        self.FZP = 0
        self.HOLO_R1 = 300.000000000000
        self.HOLO_R2 = 300.000000000000
        self.HOLO_DEL = -20.0000000000000
        self.HOLO_GAM = -20.0000000000000
        self.HOLO_W = 4879.86000000000
        self.HOLO_RT1 = 0.00000000000000
        self.HOLO_RT2 = 0.00000000000000
        self.AZIM_FAN = 0.00000000000000
        self.DIST_FAN = 0.00000000000000
        self.COMA_FAC = 0.00000000000000
        self.ALFA = 0.00000000000000
        self.GAMMA = 0.00000000000000
        self.R_IND_OBJ = 1.00000000000000
        self.R_IND_IMA = 1.00000000000000
        self.R_ATTENUATION_OBJ = 0.00000000000000
        self.R_ATTENUATION_IMA = 0.00000000000000
        self.F_R_IND = 0
        self.FILE_R_IND_OBJ = ""
        self.FILE_R_IND_IMA = ""
        self.RUL_A1 = 0.00000000000000
        self.RUL_A2 = 0.00000000000000
        self.RUL_A3 = 0.00000000000000
        self.RUL_A4 = 0.00000000000000
        self.F_POLSEL = 4
        self.F_FACET = 0
        self.F_FAC_ORIENT = 0
        self.F_FAC_LATT = 0
        self.RFAC_LENX = 10.0000000000000
        self.RFAC_LENY = 10.0000000000000
        self.RFAC_PHAX = 0.00000000000000
        self.RFAC_PHAY = 0.00000000000000
        self.RFAC_DELX1 = 0.00000000000000
        self.RFAC_DELX2 = 0.00000000000000
        self.RFAC_DELY1 = 0.00000000000000
        self.RFAC_DELY2 = 0.00000000000000
        self.FILE_FAC = ""
        self.F_SEGMENT = 0
        self.ISEG_XNUM = 1
        self.ISEG_YNUM = 1
        self.FILE_SEGMENT = ""
        self.FILE_SEGP = ""
        self.SEG_LENX = 0.00000000000000
        self.SEG_LENY = 0.00000000000000
        self.F_KOMA = 0
        self.FILE_KOMA = ""
        self.F_EXIT_SHAPE = 0
        self.F_INC_MNOR_ANG = 0
        self.ZKO_LENGTH = 0.00000000000000
        self.RKOMA_CX = 0.00000000000000
        self.RKOMA_CY = 0.00000000000000
        self.F_KOMA_CA = 0
        self.FILE_KOMA_CA = ""
        self.F_KOMA_BOUNCE = 0
        self.X_RIP_AMP = 0.00000000000000
        self.X_RIP_WAV = 0.00000000000000
        self.X_PHASE = 0.00000000000000
        self.Y_RIP_AMP = 0.00000000000000
        self.Y_RIP_WAV = 0.00000000000000
        self.Y_PHASE = 0.00000000000000
        self.N_RIP = 0
        self.ROUGH_X = 0.00000000000000
        self.ROUGH_Y = 0.00000000000000
        self.OE_NUMBER = 0
        self.IDUMMY = 0
        self.DUMMY = 0.00000000000000

        self.CX_SLIT = numpy.zeros(11)
        self.CX_SLIT[1] = 0.00000000000000
        self.CX_SLIT[2] = 0.00000000000000
        self.CX_SLIT[3] = 0.00000000000000
        self.CX_SLIT[4] = 0.00000000000000
        self.CX_SLIT[5] = 0.00000000000000
        self.CX_SLIT[6] = 0.00000000000000
        self.CX_SLIT[7] = 0.00000000000000
        self.CX_SLIT[8] = 0.00000000000000
        self.CX_SLIT[9] = 0.00000000000000
        self.CX_SLIT[10] = 0.00000000000000

        self.CZ_SLIT = numpy.zeros(11)
        self.CZ_SLIT[1] = 0.00000000000000
        self.CZ_SLIT[2] = 0.00000000000000
        self.CZ_SLIT[3] = 0.00000000000000
        self.CZ_SLIT[4] = 0.00000000000000
        self.CZ_SLIT[5] = 0.00000000000000
        self.CZ_SLIT[6] = 0.00000000000000
        self.CZ_SLIT[7] = 0.00000000000000
        self.CZ_SLIT[8] = 0.00000000000000
        self.CZ_SLIT[9] = 0.00000000000000
        self.CZ_SLIT[10] = 0.00000000000000

        self.D_PLATE = numpy.zeros(11)
        self.D_PLATE[1] = 0.00000000000000
        self.D_PLATE[2] = 0.00000000000000
        self.D_PLATE[3] = 0.00000000000000
        self.D_PLATE[4] = 0.00000000000000
        self.D_PLATE[5] = 0.00000000000000
        self.D_PLATE[6] = 0.00000000000000
        self.D_PLATE[7] = 0.00000000000000
        self.D_PLATE[8] = 0.00000000000000
        self.D_PLATE[9] = 0.00000000000000
        self.D_PLATE[10] = 0.00000000000000

        self.FILE_ABS = numpy.chararray(11)
        self.FILE_ABS[1] = ""
        self.FILE_ABS[2] = ""
        self.FILE_ABS[3] = ""
        self.FILE_ABS[4] = ""
        self.FILE_ABS[5] = ""
        self.FILE_ABS[6] = ""
        self.FILE_ABS[7] = ""
        self.FILE_ABS[8] = ""
        self.FILE_ABS[9] = ""
        self.FILE_ABS[10] = ""

        self.FILE_SCR_EXT = numpy.chararray(11)
        self.FILE_SCR_EXT[1] = ""
        self.FILE_SCR_EXT[2] = ""
        self.FILE_SCR_EXT[3] = ""
        self.FILE_SCR_EXT[4] = ""
        self.FILE_SCR_EXT[5] = ""
        self.FILE_SCR_EXT[6] = ""
        self.FILE_SCR_EXT[7] = ""
        self.FILE_SCR_EXT[8] = ""
        self.FILE_SCR_EXT[9] = ""
        self.FILE_SCR_EXT[10] = ""

        self.I_ABS = numpy.zeros(11)
        self.I_ABS[1] = 0
        self.I_ABS[2] = 0
        self.I_ABS[3] = 0
        self.I_ABS[4] = 0
        self.I_ABS[5] = 0
        self.I_ABS[6] = 0
        self.I_ABS[7] = 0
        self.I_ABS[8] = 0
        self.I_ABS[9] = 0
        self.I_ABS[10] = 0

        self.I_SCREEN = numpy.zeros(11)
        self.I_SCREEN[1] = 0
        self.I_SCREEN[2] = 0
        self.I_SCREEN[3] = 0
        self.I_SCREEN[4] = 0
        self.I_SCREEN[5] = 0
        self.I_SCREEN[6] = 0
        self.I_SCREEN[7] = 0
        self.I_SCREEN[8] = 0
        self.I_SCREEN[9] = 0
        self.I_SCREEN[10] = 0

        self.I_SLIT = numpy.zeros(11)
        self.I_SLIT[1] = 0
        self.I_SLIT[2] = 0
        self.I_SLIT[3] = 0
        self.I_SLIT[4] = 0
        self.I_SLIT[5] = 0
        self.I_SLIT[6] = 0
        self.I_SLIT[7] = 0
        self.I_SLIT[8] = 0
        self.I_SLIT[9] = 0
        self.I_SLIT[10] = 0

        self.I_STOP = numpy.zeros(11)
        self.I_STOP[1] = 0
        self.I_STOP[2] = 0
        self.I_STOP[3] = 0
        self.I_STOP[4] = 0
        self.I_STOP[5] = 0
        self.I_STOP[6] = 0
        self.I_STOP[7] = 0
        self.I_STOP[8] = 0
        self.I_STOP[9] = 0
        self.I_STOP[10] = 0

        self.K_SLIT = numpy.zeros(11)
        self.K_SLIT[1] = 0
        self.K_SLIT[2] = 0
        self.K_SLIT[3] = 0
        self.K_SLIT[4] = 0
        self.K_SLIT[5] = 0
        self.K_SLIT[6] = 0
        self.K_SLIT[7] = 0
        self.K_SLIT[8] = 0
        self.K_SLIT[9] = 0
        self.K_SLIT[10] = 0

        self.RX_SLIT = numpy.zeros(11)
        self.RX_SLIT[1] = 0.00000000000000
        self.RX_SLIT[2] = 0.00000000000000
        self.RX_SLIT[3] = 0.00000000000000
        self.RX_SLIT[4] = 0.00000000000000
        self.RX_SLIT[5] = 0.00000000000000
        self.RX_SLIT[6] = 0.00000000000000
        self.RX_SLIT[7] = 0.00000000000000
        self.RX_SLIT[8] = 0.00000000000000
        self.RX_SLIT[9] = 0.00000000000000
        self.RX_SLIT[10] = 0.00000000000000

        self.RZ_SLIT = numpy.zeros(11)
        self.RZ_SLIT[1] = 0.00000000000000
        self.RZ_SLIT[2] = 0.00000000000000
        self.RZ_SLIT[3] = 0.00000000000000
        self.RZ_SLIT[4] = 0.00000000000000
        self.RZ_SLIT[5] = 0.00000000000000
        self.RZ_SLIT[6] = 0.00000000000000
        self.RZ_SLIT[7] = 0.00000000000000
        self.RZ_SLIT[8] = 0.00000000000000
        self.RZ_SLIT[9] = 0.00000000000000
        self.RZ_SLIT[10] = 0.00000000000000

        self.SCR_NUMBER = numpy.zeros(11)
        self.SCR_NUMBER[1] = 0
        self.SCR_NUMBER[2] = 0
        self.SCR_NUMBER[3] = 0
        self.SCR_NUMBER[4] = 0
        self.SCR_NUMBER[5] = 0
        self.SCR_NUMBER[6] = 0
        self.SCR_NUMBER[7] = 0
        self.SCR_NUMBER[8] = 0
        self.SCR_NUMBER[9] = 0
        self.SCR_NUMBER[10] = 0

        self.SL_DIS = numpy.zeros(11)
        self.SL_DIS[1] = 0.00000000000000
        self.SL_DIS[2] = 0.00000000000000
        self.SL_DIS[3] = 0.00000000000000
        self.SL_DIS[4] = 0.00000000000000
        self.SL_DIS[5] = 0.00000000000000
        self.SL_DIS[6] = 0.00000000000000
        self.SL_DIS[7] = 0.00000000000000
        self.SL_DIS[8] = 0.00000000000000
        self.SL_DIS[9] = 0.00000000000000
        self.SL_DIS[10] = 0.00000000000000

        self.THICK = numpy.zeros(11)
        self.THICK[1] = 0.00000000000000
        self.THICK[2] = 0.00000000000000
        self.THICK[3] = 0.00000000000000
        self.THICK[4] = 0.00000000000000
        self.THICK[5] = 0.00000000000000
        self.THICK[6] = 0.00000000000000
        self.THICK[7] = 0.00000000000000
        self.THICK[8] = 0.00000000000000
        self.THICK[9] = 0.00000000000000
        self.THICK[10] = 0.00000000000000

        self.CCC = numpy.zeros(11)
        self.CCC[1] = 0.00000000000000
        self.CCC[2] = 0.00000000000000
        self.CCC[3] = 0.00000000000000
        self.CCC[4] = 0.00000000000000
        self.CCC[5] = 0.00000000000000
        self.CCC[6] = 0.00000000000000
        self.CCC[7] = 0.00000000000000
        self.CCC[8] = 0.00000000000000
        self.CCC[9] = 0.00000000000000
        self.CCC[10] = 0.00000000000000

    def load(self,filename="start.01"):
        self.load_start01(filename=filename)

    def load_start01(self,filename="start.01"):
        a = GFile()
        a.load_gfile((filename))
        dict1 = a.get_as_dictionary()

        for key in dict1.keys():
            if hasattr(self,key):
                # print("assigning: ",key,dict1[key])
                setattr(self, key, dict1[key])
            else: # this is for arrays...
                try:
                    command = "self.%s = %s"%(key,repr(dict1[key]))
                    # print("command: ",command)
                    exec(command)
                except:
                    print("   error executing: ",command)
        # self.THICK[6] = 110.0

if __name__ == "__main__":
    # write start.01 file with shadow4
    import Shadow
    oe_shadow3 = Shadow.OE()
    oe_shadow3.write("start.01")

    # read it with compatibility tools
    oe1 = OE()
    oe1.load(filename="start.01")

    print(dir(oe1))

    print(">>>>>oe1.RX_SLIT",oe1.RX_SLIT)
    print(">>>>>Y_ROT",oe1.Y_ROT )
    print(">>>>>ISTAR1",oe1.ISTAR1 )
    print(">>>>>THICK",oe1.THICK )

    assert(oe1.THICK[6] == oe_shadow3.THICK[6])


import cv2,os
os.chdir("D:\\AI\\AI Projects\\RTFR")
# Import numpy for matrices calculations
import numpy as np

# Create Local Binary Patterns Histograms for face recognization
recognizer = cv2.face.LBPHFaceRecognizer_create()

# Load the trained mode
recognizer.read('trainer.yml')
fn_dir = 'datasets'

# Load prebuilt model for Frontal Face
cascadePath = "haarcascade_frontalface_default.xml"
(im_width, im_height) = (112, 92)
# Part 2: Use fisherRecognizer on camera stream
(images, lables, names, id) = ([], [], {}, 0)
for (subdirs, dirs, files) in os.walk(fn_dir):
    for subdir in dirs:
        names[id] = subdir
        subjectpath = os.path.join(fn_dir, subdir)
        for filename in os.listdir(subjectpath):
            path = subjectpath + '/' + filename
            lable = id
            images.append(cv2.imread(path, 0))
            lables.append(int(lable))
        id += 1

face_cascade = cv2.CascadeClassifier(cascadePath)
webcam = cv2.VideoCapture(0)
while True:
    (_, im) = webcam.read()
    gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
    for (x,y,w,h) in faces:
        cv2.rectangle(im,(x,y),(x+w,y+h),(255,0,0),2)
        face = gray[y:y + h, x:x + w]
        face_resize = cv2.resize(face, (im_width, im_height))
        # Try to recognize the face
        prediction = recognizer.predict(face_resize)
        cv2.rectangle(im, (x, y), (x + w, y + h), (0, 255, 0), 3)

        if prediction[1]<500:
            cv2.putText(im,'%s - %.0f' % (names[prediction[0]],prediction[1]),(x-10, y-10), cv2.FONT_HERSHEY_PLAIN,1,(0, 255, 0))
        else:
            cv2.putText(im,'not recognized',(x-10, y-10), cv2.FONT_HERSHEY_PLAIN,1,(0, 255, 0))

    cv2.imshow('OpenCV', im)
    key = cv2.waitKey(10)
    if key == 27:
        break


#!/usr/bin/env python

from __future__ import print_function

from logging import info

import random
import numpy as np
import datetime
from ltlib.evaluation import conll_summary, per_type_summary, is_iob_tagging

from keras.models import Model
from keras.layers import Input, Embedding, merge, Flatten, Dense
from keras.layers import Reshape, Convolution2D, Dropout

from ltlib import filelog
from ltlib import conlldata
from ltlib import viterbi

from ltlib.features import NormEmbeddingFeature, SennaCapsFeature
from ltlib.features import windowed_inputs
from ltlib.callbacks import token_evaluator, EpochTimer
from ltlib.layers import concat, inputs_and_embeddings
from ltlib.settings import cli_settings, log_settings
from ltlib.optimizers import get_optimizer
from ltlib.util import unique
from ltlib.output import save_token_predictions_multi_output

from config import Defaults

config = cli_settings(['datadir', 'datasets','wordvecs'], Defaults)
assert len(config.filter_nums) == len(config.filter_sizes)

datasets = config.datasets.split(',')

data = []
max_fs = []
max_vfs = []
vmappers = []
for ind, dataset in enumerate(datasets):
    data_path = config.datadir + '/' + dataset
    if ind != 0:
        config.iobes = True
    data.append(conlldata.load_dir(data_path, config))
    max_fs.append((0.0,0.0))
    max_vfs.append((0.0,0.0))

all_vocab = set()
for ind, dataset in enumerate(datasets):
    all_vocab = set().union(*[all_vocab, data[ind].vocabulary])

for ind, dataset in enumerate(datasets):
    w2v = NormEmbeddingFeature.from_file(config.wordvecs,
                                        max_rank=config.max_vocab_size,
                                        vocabulary=all_vocab,
                                        name='words-%s' % ind)
    features = [w2v]
    if config.word_features:
        features.append(SennaCapsFeature('caps'))

    data[ind].tokens.add_features(features)
    data[ind].tokens.add_inputs(windowed_inputs(config.window_size, features))

    # Log word vector feature stat summary
    info('{}: {}'.format(config.wordvecs, w2v.summary()))

    if ind == 0:
        pos_inputs, pos_embeddings = inputs_and_embeddings(features, config)
        pos_x = concat(pos_embeddings)
    if ind == 1:
        ner_inputs, ner_embeddings = inputs_and_embeddings(features, config)
        ner_x = concat(ner_embeddings)

cshapes = []
reshapes = []
# Combine and reshape for convolution
pos_cshape = (config.window_size, sum(f.output_dim for f in features))
ner_cshape = (config.window_size, sum(f.output_dim for f in features))
cshapes.append(pos_cshape)
cshapes.append(ner_cshape)

pos_reshape = Reshape((1,) + (pos_cshape), name='pos-reshape')(pos_x)
ner_reshape = Reshape((1,) + (ner_cshape), name='ner-reshape')(ner_x)
reshapes.append(pos_reshape)
reshapes.append(ner_reshape)

# Convolutions
conv_outputs = []
fully_connected = []
dropout = []
for ind, dataset in enumerate(datasets):
    conv_outputs.append([])
    for filter_size, filter_num in zip(config.filter_sizes, config.filter_nums):
        conv = Convolution2D(filter_num, filter_size, cshapes[ind][1], activation='relu', name='convolution-%d-%d' % (ind,filter_size))(reshapes[ind])
        flatten = Flatten(name='flatten-%d-%d' % (ind,filter_size))(conv)
        conv_outputs[ind].append(flatten)

    seq = concat(conv_outputs[ind])

    for size in config.hidden_sizes:
        fully_connected.append(Dense(size, activation=config.hidden_activation, name='dense-1-%d' % ind)(seq))
    dropout.append(Dropout(config.output_drop_prob, name='dropout-%d' % ind)(fully_connected[ind]))

pos_dense_out = Dense(data[0].tokens.target_dim, activation='softmax', name='pos-dense-out')(dropout[0])

ner_merged = merge([dropout[0], dropout[1]], mode='concat')
ner_dense_out = Dense(data[1].tokens.target_dim, activation='softmax', name='ner-dense-out')(ner_merged)

pos_model = Model(input=pos_inputs, output=pos_dense_out)
ner_model = Model(input=pos_inputs + ner_inputs, output=ner_dense_out)

pos_model.compile(optimizer=get_optimizer(config), loss='categorical_crossentropy', metrics=['accuracy'])
ner_model.compile(optimizer=get_optimizer(config), loss='categorical_crossentropy', metrics=['accuracy'])

models = [pos_model, ner_model]

time_str = datetime.datetime.now().isoformat()
print("Started training at: %s" % time_str)

for ind, ds in enumerate(data):
    for ep in range(1, config.epochs + 1):
        percnt_keep = config.percent_keep
        amt_keep = len(ds.train.tokens.inputs['words-%s' % ind]) * percnt_keep
        print("Total: %s. Keeping: %s" % (len(ds.train.tokens.inputs['words-%s' % ind]), amt_keep))
        start = random.randrange(int(len(ds.train.tokens.inputs['words-%s' % ind]) - amt_keep) + 1)
        end = int(start + amt_keep)
        x = ds.train.tokens.inputs['words-%s' % ind][start:end]
        if ind > 0:
            x = [x, x]
        models[ind].fit(
            x,
            ds.train.tokens.targets[start:end],
            batch_size=config.batch_size,
            nb_epoch=1,
            verbose=config.verbosity
        )

        time_str = datetime.datetime.now().isoformat()
        info("\nEvaluating. Time: {}. Epoch: {}".format(time_str, ep))

        info("Dataset: {}".format(datasets[ind]))
        if ind > 0:
            predictions = models[ind].predict([ds.test.tokens.inputs['words-%s' % ind], ds.test.tokens.inputs['words-%s' % ind]])
        else:
            predictions = models[ind].predict(ds.test.tokens.inputs['words-%s' % ind])

        if is_iob_tagging(unique(ds.tokens.target_strs)):
            ds.test.tokens.set_predictions(predictions)
            summary = conll_summary(ds.test.sentences)
            #Track Maxes
            f_score = summary.split(':')[2].split('%')[0].strip()
            try:
                f_score = float(f_score)
            except:
                print("%s is not floatable!" % f_score)
            if f_score > max_fs[ind][0]:
                max_fs[ind] = (f_score, max_fs[ind][0])
                save_token_predictions_multi_output(data[eval_ind].test, conlldata.write, predictions=pred)
            elif f_score > max_fs[ind][1]:
                max_fs[ind] = (max_fs[ind][0], f_score)
                save_token_predictions_multi_output(data[eval_ind].test, conlldata.write, predictions=pred)
            #End Track Maxes
            info("{}".format(summary))
            info("Max Fs: {}".format(str(max_fs[ind])))
            if config.viterbi:
                vmapper = viterbi.TokenPredictionMapper(ds.train.sentences)
                ds.test.sentences.map_predictions(vmapper)
                info(vmapper.summary())
                vsummary = conll_summary(ds.test.sentences)
                #Track Maxes
                vf_score = vsummary.split(':')[2].split('%')[0].strip()
                try:
                    vf_score = float(vf_score)
                except:
                    print("Viterbi %s is not floatable!" % vf_score)
                if vf_score > max_vfs[ind][0]:
                    max_vfs[ind] = (vf_score, max_vfs[ind][0])
                    save_token_predictions_multi_output(data[eval_ind].test, conlldata.write, vmapper=vmapper)
                elif vf_score > max_vfs[ind][1]:
                    max_vfs[ind] = (max_vfs[ind][0], vf_score)
                    save_token_predictions_multi_output(data[eval_ind].test, conlldata.write, vmapper=vmapper)
                #End Track Maxes
                info("{}".format(vsummary))
                info("Max Viterbi Fs: {}".format(str(max_vfs[ind])))
        else:
            ds.test.tokens.set_predictions(predictions)
            info("{}".format(per_type_summary(ds.test.tokens)))


import numpy as np
import tensorflow as tf
import cv2 as cv
from tkinter import *
import Image, ImageTk

PATH_TO_LABELS = os.path.join('data', 'mscoco_label_map.pbtxt')
num_classes = 1

# Read the graph.
with tf.gfile.FastGFile("I:\\Custom-Object-Detection\\output_inference_graph\\frozen_inference_graph.pb", 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())

with tf.Session() as sess:
    # Restore session
    sess.graph.as_default()
    tf.import_graph_def(graph_def, name='')

    label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
    categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=num_classes, use_display_name=True)
    category_index = label_map_util.create_category_index(categories)

    # Read and preprocess an image.
    img = cv.imread('I:\\Custom-Object-Detection\\test_images\\image-6.jpg')
    #img_o = cv.UMat(cv.imread('C:\\Users\\geforce\\downloads\\gochiusa.jpg', cv.IMREAD_COLOR))
    rows = img.shape[0]
    cols = img.shape[1]
    inp = cv.resize(img, (300, 300))
    inp = inp[:, :, [2, 1, 0]]  # BGR2RGB

    # Run the model
    out = sess.run([sess.graph.get_tensor_by_name('num_detections:0'),
                    sess.graph.get_tensor_by_name('detection_scores:0'),
                    sess.graph.get_tensor_by_name('detection_boxes:0'),
                    sess.graph.get_tensor_by_name('detection_classes:0')],
                   feed_dict={'image_tensor:0': inp.reshape(1, inp.shape[0], inp.shape[1], 3)})

    # Visualize detected bounding boxes.
    num_detections = int(out[0][0])

    for i in range(num_detections):
        classId = int(out[3][0][i])
        score = float(out[1][0][i])
        bbox = [float(v) for v in out[2][0][i]]
        if score > 0.3:
            x = bbox[1] * cols
            y = bbox[0] * rows
            right = bbox[3] * cols
            bottom = bbox[2] * rows
            cv.rectangle(img, (int(x), int(y)), (int(right), int(bottom)), (125, 255, 51), thickness=10)
        cv.imwrite("C:\\Users\\geforce\\downloads\\test-image"+'.jpg', img)

cv.imshow('cv', img)
cv.waitKey()


# Copyright (c) 2021 Microsoft Corporation. Licensed under the MIT license.
# Reimplemtened by Jianwei Yang (jianwyan@microsoft.com)
# Adapted from https://github.com/jwyang/graph-rcnn.pytorch (Jianwei Yang) and https://github.com/danfeiX/scene-graph-TF-release (Danfei Xu)
"""
Scene Graph Generation by Iterative Message Passing
"""
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import Parameter
from ..roi_relation_feature_extractors import make_roi_relation_feature_extractor
from ..roi_relation_box_feature_extractors import make_roi_relation_box_feature_extractor
from ..roi_relation_box_predictors import make_roi_relation_box_predictor
from ..roi_relation_predictors import make_roi_relation_predictor

class IMP(nn.Module):
    # def __init__(self, fea_size, dropout=False, gate_width=1, use_kernel_function=False):
    def __init__(self, cfg, in_channels):
        super(IMP, self).__init__()
        self.cfg = cfg
        self.dim = 512
        self.update_step = cfg.MODEL.ROI_RELATION_HEAD.IMP_FEATURE_UPDATE_STEP
        self.use_online_obj_labels = cfg.MODEL.ROI_RELATION_HEAD.USE_ONLINE_OBJ_LABELS
        self.avgpool = nn.AdaptiveAvgPool2d(1)
        # self.obj_feature_extractor = make_roi_relation_box_feature_extractor(cfg, in_channels)
        self.pred_feature_extractor = make_roi_relation_feature_extractor(
            cfg, in_channels)

        self.obj_embedding = nn.Sequential(
            nn.Linear(self.pred_feature_extractor.out_channels, self.dim),
            nn.ReLU(True),
            nn.Linear(self.dim, self.dim),
        )
        self.pred_embedding = nn.Sequential(
            nn.Linear(self.pred_feature_extractor.out_channels, self.dim),
            nn.ReLU(True),
            nn.Linear(self.dim, self.dim),
        )

        if self.update_step > 0:
            self.edge_gru = nn.GRUCell(
                input_size=self.dim, hidden_size=self.dim)
            self.node_gru = nn.GRUCell(
                input_size=self.dim, hidden_size=self.dim)

            self.subj_node_gate = nn.Sequential(
                nn.Linear(self.dim * 2, 1), nn.Sigmoid())
            self.obj_node_gate = nn.Sequential(
                nn.Linear(self.dim * 2, 1), nn.Sigmoid())

            self.subj_edge_gate = nn.Sequential(
                nn.Linear(self.dim * 2, 1), nn.Sigmoid())
            self.obj_edge_gate = nn.Sequential(
                nn.Linear(self.dim * 2, 1), nn.Sigmoid())

        self.obj_predictor = make_roi_relation_box_predictor(cfg, 512)
        self.pred_predictor = make_roi_relation_predictor(cfg, 512)

    def _get_map_idxs(self, proposals, proposal_pairs):
        rel_inds = []
        offset = 0
        for proposal, proposal_pair in zip(proposals, proposal_pairs):
            rel_ind_i = proposal_pair.get_field("idx_pairs").detach()
            rel_ind_i += offset
            offset += len(proposal)
            rel_inds.append(rel_ind_i)

        rel_inds = torch.cat(rel_inds, 0)

        subj_pred_map = rel_inds.new(sum(
            [len(proposal) for proposal in proposals]), rel_inds.shape[0]).fill_(0).float().detach()
        obj_pred_map = rel_inds.new(sum(
            [len(proposal) for proposal in proposals]), rel_inds.shape[0]).fill_(0).float().detach()

        subj_pred_map.scatter_(0, (rel_inds[:, 0].contiguous().view(1, -1)), 1)
        obj_pred_map.scatter_(0, (rel_inds[:, 1].contiguous().view(1, -1)), 1)

        return rel_inds, subj_pred_map, obj_pred_map

    def forward(self, features, proposals, proposal_pairs):
        # import pdb; pdb.set_trace()
        rel_inds, subj_pred_map, obj_pred_map = self._get_map_idxs(
            proposals, proposal_pairs)
        x_obj = torch.cat([proposal.get_field("box_features")
                          for proposal in proposals], 0)

        # x_obj = self.avgpool(self.obj_feature_extractor(features, proposals))
        x_pred, _ = self.pred_feature_extractor(
            features, proposals, proposal_pairs)
        if x_pred.ndimension() == 4:
            x_pred = self.avgpool(x_pred)
            x_pred = x_pred.view(x_pred.size(0), -1)
        x_obj = x_obj.view(x_obj.size(0), -1)

        x_obj = self.obj_embedding(x_obj)
        x_pred = self.pred_embedding(x_pred)

        # hx_obj = x_obj.clone().fill_(0).detach()
        # hx_pred = x_pred.clone().fill_(0).detach()
        # hx_obj = [self.node_gru(x_obj, hx_obj)]
        # hx_edge = [self.edge_gru(x_pred, hx_pred)]

        hx_obj = [x_obj]
        hx_edge = [x_pred]

        for t in range(self.update_step):
            sub_vert = hx_obj[t][rel_inds[:, 0]]  #
            obj_vert = hx_obj[t][rel_inds[:, 1]]

            '''update object features'''
            message_pred_to_subj = self.subj_node_gate(
                torch.cat([sub_vert, hx_edge[t]], 1)) * hx_edge[t]  # nrel x d
            message_pred_to_obj = self.obj_node_gate(
                torch.cat([obj_vert, hx_edge[t]], 1)) * hx_edge[t]    # nrel x d
            node_message = (torch.mm(subj_pred_map, message_pred_to_subj) / (subj_pred_map.sum(1, keepdim=True) + 1e-5)
                            + torch.mm(obj_pred_map, message_pred_to_obj) / (obj_pred_map.sum(1, keepdim=True) + 1e-5)) / 2.
            hx_obj.append(self.node_gru(node_message, hx_obj[t]))
            # hx_obj.append(F.relu(node_message + hx_obj[t]))

            '''update predicat features'''
            message_subj_to_pred = self.subj_edge_gate(
                torch.cat([sub_vert, hx_edge[t]], 1)) * sub_vert  # nrel x d
            message_obj_to_pred = self.obj_edge_gate(
                torch.cat([obj_vert, hx_edge[t]], 1)) * obj_vert    # nrel x d
            edge_message = (message_subj_to_pred + message_obj_to_pred) / 2.
            hx_edge.append(self.edge_gru(edge_message, hx_edge[t]))
            # hx_edge.append(F.relu(edge_message + hx_edge[t]))

        # import pdb; pdb.set_trace()
        '''compute results and losses'''
        # final classifier that converts the features into predictions
        # for object prediction, we do not do bbox regression again
        obj_class_logits = self.obj_predictor(
            hx_obj[-1].unsqueeze(2).unsqueeze(3))
        pred_class_logits = self.pred_predictor(
            hx_edge[-1].unsqueeze(2).unsqueeze(3))

        if self.use_online_obj_labels and obj_class_logits is not None:
            obj_class_labels = obj_class_logits[:, 1:].max(1)[1] + 1
        else:
            obj_class_labels = torch.cat(
                [proposal.get_field("labels") for proposal in proposals], 0)

        return (hx_obj[-1], hx_edge[-1]), obj_class_logits, pred_class_logits, obj_class_labels, rel_inds

def build_imp_model(cfg, in_channels):
    return IMP(cfg, in_channels)