#! /usr/bin/env python
# coding=utf-8

import cv2
import os
import shutil
import numpy as np
import tensorflow as tf
import core.utils as utils
import time
import argparse
from core.config import cfg
from core.yolov3 import YOLOV3

class YoloTest(object):
    def __init__(self):
        self.input_size       = cfg.TEST.INPUT_SIZE
        self.anchor_per_scale = cfg.YOLO.ANCHOR_PER_SCALE
        self.classes          = utils.read_class_names(cfg.YOLO.CLASSES)
        self.num_classes      = len(self.classes)
        self.anchors          = np.array(utils.get_anchors(cfg.YOLO.ANCHORS))
        self.score_threshold  = cfg.TEST.SCORE_THRESHOLD
        self.iou_threshold    = cfg.TEST.IOU_THRESHOLD
        self.moving_ave_decay = cfg.YOLO.MOVING_AVE_DECAY
        self.annotation_path  = cfg.TEST.ANNOT_PATH
        self.number           = cfg.TEST.NUMBER
        self.weight_file      = cfg.TEST.WEIGHT_FILE
        self.model_file       = cfg.TEST.MODEL_FILE
        self.write_image      = cfg.TEST.WRITE_IMAGE
        self.write_image_path = cfg.TEST.WRITE_IMAGE_PATH
        self.show_label       = cfg.TEST.SHOW_LABEL
        self.batch_size       = cfg.TEST.BATCH_SIZE

        self.core_version     = cfg.RUNTIME.CORE_VERSION
        self.precision        = cfg.RUNTIME.PRECISION
        self.data_parallelism = cfg.RUNTIME.DATA_PARALLELISM
        self.model_parallelism = cfg.RUNTIME.MODEL_PARALLELISM
        self.core_num          = cfg.RUNTIME.CORE_NUM

        if os.path.exists(self.model_file):
            print ("model is exit")
        else :
            print ("please check out model_file")
        graph = load_graph(self.model_file)
        self.input_data = graph.get_tensor_by_name("import/input/input_data:0" )
        self.pred_sbbox = graph.get_tensor_by_name("import/pred_sbbox/concat_2:0" )
        self.pred_mbbox = graph.get_tensor_by_name("import/pred_mbbox/concat_2:0" )
        self.pred_lbbox = graph.get_tensor_by_name("import/pred_lbbox/concat_2:0" )
        self.bbox_raw = graph.get_tensor_by_name("import/Yolov3DetectionOutput:0" )
        config = tf.ConfigProto(allow_soft_placement=True,
                    inter_op_parallelism_threads=1,
                                intra_op_parallelism_threads=1)
        config.mlu_options.data_parallelism = self.data_parallelism
        config.mlu_options.model_parallelism = self.model_parallelism
        config.mlu_options.core_num = self.core_num
        config.mlu_options.core_version = self.core_version
        config.mlu_options.precision = self.precision
        config.mlu_options.save_offline_model = True
        config.mlu_options.offline_model_name = "yolov3_int8.cambricon"
        self.sess  = tf.Session(config = config, graph = graph)

    def predict_bak(self, images):

        org_h = [0 for i in range(self.batch_size)]
        org_w = [0 for i in range(self.batch_size)]
        for i in range(self.batch_size):
            org_h[i], org_w[i], _ = images[i].shape

        image_data = utils.images_preporcess(images, [self.input_size, self.input_size])

        start = time.time()
        pred_sbbox, pred_mbbox, pred_lbbox = self.sess.run(
            [self.pred_sbbox, self.pred_mbbox, self.pred_lbbox],
            feed_dict={
                self.input_data: image_data,
            }
        )
        np.savetxt("pred_sbbox.txt",pred_sbbox.flatten())
        np.savetxt("pred_mbbox.txt",pred_mbbox.flatten())
        np.savetxt("pred_lbbox.txt",pred_lbbox.flatten())
        end = time.time()

        batch_bboxes = []
        for idx in range(self.batch_size):
            pred_bbox = np.concatenate([np.reshape(pred_sbbox[idx], (-1, 5 + self.num_classes)),
                                        np.reshape(pred_mbbox[idx], (-1, 5 + self.num_classes)),
                                        np.reshape(pred_lbbox[idx], (-1, 5 + self.num_classes))], axis=0)
            bboxes = utils.postprocess_boxes(pred_bbox, (org_h[idx], org_w[idx]), self.input_size, self.score_threshold)
            batch_bboxes.append(utils.nms(bboxes, self.iou_threshold))
        print("bbox num : ",len(batch_bboxes))
        exit(0)
        return batch_bboxes, (end - start)

    def predict(self, images):

        org_h = [0 for i in range(self.batch_size)]
        org_w = [0 for i in range(self.batch_size)]
        for i in range(self.batch_size):
            org_h[i], org_w[i], _ = images[i].shape

        image_data, dh, dw, scale = utils.images_preporcess(images, [self.input_size, self.input_size])
        start = time.time()
        bbox_raw = self.sess.run(
            self.bbox_raw,
            feed_dict={
                self.input_data: image_data,
            }
        )
        end = time.time()
        print("inference time include postprocess is: ", (end-start) * 1000)
        batch_bboxes = []
        num_batches = 1
        num_boxes = 1024 * 2
        predicts_mlu = bbox_raw.flatten()
        for batchIdx in range(num_batches):
            result_boxes = int(predicts_mlu[batchIdx * (64 + num_boxes * 7)])
            current_bboxes = []
            for i in range(result_boxes):
                batchId = predicts_mlu[i * 7 + 0 + 64 + batchIdx * (64 + num_boxes * 7)]
                classId  = predicts_mlu[i * 7 + 1 + 64 + batchIdx * (64 + num_boxes * 7)]
                score    = predicts_mlu[i * 7 + 2 + 64 + batchIdx * (64 + num_boxes * 7)]
                x1       = 1.0*(predicts_mlu[i * 7 + 3 + 64 + batchIdx * (64 + num_boxes * 7)] * self.input_size - dw)/scale
                y1       = 1.0*(predicts_mlu[i * 7 + 4 + 64 + batchIdx * (64 + num_boxes * 7)] * self.input_size - dh)/scale
                x2       = 1.0*(predicts_mlu[i * 7 + 5 + 64 + batchIdx * (64 + num_boxes * 7)] * self.input_size - dw)/scale
                y2       = 1.0*(predicts_mlu[i * 7 + 6 + 64 + batchIdx * (64 + num_boxes * 7)] * self.input_size - dh)/scale
                bbox = [x1, y1, x2, y2, score, classId]
                current_bboxes.append(np.array(bbox))
            batch_bboxes.append(current_bboxes)
        return batch_bboxes, (end - start)

    def evaluate(self):
        predicted_dir_path = self.write_image_path + '/mAP/predicted'
        ground_truth_dir_path = self.write_image_path + '/mAP/ground-truth'
        if os.path.exists(predicted_dir_path): shutil.rmtree(predicted_dir_path)
        if os.path.exists(ground_truth_dir_path): shutil.rmtree(ground_truth_dir_path)
        if os.path.exists(self.write_image_path): shutil.rmtree(self.write_image_path)
        os.makedirs(predicted_dir_path)
        os.makedirs(ground_truth_dir_path)

        batch_idx = 0
        alltime_sess = 0
        start = []
        end = []
        start_end2end = 0.0
        start_post = 0.0
        end_post = 0.0
        alltime_end2end = 0.0
        alltime_prepare = 0.0
        alltime_post = 0.0
        alltime_sess_run = 0.0

        batch_count = 0
        batch_image = []
        batch_image_name = []
        with open(self.annotation_path, 'r') as annotation_file:
            for num, line in enumerate(annotation_file):
                if batch_idx == 0:
                    start_end2end = time.time()
                annotation = line.strip().split()
                image_path = annotation[0]
                image_name = image_path.split('/')[-1]
                batch_image_name.append(image_name)
                image = cv2.imread(image_path)
                batch_image.append(image)
                bbox_data_gt = np.array([list(map(int, box.split(','))) for box in annotation[1:]])

                if len(bbox_data_gt) == 0:
                    bboxes_gt=[]
                    classes_gt=[]
                else:
                    bboxes_gt, classes_gt = bbox_data_gt[:, :4], bbox_data_gt[:, 4]
                ground_truth_path = os.path.join(ground_truth_dir_path, str(num) + '.txt')

                num_bbox_gt = len(bboxes_gt)
                with open(ground_truth_path, 'w') as f:
                    for i in range(num_bbox_gt):
                        class_name = self.classes[classes_gt[i]]
                        xmin, ymin, xmax, ymax = list(map(str, bboxes_gt[i]))
                        bbox_mess = ' '.join([class_name, xmin, ymin, xmax, ymax]) + '\n'
                        f.write(bbox_mess)
                if batch_idx < self.batch_size - 1:
                    batch_idx += 1
                    continue

                print("=> Predicting %d th batch images." % (batch_count + 1))
                start.append(time.time())
                bboxes_pr, sess_run_time = self.predict(batch_image)
                end.append(time.time())
                if batch_count > 0:
                    alltime_sess_run += sess_run_time
                    duration_time = (end[batch_count] - start[batch_count])
                    alltime_sess += duration_time
                    alltime_prepare = alltime_prepare + (start[batch_count] - start_end2end)
                if self.write_image:
                    for idx in range(self.batch_size):
                        image = utils.draw_bbox(batch_image[idx], bboxes_pr[idx], show_label=self.show_label)
                        print("######### SAVE IMAGE ,",self.write_image_path+"/"+batch_image_name[idx])
                        cv2.imwrite(self.write_image_path+"/"+batch_image_name[idx], image)

                for idx in range(self.batch_size):
                    predict_result_path = os.path.join(predicted_dir_path,
                            str(batch_count * self.batch_size + idx) + '.txt')
                    with open(predict_result_path, 'w') as f:
                        for bbox in bboxes_pr[idx]:
                            coor = np.array(bbox[:4], dtype=np.int32)
                            score = bbox[4]
                            class_ind = int(bbox[5])
                            class_name = self.classes[class_ind + 1]
                            score = '%.4f' % score
                            xmin, ymin, xmax, ymax = list(map(str, coor))
                            bbox_mess = ' '.join([class_name, score, xmin, ymin, xmax, ymax]) + '\n'
                            f.write(bbox_mess)
                if batch_count > 0:
                    temp = time.time()
                    alltime_end2end = alltime_end2end + (temp - start_end2end)
                    alltime_post = alltime_post + temp - end[batch_count]
                batch_count += 1
                if self.number < (batch_count + 1) * self.batch_size:
                    print("we have evaluated %d batch images"%(batch_count))
                    break
                batch_idx = 0
                batch_image = []
                batch_image_name = []
        if(self.number > 1):
            print('latency: %f (ms)' % (alltime_sess_run * 1000 / (batch_count - 1)))
            print('throughput: %f' % (((batch_count - 1) * self.batch_size) / alltime_sess_run))

def load_graph(model_file):
    graph = tf.Graph()
    graph_def = tf.GraphDef()
    print("model_file",model_file)
    with open(model_file, "rb") as f:
      graph_def.ParseFromString(f.read())
    with graph.as_default():
      tf.import_graph_def(graph_def)

    return graph

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--graph", help="graph/model to be executed")
    parser.add_argument("--result_path", help="result path to write")
    parser.add_argument("--records", help="records to be processed")
    parser.add_argument("--number", type=int, help="number of records to be processed")
    parser.add_argument("--core_version", type=str, help="MLU100/MLU270", default="MLU100")
    parser.add_argument("--precision", type=str, help="float/int8", default="float")
    parser.add_argument("--data_parallelism", type=int, help="data_parallelism")
    parser.add_argument("--model_parallelism", type=int, help="model_parallelism")
    parser.add_argument("--core_num", type=int, help="core_num")
    parser.add_argument("--input_size", type=int, help="choose 416 or 544", default=416)
    parser.add_argument("--batch_size", type=int, help="batch size")
    args = parser.parse_args()
    if args.graph:
        cfg.TEST.MODEL_FILE = args.graph
    if args.result_path:
        cfg.TEST.WRITE_IMAGE_PATH = args.result_path
    if args.records:
        cfg.TEST.ANNOT_PATH = args.records
    if args.number:
        cfg.TEST.NUMBER = args.number
    if args.core_version:
        cfg.RUNTIME.CORE_VERSION = args.core_version
    if args.precision:
        cfg.RUNTIME.PRECISION = args.precision
    if args.data_parallelism:
        cfg.RUNTIME.DATA_PARALLELISM = args.data_parallelism
    if args.model_parallelism:
        cfg.RUNTIME.MODEL_PARALLELISM = args.model_parallelism
    if args.core_num:
        cfg.RUNTIME.CORE_NUM = args.core_num
    if args.input_size:
        cfg.TEST.INPUT_SIZE = args.input_size
    if args.batch_size:
        cfg.TEST.BATCH_SIZE = args.batch_size

    YoloTest().evaluate()


import numpy as np
import cv2
import time
from directKeys import PressKey, W, A, S, D, SPACE, ReleaseKey
import math
import warnings
warnings.filterwarnings("ignore")

from PIL import ImageGrab
import win32gui, win32ui, win32con, win32api

import torch

########### CONTROLS START################
def go_straight():
    PressKey(W)
    ReleaseKey(A)
    ReleaseKey(D)
    time.sleep(0.5)
    ReleaseKey(W)

def turn_left():
    PressKey(A)
    PressKey(W)
    time.sleep(0.5)
    ReleaseKey(A)

def turn_right():
    PressKey(D)
    ReleaseKey(A)
    time.sleep(0.5)
    ReleaseKey(D)

def stop():
    ReleaseKey(W)
    ReleaseKey(A)
    ReleaseKey(D)
    PressKey(SPACE)
    time.sleep(0.5)
    ReleaseKey(SPACE)

def slow_down():
    ReleaseKey(W)
    ReleaseKey(A)
    ReleaseKey(D)
    PressKey(S)
    time.sleep(0.5)
    ReleaseKey(S)
################### CONTROL ENDS #####################

################### SCREEN CAPTURE ###################

def grab_screen(region=None):
    hwin = win32gui.GetDesktopWindow()

    if region:
        left, top, x2, y2 = region
        width = x2 - left + 1
        height = y2 - top + 1
    else:
        width = win32api.GetSystemMetrics(win32con.SM_CXVIRTUALSCREEN)
        height = win32api.GetSystemMetrics(win32con.SM_CYVIRTUALSCREEN)
        left = win32api.GetSystemMetrics(win32con.SM_XVIRTUALSCREEN)
        top = win32api.GetSystemMetrics(win32con.SM_YVIRTUALSCREEN)

    hwindc = win32gui.GetWindowDC(hwin)
    srcdc = win32ui.CreateDCFromHandle(hwindc)
    memdc = srcdc.CreateCompatibleDC()
    bmp = win32ui.CreateBitmap()
    bmp.CreateCompatibleBitmap(srcdc, width, height)
    memdc.SelectObject(bmp)
    memdc.BitBlt((0, 0), (width, height), srcdc, (left, top), win32con.SRCCOPY)

    signedIntsArray = bmp.GetBitmapBits(True)
    img = np.fromstring(signedIntsArray, dtype='uint8')
    img.shape = (height, width, 4)

    srcdc.DeleteDC()
    memdc.DeleteDC()
    win32gui.ReleaseDC(hwin, hwindc)
    win32gui.DeleteObject(bmp.GetHandle())

    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

######################################################

##################### PROCESSING #####################

def process_img(image, model):
    '''
    Before accelerating, it takes = 0.11526s on average
    '''
    try:
        output = model(image)
        op = output.pandas().xyxy[0]
        for index, row in op.iterrows():
            if row["confidence"] > 0.75:
                xmin = int(row["xmin"])
                ymin = int(row["ymin"])
                xmax = int(row["xmax"])
                ymax = int(row["ymax"])
                name = row["name"]

                cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)

                mid_object_x, mid_object_y = (xmin + xmax)//2, (ymin + ymax)//2
                # cv2.circle(image, (mid_object_x, mid_object_y), 5, (255,0,0), -1)
                cv2.line(image, (0, 550), (800, 550), (120, 150, 90), 3)

                mid_line_x, mid_line_y = 400, 550
                # angle = math.degrees(math.atan2(mid_object_y - mid_line_y, mid_object_x - mid_line_x))
                # print('Angle is :',angle)
                dist = math.dist([mid_object_x, mid_object_y], [mid_line_x, mid_line_y])
                print("Distance is :", dist)

                cv2.line(image, (400, 550), (mid_object_x, mid_object_y), (90, 190, 200), 2)
                # cv2.putText(image, str(dist), ((mid_object_x - 400)//2, (mid_object_y - 550)//2), cv2.FONT_HERSHEY_SIMPLEX, 5, (120, 155, 50), 2)
                cv2.putText(image, name + f',{int(dist)}', (xmin - 10, ymin - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
                print(name.lower())
                if dist < 210 and name.lower() in ['car', 'person', 'truck', 'bike', 'bicycle']:
                    print("Slowing down...")
                    slow_down()
                if name.lower() == 'traffic_light_red' and dist >290 and dist < 310:
                    print("Red light ahead")
                    print("Stopping...")
                    stop()
        return image
    except Exception as e:
        print(e)

def main():
    prev_time = 0
    processedImg_prev_time = 0
    print("Staring in ...")
    for i in range(4)[::-1]:
        print(i+1)
        time.sleep(1)
    try:
        # Load COCO pretrained model
        # model = torch.hub.load("ultralytics/yolov5", "yolov5s")
        # Load Custom trained model
        model = torch.hub.load("ultralytics/yolov5", "custom", path="best_e46_335_v2.pt")
        model.cuda()
    except exception as e:
        print("While loading model, exception occured.\n", e)

    while True:
        #screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 600)))
        screen = grab_screen(region=(0, 0, 799, 599))

        curr_time = time.time()
        # print("FPS : {0}".format(1 // (curr_time - prev_time)))
        prev_time = curr_time

        #processedImg = process_img(screen)
        # processedImg_curr_time = time.time()
        processedImg = process_img(screen, model)
        #print("Time taken by func : ", (processedImg_curr_time - processedImg_prev_time))
        #processedImg_prev_time = processedImg_curr_time

        #cv2.imshow("GTA V", processedImg)
        cv2.imshow("GTA V", cv2.cvtColor(processedImg, cv2.COLOR_BGR2RGB))
        if cv2.waitKey(25) & 0xFF == ord('k'):
            cv2.destroyAllWindows()
            break

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
from finemapper import SUSIE_Wrapper
from polyfun import configure_logger, check_package_versions
import logging
import os
import scipy.sparse as sparse

def splash_screen():
    print('*********************************************************************')
    print('* Fine-mapping Wrapper')
    print('* Version 1.0.0')
    print('* (C) 2019 Omer Weissbrod')
    print('*********************************************************************')
    print()

def load_ld(ld_prefix):

    #load SNPs info
    snps_filename_parquet = ld_prefix+'.parquet'
    snps_filename_gz = ld_prefix+'.gz'
    if os.path.exists(snps_filename_parquet):
        df_ld_snps = pd.read_parquet(snps_filename_parquet)
    elif os.path.exists(snps_filename_gz):
        df_ld_snps = pd.read_table(snps_filename_gz, delim_whitespace=True)
    else:
        raise ValueError('couldn\'t find SNPs file %s or %s'%(snps_filename_parquet, snps_filename_gz))

    #load LD matrix
    R_filename = ld_prefix+'.npz'
    if not os.path.exists(R_filename):
        raise IOError('%s not found'%(R_filename))
    R = sparse.load_npz(R_filename).toarray()
    R = R+R.T
    assert np.allclose(np.diag(R), 1.0)
    logging.info('Loaded an LD matrix for %d SNPs from %s'%(R.shape[0], R_filename))

    #sanity checks
    assert R.shape[0] == R.shape[1]
    if R.shape[0] != df_ld_snps.shape[0]:
        raise ValueError('LD matrix has a different number of SNPs than the SNPs file')

    return R, df_ld_snps

if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser()

    #general parameters
    parser.add_argument('--method', required=True, help='Fine-mapping method (currently only susie is supported)')
    parser.add_argument('--sumstats', required=True, help='Name of sumstats file')
    parser.add_argument('--chr', required=True, type=int, help='Target chromosome')
    parser.add_argument('--start', required=True, type=int, help='First base-pair in the region to finemap')
    parser.add_argument('--end', required=True, type=int, help='Last base-pair in the region to finemap')
    parser.add_argument('--n', required=True, type=int, help='Sample size')
    parser.add_argument('--geno', default=None, help='Genotypes file (plink or bgen format)')
    parser.add_argument('--ld', default=None, help='prefix of LD matrix file names')

    #LDstore related parameters
    parser.add_argument('--ldstore', default=None, help='Path to an LDstore executable file')
    parser.add_argument('--threads', type=int, default=None, help='The number of CPU cores LDstore will use (if not specified, LDstore will use the max number of CPU cores available')
    parser.add_argument('--cache-dir', default=None, help='If specified, this is a path of a directory that will cache LD matrices that have already been computed')

    parser.add_argument('--max-num-causal', required=True, type=int, help='Number of causal SNPs')
    parser.add_argument('--non-funct', action='store_true', default=False, help='Perform non-functionally informed fine-mapping')
    parser.add_argument('--hess', action='store_true', default=False, help='If specified, estimate causal effect variance via HESS')
    parser.add_argument('--verbose', action='store_true', default=False, help='If specified, show verbose output')

    parser.add_argument('--sample-file', default=None, help='BGEN files must be used together with a sample file')
    parser.add_argument('--incl-samples', default=None, help='A single-column text file specifying the ids of individuals to exclude from fine-mapping')
    parser.add_argument('--out', required=True, help='name of the output file')

    #check package versions
    check_package_versions()

    #show splash screen
    splash_screen()

    #extract args
    args = parser.parse_args()

    #configure logger
    configure_logger(args.out)

    #check params
    if args.geno is None:
        if args.ld is None:
            raise ValueError('must specify either --geno or --ld')
        if args.ldstore is not None:
            raise ValueError('cannot specify both --ld and --ldstore')
    if args.geno is not None:
        if args.ld is not None:
            raise ValueError('cannot specify both --geno and --ld')
        if args.ldstore is None:
            raise ValueError('You must specify --ldstore together with --geno')

    #load LD matrix if requested
    if args.ld is None:
        ld, df_ld_snps = None, None
    else:
        ld, df_ld_snps = load_ld(args.ld)

    #start fine-mapping
    if args.method == 'susie':
        finemap_obj = SUSIE_Wrapper(genotypes_file=args.geno, sumstats_file=args.sumstats, n=args.n, chr_num=args.chr,
                                    sample_file=args.sample_file, incl_samples=args.incl_samples,
                                    ldstore_exe=args.ldstore, n_threads=args.threads,
                                    cache_dir=args.cache_dir)
        df_finemap = finemap_obj.finemap(locus_start=args.start, locus_end=args.end, num_causal_snps=args.max_num_causal,
                     use_prior_causal_prob=not args.non_funct, prior_var=None, residual_var=None, hess=args.hess,
                     verbose=args.verbose, ld=ld, df_ld_snps=df_ld_snps)
    elif args.method == 'finemap':
        raise ValueError('FINEMAP is not yet supported')
    else:
        raise ValueError('unknown method specified in --method')
    logging.info('Writing fine-mapping results to %s'%(args.out))
    df_finemap.sort_values('PIP', ascending=False, inplace=True)
    df_finemap.to_csv(args.out, sep='\t', index=False, float_format='%0.5e')


from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
import numpy as np
import pylab
from mouse_track_parser import mouseTrackParser

timeData = [0, 7, 23, 32, 39, 43, 56, 63, 71, 80, 88, 96, 103, 111, 119, 128, 136, 147, 151, 159, 167, 171, 180, 187, 196, 203, 211, 219, 227, 235, 243, 251, 263, 271, 279, 287,
            295, 299, 308, 320, 327, 335, 343, 351, 360, 368, 375, 383, 391, 400, 407, 415, 424, 428, 435, 444, 451, 460, 467, 475, 483, 492, 499, 510, 516, 524, 533, 638, 646, 732]
trackData = [0, 1, 3, 4, 5, 8, 8, 10, 11, 13, 16, 17, 20, 23, 27, 30, 32, 34, 36, 38, 40, 41, 44, 47, 49, 52, 54, 57, 59, 63, 66, 69, 72, 76, 78, 82, 88, 93, 99,
             103, 104, 108, 112, 117, 123, 128, 133, 136, 140, 145, 152, 158, 165, 173, 181, 189, 196, 204, 211, 217, 224, 231, 237, 243, 248, 254, 258, 264, 265, 265]

class ZhilianMS:
    def __init__(self, url, chromepath='C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe', timeData=timeData, trackData=trackData):
        self.driver = webdriver.Chrome(chromepath)
        self.timeData = timeData
        self.trackData = trackData
        self.url = url
        self.action = ActionChains(self.driver)

    def get_trace(self, offset, time):
        timeRatio = time*1000/np.max(self.timeData)
        trackRatio = offset/np.max(self.trackData)

        msTime = np.array(self.timeData)*timeRatio
        msTrack = np.array(self.trackData)*trackRatio
        xx = np.arange(0, time*1000)
        yy, fits = mouseTrackParser(
            msTime, msTrack).myPolyfit(xx, 3)
        yy = np.abs(np.floor(offset/np.max(yy)*yy))

        # pylab.plot(timeData, trackData, '.')
        # pylab.plot(msTime, msTrack, '-')
        # pylab.plot(xx, yy, '.')
        # pylab.show()

        return xx, yy

    """ def get_trace(self, distance):
        '''
        :param distance: (Int)缺口离滑块的距离
        :return: (List)移动轨迹
        '''

        # 创建存放轨迹信息的列表
        trace = []
        # 设置加速的距离
        faster_distance = distance*(4/5)
        # 设置初始位置、初始速度、时间间隔
        start, v0, t = 0, 0, 0.2
        # 当尚未移动到终点时
        while start < distance:
            # 如果处于加速阶段
            if start < faster_distance:
                # 设置加速度为2
                a = 1.5
            # 如果处于减速阶段
            else:
                # 设置加速度为-3
                a = -3
            # 移动的距离公式
            move = v0 * t + 1 / 2 * a * t * t
            # 此刻速度
            v = v0 + a * t
            # 重置初速度
            v0 = v
            # 重置起点
            start += move
            # 将移动的距离加入轨迹列表
            trace.append(round(move))
        # 返回轨迹信息
        return trace """

    def isElemExist(self, cssSelector):
        flag = True
        try:
            self.driver.find_element_by_css_selector(cssSelector)
            return flag
        except:
            flag = False
            return flag

    def drag_and_drop(self, ele, offset, time):
        xx, yy = self.get_trace(offset, time)
        self.action = ActionChains(self.driver)
        self.action.click_and_hold(ele).perform()
        for i in range(0, len(yy)):
            if self.if_need_verify() == False:
                break
            try:
                self.action.move_by_offset(yy[i], 0).perform()
                # self.action.reset_actions()
            except:
                break
            # ActionChains(driver).drag_and_drop_by_offset(ele, y, 0).perform()
        self.action.release().perform()
        sleep(3)
        warnElem = self.isElemExist('.nc-container .errloading')
        if warnElem == True:
            warnElem = self.driver.find_element_by_css_selector(
                '.nc-container .errloading')
            refreshBtn = warnElem.find_element_by_tag_name('a')
            refreshBtn.click()
        else:
            return True

    def if_need_verify(self, sliderSelector='#nocaptcha .nc_iconfont.btn_slide', wrapperSelector='.nc-container .nc_scale',):
        try:
            # 滑块
            slide = self.driver.find_element_by_css_selector(sliderSelector)
            # 滑块包裹
            nc_scale = self.driver.find_element_by_css_selector(
                wrapperSelector)
        except:
            print('未找到元素')
            return False
        return True

    def start_simulate(self, sliderSelector='#nocaptcha .nc_iconfont.btn_slide', wrapperSelector='.nc-container .nc_scale', time=1, timeOut=3):
        self.driver.maximize_window()
        self.driver.get(self.url)
        self.driver.implicitly_wait(1)

        tryTime = 0
        while self.if_need_verify(sliderSelector, wrapperSelector) and tryTime < timeOut:
            tryTime += 1
            # 滑块
            slide = self.driver.find_element_by_css_selector(sliderSelector)
            # 滑块包裹
            nc_scale = self.driver.find_element_by_css_selector(
                wrapperSelector)
            slide_width = int(slide.value_of_css_property('width')[0:-2])
            nc_scale_width = int(nc_scale.value_of_css_property('width')[0:-2])
            print(slide_width, nc_scale_width)
            self.drag_and_drop(slide, nc_scale_width-slide_width, time)

            sleep(1)

        if tryTime < timeOut:
            return True, self.driver.find_element_by_xpath(
                "//*").get_attribute("outerHTML")
        return False, None

    def close(self):
        self.driver.quit()

if __name__ == '__main__':
    zlms = ZhilianMS('https://127.0.0.1:9008')
    flag, html = zlms.start_simulate()
    print(flag)


import json
import os
import random
from typing import Any, Union

import numpy as np
import tensorflow as tf

from Configs.FasterRCNN_config import Param
from Debugger import debug_print
from NN_Components import Backbone, RPN, RoI
from NN_Helper import NnDataGenerator, BboxTools

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'  # for mac os tensorflow setting

class FasterRCNN():
    def __init__(self):
        self.Backbone = Backbone(img_shape=Param.IMG_RESIZED_SHAPE, n_stage=Param.N_STAGE)
        self.IMG_SHAPE = Param.IMG_RESIZED_SHAPE
        # self.backbone_model.trainable= False
        # === RPN part ===
        self.RPN = RPN(backbone_model=self.Backbone.backbone_model,
                       lambda_factor=Param.LAMBDA_FACTOR,
                       batch=Param.BATCH_RPN,
                       lr=Param.LR)

        # === RoI part ===
        self.RoI = RoI(backbone_model=self.Backbone.backbone_model,
                       img_shape=self.IMG_SHAPE,
                       n_output_classes=Param.N_OUT_CLASS,
                       lr=Param.LR)
        self.RoI_header = self.RoI.RoI_header_model

        # === Data part ===
        self.train_data_generator = NnDataGenerator(
            file=Param.DATA_JSON_FILE,
            imagefolder_path=Param.PATH_IMAGES,
            anchor_base_size=Param.BASE_SIZE,
            ratios=Param.RATIOS,
            scales=Param.SCALES,
            n_anchors=Param.N_ANCHORS,
            img_shape_resize=Param.IMG_RESIZED_SHAPE,
            n_stage=Param.N_STAGE,
            threshold_iou_rpn=Param.THRESHOLD_IOU_RPN,
            threshold_iou_roi=Param.THRESHOLD_IOU_RoI)
        self.cocotool = self.train_data_generator.dataset_coco

        self.anchor_candidate_generator = self.train_data_generator.gen_candidate_anchors
        self.anchor_candidates = self.anchor_candidate_generator.anchor_candidates

    def test_loss_function(self):
        inputs, anchor_targets, bbox_targets = self.train_data_generator.gen_train_data_rpn_all()
        print(inputs.shape, anchor_targets.shape, bbox_targets.shape)
        input1 = np.reshape(inputs[0, :, :, :], (1, 720, 1280, 3))
        anchor1 = np.reshape(anchor_targets[0, :, :, :], (1, 23, 40, 9))
        anchor2 = tf.convert_to_tensor(anchor1)
        anchor2 = tf.dtypes.cast(anchor2, tf.int32)
        anchor2 = tf.one_hot(anchor2, 2, axis=-1)
        print(anchor1)
        bbox1 = np.reshape(bbox_targets[0, :, :, :, :], (1, 23, 40, 9, 4))
        loss = self.RPN._rpn_loss(anchor1, bbox1, anchor2, bbox1)
        print(loss)

    def faster_rcnn_output(self):
        # === prepare input images ===
        image_ids = self.train_data_generator.dataset_coco.image_ids
        inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(image_ids[0])
        print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
        image = np.reshape(inputs[0, :, :, :], (1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
        # === get proposed region boxes ===
        rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(image)
        proposed_boxes = self.RPN._proposal_boxes(rpn_anchor_pred, rpn_bbox_regression_pred,
                                                  self.anchor_candidates,
                                                  self.anchor_candidate_generator.h,
                                                  self.anchor_candidate_generator.w,
                                                  self.anchor_candidate_generator.n_anchors,
                                                  Param.ANCHOR_PROPOSAL_N,
                                                  Param.ANCHOR_THRESHOLD
                                                  )
        # === processing boxes with RoI header ===
        pred_class, pred_box_reg = self.RoI.process_image([image, proposed_boxes])
        # === processing the results ===
        pred_class_sparse = np.argmax(a=pred_class[:, :], axis=1)
        pred_class_sparse_value = np.max(a=pred_class[:, :], axis=1)
        print(pred_class, pred_box_reg)
        print(pred_class_sparse, pred_class_sparse_value)
        print(np.max(proposed_boxes), np.max(pred_box_reg))
        final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes, regs=pred_box_reg)
        final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
        # === output to official coco bbox result json file ===
        temp_output_to_file = []
        for i in range(pred_class_sparse.shape[0]):
            temp_category = self.train_data_generator.dataset_coco.get_category_from_sparse(pred_class_sparse[i])
            temp_output_to_file.append({
                "image_id": f"{image_ids[0]}",
                "bbox": [final_box[i][0].item(), final_box[i][1].item(), final_box[i][2].item(),
                         final_box[i][3].item()],
                "score": pred_class_sparse_value[i].item(),
                "category": f"{temp_category}"
            })
        with open("results.pkl.bbox.json", "w") as f:
            json.dump(temp_output_to_file, f, indent=4)
        # print(final_box)
        print(final_box[pred_class_sparse_value > 0.9])
        final_box = final_box[pred_class_sparse_value > 0.9]
        self.cocotool.draw_bboxes(original_image=image[0], bboxes=final_box.tolist(), show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='6PredRoISBoxes')

        # === Non maximum suppression ===
        final_box_temp = np.array(final_box).astype(np.int)
        nms_boxes_list = []
        while final_box_temp.shape[0] > 0:
            ious = self.nms_loop_np(final_box_temp)
            nms_boxes_list.append(
                final_box_temp[0, :])  # since it's sorted by the value, here we can pick the first one each time.
            final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
        debug_print('number of box after nms', len(nms_boxes_list))
        self.cocotool.draw_bboxes(original_image=image[0], bboxes=nms_boxes_list, show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='7PredRoINMSBoxes')

    def test_proposal_visualization(self):
        # === Prediction part ===
        image_ids = self.train_data_generator.dataset_coco.image_ids
        inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(image_ids[0])
        print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
        input1 = np.reshape(inputs[0, :, :, :], (1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
        rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(input1)
        print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)

        # === Selection part ===
        # top_values, top_indices = tf.math.top_k()
        rpn_anchor_pred = tf.slice(rpn_anchor_pred, [0, 0, 0, 0, 1],
                                   [1, self.anchor_candidate_generator.h, self.anchor_candidate_generator.w,
                                    self.anchor_candidate_generator.n_anchors, 1])  # second channel is foreground
        print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
        # squeeze the pred of anchor and bbox_reg
        rpn_anchor_pred = tf.squeeze(rpn_anchor_pred)
        rpn_bbox_regression_pred = tf.squeeze(rpn_bbox_regression_pred)
        shape1 = tf.shape(rpn_anchor_pred)
        print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
        # flatten the pred of anchor to get top N values and indices
        rpn_anchor_pred = tf.reshape(rpn_anchor_pred, (-1,))
        n_anchor_proposal = Param.ANCHOR_PROPOSAL_N
        top_values, top_indices = tf.math.top_k(rpn_anchor_pred,
                                                n_anchor_proposal)  # top_k has sort function. it's important here
        top_indices = tf.gather_nd(top_indices, tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD)))
        top_values = tf.gather_nd(top_values, tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD)))

        debug_print('top values', top_values)

        # test_indices = tf.where(tf.greater(tf.reshape(RPN_Anchor_Pred, (-1,)), 0.9))
        # print(test_indices)

        top_indices = tf.reshape(top_indices, (-1, 1))
        debug_print('top indices', top_indices)
        update_value = tf.math.add(top_values, 1)
        rpn_anchor_pred = tf.tensor_scatter_nd_update(rpn_anchor_pred, top_indices, update_value)
        rpn_anchor_pred = tf.reshape(rpn_anchor_pred, shape1)

        # --- find the base boxes ---
        anchor_pred_top_indices = tf.where(tf.greater(rpn_anchor_pred, 1))
        debug_print('original_indices shape', anchor_pred_top_indices.shape)
        debug_print('original_indices', anchor_pred_top_indices)
        base_boxes = tf.gather_nd(self.anchor_candidates, anchor_pred_top_indices)
        debug_print('base_boxes shape', base_boxes.shape)
        debug_print('base_boxes', base_boxes)
        base_boxes = np.array(base_boxes)

        # --- find the bbox_regs ---
        # flatten the bbox_reg by last dim to use top_indices to get final_box_reg
        rpn_bbox_regression_pred_shape = tf.shape(rpn_bbox_regression_pred)
        rpn_bbox_regression_pred = tf.reshape(rpn_bbox_regression_pred, (-1, rpn_bbox_regression_pred_shape[-1]))
        debug_print('RPN_BBOX_Regression_Pred shape', rpn_bbox_regression_pred.shape)
        final_box_reg = tf.gather_nd(rpn_bbox_regression_pred, top_indices)
        debug_print('final box reg values', final_box_reg)

        # Convert to numpy to plot
        final_box_reg = np.array(final_box_reg)
        debug_print('final box reg shape', final_box_reg.shape)
        debug_print('max value of final box reg', np.max(final_box_reg))
        final_box = BboxTools.bbox_reg2truebox(base_boxes=base_boxes, regs=final_box_reg)

        # === Non maximum suppression ===
        final_box_temp = np.array(final_box).astype(np.int)
        nms_boxes_list = []
        while final_box_temp.shape[0] > 0:
            ious = self.nms_loop_np(final_box_temp)
            nms_boxes_list.append(
                final_box_temp[0, :])  # since it's sorted by the value, here we can pick the first one each time.
            final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
        debug_print('number of box after nms', len(nms_boxes_list))

        # Need to convert above instructions to tf operations

        # === visualization part ===
        # clip the boxes to make sure they are legal boxes
        debug_print('max value of final box', np.max(final_box))
        final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)

        original_boxes = self.cocotool.get_original_bboxes_list(image_id=self.cocotool.image_ids[0])
        self.cocotool.draw_bboxes(original_image=input1[0], bboxes=original_boxes, show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='1GroundTruthBoxes')
        target_anchor_boxes, target_classes = self.train_data_generator.gen_target_anchor_bboxes_classes_for_debug(
            image_id=self.cocotool.image_ids[0])
        self.cocotool.draw_bboxes(original_image=input1[0], bboxes=target_anchor_boxes, show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='2TrueAnchorBoxes')
        self.cocotool.draw_bboxes(original_image=input1[0], bboxes=base_boxes.tolist(), show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='3PredAnchorBoxes')
        self.cocotool.draw_bboxes(original_image=input1[0], bboxes=final_box.tolist(), show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='4PredRegBoxes')
        self.cocotool.draw_bboxes(original_image=input1[0], bboxes=nms_boxes_list, show=True, save_file=True,
                                  path=Param.PATH_DEBUG_IMG, save_name='5PredNMSBoxes')

    def test_total_visualization(self):
        # === prediction part ===
        input_images, target_anchor_bboxes, target_classes = self.train_data_generator.gen_train_data_roi_one(
            self.train_data_generator.dataset_coco.image_ids[0],
            self.train_data_generator.gen_candidate_anchors.anchor_candidates_list)
        input_images, target_anchor_bboxes, target_classes = np.asarray(input_images).astype(np.float), np.asarray(
            target_anchor_bboxes), np.asarray(target_classes)
        # TODO:check tf.image.crop_and_resize
        input_images2 = input_images[:1].astype(np.float)
        print(input_images2.shape)
        target_anchor_bboxes2 = target_anchor_bboxes[:1].astype(np.float)
        print(target_anchor_bboxes2.shape)
        class_header, box_reg_header = self.RoI.process_image([input_images2, target_anchor_bboxes2])
        print(class_header.shape, box_reg_header.shape)
        print(class_header)

    def nms_loop_np(self, boxes):
        # boxes : (N, 4), box_1target : (4,)
        # box axis format: (x1,y1,x2,y2)

        box_1target = np.ones(shape=boxes.shape)
        zeros = np.zeros(shape=boxes.shape)
        box_1target = box_1target * boxes[0, :]
        box_b_area = (box_1target[:, 2] - box_1target[:, 0] + 1) * (box_1target[:, 3] - box_1target[:, 1] + 1)
        # --- determine the (x, y)-coordinates of the intersection rectangle ---
        x_a = np.max(np.array([boxes[:, 0], box_1target[:, 0]]), axis=0)
        y_a = np.max(np.array([boxes[:, 1], box_1target[:, 1]]), axis=0)
        x_b = np.min(np.array([boxes[:, 2], box_1target[:, 2]]), axis=0)
        y_b = np.min(np.array([boxes[:, 3], box_1target[:, 3]]), axis=0)

        # --- compute the area of intersection rectangle ---
        inter_area = np.max(np.array([zeros[:, 0], x_b - x_a + 1]), axis=0) * np.max(
            np.array([zeros[:, 0], y_b - y_a + 1]),
            axis=0)

        # --- compute the area of both the prediction and ground-truth ---
        # rectangles
        box_a_area: Union[int, Any] = (boxes[:, 2] - boxes[:, 0] + 1) * (boxes[:, 3] - boxes[:, 1] + 1)

        # compute the intersection over union by taking the intersection
        # area and dividing it by the sum of prediction + ground-truth
        # areas - the intersection area
        ious = (inter_area / (box_a_area + box_b_area - inter_area))

        # return the intersection over union value
        return ious

    def train_rpn_roi(self, ):
        # TODO: use the output of RPN to train RoI
        image_ids = self.train_data_generator.dataset_coco.image_ids
        for epoch in range(Param.EPOCH):
            print(f'epoch : {epoch}')
            temp_image_ids = random.choices(population=image_ids, weights=None, k=8)

            for image_id in image_ids:
                # --- train RoI ---
                input_img, input_box_filtered_by_iou, target_class, target_bbox_reg = \
                    self.train_data_generator.gen_train_data_roi_one(
                        image_id, self.train_data_generator.gen_candidate_anchors.anchor_candidates_list)
                n_box = input_img.shape[0]
                # --- train RoI with backbone once, balance with the RPN train ---
                # j = random.randint(a=0,
                #                    b=n_box - 1)
                # model with backbone only be trained once to balance RPN and RoI training
                for j in range(n_box):
                    self.RoI.train_step_with_backbone(input_img[j:j + 1], input_box_filtered_by_iou[j:j + 1],
                                                      target_class[j:j + 1], target_bbox_reg[j:j + 1])

                # --- train RPN with backbone---
                inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(image_id)
                self.RPN.train_step_with_backbone(inputs, anchor_targets, bbox_reg_targets)

                # --- train RoI header secondly ---
                for j in range(n_box):
                    self.RoI.train_step_header(input_img[0:1], input_box_filtered_by_iou[j:j + 1],
                                               target_class[j:j + 1], target_bbox_reg[j:j + 1])

                # --- train RoI with RPN proposed boxes ---
                if epoch > 10:
                    rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(input_img[:1])
                    proposed_boxes = self.RPN._proposal_boxes(rpn_anchor_pred, rpn_bbox_regression_pred,
                                                              self.anchor_candidates,
                                                              self.anchor_candidate_generator.h,
                                                              self.anchor_candidate_generator.w,
                                                              self.anchor_candidate_generator.n_anchors,
                                                              Param.ANCHOR_PROPOSAL_N,
                                                              Param.ANCHOR_THRESHOLD
                                                              )
                    if len(list(proposed_boxes.tolist())) == 0:
                        continue
                    input_img, input_box_filtered_by_iou, target_class, target_bbox_reg = \
                        self.train_data_generator.gen_train_data_roi_one(
                            image_id, proposed_boxes.tolist())
                    for j in range(n_box):
                        self.RoI.train_step_header(input_img[j:j + 1], input_box_filtered_by_iou[j:j + 1],
                                                   target_class[j:j + 1], target_bbox_reg[j:j + 1])

            # --- train RPN header first ---
            for image_id in image_ids:
                inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
                    image_id)
                self.RPN.train_step_header(inputs, anchor_targets, bbox_reg_targets)

    def save_weight(self):
        self.RPN.save_model(Param.PATH_MODEL)
        self.RoI.save_header(Param.PATH_MODEL)
        self.Backbone.save_weight(Param.PATH_MODEL)

    def load_weight(self):
        self.RPN.load_model(Param.PATH_MODEL)
        self.RoI.load_header(Param.PATH_MODEL)
        self.Backbone.load_weight(Param.PATH_MODEL)

if __name__ == '__main__':
    f1 = FasterRCNN()

    # f1.train_rpn_roi()
    # f1.save_weight()
    f1.load_weight()
    f1.test_proposal_visualization()
    f1.faster_rcnn_output()


import logging
import os
import requests
import argparse
from pathlib import Path
import numpy as np
import cv2
import torch
import torch.backends.cudnn as cudnn
import os
from datetime import datetime
from time import time
import imutils
from models.common import DetectMultiBackend
from utils.general import ( check_img_size,non_max_suppression, scale_coords,LOGGER)
from utils.torch_utils import select_device, time_sync
from utils.augmentations import letterbox
from utils import all_utils
from model import model_load

FRAMES_TO_PERSIST = 10 # number of frames after compare
MIN_SIZE_FOR_MOVEMENT = 20000000 # movement size
MOVEMENT_DETECTED_PERSISTENCE = 100

def preprocess_image(im, half, device):
    """it will preprocess image for model

    Args:
        im ([image]): [description]
        half ([type]): [description]
        device ([type]): [description]

    Returns:
        image: [description]
    """
    im = letterbox(im,640,32)[0]
    im = im.transpose((2, 0, 1))[::-1]
    im = np.ascontiguousarray(im)
    im = torch.from_numpy(im).to(device)
    im = im.half() if half else im.float()  # uint8 to fp16/32
    im /= 255  # 0 - 255 to 0.0 - 1.0
    if len(im.shape) == 3:
        im = im[None]  # expand for batch dim

    return im

def filter(det):
    """this function will filter out gurad from prediction

    Args:
        det ([type]): [description]

    Returns:
        tensor: [description]
    """
    new = [t for t in det if t[-1]==0]
    try:
        new_t=torch.stack(new)
    except:
        new_t = torch.tensor([]).to("cpu")
    return new_t
@torch.no_grad()
def run(weights="",
        source = "s.mp4",  # model.pt path(s)
        imgsz=640,  # inference size (pixels)
        conf_thres=0.25,  # confidence threshold
        iou_thres=0.45,  # NMS IOU threshold
        max_det=1000,  # maximum detections per image
        device='',  # cuda device, i.e. 0 or 0,1,2,3 or cpu
        classes=None,  # filter by class: --class 0, or --class 0 2 3
        agnostic_nms=False,  # class-agnostic NMS
        augment=False,  # augmented inference
        visualize=False,  # visualize features
        half=False,  # use FP16 half-precision inference
        dnn=False,  # use OpenCV DNN for ONNX inference
        ):
    FRAMES_TO_PERSIST = 10 # number of frames after compare
    MIN_SIZE_FOR_MOVEMENT = 20000 # movement size
    MOVEMENT_DETECTED_PERSISTENCE = 100
    first_frame = None
    next_frame = None
    font = cv2.FONT_HERSHEY_SIMPLEX
    delay_counter = 0
    movement_persistent_counter = 0

    model,device = model_load(device=device,weights=weights,dnn=dnn,imgsz=imgsz,half=half)
    cap = cv2.VideoCapture(source)
    frame_width = int(cap.get(3))
    frame_height = int(cap.get(4))
    out_cam=cv2.VideoWriter("output.mp4",0x00000021, 10, (frame_width,frame_height))
    try:
        while True:
            _,frame = cap.read()
            im = frame.copy()
            im0 = frame.copy()
            im0s = frame.copy()
            im = preprocess_image(im,half,device)
            pred = model(im,augment=augment,visualize=False)

            pred = non_max_suppression(pred,conf_thres,iou_thres,classes,agnostic_nms,max_det=max_det)

            for det in pred:
                det = filter(det)
                if len(det):
                    det[:, :4] = scale_coords(im.shape[2:], det[:, :4], im0.shape).round()

                    for *xyxy, conf, cls in reversed(det):
                        c1,c2 = (int(xyxy[0]),int(xyxy[1])), (int(xyxy[2]),int(xyxy[3]))
                        img12 = im0[c1[1]:c2[1],c1[0]:c2[0]]
                        frame = imutils.resize(img12,width = 350)
                        frame = cv2.resize(frame, (350,250))
                        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
                        gray = cv2.GaussianBlur(gray, (21, 21), 0)
                        if first_frame is None: first_frame = gray
                        delay_counter += 1
                        if delay_counter > FRAMES_TO_PERSIST:
                            delay_counter = 0
                            first_frame = next_frame
                        next_frame = gray
                        frame_delta = cv2.absdiff(first_frame, next_frame)
                        thresh = cv2.threshold(frame_delta, 25, 255, cv2.THRESH_BINARY)[1]
                        thresh = cv2.dilate(thresh, None, iterations = 2)
                        cnts, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
                        transient_movement_flag = False
                        for cn in cnts:
                            (x, y, w, h) = cv2.boundingRect(cn)
                            print("-"*50)
                            print(cv2.contourArea(cn))
                            transient_movement_flag = cv2.contourArea(cn) > MIN_SIZE_FOR_MOVEMENT

                        if transient_movement_flag:
                            movement_persistent_counter = MOVEMENT_DETECTED_PERSISTENCE

                        if movement_persistent_counter > 0:
                            text = "Guard: " + "not sleeping"
                            movement_persistent_counter -= 1
                        else: # No movement detected
                            text = "Guard :- sleeping"
                        cv2.putText(im0, str(text), (10,35), font, 0.75, (0,0,255), 2, cv2.LINE_AA)
                        frame_delta = cv2.cvtColor(frame_delta, cv2.COLOR_GRAY2BGR)
                        cv2.rectangle(im0,c1,c2,color=(0,0,255))
                        cv2.putText(im0, "guard", (int(xyxy[0]),int(xyxy[1])), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 1, cv2.LINE_AA)

                else:
                    pass
                out_cam.write(im0)
                cv2.imshow("frame",im0)
                if cv2.waitKey(1) & 0xFF == ord("q"):
                    break
    except:
        print("complete")
def parse_opt():
    parser = argparse.ArgumentParser()
    parser.add_argument('--weights', nargs='+', type=str, default= os.path.join(os.getcwd(),"yolov5s.pt"), help='model path(s)')
    parser.add_argument('--source', type=str, default='s.mp4', help='file/dir/URL/glob, 0 for webcam')
    parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w')
    opt = parser.parse_args()
    opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1  # expand
    return opt

def main(opt):
    run(**vars(opt))

if __name__ == "__main__":
    print("Inside Main")

    opt = parse_opt()
    main(opt)


# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A demo that runs object detection on camera frames using OpenCV.

TEST_DATA=../all_models

Run face detection model:
python3 detect.py \
  --model ${TEST_DATA}/mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite

Run coco model:
python3 detect.py \
  --model ${TEST_DATA}/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite \
  --labels ${TEST_DATA}/coco_labels.txt

"""
from __future__ import division, print_function, absolute_import
import argparse
import cv2
import os
from timeit import time
import warnings
import numpy as np
from PIL import Image

from deep_sort import preprocessing
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
#from tools import generate_detections as gdet
import imutils.video

from pycoral.adapters import classify
from pycoral.adapters import common
from pycoral.adapters.common import input_size
from pycoral.adapters.detect import get_objects
from pycoral.utils.dataset import read_label_file
from pycoral.utils.edgetpu import make_interpreter
from pycoral.utils.edgetpu import run_inference

def main():
    default_model_dir = '../models'
    default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
    default_model_extract_em = 'resnet_edgetpu.tflite' #'resnet_edgetpu.tflite'#'efficientnet-edgetpu-M_quant_embedding_extractor_edgetpu.tflite'
    default_labels = 'coco_labels.txt'
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', help='.tflite model path',
                        default=os.path.join(default_model_dir,default_model))
    parser.add_argument('--model_extract_em', help='.tflite model path',
                        default=os.path.join(default_model_dir,default_model_extract_em))
    parser.add_argument('--labels', help='label file path',
                        default=os.path.join(default_model_dir, default_labels))
    parser.add_argument('--top_k', type=int, default=3,
                        help='number of categories with highest score to display')
    parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 0)
    parser.add_argument('--threshold', type=float, default=0.1,
                        help='classifier score threshold')
    args = parser.parse_args()
    #Definition of the parameters for tracking
    max_cosine_distance = 12
    nn_budget = None
    nms_max_overlap = 1.0

    metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
    tracker = Tracker(metric)
    tracking = True

    #
    print('Loading {} with {} labels.'.format(args.model, args.labels))
    interpreter = make_interpreter(args.model)
    interpreter.allocate_tensors()
    labels = read_label_file(args.labels)
    inference_size = input_size(interpreter)
    interpreter_extactor = make_interpreter(args.model_extract_em, device=':0')
    interpreter_extactor.allocate_tensors()

    cap = cv2.VideoCapture(args.camera_idx)

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        cv2_im = frame

        cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB)
        cv2_im_rgb = cv2.resize(cv2_im_rgb, inference_size)
        run_inference(interpreter, cv2_im_rgb.tobytes())
        objs = get_objects(interpreter, args.threshold)[:args.top_k]
        cv2_im,boxes,scores,classes,features = append_objs_to_img(cv2_im,interpreter_extactor, inference_size, objs, labels)
        print("boxes",len(boxes))
        print("classes",len(classes))
        print("features",len(features))
        detections = [Detection(bbox, confidence, clss, feature[0]) for bbox, confidence, clss, feature in
                      zip(boxes, scores, classes, features)]
        for det in detections:
            bbox = det.to_tlbr()
            print("bbox",bbox)
            score = "%.2f" % round(det.confidence * 100, 2) + "%"
            cv2.rectangle(frame, (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])), (255, 0, 0), 2)
            if len(classes) > 0:
                cls = det.cls
                cv2.putText(frame, str(cls) + " " + score, (int(bbox[0]), int(bbox[3])), 0,
                            1e-3 * frame.shape[0], (0, 255, 0), 1)

        #indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores)
        #detections = [detections[i] for i in indices]
        print("detections",detections)

        if tracking:
            # Call the tracker
            tracker.predict()
            tracker.update(detections)

            for track in tracker.tracks:
                if not track.is_confirmed() or track.time_since_update > 1:
                    continue
                bbox = track.to_tlbr()
                cv2.rectangle(frame, (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])), (255, 255, 255), 2)
                cv2.putText(frame, "ID: " + str(track.track_id), (int(bbox[0]), int(bbox[1])), 0,
                            1e-3 * frame.shape[0], (0, 255, 0), 1)

        #cv2.imshow('', frame)
        height, width, channels = cv2_im.shape
        cv2_im = cv2.resize(cv2_im, (width *3,height*3))
        cv2.imshow('frame', cv2_im)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()

def extract_embeddings(img, interpreter):
    """Uses model to process images as embeddings.
    Reads image, resizes and feeds to model to get feature embeddings. Original
    image is discarded to keep maximum memory consumption low.
    Args:
        image_paths: ndarray, represents a list of image paths.
        interpreter: TFLite interpreter, wraps embedding extractor model.
    Returns:
        ndarray of length image_paths.shape[0] of embeddings.
    """
    input_size = common.input_size(interpreter)
    feature_dim = classify.num_classes(interpreter)

    embeddings = np.empty((1, feature_dim), dtype=np.float32)
    print("embeddings",embeddings,feature_dim)
    #embeddings[0,:] = classify.get_scores(interpreter)
    #print("embeddings1",embeddings)
    #common.set_input(interpreter, img.resize(input_size, Image.NEAREST))
    #interpreter.invoke()
    #embeddings[count,:] = classify.get_scores(interpreter)

    return embeddings

def append_objs_to_img(cv2_im,interpreter_extactor, inference_size, objs, labels):
    features = [] #!
    image_patches = [] #1
    boxes = []
    scores = []
    classes = []
    height, width, channels = cv2_im.shape
    scale_x, scale_y = width / inference_size[0], height / inference_size[1]
    for obj in objs: #0.1
        bbox = obj.bbox.scale(scale_x, scale_y)

        #end1
        x0, y0 = int(bbox.xmin), int(bbox.ymin)
        x1, y1 = int(bbox.xmax), int(bbox.ymax)
        #print("x0, y0,x1, y1",(x0/scale_x)/inference_size[0], (y0/scale_y)/inference_size[1],(x1/scale_x)/inference_size[0], (y1/scale_y)/inference_size[1])
        #cv2.imshow('frame_cut', cv2_im[y0:y1, x0:x1])
        feature = extract_embeddings(cv2_im[y0:y1, x0:x1], interpreter_extactor)
        #print("feature",feature)
        features.append(feature)

        percent = int(100 * obj.score)
        label = '{}% {}'.format(percent, labels.get(obj.id, obj.id))
        boxes.append(bbox)
        scores.append(obj.score)
        classes.append(labels.get(obj.id, obj.id))
        cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2)
        cv2_im = cv2.putText(cv2_im, label, (x0, y0+30),
                             cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2)
    return cv2_im,np.array(boxes),np.array(scores),np.array(classes),np.array(features)

if __name__ == '__main__':
    main()


from ._cfunctions import MKL, mkl_library_name
from ._constants import *
from ._structs import sparse_matrix_t, MKL_Complex8, MKL_Complex16

import numpy as _np
import ctypes as _ctypes
from scipy import sparse as _spsparse
import warnings
import time

from numpy.ctypeslib import as_array

# Use mkl-service to check version if it's installed
# Since it's not on PyPi I don't want to make this an actual package dependency
# So without it just create mock functions and don't do version checking
try:
    from mkl import get_version, get_version_string, get_max_threads
except ImportError:
    def get_version():
        return None

    def get_version_string():
        return None

    def get_max_threads():
        return None

if get_version() is not None and get_version()["MajorVersion"] < 2020:
    _verr_msg = "Loaded version of MKL is out of date: {v}".format(v=get_version_string())
    warnings.warn(_verr_msg)

# Dict keyed by ('sparse_type_str', 'double_precision_bool', 'complex_bool')
_create_functions = {('csr', False, False): MKL._mkl_sparse_s_create_csr,
                     ('csr', True, False): MKL._mkl_sparse_d_create_csr,
                     ('csr', False, True): MKL._mkl_sparse_c_create_csr,
                     ('csr', True, True): MKL._mkl_sparse_z_create_csr,
                     ('csc', False, False): MKL._mkl_sparse_s_create_csc,
                     ('csc', True, False): MKL._mkl_sparse_d_create_csc,
                     ('csc', False, True): MKL._mkl_sparse_c_create_csc,
                     ('csc', True, True): MKL._mkl_sparse_z_create_csc,
                     ('bsr', False, False): MKL._mkl_sparse_s_create_bsr,
                     ('bsr', True, False): MKL._mkl_sparse_d_create_bsr,
                     ('bsr', False, True): MKL._mkl_sparse_c_create_bsr,
                     ('bsr', True, True): MKL._mkl_sparse_z_create_bsr}

# Dict keyed by ('sparse_type_str', 'double_precision_bool', 'complex_bool')
_export_functions = {('csr', False, False): MKL._mkl_sparse_s_export_csr,
                     ('csr', True, False): MKL._mkl_sparse_d_export_csr,
                     ('csr', False, True): MKL._mkl_sparse_c_export_csr,
                     ('csr', True, True): MKL._mkl_sparse_z_export_csr,
                     ('csc', False, False): MKL._mkl_sparse_s_export_csc,
                     ('csc', True, False): MKL._mkl_sparse_d_export_csc,
                     ('csc', False, True): MKL._mkl_sparse_c_export_csc,
                     ('csc', True, True): MKL._mkl_sparse_z_export_csc,
                     ('bsr', False, False): MKL._mkl_sparse_s_export_bsr,
                     ('bsr', True, False): MKL._mkl_sparse_d_export_bsr,
                     ('bsr', False, True): MKL._mkl_sparse_c_export_bsr,
                     ('bsr', True, True): MKL._mkl_sparse_z_export_bsr}

# Dict keyed by ('double_precision_bool', 'complex_bool')
_output_dtypes = {(False, False): _np.float32,
                 (True, False): _np.float64,
                 (False, True): _np.csingle,
                 (True, True): _np.cdouble}

NUMPY_FLOAT_DTYPES = [_np.float32, _np.float64]
NUMPY_COMPLEX_DTYPES = [_np.csingle, _np.cdouble]

def set_debug_mode(debug_bool):
    """
    Activate or deactivate debug mode

    :param debug_bool: True to be printy. False to be quiet.
    :type debug_bool: bool
    """

    MKL.MKL_DEBUG = debug_bool

def print_mkl_debug():
    """
    Print the MKL interface status if debug mode is on
    """

    if not MKL.MKL_DEBUG:
        return

    if get_version_string() is None:
        print("mkl-service must be installed to get full debug messaging")
    else:
        print(get_version_string())
        print("MKL Number of Threads: {n}".format(n=get_max_threads()))

    print("MKL linked: {fn}".format(fn=mkl_library_name()))
    print("MKL interface {_np} | {c}".format(_np=MKL.MKL_INT_NUMPY, c=MKL.MKL_INT))
    print("Set int32 interface with env MKL_INTERFACE_LAYER=LP64")
    print("Set int64 interface with env MKL_INTERFACE_LAYER=ILP64")

def debug_print(msg):
    """
    Print a message if debug mode is on
    :param msg: Message
    :type msg: str
    """

    if not MKL.MKL_DEBUG:
        return
    else:
        print(msg)

def debug_timer(msg=None, old_time=None):
    """
    Print a message with timing information if debug mode is on
    :param msg: Message
    :type msg: str
    :param old_time: Time to calculate difference for
    :type old_time: float
    """

    if not MKL.MKL_DEBUG:
        return

    t0 = time.time()

    if old_time is not None and msg is not None:
        print(msg + ": {0:.6f} seconds".format(t0 - old_time))

    return t0

def _check_scipy_index_typing(sparse_matrix):
    """
    Ensure that the sparse matrix indicies are in the correct integer type

    :param sparse_matrix: Scipy matrix in CSC or CSR format
    :type sparse_matrix: scipy.sparse.spmatrix
    """

    int_max = _np.iinfo(MKL.MKL_INT_NUMPY).max
    if (sparse_matrix.nnz > int_max) or (max(sparse_matrix.shape) > int_max):
        msg = "MKL interface is {t} and cannot hold matrix {m}\n".format(m=repr(sparse_matrix), t=MKL.MKL_INT_NUMPY)
        msg += "Try changing MKL to int64 with the environment variable MKL_INTERFACE_LAYER=ILP64"
        raise ValueError(msg)

    # Cast indexes to MKL_INT type
    if sparse_matrix.indptr.dtype != MKL.MKL_INT_NUMPY:
        sparse_matrix.indptr = sparse_matrix.indptr.astype(MKL.MKL_INT_NUMPY)
    if sparse_matrix.indices.dtype != MKL.MKL_INT_NUMPY:
        sparse_matrix.indices = sparse_matrix.indices.astype(MKL.MKL_INT_NUMPY)

def _get_numpy_layout(numpy_arr, second_arr=None):
    """
    Get the array layout code for a dense array in C or F order.
    Raises a ValueError if the array is not contiguous.

    :param numpy_arr: Numpy dense array
    :type numpy_arr: _np.ndarray
    :param second_arr: Numpy dense array; if numpy_arr is 1d (and therefore both C and F order), use this order
    :type second_arr: _np.ndarray, None
    :return: The layout code for MKL and the leading dimension
    :rtype: int, int
    """

    # Return the second array order if the first is ambiguous
    if numpy_arr.flags.c_contiguous and numpy_arr.flags.f_contiguous and second_arr is not None:
        if second_arr.flags.c_contiguous:
            return LAYOUT_CODE_C, numpy_arr.shape[1]
        elif second_arr.flags.f_contiguous:
            return LAYOUT_CODE_F, numpy_arr.shape[0]

    # Return the first order array otherwise
    elif numpy_arr.flags.c_contiguous:
        return LAYOUT_CODE_C, numpy_arr.shape[1]
    elif numpy_arr.flags.f_contiguous:
        return LAYOUT_CODE_F, numpy_arr.shape[0]
    elif not numpy_arr.flags.contiguous:
        raise ValueError("Array is not contiguous")
    else:
        raise ValueError("Array layout check has failed for unknown reason")

def _create_mkl_sparse(matrix):
    """
    Create MKL internal representation

    :param matrix: Sparse data in CSR or CSC format
    :type matrix: scipy.sparse.spmatrix

    :return ref, double_precision: Handle for the MKL internal representation and boolean for
    double precision and for complex dtype
    :rtype: sparse_matrix_t, bool, bool
    """

    double_precision, complex_type = _is_double(matrix)

    # Figure out which matrix creation function to use
    if _spsparse.isspmatrix_csr(matrix):
        _check_scipy_index_typing(matrix)
        assert matrix.data.shape[0] == matrix.indices.shape[0]
        assert matrix.indptr.shape[0] == matrix.shape[0] + 1
        handle_func = _create_functions[('csr', double_precision, complex_type)]

    elif _spsparse.isspmatrix_csc(matrix):
        _check_scipy_index_typing(matrix)
        assert matrix.data.shape[0] == matrix.indices.shape[0]
        assert matrix.indptr.shape[0] == matrix.shape[1] + 1
        handle_func = _create_functions[('csc', double_precision, complex_type)]

    elif _spsparse.isspmatrix_bsr(matrix):
        _check_scipy_index_typing(matrix)
        return _create_mkl_sparse_bsr(matrix), double_precision, complex_type

    else:
        raise ValueError("Matrix is not CSC, CSR, or BSR")

    return _pass_mkl_handle_csr_csc(matrix, handle_func), double_precision, complex_type

def _pass_mkl_handle_csr_csc(data, handle_func):
    """
    Create MKL internal representation for CSR or CSC matrix

    :param data: Sparse data
    :type data: scipy.sparse.spmatrix
    :return ref: Handle for the MKL internal representation
    :rtype: sparse_matrix_t
    """

    # Create a pointer for the output matrix
    ref = sparse_matrix_t()

    # Load into a MKL data structure and check return
    ret_val = handle_func(_ctypes.byref(ref),
                          _ctypes.c_int(SPARSE_INDEX_BASE_ZERO),
                          MKL.MKL_INT(data.shape[0]),
                          MKL.MKL_INT(data.shape[1]),
                          data.indptr[0:-1],
                          data.indptr[1:],
                          data.indices,
                          data.data)

    # Check return
    _check_return_value(ret_val, handle_func.__name__)

    return ref

def _create_mkl_sparse_bsr(matrix):
    """
    Create MKL internal representation for BSR matrix

    :param matrix: Sparse data
    :type matrix: scipy.sparse.bsr_matrix
    :return ref: Handle for the MKL internal representation
    :rtype: sparse_matrix_t
    """

    double_precision, complex_type = _is_double(matrix)
    handle_func = _create_functions[('bsr', double_precision, complex_type)]

    # Get the blocksize and check that the blocks are square
    _blocksize = matrix.blocksize[0]

    if _blocksize != matrix.blocksize[1]:
        _err = "MKL BSR representation requires square blocks; {n} blocks provided".format(n=matrix.blocksize)
        raise ValueError(_err)

    if (matrix.shape[0] % _blocksize != 0) or (matrix.shape[1] % _blocksize != 0):
        _err = "BSR blocks {n} do not align with dims {m}".format(n=matrix.blocksize, m=matrix.shape)
        raise ValueError(_err)

    _block_rows = int(matrix.shape[0] / _blocksize)
    _block_cols = int(matrix.shape[1] / _blocksize)

    # Get the data block array structure
    _layout, _ = _get_numpy_layout(matrix.data)

    # Create a pointer for the output matrix
    ref = sparse_matrix_t()

    # Load into a MKL data structure and check return
    ret_val = handle_func(_ctypes.byref(ref),
                          _ctypes.c_int(SPARSE_INDEX_BASE_ZERO),
                          _ctypes.c_int(_layout),
                          MKL.MKL_INT(_block_rows),
                          MKL.MKL_INT(_block_cols),
                          MKL.MKL_INT(_blocksize),
                          matrix.indptr[0:-1],
                          matrix.indptr[1:],
                          matrix.indices,
                          matrix.data)

    # Check return
    _check_return_value(ret_val, handle_func.__name__)

    return ref

def _export_mkl(csr_mkl_handle, double_precision, complex_type=False, output_type="csr"):
    """
    Export a MKL sparse handle of CSR or CSC type

    :param csr_mkl_handle: Handle for the MKL internal representation
    :type csr_mkl_handle: sparse_matrix_t
    :param double_precision: Use float64 if True, float32 if False. This MUST match the underlying float type - this
        defines a memory view, it does not cast.
    :type double_precision: bool
    :param output_type: The structure of the MKL handle (and therefore the type of scipy sparse to create)
    :type output_type: str

    :return: Sparse matrix in scipy format
    :rtype: scipy.spmatrix
    """

    output_type = output_type.lower()

    if output_type == "bsr":
        return _export_mkl_sparse_bsr(csr_mkl_handle, double_precision, complex_type=complex_type)
    elif output_type == "csr" or output_type == "csc":
        out_func = _export_functions[(output_type, double_precision, complex_type)]
        sp_matrix_constructor = _spsparse.csr_matrix if output_type == "csr" else _spsparse.csc_matrix
    else:
        raise ValueError("Only CSR, CSC, and BSR output types are supported")

    # Allocate for output
    ordering, nrows, ncols, indptrb, indptren, indices, data = _allocate_for_export(double_precision)
    final_dtype = _output_dtypes[(double_precision, complex_type)]

    ret_val = out_func(csr_mkl_handle,
                       _ctypes.byref(ordering),
                       _ctypes.byref(nrows),
                       _ctypes.byref(ncols),
                       _ctypes.byref(indptrb),
                       _ctypes.byref(indptren),
                       _ctypes.byref(indices),
                       _ctypes.byref(data))

    # Check return
    _check_return_value(ret_val, out_func.__name__)

    # Check ordering
    if ordering.value != 0:
        raise ValueError("1-indexing (F-style) is not supported")

    # Get matrix dims
    ncols, nrows = ncols.value, nrows.value

    # If any axis is 0 return an empty matrix
    if nrows == 0 or ncols == 0:
        return sp_matrix_constructor((nrows, ncols), dtype=final_dtype)

    # Get the index dimension
    index_dim = nrows if output_type == "csr" else ncols

    # Construct a numpy array and add 0 to first position for scipy.sparse's 3-array indexing
    indptrb = as_array(indptrb, shape=(index_dim,))
    indptren = as_array(indptren, shape=(index_dim,))

    indptren = _np.insert(indptren, 0, indptrb[0])
    nnz = indptren[-1] - indptrb[0]

    # If there are no non-zeros, return an empty matrix
    # If the number of non-zeros is insane, raise a ValueError
    if nnz == 0:
        return sp_matrix_constructor((nrows, ncols), dtype=final_dtype)
    elif nnz < 0 or nnz > ncols * nrows:
        raise ValueError("Matrix ({m} x {n}) is attempting to index {z} elements".format(m=nrows, n=ncols, z=nnz))

    # Construct numpy arrays from data pointer and from indicies pointer
    data = _np.array(as_array(data, shape=(nnz * 2 if complex_type else nnz,)), copy=True)
    indices = _np.array(as_array(indices, shape=(nnz,)), copy=True)

    # Pack and return the matrix
    return sp_matrix_constructor((data.view(final_dtype) if complex_type else data, indices, indptren),
                                 shape=(nrows, ncols))

def _export_mkl_sparse_bsr(bsr_mkl_handle, double_precision, complex_type=False):
    """
    Export a BSR matrix from MKL's internal representation to scipy

    :param bsr_mkl_handle: MKL internal representation
    :type bsr_mkl_handle: sparse_matrix_t
    :param double_precision: Use float64 if True, float32 if False. This MUST match the underlying float type - this
        defines a memory view, it does not cast.
    :type double_precision: bool
    :return: Sparse BSR matrix
    :rtype:
    """

    # Allocate for output
    ordering, nrows, ncols, indptrb, indptren, indices, data = _allocate_for_export(double_precision)
    block_layout = _ctypes.c_int()
    block_size = MKL.MKL_INT()

    # Set output
    out_func = _export_functions[('bsr', double_precision, complex_type)]
    final_dtype = _output_dtypes[(double_precision, complex_type)]

    ret_val = out_func(bsr_mkl_handle,
                       _ctypes.byref(ordering),
                       _ctypes.byref(block_layout),
                       _ctypes.byref(nrows),
                       _ctypes.byref(ncols),
                       _ctypes.byref(block_size),
                       _ctypes.byref(indptrb),
                       _ctypes.byref(indptren),
                       _ctypes.byref(indices),
                       _ctypes.byref(data))

    # Check return
    _check_return_value(ret_val, out_func.__name__)

    # Get matrix dims
    ncols, nrows, block_size = ncols.value, nrows.value, block_size.value
    index_dim, block_dims = nrows, (block_size, block_size)
    ncols, nrows = ncols * block_size, nrows * block_size

    # If any axis is 0 return an empty matrix
    if nrows == 0 or ncols == 0:
        return _spsparse.bsr_matrix((nrows, ncols), dtype=final_dtype, blocksize=block_dims)

    ordering = "F" if ordering.value == LAYOUT_CODE_F else "C"

    # Construct a numpy array and add 0 to first position for scipy.sparse's 3-array indexing
    indptrb = as_array(indptrb, shape=(index_dim,))
    indptren = as_array(indptren, shape=(index_dim,))

    indptren = _np.insert(indptren, 0, indptrb[0])

    nnz_blocks = (indptren[-1] - indptrb[0])
    nnz = nnz_blocks * (block_size ** 2)

    # If there's no non-zero data, return an empty matrix
    if nnz_blocks == 0:
        return _spsparse.bsr_matrix((nrows, ncols), dtype=final_dtype, blocksize=block_dims)

    elif nnz_blocks < 0 or nnz > ncols * nrows:
        _err = "Matrix ({m} x {n}) is attempting to index {z} elements as {b} {bs} blocks"
        _err = _err.format(m=nrows, n=ncols, z=nnz, b=nnz_blocks, bs=(block_size, block_size))
        raise ValueError(_err)

    nnz_row_block = block_size if not complex_type else block_size * 2
    data = _np.array(as_array(data, shape=(nnz_blocks, block_size, nnz_row_block)), copy=True, order=ordering)
    indices = _np.array(as_array(indices, shape=(nnz_blocks,)), copy=True)

    return _spsparse.bsr_matrix((data.view(final_dtype) if complex_type else data, indices, indptren),
                                shape=(nrows, ncols), blocksize=block_dims)

def _allocate_for_export(double_precision):
    """
    Get pointers for output from MKL internal representation
    :param double_precision: Allocate an output pointer of doubles
    :type double_precision: bool
    :return: ordering, nrows, ncols, indptrb, indptren, indices, data
    :rtype: c_int, MKL_INT, MKL_INT, MKL_INT*, MKL_INT*, MKL_INT*, c_float|c_double*
    """
    # Create the pointers for the output data
    indptrb = _ctypes.POINTER(MKL.MKL_INT)()
    indptren = _ctypes.POINTER(MKL.MKL_INT)()
    indices = _ctypes.POINTER(MKL.MKL_INT)()

    ordering = _ctypes.c_int()
    nrows = MKL.MKL_INT()
    ncols = MKL.MKL_INT()

    data = _ctypes.POINTER(_ctypes.c_double)() if double_precision else _ctypes.POINTER(_ctypes.c_float)()

    return ordering, nrows, ncols, indptrb, indptren, indices, data

def _check_return_value(ret_val, func_name):
    """
    Check the return value from a sparse function

    :param ret_val:
    :param func_name:
    :return:
    """

    if ret_val != 0:
        err_msg = "{fn} returned {v} ({e})".format(fn=func_name, v=ret_val, e=RETURN_CODES[ret_val])
        if ret_val == 2:
            err_msg += "; " + ILP64_MSG
        raise ValueError(err_msg)
    elif MKL.MKL_DEBUG:
        print("{fn} returned {v} ({e})".format(fn=func_name, v=ret_val, e=RETURN_CODES[ret_val]))
    else:
        return

def _destroy_mkl_handle(ref_handle):
    """
    Deallocate a MKL sparse handle

    :param ref_handle:
    :type ref_handle: sparse_matrix_t
    """

    ret_val = MKL._mkl_sparse_destroy(ref_handle)
    _check_return_value(ret_val, "mkl_sparse_destroy")

def _order_mkl_handle(ref_handle):
    """
    Reorder indexes in a MKL sparse handle

    :param ref_handle:
    :type ref_handle: sparse_matrix_t
    """

    ret_val = MKL._mkl_sparse_order(ref_handle)
    _check_return_value(ret_val, "mkl_sparse_order")

def _convert_to_csr(ref_handle, destroy_original=False):
    """
    Convert a MKL sparse handle to CSR format

    :param ref_handle:
    :type ref_handle: sparse_matrix_t
    :return:
    """

    csr_ref = sparse_matrix_t()
    ret_val = MKL._mkl_sparse_convert_csr(ref_handle, _ctypes.c_int(10), _ctypes.byref(csr_ref))

    try:
        _check_return_value(ret_val, "mkl_sparse_convert_csr")
    except ValueError:
        try:
            _destroy_mkl_handle(csr_ref)
        except ValueError:
            pass

        raise

    if destroy_original:
        _destroy_mkl_handle(ref_handle)

    return csr_ref

def _sanity_check(matrix_a, matrix_b, allow_vector=False):
    """
    Check matrix dimensions
    :param matrix_a: sp.sparse or numpy array
    :param matrix_b: sp.sparse or numpy array
    """

    a_2d, b_2d = matrix_a.ndim == 2, matrix_b.ndim == 2
    a_vec, b_vec = _is_dense_vector(matrix_a), _is_dense_vector(matrix_b)

    # Check to make sure that both matrices are 2-d
    if not allow_vector and (not a_2d or not b_2d):
        err_msg = "Matrices must be 2d: {m1} * {m2} is not valid".format(m1=matrix_a.shape, m2=matrix_b.shape)
        raise ValueError(err_msg)

    invalid_ndims = not (a_2d or a_vec) or not (b_2d or b_vec)
    invalid_align = (matrix_a.shape[1] if not matrix_a.ndim == 1 else matrix_a.shape[0]) != matrix_b.shape[0]

    # Check to make sure that this multiplication can work
    if invalid_align or invalid_ndims:
        err_msg = "Matrix alignment error: {m1} * {m2} is not valid".format(m1=matrix_a.shape, m2=matrix_b.shape)
        raise ValueError(err_msg)

def _cast_to(matrix, dtype):
    """ Make a copy of the array as double precision floats or return the reference if it already is"""
    return matrix.astype(dtype) if matrix.dtype != dtype else matrix

def _is_valid_dtype(matrix, complex_dtype=False, all_dtype=False):
    """ Check to see if it's a usable float dtype """
    if all_dtype:
        return matrix.dtype in NUMPY_FLOAT_DTYPES + NUMPY_COMPLEX_DTYPES
    elif complex_dtype:
        return matrix.dtype in NUMPY_COMPLEX_DTYPES
    else:
        return matrix.dtype in NUMPY_FLOAT_DTYPES

def _type_check(matrix_a, matrix_b=None, cast=False):
    """
    Make sure that both matrices are single precision floats or both are double precision floats
    If not, convert to double precision floats if cast is True, or raise an error if cast is False
    """

    _n_complex = _np.iscomplexobj(matrix_a) + _np.iscomplexobj(matrix_b)

    # If there's no matrix B and matrix A is valid dtype, return it
    if matrix_b is None and _is_valid_dtype(matrix_a, all_dtype=True):
        return matrix_a
    # If matrix A is complex but not csingle or cdouble, and cast is True, convert it to a cdouble
    elif matrix_b is None and cast and _n_complex == 1:
        return _cast_to(matrix_a, _np.cdouble)
    # If matrix A is real but not float32 or float64, and cast is True, convert it to a float64
    elif matrix_b is None and cast:
        return _cast_to(matrix_a, _np.float64)
    # Raise an error - the dtype is invalid and cast is False
    elif matrix_b is None:
        _err_msg = f"Matrix data type must be float32, float64, csingle, or cdouble; {matrix_a.dtype} provided"
        raise ValueError(_err_msg)

    # If Matrix A & B have the same valid dtype, return them
    if _is_valid_dtype(matrix_a, all_dtype=True) and matrix_a.dtype == matrix_b.dtype:
        return matrix_a, matrix_b

    # If neither matrix is complex and cast is True, convert to float64s and return them
    elif cast and _n_complex == 0:
        debug_print(f"Recasting matrix data types {matrix_a.dtype} and {matrix_b.dtype} to _np.float64")
        return _cast_to(matrix_a, _np.float64), _cast_to(matrix_b, _np.float64)

    # If both matrices are complex and cast is True, convert to cdoubles and return them
    elif cast and _n_complex == 2:
        debug_print(f"Recasting matrix data types {matrix_a.dtype} and {matrix_b.dtype} to _np.cdouble")
        return _cast_to(matrix_a, _np.cdouble), _cast_to(matrix_b, _np.cdouble)

    # Cast reals and complex matrices together
    elif cast and _n_complex == 1 and _is_valid_dtype(matrix_a, complex_dtype=True):
        debug_print(f"Recasting matrix data type {matrix_b.dtype} to {matrix_a.dtype}")
        return matrix_a, _cast_to(matrix_b, matrix_a.dtype)
    elif cast and _n_complex == 1 and _is_valid_dtype(matrix_b, complex_dtype=True):
        debug_print(f"Recasting matrix data type {matrix_a.dtype} to {matrix_b.dtype}")
        return _cast_to(matrix_a, matrix_b.dtype), matrix_b
    elif cast and _n_complex == 1:
        debug_print(f"Recasting matrix data type {matrix_a.dtype} and {matrix_b.dtype} to _np.cdouble")
        return _cast_to(matrix_a, _np.cdouble), _cast_to(matrix_b, _np.cdouble)

    # If cast is False, can't cast anything together
    elif not cast:
        raise ValueError(
            "Matrix data type must be float32, float64, csingle, or cdouble, " +
            "and must be the same if cast=False; " +
            f"{matrix_a.dtype} & {matrix_b.dtype} provided"
        )

def _mkl_scalar(scalar, complex_type, double_precision):
    """Turn complex scalars into appropriate precision MKL scalars or leave floats as floats"""

    scalar = 1. if scalar is None else scalar

    if complex_type and double_precision:
        return MKL_Complex16(complex(scalar))
    elif complex_type:
        return MKL_Complex8(complex(scalar))
    else:
        return float(scalar)

def _out_matrix(shape, dtype, order="C", out_arr=None, out_t=False):
    """
    Create an all-zero matrix or check to make sure that the provided output array matches

    :param shape: Required output shape
    :type shape: tuple(int)
    :param dtype: Required output data type
    :type dtype: _np.dtype
    :param order: Array order (row or column-major)
    :type order: str
    :param out_arr: Provided output array
    :type out_arr: _np.ndarray
    :param out_t: Out array has been transposed
    :type out_t: bool
    :return: Array
    :rtype: _np.ndarray
    """

    out_t = False if out_t is None else out_t

    # If there's no output array allocate a new array and return it
    if out_arr is None:
        return _np.zeros(shape, dtype=dtype, order=order)

    # Check and make sure the order is correct
    # Note 1d arrays have both flags set
    _order_match = out_arr.flags['C_CONTIGUOUS'] if order == "C" else out_arr.flags['F_CONTIGUOUS']

    # If there are any incompatible parameters, raise an error with the provided and required array parameters
    # Flip them if out_T is set so that the original values and the values which would have to be provided are correct
    if shape != out_arr.shape or dtype != out_arr.dtype or not _order_match or not out_arr.data.contiguous:
        if not out_t or out_arr.ndim == 1:
            _err_shape, _req_shape = out_arr.shape, shape
            _err_order, _req_order = "C" if out_arr.flags['C_CONTIGUOUS'] else "F", order
        else:
            _err_shape, _req_shape = out_arr.shape[::-1], shape[::-1]
            _err_order = "F" if out_arr.flags['C_CONTIGUOUS'] and not out_arr.flags['F_CONTIGUOUS'] else "C"
            _req_order = "C" if order == "F" else "F"

        try:
            _req_dtype = dtype.__name__
        except AttributeError:
            _req_dtype = dtype.name

        _err_msg = "Provided out array is "
        _err_msg += "{s} {d} [{o}_{c}]".format(s=_err_shape, d=out_arr.dtype, o=_err_order,
                                               c="CONTIGUOUS" if out_arr.data.contiguous else "NONCONTIGUOUS")

        _err_msg += "; product requires {s} {d} [{o}_{c}]".format(s=_req_shape, d=_req_dtype, o=_req_order,
                                                                  c="CONTIGUOUS")
        raise ValueError(_err_msg)

    else:
        return out_arr

def _is_dense_vector(m_or_v):
    return not _spsparse.issparse(m_or_v) and ((m_or_v.ndim == 1) or ((m_or_v.ndim == 2) and min(m_or_v.shape) == 1))

def _is_double(arr):
    """
    Return true if the array is doubles, false if singles, and raise an error if it's neither.

    :param arr:
    :type arr: _np.ndarray, scipy.sparse.spmatrix
    :return:
    :rtype: bool
    """

    # Figure out which dtype for data
    if arr.dtype == _np.float32:
        return False, False
    elif arr.dtype == _np.float64:
        return True, False
    elif arr.dtype == _np.csingle:
        return False, True
    elif arr.dtype == _np.cdouble:
        return True, True
    else:
        raise ValueError("Only float32, float64, csingle, and cdouble dtypes are supported")

def _is_allowed_sparse_format(matrix):
    """
    Return True if the matrix is dense or a sparse format we can turn into an MKL object. False otherwise.
    :param matrix:
    :return:
    :rtype: bool
    """
    if _spsparse.isspmatrix(matrix):
        return _spsparse.isspmatrix_csr(matrix) or _spsparse.isspmatrix_csc(matrix) or _spsparse.isspmatrix_bsr(matrix)
    else:
        return True

def _empty_output_check(matrix_a, matrix_b):
    """Check for trivial cases where an empty array should be produced"""

    # One dimension is zero
    if min([*matrix_a.shape, *matrix_b.shape]) == 0:
        return True

    # The sparse array is empty
    elif _spsparse.issparse(matrix_a) and min(matrix_a.data.size, matrix_a.indices.size) == 0:
        return True
    elif _spsparse.issparse(matrix_b) and min(matrix_b.data.size, matrix_b.indices.size) == 0:
        return True

    # Neither trivial condition
    else:
        return False