from __future__ import division
from __init__ import portL, portR, baud, testDir, simDir, EXPDIR, learnDir, keyboard_portname
from port_read import ReadSerial, ParseSerial, printSens, ReadKeyboard, ParseFile
from data_analysis import ReadWrite, Analyze, Analyze2Hands
from senstonote import WeighTrig_ai, WeighTrig
import senstonote
import time
import numpy as np
import random
from threading import Thread
import binascii
from struct import *
from collections import namedtuple
import matplotlib.pyplot as plt
import math
import queue
from collections import deque
import csv, sys, os

class RunGlove:
    def __init__(self, directory=testDir, filename='test0'):
        self.directory = directory
        self.filename = filename
        self.time0 = time.time()

    def runKeyboard(self):
        self.keyboardThread = ReadKeyboard(keyboard_portname, self.time0)
        self.keyboardThread.start()

    def plotRun(self):
        dataplotting = Analyze(self.directory, self.filename)
        dataplotting.plotKeyAndGlove('basic')
        dataplotting.plot_sampling_frequency()

    def saveRun(self, sensThread, keyboardThread):
        datawriter = ReadWrite(self.directory, self.filename)
        datawriter.saveData(sensThread.getQ(), key_dataq = keyboardThread.getQ())
        self.directory, self.filename = datawriter.getSaveLocation()

    def setSaveLocation(self, directory, filename):
        self.directory = directory
        self.filename = filename

    def getSaveLocation(self):
        return self.directory, self.filename

    def recordKeyboardAndGlove(self, port=portR):
        sensThread = ReadSerial(portR, baud)
        parseThread = ParseSerial(sensThread.sensq, self.time0)
        parseThread.readFullData = True

        self.runKeyboard()
        sensThread.start()
        parseThread.start()

        key = input('press any key to finish \n')
        print('End of run')

        self.saveRun(parseThread, self.keyboardThread)
        self.plotRun()

    def printAllSens(self, hands='right', printit=True, save=False, plot=False, fromfile=False): # can function be split in 2? (right-left hands)
        righthand = False
        lefthand = False
        if hands == 'right':
            righthand = True
        elif hands == 'left':
            lefthand = True
        elif hands == 'both':
            righthand = True
            lefthand = True
        else:
            print('Incorrect Option: set variable "hands" to right, left or both')

        if righthand:
            sensThreadR = ReadSerial(portR, baud)
            parseThreadR = ParseSerial(sensThreadR.sensq, self.time0)
            parseThreadR.readFullData = True
            sensThreadR.start()
            parseThreadR.start()

            if printit:
                printThreadR = printSens(parseThreadR.dataq, info='R')
                printThreadR.start()

        if lefthand:
            sensThreadL = ReadSerial(portL, baud)
            parseThreadL = ParseSerial(sensThreadL.sensq, self.time0)
            parseThreadL.readFullData = True
            sensThreadL.start()
            parseThreadL.start()

            if printit:
                printThreadL = printSens(parseThreadL.dataq, info='L')
                printThreadL.start()

        key = input('press any key to finish \n')
        print('End of run')

       # Better to separate into other functions?
        if save:
            if not printit:
                datawriter = ReadWrite(self.directory, self.filename)
                datawriter.makeDir(saveoption='addnew')
                if lefthand:
                    datawriter.saveData(parseThreadL.dataq, hand='L', saveoption='overwrite') #redundant saveoption
                if righthand:
                    datawriter.saveData(parseThreadR.dataq, hand='R', saveoption='overwrite')
                new_filename = datawriter.get_filename()
            else:
                print('Not Saved! You can either print or save - Fix sensq.get(block) in printSens to do both simultaneously')

        if plot:
            plotter = Analyze2Hands(self.directory, new_filename)
            plotter.plotSensors(savename='all_sensors')

#####################################
#####################################

    def simpleTrigger(self, notes, pressTrigThresh):
        sensThread = ReadSerial(portR, baud)
        parseThread = ParseSerial(sensThread.sensq, self.time0)
        parseThread.readPress = True
        PressTrigP = WeighTrig(parseThread.pressq, pressTrigThresh, notes)

        sensThread.start()
        parseThread.start()
        PressTrigP.start()

        key = input('press any key to finish \n')
        print('End of run')

    def aiTrigger(self):
        sensThread = ReadSerial(portR, baud)
        parseThread = ParseSerial(sensThread.sensq, self.time0)
        parseThread.readPress = True
        parseThread.readFlex = True
        pre_flexsize = 12
        pressTrigThresh = 15
        dshmidt = 5
        PressTrigP = WeighTrig_ai(parseThread.pressq, parseThread.flexq, pre_flexsize, pressTrigThresh, dshmidt)

        sensThread.start()
        parseThread.start()
        PressTrigP.start()

        key = input('press any key to finish \n')
        print('End of run')

    def twoHandInstrument(self, fromfile=False):
        # Uses Pressure in right and flex in left for full keyboard access.

        if fromfile:
            parseThreadR = ParseFile(directory=self.directory, filename=self.filename, hand='R')
            parseThreadL = ParseFile(directory=self.directory, filename=self.filename, hand='L')
            parseThreadR.readPress = True
            parseThreadL.readFlex = True
            parseThreadR.start()
            parseThreadL.start()

        else:
            sensThreadR = ReadSerial(portR, baud)
            sensThreadL = ReadSerial(portL, baud)
            parseThreadR = ParseSerial(sensThreadR.sensq, self.time0)
            parseThreadL = ParseSerial(sensThreadL.sensq, self.time0)
            parseThreadR.readPress = True
            parseThreadL.readFlex = True

            sensThreadR.start()
            parseThreadR.start()
            sensThreadL.start()
            parseThreadL.start()

        # Fix that part
        collectq = queue.Queue(maxsize=20)
        pressTrigThresh = 15
        dshmidt = 5
        basenote = 'C3'
        key = 'C'

        pressTrigPR = senstonote.WeighTrig2h(parseThreadR.pressq, collectq, 'R', pressTrigThresh, dshmidt)
        #pressTrigPL = senstonote.WeighTrig2h(sensThreadL.pressq, collectq, 'L', 40, 10)
        pressTrigPL = senstonote.WeighTrig2h(parseThreadL.flexq, collectq, 'L', 100, 20)

        collectSend = senstonote.CombHandsSendMidi(collectq, basenote, key)

        pressTrigPR.start()
        pressTrigPL.start()
        collectSend.start()

        key = input('press any key to finish \n')
        print('End of run')


#!/usr/bin/env python3
# coding: utf-8

import sys
import numpy as np
import pandas as pd

def main():
    dataset = pd.read_csv(sys.argv[1])
    X = dataset.iloc[:, 2:-2].values
    y = dataset.iloc[:, -2].values

    from sklearn.ensemble import RandomForestClassifier
    classifier = RandomForestClassifier(n_estimators = 10, criterion = 'entropy')
    classifier.fit(X, y)

    toPredict = pd.read_csv(sys.argv[2])
    toPredictX = toPredict.iloc[:, 2:-1].values

    toPredicty = classifier.predict(toPredictX)

    for value in toPredicty:
        print(value)

    toPredict['is_attack'] = toPredicty
    pd.DataFrame(toPredict).to_csv(sys.argv[2], index=False)

main()


#!/usr/bin/env python3

import numpy as np
import os
import sys
import argparse
import glob
import time
import cv2
import caffe
from cvi_toolkit.utils.yolov3_util import preprocess, postprocess, draw
from cvi_toolkit.utils.alphapose_util import preprocess as pose_preprocess, postprocess as pose_postprocess,draw as draw_pose
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import json
from pathlib import Path
from tqdm import tqdm
import pymlir

def parse_args():
    parser = argparse.ArgumentParser(description='Eval YOLO networks.')
    parser.add_argument('--pose_model', type=str, default='',
                        help="MLIR Open Pose Model file")
    parser.add_argument('--yolov3_model', type=str, default='',
                        help="MLIR Yolov3 Model file")
    parser.add_argument("--net_input_dims", default='416,416',
                        help="'height,width' dimensions of net input tensors.")
    parser.add_argument("--pose_net_input_dims", default='256,192',
                    help="'height,width' dimensions of pose detect input tensors.")
    parser.add_argument("--input_file", type=str, default='',
                        help="Input image for testing")
    parser.add_argument("--label_file", type=str, default='',
                        help="coco lable file in txt format")
    parser.add_argument("--draw_image", type=str, default='',
                        help="Draw results on image")
    parser.add_argument("--dump_blobs",
                        help="Dump all blobs into a file in npz format")
    parser.add_argument("--obj_threshold", type=float, default=0.3,
                        help="Object confidence threshold")
    parser.add_argument("--nms_threshold", type=float, default=0.5,
                        help="NMS threshold")
    parser.add_argument("--dataset", type=str, default='./val2017_5k.txt',
                        help="dataset image list file")
    parser.add_argument("--annotations", type=str, default='./instances_val2017.json',
                        help="annotations file")
    parser.add_argument("--result_json", type=str,
                        help="Result json file")
    parser.add_argument("--pre_result_json", type=str,
                        help="when present, use pre detected result file, skip detection")
    parser.add_argument("--count", type=int, default=-1)

    args = parser.parse_args()
    return args

def yolov3_detect(module, image, net_input_dims, obj_threshold, nms_threshold):
    x = preprocess(image, net_input_dims)
    x = np.expand_dims(x, axis=0)
    res = module.run(x)
    data = module.get_all_tensor()

    out_feat = {}
    if ('layer82-conv' in res.keys()):
      out_feat['layer82-conv'] = res['layer82-conv']
      out_feat['layer94-conv'] = res['layer94-conv']
      out_feat['layer106-conv'] = res['layer106-conv']
    elif ('layer82-conv_dequant' in res.keys()):
      out_feat['layer82-conv'] = res['layer82-conv_dequant']
      out_feat['layer94-conv'] = res['layer94-conv_dequant']
      out_feat['layer106-conv'] = res['layer106-conv_dequant']
    else:
      assert(False)

    batched_predictions = postprocess(out_feat, image.shape, net_input_dims,
                              obj_threshold, nms_threshold, batch=1)
    # batch = 1
    predictions = batched_predictions[0]
    return predictions

def alphapose_detect(yolo_module,pose_module, image,net_input_dims,pose_net_input_dims,
                  obj_threshold, nms_threshold,result_image):

    predictions = yolov3_detect(yolo_module, image, net_input_dims,
                                obj_threshold, nms_threshold)

    human_preds = []

    for i in range(len(predictions)):
        preds = predictions[i]
        if preds[2] == 0:
            human_preds.append(preds)

    for idx in range(len(human_preds)):
        pose_pred = fastpose_detect_one(pose_module, image, human_preds[idx], pose_net_input_dims)
        image = draw_pose(image, pose_pred)

    cv2.imshow('AlphaPose result', image)
    cv2.waitKey(0)
    cv2.imwrite(result_image, image)
    print('##### Pose detection result is saved to  {} #####'.format(result_image))

def keep_human_bbox(yolo_preds):
    human_preds = []
    for i in range(len(yolo_preds)):
        pred = yolo_preds[i]
        if pred[2] == 0:
            human_preds.append(pred)
    return human_preds

def fastpose_detect(module, bgr_img, yolo_preds,pose_net_input_dims):

    y = []
    align_bbox_list = []

    for yolo_pred in yolo_preds:
        bbox = yolo_pred[0]
        x, align_bbox= pose_preprocess(bgr_img, bbox, pose_net_input_dims[0], pose_net_input_dims[1])
        x = np.expand_dims(x, axis=0)
        res = module.run(x)
        data = module.get_all_tensor()

        if ('Convolution56' in res.keys()):
            pose_pred = res['Convolution56']
        else:
            assert(False)

        y.append(pose_pred.copy())
        align_bbox_list.append(align_bbox)

    return pose_postprocess(y, align_bbox_list, yolo_preds)

def fastpose_detect_one(module, bgr_img, yolo_pred,pose_net_input_dims):

    y = []
    align_bbox_list = []

    bbox = yolo_pred[0]
    # print(bbox)
    x, align_bbox= pose_preprocess(bgr_img, bbox, pose_net_input_dims[0], pose_net_input_dims[1])
    # print(align_bbox)
    x = np.expand_dims(x, axis=0)
    res = module.run(x)
    data = module.get_all_tensor()

    if ('output_Conv' in res.keys()):
        pose_pred = res['output_Conv']
    else:
        assert(False)

    y.append(pose_pred.copy())
    align_bbox_list.append(align_bbox)

    yolo_preds = []
    yolo_preds.append(yolo_pred)

    return pose_postprocess(y, align_bbox_list, yolo_preds)

def get_image_id_in_path(image_path):
    stem = Path(image_path).stem
    # in val2014, name would be like COCO_val2014_000000xxxxxx.jpg
    # in val2017, name would be like 000000xxxxxx.jpg
    if (stem.rfind('_') == -1):
        id = int(stem)
    else:
        id = int(stem[stem.rfind('_') + 1:])
    return id

def clip_box(box, image_shape):
    x, y, w, h = box
    xmin = max(0, x)
    ymin = max(0, y)
    xmax = min(image_shape[1], x + w)
    ymax = min(image_shape[0], y + h)

    bx = xmin
    by = ymin
    bw = xmax - xmin
    bh = ymax - ymin
    return np.array([bx, by, bw, bh])

def eval_detector(yolo_module, pose_module, result_json_file, dataset_path, net_input_dims,pose_net_input_dims,
                  obj_threshold, nms_threshold, count=-1):
    coco_ids= [ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 13, 14, 15, 16, 17,
               18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36,
               37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53,
               54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73,
               74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]

    jdict = []
    image_list = os.listdir(dataset_path)
    counter=0

    with tqdm(total=count if count > 0 else len(image_list)) as pbar:
        for image_name in image_list:
            image_path = os.path.join(dataset_path, image_name)

            # print(image_path)
            if (not os.path.exists(image_path)):
                print("image not exit,", image_path)
                exit(-1)
            image_id = get_image_id_in_path(image_path)
            image = cv2.imread(image_path)
            predictions = yolov3_detect(yolo_module, image, net_input_dims,
                                        obj_threshold, nms_threshold)

            human_preds = []
            for i in range(len(predictions)):
                preds = predictions[i]
                if preds[2] == 0:
                    human_preds.append(preds)
            for idx in range(len(human_preds)):
                pose_pred = fastpose_detect_one(pose_module, image, human_preds[idx], pose_net_input_dims)
                for human in pose_pred:
                    kp_preds = human['keypoints']
                    # print(kp_preds)
                    kp_scores = human['kp_score']
                    keypoints = np.concatenate((kp_preds, kp_scores), axis=1)

                    keypoints = keypoints.reshape(-1).tolist()
                    #nothing to do ,  just call to get align_bbox
                    x_, align_bbox= pose_preprocess(image, human_preds[idx][0], pose_net_input_dims[0], pose_net_input_dims[1])

                    clipped_box = clip_box(human_preds[idx][0], image.shape)
                    x, y, w, h = align_bbox

                    # print(np.mean(kp_scores) + np.max(kp_scores))
                    #Not sure if human_detect score should be added.
                    # human_detect_score = human_preds[idx][1]
                    jdict.append({
                        "bbox": [x, y, w, h],
                        "image_id": image_id,
                        "score":float(np.mean(kp_scores) + np.max(kp_scores)),
                        "category_id": 1,
                        "keypoints":keypoints
                    })

            counter += 1
            if (counter == count):
                break
            pbar.update(1)

    if os.path.exists(result_json_file):
        os.remove(result_json_file)
    with open(result_json_file, 'w') as file:
        json.dump(jdict, file)

def get_img_id(file_name):
    ls = []
    myset = []
    annos = json.load(open(file_name, 'r'))
    for anno in annos:
        ls.append(anno['image_id'])
    myset = {}.fromkeys(ls).keys()
    return myset

def evaluate_mAP(annotations, res_file, ann_type='bbox', ann_file='person_keypoints_val2017.json', silence=True):
    """Evaluate mAP result for coco dataset.

    Parameters
    ----------
    res_file: str
        Path to result json file.
    ann_type: str
        annotation type, including: `bbox`, `segm`, `keypoints`.
    ann_file: str
        Path to groundtruth file.
    silence: bool
        True: disable running log.

    """
    class NullWriter(object):
        def write(self, arg):
            pass

    if silence:
        nullwrite = NullWriter()
        oldstdout = sys.stdout
        sys.stdout = nullwrite  # disable output

    cocoGt = COCO(annotations)
    cocoDt = cocoGt.loadRes(res_file)

    cocoEval = COCOeval(cocoGt, cocoDt, ann_type)
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()

    if silence:
        sys.stdout = oldstdout  # enable output

    stats_names = ['AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)',
                   'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)']
    info_str = {}
    for ind, name in enumerate(stats_names):
        info_str[name] = cocoEval.stats[ind]

    return info_str

def main(argv):
    args = parse_args()

    if (args.pre_result_json) :
        detbox_AP = evaluate_mAP(args.annotations,args.pre_result_json, ann_type='keypoints')
        print('##### det box: {} mAP #####'.format(detbox_AP['AP']))
        return

    # Make Detector
    net_input_dims = [int(s) for s in args.net_input_dims.split(',')]
    pose_net_input_dims = [int(s) for s in args.pose_net_input_dims.split(',')]
    obj_threshold = float(args.obj_threshold)
    nms_threshold = float(args.nms_threshold)
    print("yolov3 net_input_dims", net_input_dims)
    print("alpha pose net_input_dims", pose_net_input_dims)
    print("obj_threshold", obj_threshold)
    print("nms_threshold", nms_threshold)

    yolo_module = pymlir.module()
    print('load module ', args.yolov3_model)
    yolo_module.load(args.yolov3_model)
    print('load module done')

    pose_module = pymlir.module()
    print('load module ', args.pose_model)
    pose_module.load(args.pose_model)
    print('load module done')

    # Load image
    if (args.input_file != '') :
        image = cv2.imread(args.input_file)
        alphapose_detect(yolo_module,pose_module, image,net_input_dims,pose_net_input_dims,
                  obj_threshold, nms_threshold,args.draw_image)
        return

    # eval
    result_json_file = args.result_json
    eval_detector(yolo_module,pose_module, result_json_file, args.dataset, net_input_dims,pose_net_input_dims,
                  obj_threshold, nms_threshold, count=args.count)

    detbox_AP = evaluate_mAP(args.annotations,result_json_file, ann_type='keypoints')
    # print('##### det box: {} mAP #####'.format(detbox_AP))
    print('AP:     {}   AR:     {}'.format(detbox_AP['AP'],detbox_AP['AR']))
    print('AP .5:  {}   AR .5:   {}'.format(detbox_AP['AP .5'],detbox_AP['AR .5']))
    print('AP .75: {}   AR .75:  {}'.format(detbox_AP['AP .5'],detbox_AP['AR .5']))
    print('AP (M): {}   AR (M): {}'.format(detbox_AP['AP (M)'],detbox_AP['AR (M)']))
    print('AP (L): {}  AR (L): {}'.format(detbox_AP['AR (M)'],detbox_AP['AR (L)']))

if __name__ == '__main__':
    main(sys.argv)


# -*- coding utf-8 -*-
import cv2
import os
import numpy as np
from sklearn.model_selection import train_test_split
import random
import tensorflow as tf

def read_data(img_path, image_h = 64, image_w = 64):
    image_data = []
    label_data = []
    image = cv2.imread(img_path)
    #cv2.namedWindow("Image")
    #cv2.imshow("Image",image)
    #cv2.waitKey(0)
    h,w,_ = image.shape
    longest_edge = max(h,w)
    top, bottom, left, right = (0, 0, 0, 0)
    dh,dw = (0,0)
    if h < longest_edge:
        dh = longest_edge - h
        top = dh // 2
        bottom = dh - top
    elif w < longest_edge:
        dw = longest_edge - w
        left = dw // 2
        right = dw - left
    else:
        pass
    image_pad = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0])
    image = cv2.resize(image_pad, (image_h, image_w))

    image_data.append(image)
    label_data.append(img_path)
    image_data = np.array(image_data)
    train_x, test_x, train_y, test_y = train_test_split(image_data, label_data, test_size=0.05,
                                                        random_state=random.randint(0, 100))
    X = tf.placeholder(tf.float32,[None, 64, 64, 3])
    Y = tf.placeholder(tf.float32, [None, 2])
    return Y

#img_path = '4833.jpg'
#print(read_data(img_path))

x_data = np.float32(np.random.rand(2,100))
y_data = np.dot([0.100, 0.200], x_data) + 0.300

b = tf.Variable(tf.zeros([1]), name='B')
W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0), name='W')
y = tf.add(tf.matmul(W, x_data, name='MatMul'), b ,name='add')

loss = tf.reduce_mean(tf.square(tf.subtract(y, y_data, name='Sub'), name='Square'), name='ReduceMean')
optimizer = tf.train.GradientDescentOptimizer(0.001, name='Optimizer')
train = optimizer.minimize(loss, name='minimize')

summaries = [tf.summary.histogram('W',W), tf.summary.histogram('b', b), tf.summary.scalar('loss', loss)]
summary_op = tf.summary.merge(summaries)
print(summary_op)


# Copyright Howto100M authors.
# Copyright (c) Facebook, Inc. All Rights Reserved

import torch as th
import torch.nn.functional as F
import math
import numpy as np
import argparse

from torch.utils.data import DataLoader
from model import get_model
from preprocessing import Preprocessing
from random_sequence_shuffler import RandomSequenceSampler

from tqdm import tqdm
from pathbuilder import PathBuilder
from videoreader import VideoLoader

parser = argparse.ArgumentParser(description='Easy video feature extractor')

parser.add_argument('--vdir', type=str)
parser.add_argument('--fdir', type=str)
parser.add_argument('--hflip', type=int, default=0)

parser.add_argument('--batch_size', type=int, default=64,
                            help='batch size')
parser.add_argument('--type', type=str, default='2d',
                            help='CNN type')
parser.add_argument('--half_precision', type=int, default=0,
                            help='output half precision float')
parser.add_argument('--num_decoding_thread', type=int, default=4,
                            help='Num parallel thread for video decoding')
parser.add_argument('--l2_normalize', type=int, default=1,
                            help='l2 normalize feature')
parser.add_argument('--resnext101_model_path', type=str, default='model/resnext101.pth',
                            help='Resnext model path')
parser.add_argument('--vmz_model_path', type=str, default='model/r2plus1d_34_clip8_ig65m_from_scratch-9bae36ae.pth',
                            help='vmz model path')

args = parser.parse_args()

# TODO: refactor all args into config. (current code is from different people.)
CONFIGS = {
    "2d": {
        "fps": 1,
        "size": 224,
        "centercrop": False,
        "shards": 0,
    },
    "3d": {
        "fps": 24,
        "size": 112,
        "centercrop": True,
        "shards": 0,
    },
    "s3d": {
        "fps": 30,
        "size": 224,
        "centercrop": True,
        "shards": 0,
    },
    "vmz": {
        "fps": 24,
        "size": 112,
        "centercrop": True,
        "shards": 0,
    },
    "vae": {
        "fps": 2,
        "size": 256,
        "centercrop": True,
        "shards": 100,
    }
}

config = CONFIGS[args.type]

video_dirs = args.vdir
feature_dir = args.fdir

video_dict = PathBuilder.build(video_dirs, feature_dir, ".npy", config["shards"])

dataset = VideoLoader(
    video_dict=video_dict,
    framerate=config["fps"],
    size=config["size"],
    centercrop=config["centercrop"],
    hflip=args.hflip
)
n_dataset = len(dataset)
sampler = RandomSequenceSampler(n_dataset, 10)
loader = DataLoader(
    dataset,
    batch_size=1,
    shuffle=False,
    num_workers=args.num_decoding_thread,
    sampler=sampler if n_dataset > 10 else None,
)
preprocess = Preprocessing(args.type)
model = get_model(args)

with th.no_grad():
    for k, data in tqdm(enumerate(loader), total=loader.__len__(), ascii=True):
        input_file = data['input'][0]
        output_file = data['output'][0]
        if len(data['video'].shape) > 3:
            video = data['video'].squeeze()
            if len(video.shape) == 4:
                video = preprocess(video)
                n_chunk = len(video)
                if args.type == 'vmz':
                    n_chunk = math.ceil(n_chunk/float(3))
                    features = th.cuda.FloatTensor(n_chunk, 512).fill_(0)
                elif args.type == 's3d':
                    features = th.cuda.FloatTensor(n_chunk, 512).fill_(0)
                elif args.type == "vae":
                    features = th.cuda.LongTensor(n_chunk, 1024).fill_(0)
                else:
                    features = th.cuda.FloatTensor(n_chunk, 2048).fill_(0)
                n_iter = int(math.ceil(n_chunk / float(args.batch_size)))
                for i in range(n_iter):
                    factor = 1
                    if args.type == 'vmz':
                        factor = 3
                    min_ind = factor * i * args.batch_size
                    max_ind = factor * (i + 1) * args.batch_size
                    video_batch = video[min_ind:max_ind:factor].cuda()
                    if args.type == '2d':
                        batch_features = model(video_batch) # (51, 487), (51, 512)
                    elif args.type == 's3d':
                        batch_features = model(video_batch)
                        batch_features = batch_features['video_embedding']
                    elif args.type == "vae":
                        # image_code.
                        batch_features = model(video_batch)
                    else:
                        batch_pred, batch_features = model(video_batch) # (51, 487), (51, 512)
                    if args.l2_normalize:
                        batch_features = F.normalize(batch_features, dim=1)
                    features[i*args.batch_size:(i+1)*args.batch_size] = batch_features
                features = features.cpu().numpy()
                if args.half_precision:
                    if args.type == "vae":
                        features = features.astype(np.int16)
                    else:
                        features = features.astype('float16')
                else:
                    if args.type == "vae":
                        features = features.astype(np.int32)
                    else:
                        features = features.astype('float32')
                np.save(output_file, features)
        else:
            print('Video {} error.'.format(input_file))


from __future__ import annotations
import os
import time

import click
import git
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm

from pelutils import get_repo
from pelutils.ds.plots import figsize_std, update_rc_params, rc_params_small

_default_extensions = ", ".join((
    ".py", ".pyw",
    ".c", ".cpp", ".cc", ".h",
    ".tex",
    ".html", ".css", ".js", ".jsx", ".ts",
    ".java", ".kt", ".dart", ".swift",
    ".rs", ".go",
    ".r", ".m",
))

def _count(repo: git.Repo, branch: git.Head, exts: list[str]) -> tuple[np.ndarray, dict[str, np.ndarray]]:
    """ Counts lines of all files recursively in the current working directory
    Returns an array of commit epoch times and a dict mapping file extensions to line counts """
    commits = list(reversed(list(repo.iter_commits())))  # List of commit from oldest to newest
    times = np.array([c.committed_date for c in commits])
    lines = { ext: np.zeros_like(times) for ext in exts }
    for i, commit in enumerate(tqdm(commits)):
        try:
            repo.git.checkout(str(commit))
        except git.GitCommandError as e:
            # This error happens if there is a file that has been moved in and out of .gitignore
            # For safety, the commit is ignored
            if "error: The following untracked working tree files would be overwritten by checkout:" in str(e):
                continue
            raise
        for path, __ in repo.index.entries:
            try:
                ext = next(e for e in exts if path.endswith(e) and os.path.isfile(path))
            except StopIteration:
                continue
            with open(path) as f:
                lines[ext][i] += len([line for line in f.readlines() if line.strip()])

    repo.git.checkout(branch)
    return times, lines

def _fuse_times(all_times: list[np.ndarray]) -> np.ndarray:
    """ Merges all time arrays together into a single array that is also sorted """
    n = sum(times.size for times in all_times)
    times = np.empty(n, dtype=int)
    idcs = [0] * len(all_times)
    for i in range(n):
        current_times = [time[idx] if idx < len(time) else float("inf") for time, idx in zip(all_times, idcs)]
        min_time = np.argmin(current_times)
        times[i] = current_times[min_time]
        idcs[min_time] += 1
    return times

def _last_initial_zero(values: np.ndarray) -> int:
    if values[0] != 0:
        return 0
    return np.where(values!=0)[0][0] - 1

@click.command()
@click.argument("repos", nargs=-1)
@click.option("-e", "--extensions", default=_default_extensions, help="Comma seperated list of file extensions to look for")
@click.option("-o", "--outplot-path", default="linecount.png")
@click.option("-d", "--date-format", default="%d-%m-%y", help="How to format axis labels")
@click.option("-n", "--no-repo-name", is_flag=True)
def linecounter(repos: list[str], extensions: str, outplot_path: str, date_format: str, no_repo_name: bool):
    extensions = [x.strip() for x in extensions.split(",")]
    extensions = [x if x.startswith(".") else "." + x for x in extensions]
    wd = os.getcwd()
    repo_names, all_times, all_counts = list(), list(), list()
    for repo in repos:
        repo_path, __ = get_repo(repo)
        if repo_path is None:
            raise ValueError("%s is not a git repository" % repo_path)
        os.chdir(repo_path)

        try:
            repo = git.Repo()
            branch = repo.active_branch
            times, lines = _count(repo, branch, extensions)
        except:
            # Reset repo
            repo.git.checkout(branch)
            raise

        repo_names.append(os.path.split(repo_path)[-1])
        all_times.append(times)
        all_counts.append(lines)
        os.chdir(wd)

    plt.figure(figsize=figsize_std)
    for i, (repo_name, times, counts) in enumerate(zip(repo_names, all_times, all_counts)):
        for ext, l in counts.items():
            if not l.any():
                continue
            non_zero = l != 0
            non_zero[_last_initial_zero(l)] = 1
            all_times[i] = times[non_zero]
            all_counts[i] = l[non_zero]
            lab = ext+(f" ({repo_name})" if len(repo_names) > 1 and not no_repo_name else "")
            plt.plot(all_times[i], all_counts[i], marker=".", ms=8, lw=2, label=lab)

    fused_times = _fuse_times(all_times)
    xticks = np.linspace(0, len(fused_times)-1, 10, dtype=int)
    xticklabels = [time.strftime(date_format, time.gmtime(x)) for i, x in enumerate(fused_times) if i in xticks]
    plt.xticks([fused_times[i] for i in xticks], xticklabels, rotation=60)
    title = "Line counts over time" + (f" ({repo_name})" if len(repo_names) == 1 and not no_repo_name else "")
    plt.title(title)
    plt.xlabel("Date of commit")
    plt.ylabel("Number of lines (excluding empty lines)")
    plt.legend(loc=2)
    plt.grid()
    plt.tight_layout()
    plt.savefig(outplot_path)

if __name__ == "__main__":
    update_rc_params(rc_params_small)
    linecounter()


# %%
import os
import glob
import ogr
import gdal
import sys
from collections import defaultdict
import numpy as np
import cv2
import matplotlib.pyplot as plt
from math import sqrt
from mytools.building_jinan.utils import walk, geo2cr, check_tifs
ogr.RegisterAll()
gdal.SetConfigOption('SHAPE_ENCODING', "UTF8")
gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES")

min_box_size = 4

input_dir = '/data/zfp/data/济南大学'
dir1 = '融合图像'
dir2 = '高分图像'
dir3 = '多光谱'
fusion_dir = os.path.join(input_dir, dir1)
pan_dir = os.path.join(input_dir, dir2)
ms_dir = os.path.join(pan_dir, dir3)

fusion_files = sorted(glob.glob(os.path.join(fusion_dir, '*.tif')))
tif_filenames = [os.path.splitext(os.path.basename(fusion_file))[0] for fusion_file in fusion_files]

files = {}
for tif_filename, fusion_file in zip(tif_filenames, fusion_files):
    files[tif_filename] = {}
    files[tif_filename]['fusion'] = fusion_file

for tif_filename in tif_filenames:
    tif_file, shp_file = walk(os.path.join(pan_dir, tif_filename))
    files[tif_filename]['pan'] = tif_file
    files[tif_filename]['shp'] = shp_file
    ms_file = os.path.join(ms_dir, '%s.tif' % tif_filename)
    assert os.path.exists(ms_file), '{} doesn\'t exist'
    files[tif_filename]['ms'] = ms_file

files = check_tifs(files)

for k, v in files.items():
    print(k)
    print(v)
    break

# %%
cate2 = {'住房': 'Hb', '厂房': 'Fb', '大棚': 'Gh', '其他1': 'OI', '其他2': 'OII', None: 'OII'}
file_polys = {}
files_fail = []
polys_fail = defaultdict(int)
num_per_cate = defaultdict(int)
for k, v in files.items():
    print(k)
    file_polys[k] = {}
    file_polys[k]['points'] = []
    file_polys[k]['box'] = []
    file_polys[k]['rbox'] = []

    pan_tif_file = v['pan']
    ms_tif_file = v['ms']
    scale_w, scale_h = 1, 1
    # scale_w, scale_h = get_scale(pan_tif_file, ms_tif_file)

    pan_tif_file = v['pan']
    dataset = gdal.Open(pan_tif_file)
    if dataset is None:
        print('fail to open {}'.format(pan_tif_file))
        files_fail.append(k)
        continue

    width = dataset.RasterXSize
    height = dataset.RasterYSize
    geoTransform = dataset.GetGeoTransform()
    if geoTransform is None:
        print('{}: geoTransfrom is None'.format(pan_tif_file))
        files_fail.append(k)
        continue

    shp_file = v['shp']
    dataSource = ogr.Open(shp_file)
    if dataSource is None:
        print('fail to open {}'.format(shp_file))
        files_fail.append(k)
        continue
    daLayer = dataSource.GetLayer(0)
    featureCount = daLayer.GetFeatureCount()

    daLayer.ResetReading()
    for _ in range(featureCount):
        feature = daLayer.GetNextFeature()
        fieldName = feature.GetField('类型')
        if fieldName == '其它2':
            fieldName = '其他2'
        geometry = feature.GetGeometryRef()
        if geometry is None:
            print('{}: geometry is None'.format(shp_file))
            polys_fail[k] += 1
            continue
        geometryType = geometry.GetGeometryType()
        if geometryType != ogr.wkbPolygon:
            print('{}: the FID is {}, the type of geometry is {}'.format(shp_file, feature.GetFID(), geometryType))
            polys_fail[k] += 1
            continue
        geometryCount = geometry.GetGeometryCount()
        if geometryCount != 1:
            print('{}: poly has {} rings'.format(shp_file, geometryCount))
            polys_fail[k] += 1

        ring = geometry.GetGeometryRef(0)
        numPoints = ring.GetPointCount()
        if numPoints < 4:
            print('{} : the num of ring is less than 3'.format(shp_file))
            polys_fail[k] += 1
            continue

        points = []
        max_y = 0
        max_x = 0
        min_y = height - 1
        min_x = width - 1
        for i in range(numPoints - 1):
            x, y = geo2cr(geoTransform, ring.GetX(i), ring.GetY(i))

            x = max(min(x, width - 1), 0)
            y = max(min(y, height - 1), 0)

            points.extend([x, y])

        points = np.array(points).reshape(-1, 2) * [scale_w, scale_h]
        x_min, y_min = points.min(axis=0)
        x_max, y_max = points.max(axis=0)
        w = x_max - x_min + 1
        h = y_max - y_min + 1
        box = [x_min, y_min, w, h]

        rect = cv2.minAreaRect(points.astype(np.int))
        x_c, y_c = rect[0]
        w, h = rect[1]
        a = rect[2]
        rbox = [x_c, y_c, w, h, a]

        if min(box[2], box[3]) < min_box_size:
            print('{}: the poly is so small, FID:{}, width:{}, height:{}'.format(shp_file, feature.GetFID(), box[2], box[3]))
            polys_fail[k] += 1
            continue
        num_per_cate[cate2[fieldName]] += 1
        file_polys[k]['box'].append(box)
        file_polys[k]['points'].append(points)
        file_polys[k]['rbox'].append(rbox)
# %%
print(num_per_cate)
x=np.arange(len(num_per_cate.keys()))
plt.bar(x, list(num_per_cate.values()))
ax=plt.gca()
labels = list(num_per_cate.keys())
labels.insert(0, '1')

ax.set_xticklabels(labels)
plt.savefig(os.path.abspath('cate.png'))
plt.show()
print('error with the num of files:', len(files_fail))
print('error with the num of polys:', sum(polys_fail.values()))
num_building = 0
for k, v in file_polys.items():
    num_building += len(file_polys[k]['points'])
print('the num of polys:', num_building)
# error with the num of files: 0
# error with the num of polys: 72
# the num of polys: 62177
# %%
# the angle of instance
angles = []
for k, v in file_polys.items():
    for rbox in v['rbox']:
        w, h, angle  = rbox[-3:]
        if w < h:
            angle = angle + 90
        angles.append(angle)
plt.hist(angles, rwidth=0.9)
plt.savefig(os.path.abspath('angle.png'))
plt.show()

# %%
import bisect
sizes_range = [0, 32, 96, 1000]
sizes_num = [0 for _ in range(len(sizes_range))]
sizes = []
for k, v in file_polys.items():
    sizes_per_image = [sqrt(box[2] * box[3]) for box in v['box']]
    pos_per_image = [bisect.bisect_right(sizes_range, size) for size in sizes_per_image]
    for pos in pos_per_image:
        sizes_num[pos - 1] += 1
    sizes.extend(sizes_per_image)
print(sizes_num)
plt.hist(sizes, bins=10, rwidth=0.8)
plt.savefig(os.path.abspath('instance_size.png'))
plt.show()

# %%
img_sizes = []
for k, v in files.items():
    pan_file = v['pan']
    dataset = gdal.Open(pan_file)
    img_sizes.append(sqrt(dataset.RasterXSize * dataset.RasterYSize))
plt.hist(img_sizes, bins=10, rwidth=0.8)
plt.savefig(os.path.abspath('img_size.png'))
plt.show()
# %%
bbox_ratios = []
rbox_ratios = []
for k, v in file_polys.items():
    for box in v['box']:
        w = box[2]
        h = box[3]
        big = max(w, h)
        small = min(w, h)
        if big / small > 15:
            continue
        bbox_ratios.append(big / small)

    for rbox in v['rbox']:
        w = rbox[2]
        h = rbox[3]
        big = max(w, h)
        small = min(w, h)
        if big / small > 15:
            continue
        rbox_ratios.append(big / small)
plt.hist(bbox_ratios, bins=15, rwidth=0.8)
plt.savefig(os.path.abspath('bbox_ratio.png'))
plt.show()
plt.hist(rbox_ratios, bins=15, rwidth=0.8)
plt.savefig(os.path.abspath('rbox_ratio.png'))
plt.show()

# %%


#!/usr/bin/python
import time
import datetime
import pytz
import numpy
import random
import gzip
import zipfile
import sys
import argparse
import math
from faker import Faker
from random import randrange
from tzlocal import get_localzone
local = get_localzone()

parser = argparse.ArgumentParser(
    __file__, description="Fake Apache Log Generator")
parser.add_argument(
    "--output",
    "-o",
    dest='output_type',
    help="Write to a Log file, a gzip file or to STDOUT",
    choices=['LOG', 'GZ', 'CONSOLE'])
parser.add_argument(
    "--log-format",
    "-l",
    dest='log_format',
    help="Log format, Common or Extended Log Format ",
    choices=['CLF', 'ELF'],
    default="ELF")
parser.add_argument(
    "--num",
    "-n",
    dest='num_lines',
    help="Number of lines to generate (0 for infinite)",
    type=int,
    default=1)
parser.add_argument(
    "--prefix",
    "-p",
    dest='file_prefix',
    help="Prefix the output file name",
    type=str)
parser.add_argument(
    "--sleep",
    "-s",
    help="Sleep this long between lines (in seconds)",
    default=0.0,
    type=float)

args = parser.parse_args()

log_lines = args.num_lines
file_prefix = args.file_prefix
output_type = args.output_type
log_format = args.log_format

if not log_lines:
    log_lines = math.inf

faker = Faker()

timestr = time.strftime("%Y%m%d-%H%M%S")
otime = datetime.datetime.now()

outFileName = 'access_log_' + timestr + \
    '.log' if not file_prefix else file_prefix + '_access_log_' + timestr + '.log'

if output_type == 'LOG':
    f = open(outFileName, 'w')
if output_type == 'GZ':
    f = gzip.open(outFileName + '.gz', 'w')
else:
    f = sys.stdout

RESPONSES = ["200", "404", "500", "301"]

VERBS = ["GET", "POST", "DELETE", "PUT"]

RESOURCES = [
    "/list", "/wp-content", "/wp-admin", "/explore", "/search/tag/list",
    "/app/main/posts", "/posts/posts/explore", "/apps/cart.jsp?appID="
]

BROWSERS = [
    faker.firefox, faker.chrome, faker.safari, faker.internet_explorer,
    faker.opera
]

flag = True
while log_lines:
    if args.sleep:
        increment = datetime.timedelta(seconds=args.sleep)
    else:
        increment = datetime.timedelta(seconds=random.randint(30, 300))
    otime += increment

    ip = faker.ipv4()
    dt = otime.strftime('%d/%b/%Y:%H:%M:%S')
    tz = datetime.datetime.now(local).strftime('%z')
    vrb = numpy.random.choice(VERBS, p=[0.6, 0.1, 0.1, 0.2])

    uri = random.choice(RESOURCES)
    if uri.find("apps") > 0:
        uri += str(random.randint(1000, 10000))

    resp = numpy.random.choice(RESPONSES, p=[0.9, 0.04, 0.02, 0.04])
    byt = int(random.gauss(5000, 50))
    referer = faker.uri()
    useragent = numpy.random.choice(BROWSERS, p=[0.5, 0.3, 0.1, 0.05, 0.05])()
    if log_format == "CLF":
        f.write('%s - - [%s %s] "%s %s HTTP/1.0" %s %s\n' % (ip, dt, tz, vrb,
                                                             uri, resp, byt))
    elif log_format == "ELF":
        f.write('%s - - [%s %s] "%s %s HTTP/1.0" %s %s "%s" "%s"\n' %
                (ip, dt, tz, vrb, uri, resp, byt, referer, useragent))
    f.flush()

    log_lines = log_lines - 1
    if args.sleep:
        time.sleep(args.sleep)


# -*- coding: utf-8 -*-
"""
Created in 2017

@author: Jon Francombe
"""

import numpy
import math

from metadapter import SequenceProcessorInterface

#class FakeGroupsProcessor(MetadataProcessorInterface):
class FakeGroupsProcessor(SequenceProcessorInterface):
    def __init__(self, arguments ):
        #MetadataProcessorInterface.__init__(self, arguments)
        SequenceProcessorInterface.__init__(self, arguments)
        #leveldB = float(arguments.attrib['volumeAdjust'])
        #self.volumeChange = numpy.power( 10.0, leveldB/20.0 )
        self.programme = 'None'

    def processObjectVector( self, objectVector):
        # Function to be implemented by all derived MetaDataProcessor interfaces.

        ### GROUPS: ###
        # 1: Clean speech
        # 2: Non-diegetic music
        # 3: Foreground action sounds
        # 4: Background action sounds
        # 5: Reverb and effects

        debug = False

        if self.programme == 'Forest':

            # This version uses the background sounds group:
            groups = [1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]

            # This version uses the reverb and effects group:
            #groups = [1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5]

        elif self.programme == 'Frame':
            groups = [1,4,4,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,4,4,4,4,4,4]

        elif self.programme == 'None':
            return objectVector

        if debug is True:
            print ("Number of objects: {}".format(len(objectVector)))
            print ("Number of groups specified: {}".format(len(groups)))

        # Add the groups
        for i,obj in enumerate(objectVector):

            if i == 2 and debug is True:
                print (obj)

            # Set the group
            obj['group'] = unicode(str(groups[i]))

            # For testing — write junk into the metadata, see if the renderer complains
            # obj['myattribute'] = u'ThisIsAString'
            # print obj
            # It doesn't seem to complain

            if i == 2 and debug is True:
                print (obj)
                print()

        return objectVector

    def setParameter( self, key, valueList ):
        """ Set the parameter to a given value."""

        if (key != "programme") or (len( valueList ) != 1):
            raise KeyError( "FakeGroupsProcessor supports only the parameter set command \"programme\" consisting of a single string" )

        else: # key is programme
            self.programme = valueList[0]

            print( "FakeGroups: Programme set to: {}".format(self.programme))


import numpy as np
from ...FileReaders.GeomReader.VertexComponents import vertex_components_from_names

##################################
#  Polygon data type converters  #
##################################
def triangle_strips_to_polys(idxs):
    triangles = []
    for i, tri in enumerate(zip(idxs, idxs[1:], idxs[2:])):
        order = i % 2
        tri = (tri[0 + order], tri[1 - order], tri[2])
        triangle = set(tri)
        if not (len(triangle) != 3 or tri in triangles):
            triangles.append(tri)
    return triangles

def triangles_to_polys(idxs):
    triangles = []
    for tri, (idx_a, idx_b, idx_c) in enumerate(zip(idxs[::3], idxs[1::3], idxs[2::3])):
        triangles.append((idx_a, idx_b, idx_c))
    return triangles

triangle_converters = {'Triangles': triangles_to_polys,
                       'TriangleStrips': triangle_strips_to_polys}

class MeshInterface:
    def __init__(self):
        self.unknown_0x31 = None
        self.unknown_0x34 = None
        self.unknown_0x36 = None
        self.unknown_0x4C = None

        self.vertices = []
        self.vertex_group_bone_idxs = []
        self.polygons = []
        self.material_id = None

        self.unknown_data = {}

    @classmethod
    def from_subfile(cls, meshReader):
        interface = cls()
        interface.unknown_0x31 = meshReader.unknown_0x31
        interface.unknown_0x34 = meshReader.unknown_0x34
        interface.unknown_0x36 = meshReader.unknown_0x36
        interface.unknown_0x4C = meshReader.unknown_0x4C

        interface.vertices = process_posweights(meshReader.vertex_data, meshReader.max_vertex_groups_per_vertex)
        interface.vertex_group_bone_idxs = meshReader.weighted_bone_idxs
        interface.polygons = triangle_converters[meshReader.polygon_data_type](meshReader.polygon_data)
        interface.material_id = meshReader.material_id

        interface.unknown_data['bb'] = meshReader.bounding_box_lengths
        interface.unknown_data['bbc'] = meshReader.mesh_centre

        return interface

    def to_subfile(self, meshReader, virtual_pos):
        meshReader.vertex_data_start_ptr = virtual_pos

        vgroup_idxs = self.vertex_group_bone_idxs
        meshReader.bytes_per_vertex, meshReader.vertex_components = calculate_vertex_properties(self.vertices, vgroup_idxs)
        vertex_generators = [vc.generator for vc in meshReader.vertex_components]

        meshReader.vertex_data = generate_vertex_data(self.vertices, vertex_generators)

        virtual_pos += meshReader.bytes_per_vertex * len(meshReader.vertex_data)
        meshReader.weighted_bone_data_start_ptr = virtual_pos
        meshReader.weighted_bone_idxs = vgroup_idxs
        virtual_pos += 4 * len(meshReader.weighted_bone_idxs)
        meshReader.polygon_data_start_ptr = virtual_pos
        meshReader.polygon_data = polys_to_triangles(self.polygons)
        virtual_pos += 2 * len(meshReader.polygon_data)
        virtual_pos += (4 - (virtual_pos % 4)) % 4  # Fix ragged chunk of size 4
        meshReader.padding_0x18 = 0

        meshReader.vertex_components_start_ptr = virtual_pos
        virtual_pos += 8 * len(meshReader.vertex_components)
        meshReader.num_weighted_bone_idxs = len(meshReader.weighted_bone_idxs)
        meshReader.num_vertex_components = len(meshReader.vertex_components)
        meshReader.always_5123 = meshReader.header_breaker

        meshReader.max_vertex_groups_per_vertex = max([len(vtx['WeightedBoneID']) if vtx['WeightedBoneID'] is not None else 0 for vtx in self.vertices])
        meshReader.max_vertex_groups_per_vertex = 0 if len(meshReader.weighted_bone_idxs) == 1 else meshReader.max_vertex_groups_per_vertex
        meshReader.unknown_0x31 = self.unknown_0x31
        meshReader.polygon_numeric_data_type = 4  # Can only write to triangles atm
        meshReader.unknown_0x34 = self.unknown_0x34
        meshReader.unknown_0x36 = self.unknown_0x36
        meshReader.material_id = self.material_id
        meshReader.num_vertices = len(meshReader.vertex_data)

        meshReader.num_polygon_idxs = len(meshReader.polygon_data)
        meshReader.padding_0x44 = 0
        meshReader.padding_0x48 = 0
        meshReader.unknown_0x4C = self.unknown_0x4C

        vertices = np.array([v['Position'] for v in self.vertices])
        minvs = np.min(vertices, axis=0)
        maxvs = np.max(vertices, axis=0)
        assert len(maxvs) == 3

        meshReader.mesh_centre = (maxvs + minvs) / 2
        meshReader.bounding_box_lengths = (maxvs - minvs) / 2
        return virtual_pos

def process_posweights(vertices, max_vertex_groups_per_vertex):
    example_vertex = vertices[0]
    if 'WeightedBoneID' in example_vertex:
        for vtx in vertices:
            res_ids = []
            res_wghts = []
            for id, wght in zip(vtx['WeightedBoneID'], vtx['BoneWeight']):
                if wght != 0.:
                    res_ids.append(id//3)
                    res_wghts.append(wght)
            vtx['WeightedBoneID'] = res_ids
            vtx['BoneWeight'] = res_wghts
            if len(vtx['WeightedBoneID']) != len(vtx['BoneWeight']):
                assert 0, f"{vtx['WeightedBoneID']} {vtx['BoneWeight']}"
    elif max_vertex_groups_per_vertex == 0:
        for vtx in vertices:
            vtx['WeightedBoneID'] = [0]
            vtx['BoneWeight'] = [1]
    elif max_vertex_groups_per_vertex == 1:
        for vtx in vertices:
            vtx['WeightedBoneID'] = [int(vtx['Position'][3]) // 3]
            vtx['BoneWeight'] = [1]
            vtx['Position'] = vtx['Position'][:3]
    else:
        assert 0, "Something went seriously wrong when processing posweights."

    return vertices

def calculate_vertex_properties(vertices, num_vertex_groups):
    """
    Still a bit messy but much nicer than before.
    """
    bytes_per_vertex = 0
    vertex_components = []
    example_vertex = vertices[0]
    max_vtx_groups = max([len(vtx['WeightedBoneID']) for vtx in vertices])
    if 'Position' in example_vertex:
        if max_vtx_groups == 1 and len(num_vertex_groups) > 1:
            vertex_components.append(vertex_components_from_names['PosWeight'](bytes_per_vertex))
            bytes_per_vertex += 16
        else:
            vertex_components.append(vertex_components_from_names['Position'](bytes_per_vertex))
            bytes_per_vertex += 12
    if 'Normal' in example_vertex:
        vertex_components.append(vertex_components_from_names['Normal'](bytes_per_vertex))
        bytes_per_vertex += 8
    if 'UV' in example_vertex:
        vertex_components.append(vertex_components_from_names['UV'](bytes_per_vertex))
        bytes_per_vertex += 4
    if 'UV2' in example_vertex:
        vertex_components.append(vertex_components_from_names['UV2'](bytes_per_vertex))
        bytes_per_vertex += 4
    if 'UV3' in example_vertex:
        vertex_components.append(vertex_components_from_names['UV3'](bytes_per_vertex))
        bytes_per_vertex += 4
    if 'Colour' in example_vertex:
        vertex_components.append(vertex_components_from_names['Colour'](bytes_per_vertex))
        bytes_per_vertex += 8
    if 'Tangent' in example_vertex:
        vertex_components.append(vertex_components_from_names['Tangent'](bytes_per_vertex))
        bytes_per_vertex += 8
    if 'Binormal' in example_vertex:
        vertex_components.append(vertex_components_from_names['Binormal'](bytes_per_vertex))
        bytes_per_vertex += 8
    if 'WeightedBoneID' in example_vertex and max_vtx_groups >= 2:
        num_grps = max_vtx_groups
        vertex_components.append(vertex_components_from_names[f'Indices{num_grps}'](bytes_per_vertex))
        nominal_bytes = num_grps
        bytes_per_vertex += nominal_bytes + ((4 - (nominal_bytes % 4)) % 4)

        vertex_components.append(vertex_components_from_names[f'Weights{num_grps}'](bytes_per_vertex))
        nominal_bytes = 2*num_grps
        bytes_per_vertex += nominal_bytes + ((4 - (nominal_bytes % 4)) % 4)

    return bytes_per_vertex, vertex_components

def generate_vertex_data(vertices, generators):
    retval = []
    for vertex in vertices:
        vdata = [generator(vertex) for generator in generators]
        retval.append({k: v for d in vdata for k, v in d.items()})
    return retval

def polys_to_triangles(polys):
    return [sublist for lst in polys for sublist in lst]

class MissingWeightsError(TypeError):
    pass