clean

yolov7-tracker-example/tracker/trackeval/__init__.py

@@ -0,0 +1,5 @@
+from .eval import Evaluator
+from . import datasets
+from . import metrics
+from . import plotting
+from . import utils

yolov7-tracker-example/tracker/trackeval/_timing.py

@@ -0,0 +1,65 @@
+from functools import wraps
+from time import perf_counter
+import inspect
+
+DO_TIMING = False
+DISPLAY_LESS_PROGRESS = False
+timer_dict = {}
+counter = 0
+
+
+def time(f):
+    @wraps(f)
+    def wrap(*args, **kw):
+        if DO_TIMING:
+            # Run function with timing
+            ts = perf_counter()
+            result = f(*args, **kw)
+            te = perf_counter()
+            tt = te-ts
+
+            # Get function name
+            arg_names = inspect.getfullargspec(f)[0]
+            if arg_names[0] == 'self' and DISPLAY_LESS_PROGRESS:
+                return result
+            elif arg_names[0] == 'self':
+                method_name = type(args[0]).__name__ + '.' + f.__name__
+            else:
+                method_name = f.__name__
+
+            # Record accumulative time in each function for analysis
+            if method_name in timer_dict.keys():
+                timer_dict[method_name] += tt
+            else:
+                timer_dict[method_name] = tt
+
+            # If code is finished, display timing summary
+            if method_name == "Evaluator.evaluate":
+                print("")
+                print("Timing analysis:")
+                for key, value in timer_dict.items():
+                    print('%-70s %2.4f sec' % (key, value))
+            else:
+                # Get function argument values for printing special arguments of interest
+                arg_titles = ['tracker', 'seq', 'cls']
+                arg_vals = []
+                for i, a in enumerate(arg_names):
+                    if a in arg_titles:
+                        arg_vals.append(args[i])
+                arg_text = '(' + ', '.join(arg_vals) + ')'
+
+                # Display methods and functions with different indentation.
+                if arg_names[0] == 'self':
+                    print('%-74s %2.4f sec' % (' '*4 + method_name + arg_text, tt))
+                elif arg_names[0] == 'test':
+                    pass
+                else:
+                    global counter
+                    counter += 1
+                    print('%i %-70s %2.4f sec' % (counter, method_name + arg_text, tt))
+
+            return result
+        else:
+            # If config["TIME_PROGRESS"] is false, or config["USE_PARALLEL"] is true, run functions normally without timing.
+            return f(*args, **kw)
+    return wrap

yolov7-tracker-example/tracker/trackeval/baselines/__init__.py

@@ -0,0 +1,6 @@
+import baseline_utils
+import stp
+import non_overlap
+import pascal_colormap
+import thresholder
+import vizualize

yolov7-tracker-example/tracker/trackeval/baselines/baseline_utils.py

@@ -0,0 +1,321 @@
+
+import os
+import csv
+import numpy as np
+from copy import deepcopy
+from PIL import Image
+from pycocotools import mask as mask_utils
+from scipy.optimize import linear_sum_assignment
+from trackeval.baselines.pascal_colormap import pascal_colormap
+
+
+def load_seq(file_to_load):
+    """ Load input data from file in RobMOTS format (e.g. provided detections).
+    Returns: Data object with the following structure (see STP :
+        data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    """
+    fp = open(file_to_load)
+    dialect = csv.Sniffer().sniff(fp.readline(), delimiters=' ')
+    dialect.skipinitialspace = True
+    fp.seek(0)
+    reader = csv.reader(fp, dialect)
+    read_data = {}
+    num_timesteps = 0
+    for i, row in enumerate(reader):
+        if row[-1] in '':
+            row = row[:-1]
+        t = int(row[0])
+        cid = row[1]
+        c = int(row[2])
+        s = row[3]
+        h = row[4]
+        w = row[5]
+        rle = row[6]
+
+        if t >= num_timesteps:
+            num_timesteps = t + 1
+
+        if c in read_data.keys():
+            if t in read_data[c].keys():
+                read_data[c][t]['ids'].append(cid)
+                read_data[c][t]['scores'].append(s)
+                read_data[c][t]['im_hs'].append(h)
+                read_data[c][t]['im_ws'].append(w)
+                read_data[c][t]['mask_rles'].append(rle)
+            else:
+                read_data[c][t] = {}
+                read_data[c][t]['ids'] = [cid]
+                read_data[c][t]['scores'] = [s]
+                read_data[c][t]['im_hs'] = [h]
+                read_data[c][t]['im_ws'] = [w]
+                read_data[c][t]['mask_rles'] = [rle]
+        else:
+            read_data[c] = {t: {}}
+            read_data[c][t]['ids'] = [cid]
+            read_data[c][t]['scores'] = [s]
+            read_data[c][t]['im_hs'] = [h]
+            read_data[c][t]['im_ws'] = [w]
+            read_data[c][t]['mask_rles'] = [rle]
+    fp.close()
+
+    data = {}
+    for c in read_data.keys():
+        data[c] = [{} for _ in range(num_timesteps)]
+        for t in range(num_timesteps):
+            if t in read_data[c].keys():
+                data[c][t]['ids'] = np.atleast_1d(read_data[c][t]['ids']).astype(int)
+                data[c][t]['scores'] = np.atleast_1d(read_data[c][t]['scores']).astype(float)
+                data[c][t]['im_hs'] = np.atleast_1d(read_data[c][t]['im_hs']).astype(int)
+                data[c][t]['im_ws'] = np.atleast_1d(read_data[c][t]['im_ws']).astype(int)
+                data[c][t]['mask_rles'] = np.atleast_1d(read_data[c][t]['mask_rles']).astype(str)
+            else:
+                data[c][t]['ids'] = np.empty(0).astype(int)
+                data[c][t]['scores'] = np.empty(0).astype(float)
+                data[c][t]['im_hs'] = np.empty(0).astype(int)
+                data[c][t]['im_ws'] = np.empty(0).astype(int)
+                data[c][t]['mask_rles'] = np.empty(0).astype(str)
+    return data
+
+
+def threshold(tdata, thresh):
+    """ Removes detections below a certian threshold ('thresh') score. """
+    new_data = {}
+    to_keep = tdata['scores'] > thresh
+    for field in ['ids', 'scores', 'im_hs', 'im_ws', 'mask_rles']:
+        new_data[field] = tdata[field][to_keep]
+    return new_data
+
+
+def create_coco_mask(mask_rles, im_hs, im_ws):
+    """ Converts mask as rle text (+ height and width) to encoded version used by pycocotools. """
+    coco_masks = [{'size': [h, w], 'counts': m.encode(encoding='UTF-8')}
+                  for h, w, m in zip(im_hs, im_ws, mask_rles)]
+    return coco_masks
+
+
+def mask_iou(mask_rles1, mask_rles2, im_hs, im_ws, do_ioa=0):
+    """ Calculate mask IoU between two masks.
+    Further allows 'intersection over area' instead of IoU (over the area of mask_rle1).
+    Allows either to pass in 1 boolean for do_ioa for all mask_rles2 or also one for each mask_rles2.
+    It is recommended that mask_rles1 is a detection and mask_rles2 is a groundtruth.
+    """
+    coco_masks1 = create_coco_mask(mask_rles1, im_hs, im_ws)
+    coco_masks2 = create_coco_mask(mask_rles2, im_hs, im_ws)
+
+    if not hasattr(do_ioa, "__len__"):
+        do_ioa = [do_ioa]*len(coco_masks2)
+    assert(len(coco_masks2) == len(do_ioa))
+    if len(coco_masks1) == 0 or len(coco_masks2) == 0:
+        iou = np.zeros(len(coco_masks1), len(coco_masks2))
+    else:
+        iou = mask_utils.iou(coco_masks1, coco_masks2, do_ioa)
+    return iou
+
+
+def sort_by_score(t_data):
+    """ Sorts data by score """
+    sort_index = np.argsort(t_data['scores'])[::-1]
+    for k in t_data.keys():
+        t_data[k] = t_data[k][sort_index]
+    return t_data
+
+
+def mask_NMS(t_data, nms_threshold=0.5, already_sorted=False):
+    """ Remove redundant masks by performing non-maximum suppression (NMS) """
+
+    # Sort by score
+    if not already_sorted:
+        t_data = sort_by_score(t_data)
+
+    #  Calculate the mask IoU between all detections in the timestep.
+    mask_ious_all = mask_iou(t_data['mask_rles'], t_data['mask_rles'], t_data['im_hs'], t_data['im_ws'])
+
+    # Determine which masks NMS should remove
+    # (those overlapping greater than nms_threshold with another mask that has a higher score)
+    num_dets = len(t_data['mask_rles'])
+    to_remove = [False for _ in range(num_dets)]
+    for i in range(num_dets):
+        if not to_remove[i]:
+            for j in range(i + 1, num_dets):
+                if mask_ious_all[i, j] > nms_threshold:
+                    to_remove[j] = True
+
+    # Remove detections which should be removed
+    to_keep = np.logical_not(to_remove)
+    for k in t_data.keys():
+        t_data[k] = t_data[k][to_keep]
+
+    return t_data
+
+
+def non_overlap(t_data, already_sorted=False):
+    """ Enforces masks to be non-overlapping in an image, does this by putting masks 'on top of one another',
+    such that higher score masks 'occlude' and thus remove parts of lower scoring masks.
+
+    Help wanted: if anyone knows a way to do this WITHOUT converting the RLE to the np.array let me know, because that
+    would be MUCH more efficient. (I have tried, but haven't yet had success).
+    """
+
+    # Sort by score
+    if not already_sorted:
+        t_data = sort_by_score(t_data)
+
+    # Get coco masks
+    coco_masks = create_coco_mask(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws'])
+
+    # Create a single np.array to hold all of the non-overlapping mask
+    masks_array = np.zeros((t_data['im_hs'][0], t_data['im_ws'][0]), 'uint8')
+
+    # Decode each mask into a np.array, and place it into the overall array for the whole frame.
+    # Since masks with the lowest score are placed first, they are 'partially overridden' by masks with a higher score
+    # if they overlap.
+    for i, mask in enumerate(coco_masks[::-1]):
+        masks_array[mask_utils.decode(mask).astype('bool')] = i + 1
+
+    # Encode the resulting np.array back into a set of coco_masks which are now non-overlapping.
+    num_dets = len(coco_masks)
+    for i, j in enumerate(range(1, num_dets + 1)[::-1]):
+        coco_masks[i] = mask_utils.encode(np.asfortranarray(masks_array == j, dtype=np.uint8))
+
+    # Convert from coco_mask back into our mask_rle format.
+    t_data['mask_rles'] = [m['counts'].decode("utf-8") for m in coco_masks]
+
+    return t_data
+
+
+def masks2boxes(mask_rles, im_hs, im_ws):
+    """ Extracts bounding boxes which surround a set of masks. """
+    coco_masks = create_coco_mask(mask_rles, im_hs, im_ws)
+    boxes = np.array([mask_utils.toBbox(x) for x in coco_masks])
+    if len(boxes) == 0:
+        boxes = np.empty((0, 4))
+    return boxes
+
+
+def box_iou(bboxes1, bboxes2, box_format='xywh', do_ioa=False, do_giou=False):
+    """ Calculates the IOU (intersection over union) between two arrays of boxes.
+    Allows variable box formats ('xywh' and 'x0y0x1y1').
+    If do_ioa (intersection over area), then calculates the intersection over the area of boxes1 - this is commonly
+    used to determine if detections are within crowd ignore region.
+    If do_giou (generalized intersection over union, then calculates giou.
+    """
+    if len(bboxes1) == 0 or len(bboxes2) == 0:
+        ious = np.zeros((len(bboxes1), len(bboxes2)))
+        return ious
+    if box_format in 'xywh':
+        # layout: (x0, y0, w, h)
+        bboxes1 = deepcopy(bboxes1)
+        bboxes2 = deepcopy(bboxes2)
+
+        bboxes1[:, 2] = bboxes1[:, 0] + bboxes1[:, 2]
+        bboxes1[:, 3] = bboxes1[:, 1] + bboxes1[:, 3]
+        bboxes2[:, 2] = bboxes2[:, 0] + bboxes2[:, 2]
+        bboxes2[:, 3] = bboxes2[:, 1] + bboxes2[:, 3]
+    elif box_format not in 'x0y0x1y1':
+        raise (Exception('box_format %s is not implemented' % box_format))
+
+    # layout: (x0, y0, x1, y1)
+    min_ = np.minimum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
+    max_ = np.maximum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
+    intersection = np.maximum(min_[..., 2] - max_[..., 0], 0) * np.maximum(min_[..., 3] - max_[..., 1], 0)
+    area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1])
+
+    if do_ioa:
+        ioas = np.zeros_like(intersection)
+        valid_mask = area1 > 0 + np.finfo('float').eps
+        ioas[valid_mask, :] = intersection[valid_mask, :] / area1[valid_mask][:, np.newaxis]
+
+        return ioas
+    else:
+        area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1])
+        union = area1[:, np.newaxis] + area2[np.newaxis, :] - intersection
+        intersection[area1 <= 0 + np.finfo('float').eps, :] = 0
+        intersection[:, area2 <= 0 + np.finfo('float').eps] = 0
+        intersection[union <= 0 + np.finfo('float').eps] = 0
+        union[union <= 0 + np.finfo('float').eps] = 1
+        ious = intersection / union
+
+    if do_giou:
+        enclosing_area = np.maximum(max_[..., 2] - min_[..., 0], 0) * np.maximum(max_[..., 3] - min_[..., 1], 0)
+        eps = 1e-7
+        # giou
+        ious = ious - ((enclosing_area - union) / (enclosing_area + eps))
+
+    return ious
+
+
+def match(match_scores):
+    match_rows, match_cols = linear_sum_assignment(-match_scores)
+    return match_rows, match_cols
+
+
+def write_seq(output_data, out_file):
+    out_loc = os.path.dirname(out_file)
+    if not os.path.exists(out_loc):
+        os.makedirs(out_loc, exist_ok=True)
+    fp = open(out_file, 'w', newline='')
+    writer = csv.writer(fp, delimiter=' ')
+    for row in output_data:
+        writer.writerow(row)
+    fp.close()
+
+
+def combine_classes(data):
+    """ Converts data from a class-separated to a class-combined format.
+    Input format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    Output format: data[t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles', 'cls'}
+    """
+    output_data = [{} for _ in list(data.values())[0]]
+    for cls, cls_data in data.items():
+        for timestep, t_data in enumerate(cls_data):
+            for k in t_data.keys():
+                if k in output_data[timestep].keys():
+                    output_data[timestep][k] += list(t_data[k])
+                else:
+                    output_data[timestep][k] = list(t_data[k])
+            if 'cls' in output_data[timestep].keys():
+                output_data[timestep]['cls'] += [cls]*len(output_data[timestep]['ids'])
+            else:
+                output_data[timestep]['cls'] = [cls]*len(output_data[timestep]['ids'])
+
+    for timestep, t_data in enumerate(output_data):
+        for k in t_data.keys():
+            output_data[timestep][k] = np.array(output_data[timestep][k])
+
+    return output_data
+
+
+def save_as_png(t_data, out_file, im_h, im_w):
+    """ Save a set of segmentation masks into a PNG format, the same as used for the DAVIS dataset."""
+
+    if len(t_data['mask_rles']) > 0:
+        coco_masks = create_coco_mask(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws'])
+
+        list_of_np_masks = [mask_utils.decode(mask) for mask in coco_masks]
+
+        png = np.zeros((t_data['im_hs'][0], t_data['im_ws'][0]))
+        for mask, c_id in zip(list_of_np_masks, t_data['ids']):
+            png[mask.astype("bool")] = c_id + 1
+    else:
+        png = np.zeros((im_h, im_w))
+
+    if not os.path.exists(os.path.dirname(out_file)):
+        os.makedirs(os.path.dirname(out_file))
+
+    colmap = (np.array(pascal_colormap) * 255).round().astype("uint8")
+    palimage = Image.new('P', (16, 16))
+    palimage.putpalette(colmap)
+    im = Image.fromarray(np.squeeze(png.astype("uint8")))
+    im2 = im.quantize(palette=palimage)
+    im2.save(out_file)
+
+
+def get_frame_size(data):
+    """ Gets frame height and width from data. """
+    for cls, cls_data in data.items():
+        for timestep, t_data in enumerate(cls_data):
+            if len(t_data['im_hs'] > 0):
+                im_h = t_data['im_hs'][0]
+                im_w = t_data['im_ws'][0]
+                return im_h, im_w
+    return None

yolov7-tracker-example/tracker/trackeval/baselines/non_overlap.py

@@ -0,0 +1,92 @@
+"""
+Non-Overlap: Code to take in a set of raw detections and produce a set of non-overlapping detections from it.
+
+Author: Jonathon Luiten
+"""
+
+import os
+import sys
+from multiprocessing.pool import Pool
+from multiprocessing import freeze_support
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from trackeval.baselines import baseline_utils as butils
+from trackeval.utils import get_code_path
+
+code_path = get_code_path()
+config = {
+    'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/raw_supplied/data/'),
+    'OUTPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'),
+    'SPLIT': 'train',  # valid: 'train', 'val', 'test'.
+    'Benchmarks': None,  # If None, all benchmarks in SPLIT.
+
+    'Num_Parallel_Cores': None,  # If None, run without parallel.
+
+    'THRESHOLD_NMS_MASK_IOU': 0.5,
+}
+
+
+def do_sequence(seq_file):
+
+    # Load input data from file (e.g. provided detections)
+    # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    data = butils.load_seq(seq_file)
+
+    # Converts data from a class-separated to a class-combined format.
+    # data[t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles', 'cls'}
+    data = butils.combine_classes(data)
+
+    # Where to accumulate output data for writing out
+    output_data = []
+
+    # Run for each timestep.
+    for timestep, t_data in enumerate(data):
+
+        # Remove redundant masks by performing non-maximum suppression (NMS)
+        t_data = butils.mask_NMS(t_data, nms_threshold=config['THRESHOLD_NMS_MASK_IOU'])
+
+        # Perform non-overlap, to get non_overlapping masks.
+        t_data = butils.non_overlap(t_data, already_sorted=True)
+
+        # Save result in output format to write to file later.
+        # Output Format = [timestep ID class score im_h im_w mask_RLE]
+        for i in range(len(t_data['ids'])):
+            row = [timestep, int(t_data['ids'][i]), t_data['cls'][i], t_data['scores'][i], t_data['im_hs'][i],
+                   t_data['im_ws'][i], t_data['mask_rles'][i]]
+            output_data.append(row)
+
+    # Write results to file
+    out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']),
+                                config['OUTPUT_FOL'].format(split=config['SPLIT']))
+    butils.write_seq(output_data, out_file)
+
+    print('DONE:', seq_file)
+
+
+if __name__ == '__main__':
+
+    # Required to fix bug in multiprocessing on windows.
+    freeze_support()
+
+    # Obtain list of sequences to run tracker for.
+    if config['Benchmarks']:
+        benchmarks = config['Benchmarks']
+    else:
+        benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao']
+        if config['SPLIT'] != 'train':
+            benchmarks += ['waymo', 'mots_challenge']
+    seqs_todo = []
+    for bench in benchmarks:
+        bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench)
+        seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)]
+
+    # Run in parallel
+    if config['Num_Parallel_Cores']:
+        with Pool(config['Num_Parallel_Cores']) as pool:
+            results = pool.map(do_sequence, seqs_todo)
+
+    # Run in series
+    else:
+        for seq_todo in seqs_todo:
+            do_sequence(seq_todo)
+

yolov7-tracker-example/tracker/trackeval/baselines/pascal_colormap.py

@@ -0,0 +1,257 @@
+pascal_colormap = [
+    0     ,         0,         0,
+    0.5020,         0,         0,
+         0,    0.5020,         0,
+    0.5020,    0.5020,         0,
+         0,         0,    0.5020,
+    0.5020,         0,    0.5020,
+         0,    0.5020,    0.5020,
+    0.5020,    0.5020,    0.5020,
+    0.2510,         0,         0,
+    0.7529,         0,         0,
+    0.2510,    0.5020,         0,
+    0.7529,    0.5020,         0,
+    0.2510,         0,    0.5020,
+    0.7529,         0,    0.5020,
+    0.2510,    0.5020,    0.5020,
+    0.7529,    0.5020,    0.5020,
+         0,    0.2510,         0,
+    0.5020,    0.2510,         0,
+         0,    0.7529,         0,
+    0.5020,    0.7529,         0,
+         0,    0.2510,    0.5020,
+    0.5020,    0.2510,    0.5020,
+         0,    0.7529,    0.5020,
+    0.5020,    0.7529,    0.5020,
+    0.2510,    0.2510,         0,
+    0.7529,    0.2510,         0,
+    0.2510,    0.7529,         0,
+    0.7529,    0.7529,         0,
+    0.2510,    0.2510,    0.5020,
+    0.7529,    0.2510,    0.5020,
+    0.2510,    0.7529,    0.5020,
+    0.7529,    0.7529,    0.5020,
+         0,         0,    0.2510,
+    0.5020,         0,    0.2510,
+         0,    0.5020,    0.2510,
+    0.5020,    0.5020,    0.2510,
+         0,         0,    0.7529,
+    0.5020,         0,    0.7529,
+         0,    0.5020,    0.7529,
+    0.5020,    0.5020,    0.7529,
+    0.2510,         0,    0.2510,
+    0.7529,         0,    0.2510,
+    0.2510,    0.5020,    0.2510,
+    0.7529,    0.5020,    0.2510,
+    0.2510,         0,    0.7529,
+    0.7529,         0,    0.7529,
+    0.2510,    0.5020,    0.7529,
+    0.7529,    0.5020,    0.7529,
+         0,    0.2510,    0.2510,
+    0.5020,    0.2510,    0.2510,
+         0,    0.7529,    0.2510,
+    0.5020,    0.7529,    0.2510,
+         0,    0.2510,    0.7529,
+    0.5020,    0.2510,    0.7529,
+         0,    0.7529,    0.7529,
+    0.5020,    0.7529,    0.7529,
+    0.2510,    0.2510,    0.2510,
+    0.7529,    0.2510,    0.2510,
+    0.2510,    0.7529,    0.2510,
+    0.7529,    0.7529,    0.2510,
+    0.2510,    0.2510,    0.7529,
+    0.7529,    0.2510,    0.7529,
+    0.2510,    0.7529,    0.7529,
+    0.7529,    0.7529,    0.7529,
+    0.1255,         0,         0,
+    0.6275,         0,         0,
+    0.1255,    0.5020,         0,
+    0.6275,    0.5020,         0,
+    0.1255,         0,    0.5020,
+    0.6275,         0,    0.5020,
+    0.1255,    0.5020,    0.5020,
+    0.6275,    0.5020,    0.5020,
+    0.3765,         0,         0,
+    0.8784,         0,         0,
+    0.3765,    0.5020,         0,
+    0.8784,    0.5020,         0,
+    0.3765,         0,    0.5020,
+    0.8784,         0,    0.5020,
+    0.3765,    0.5020,    0.5020,
+    0.8784,    0.5020,    0.5020,
+    0.1255,    0.2510,         0,
+    0.6275,    0.2510,         0,
+    0.1255,    0.7529,         0,
+    0.6275,    0.7529,         0,
+    0.1255,    0.2510,    0.5020,
+    0.6275,    0.2510,    0.5020,
+    0.1255,    0.7529,    0.5020,
+    0.6275,    0.7529,    0.5020,
+    0.3765,    0.2510,         0,
+    0.8784,    0.2510,         0,
+    0.3765,    0.7529,         0,
+    0.8784,    0.7529,         0,
+    0.3765,    0.2510,    0.5020,
+    0.8784,    0.2510,    0.5020,
+    0.3765,    0.7529,    0.5020,
+    0.8784,    0.7529,    0.5020,
+    0.1255,         0,    0.2510,
+    0.6275,         0,    0.2510,
+    0.1255,    0.5020,    0.2510,
+    0.6275,    0.5020,    0.2510,
+    0.1255,         0,    0.7529,
+    0.6275,         0,    0.7529,
+    0.1255,    0.5020,    0.7529,
+    0.6275,    0.5020,    0.7529,
+    0.3765,         0,    0.2510,
+    0.8784,         0,    0.2510,
+    0.3765,    0.5020,    0.2510,
+    0.8784,    0.5020,    0.2510,
+    0.3765,         0,    0.7529,
+    0.8784,         0,    0.7529,
+    0.3765,    0.5020,    0.7529,
+    0.8784,    0.5020,    0.7529,
+    0.1255,    0.2510,    0.2510,
+    0.6275,    0.2510,    0.2510,
+    0.1255,    0.7529,    0.2510,
+    0.6275,    0.7529,    0.2510,
+    0.1255,    0.2510,    0.7529,
+    0.6275,    0.2510,    0.7529,
+    0.1255,    0.7529,    0.7529,
+    0.6275,    0.7529,    0.7529,
+    0.3765,    0.2510,    0.2510,
+    0.8784,    0.2510,    0.2510,
+    0.3765,    0.7529,    0.2510,
+    0.8784,    0.7529,    0.2510,
+    0.3765,    0.2510,    0.7529,
+    0.8784,    0.2510,    0.7529,
+    0.3765,    0.7529,    0.7529,
+    0.8784,    0.7529,    0.7529,
+         0,    0.1255,         0,
+    0.5020,    0.1255,         0,
+         0,    0.6275,         0,
+    0.5020,    0.6275,         0,
+         0,    0.1255,    0.5020,
+    0.5020,    0.1255,    0.5020,
+         0,    0.6275,    0.5020,
+    0.5020,    0.6275,    0.5020,
+    0.2510,    0.1255,         0,
+    0.7529,    0.1255,         0,
+    0.2510,    0.6275,         0,
+    0.7529,    0.6275,         0,
+    0.2510,    0.1255,    0.5020,
+    0.7529,    0.1255,    0.5020,
+    0.2510,    0.6275,    0.5020,
+    0.7529,    0.6275,    0.5020,
+         0,    0.3765,         0,
+    0.5020,    0.3765,         0,
+         0,    0.8784,         0,
+    0.5020,    0.8784,         0,
+         0,    0.3765,    0.5020,
+    0.5020,    0.3765,    0.5020,
+         0,    0.8784,    0.5020,
+    0.5020,    0.8784,    0.5020,
+    0.2510,    0.3765,         0,
+    0.7529,    0.3765,         0,
+    0.2510,    0.8784,         0,
+    0.7529,    0.8784,         0,
+    0.2510,    0.3765,    0.5020,
+    0.7529,    0.3765,    0.5020,
+    0.2510,    0.8784,    0.5020,
+    0.7529,    0.8784,    0.5020,
+         0,    0.1255,    0.2510,
+    0.5020,    0.1255,    0.2510,
+         0,    0.6275,    0.2510,
+    0.5020,    0.6275,    0.2510,
+         0,    0.1255,    0.7529,
+    0.5020,    0.1255,    0.7529,
+         0,    0.6275,    0.7529,
+    0.5020,    0.6275,    0.7529,
+    0.2510,    0.1255,    0.2510,
+    0.7529,    0.1255,    0.2510,
+    0.2510,    0.6275,    0.2510,
+    0.7529,    0.6275,    0.2510,
+    0.2510,    0.1255,    0.7529,
+    0.7529,    0.1255,    0.7529,
+    0.2510,    0.6275,    0.7529,
+    0.7529,    0.6275,    0.7529,
+         0,    0.3765,    0.2510,
+    0.5020,    0.3765,    0.2510,
+         0,    0.8784,    0.2510,
+    0.5020,    0.8784,    0.2510,
+         0,    0.3765,    0.7529,
+    0.5020,    0.3765,    0.7529,
+         0,    0.8784,    0.7529,
+    0.5020,    0.8784,    0.7529,
+    0.2510,    0.3765,    0.2510,
+    0.7529,    0.3765,    0.2510,
+    0.2510,    0.8784,    0.2510,
+    0.7529,    0.8784,    0.2510,
+    0.2510,    0.3765,    0.7529,
+    0.7529,    0.3765,    0.7529,
+    0.2510,    0.8784,    0.7529,
+    0.7529,    0.8784,    0.7529,
+    0.1255,    0.1255,         0,
+    0.6275,    0.1255,         0,
+    0.1255,    0.6275,         0,
+    0.6275,    0.6275,         0,
+    0.1255,    0.1255,    0.5020,
+    0.6275,    0.1255,    0.5020,
+    0.1255,    0.6275,    0.5020,
+    0.6275,    0.6275,    0.5020,
+    0.3765,    0.1255,         0,
+    0.8784,    0.1255,         0,
+    0.3765,    0.6275,         0,
+    0.8784,    0.6275,         0,
+    0.3765,    0.1255,    0.5020,
+    0.8784,    0.1255,    0.5020,
+    0.3765,    0.6275,    0.5020,
+    0.8784,    0.6275,    0.5020,
+    0.1255,    0.3765,         0,
+    0.6275,    0.3765,         0,
+    0.1255,    0.8784,         0,
+    0.6275,    0.8784,         0,
+    0.1255,    0.3765,    0.5020,
+    0.6275,    0.3765,    0.5020,
+    0.1255,    0.8784,    0.5020,
+    0.6275,    0.8784,    0.5020,
+    0.3765,    0.3765,         0,
+    0.8784,    0.3765,         0,
+    0.3765,    0.8784,         0,
+    0.8784,    0.8784,         0,
+    0.3765,    0.3765,    0.5020,
+    0.8784,    0.3765,    0.5020,
+    0.3765,    0.8784,    0.5020,
+    0.8784,    0.8784,    0.5020,
+    0.1255,    0.1255,    0.2510,
+    0.6275,    0.1255,    0.2510,
+    0.1255,    0.6275,    0.2510,
+    0.6275,    0.6275,    0.2510,
+    0.1255,    0.1255,    0.7529,
+    0.6275,    0.1255,    0.7529,
+    0.1255,    0.6275,    0.7529,
+    0.6275,    0.6275,    0.7529,
+    0.3765,    0.1255,    0.2510,
+    0.8784,    0.1255,    0.2510,
+    0.3765,    0.6275,    0.2510,
+    0.8784,    0.6275,    0.2510,
+    0.3765,    0.1255,    0.7529,
+    0.8784,    0.1255,    0.7529,
+    0.3765,    0.6275,    0.7529,
+    0.8784,    0.6275,    0.7529,
+    0.1255,    0.3765,    0.2510,
+    0.6275,    0.3765,    0.2510,
+    0.1255,    0.8784,    0.2510,
+    0.6275,    0.8784,    0.2510,
+    0.1255,    0.3765,    0.7529,
+    0.6275,    0.3765,    0.7529,
+    0.1255,    0.8784,    0.7529,
+    0.6275,    0.8784,    0.7529,
+    0.3765,    0.3765,    0.2510,
+    0.8784,    0.3765,    0.2510,
+    0.3765,    0.8784,    0.2510,
+    0.8784,    0.8784,    0.2510,
+    0.3765,    0.3765,    0.7529,
+    0.8784,    0.3765,    0.7529,
+    0.3765,    0.8784,    0.7529,
+    0.8784,    0.8784,    0.7529]

yolov7-tracker-example/tracker/trackeval/baselines/stp.py

@@ -0,0 +1,144 @@
+"""
+STP: Simplest Tracker Possible
+
+Author: Jonathon Luiten
+
+This simple tracker, simply assigns track IDs which maximise the 'bounding box IoU' between previous tracks and current
+detections. It is also able to match detections to tracks at more than one timestep previously.
+"""
+
+import os
+import sys
+import numpy as np
+from multiprocessing.pool import Pool
+from multiprocessing import freeze_support
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from trackeval.baselines import baseline_utils as butils
+from trackeval.utils import get_code_path
+
+code_path = get_code_path()
+config = {
+    'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'),
+    'OUTPUT_FOL': os.path.join(code_path, 'data/trackers/rob_mots/{split}/STP/data/'),
+    'SPLIT': 'train',  # valid: 'train', 'val', 'test'.
+    'Benchmarks': None,  # If None, all benchmarks in SPLIT.
+
+    'Num_Parallel_Cores': None,  # If None, run without parallel.
+
+    'DETECTION_THRESHOLD': 0.5,
+    'ASSOCIATION_THRESHOLD': 1e-10,
+    'MAX_FRAMES_SKIP': 7
+}
+
+
+def track_sequence(seq_file):
+
+    # Load input data from file (e.g. provided detections)
+    # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    data = butils.load_seq(seq_file)
+
+    # Where to accumulate output data for writing out
+    output_data = []
+
+    # To ensure IDs are unique per object across all classes.
+    curr_max_id = 0
+
+    # Run tracker for each class.
+    for cls, cls_data in data.items():
+
+        # Initialize container for holding previously tracked objects.
+        prev = {'boxes': np.empty((0, 4)),
+                'ids': np.array([], np.int),
+                'timesteps': np.array([])}
+
+        # Run tracker for each timestep.
+        for timestep, t_data in enumerate(cls_data):
+
+            # Threshold detections.
+            t_data = butils.threshold(t_data, config['DETECTION_THRESHOLD'])
+
+            # Convert mask dets to bounding boxes.
+            boxes = butils.masks2boxes(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws'])
+
+            # Calculate IoU between previous and current frame dets.
+            ious = butils.box_iou(prev['boxes'], boxes)
+
+            # Score which decreases quickly for previous dets depending on how many timesteps before they come from.
+            prev_timestep_scores = np.power(10, -1 * prev['timesteps'])
+
+            # Matching score is such that it first tries to match 'most recent timesteps',
+            # and within each timestep maximised IoU.
+            match_scores = prev_timestep_scores[:, np.newaxis] * ious
+
+            # Find best matching between current dets and previous tracks.
+            match_rows, match_cols = butils.match(match_scores)
+
+            # Remove matches that have an IoU below a certain threshold.
+            actually_matched_mask = ious[match_rows, match_cols] > config['ASSOCIATION_THRESHOLD']
+            match_rows = match_rows[actually_matched_mask]
+            match_cols = match_cols[actually_matched_mask]
+
+            # Assign the prev track ID to the current dets if they were matched.
+            ids = np.nan * np.ones((len(boxes),), np.int)
+            ids[match_cols] = prev['ids'][match_rows]
+
+            # Create new track IDs for dets that were not matched to previous tracks.
+            num_not_matched = len(ids) - len(match_cols)
+            new_ids = np.arange(curr_max_id + 1, curr_max_id + num_not_matched + 1)
+            ids[np.isnan(ids)] = new_ids
+
+            # Update maximum ID to ensure future added tracks have a unique ID value.
+            curr_max_id += num_not_matched
+
+            # Drop tracks from 'previous tracks' if they have not been matched in the last MAX_FRAMES_SKIP frames.
+            unmatched_rows = [i for i in range(len(prev['ids'])) if
+                              i not in match_rows and (prev['timesteps'][i] + 1 <= config['MAX_FRAMES_SKIP'])]
+
+            # Update the set of previous tracking results to include the newly tracked detections.
+            prev['ids'] = np.concatenate((ids, prev['ids'][unmatched_rows]), axis=0)
+            prev['boxes'] = np.concatenate((np.atleast_2d(boxes), np.atleast_2d(prev['boxes'][unmatched_rows])), axis=0)
+            prev['timesteps'] = np.concatenate((np.zeros((len(ids),)), prev['timesteps'][unmatched_rows] + 1), axis=0)
+
+            # Save result in output format to write to file later.
+            # Output Format = [timestep ID class score im_h im_w mask_RLE]
+            for i in range(len(t_data['ids'])):
+                row = [timestep, int(ids[i]), cls, t_data['scores'][i], t_data['im_hs'][i], t_data['im_ws'][i],
+                       t_data['mask_rles'][i]]
+                output_data.append(row)
+
+    # Write results to file
+    out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']),
+                                config['OUTPUT_FOL'].format(split=config['SPLIT']))
+    butils.write_seq(output_data, out_file)
+
+    print('DONE:', seq_file)
+
+
+if __name__ == '__main__':
+
+    # Required to fix bug in multiprocessing on windows.
+    freeze_support()
+
+    # Obtain list of sequences to run tracker for.
+    if config['Benchmarks']:
+        benchmarks = config['Benchmarks']
+    else:
+        benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao']
+        if config['SPLIT'] != 'train':
+            benchmarks += ['waymo', 'mots_challenge']
+    seqs_todo = []
+    for bench in benchmarks:
+        bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench)
+        seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)]
+
+    # Run in parallel
+    if config['Num_Parallel_Cores']:
+        with Pool(config['Num_Parallel_Cores']) as pool:
+            results = pool.map(track_sequence, seqs_todo)
+
+    # Run in series
+    else:
+        for seq_todo in seqs_todo:
+            track_sequence(seq_todo)
+

yolov7-tracker-example/tracker/trackeval/baselines/thresholder.py

@@ -0,0 +1,92 @@
+"""
+Thresholder
+
+Author: Jonathon Luiten
+
+Simply reads in a set of detection, thresholds them at a certain score threshold, and writes them out again.
+"""
+
+import os
+import sys
+from multiprocessing.pool import Pool
+from multiprocessing import freeze_support
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from trackeval.baselines import baseline_utils as butils
+from trackeval.utils import get_code_path
+
+THRESHOLD = 0.2
+
+code_path = get_code_path()
+config = {
+    'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'),
+    'OUTPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/threshold_' + str(100*THRESHOLD) + '/data/'),
+    'SPLIT': 'train',  # valid: 'train', 'val', 'test'.
+    'Benchmarks': None,  # If None, all benchmarks in SPLIT.
+
+    'Num_Parallel_Cores': None,  # If None, run without parallel.
+
+    'DETECTION_THRESHOLD': THRESHOLD,
+}
+
+
+def do_sequence(seq_file):
+
+    # Load input data from file (e.g. provided detections)
+    # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    data = butils.load_seq(seq_file)
+
+    # Where to accumulate output data for writing out
+    output_data = []
+
+    # Run for each class.
+    for cls, cls_data in data.items():
+
+        # Run for each timestep.
+        for timestep, t_data in enumerate(cls_data):
+
+            # Threshold detections.
+            t_data = butils.threshold(t_data, config['DETECTION_THRESHOLD'])
+
+            # Save result in output format to write to file later.
+            # Output Format = [timestep ID class score im_h im_w mask_RLE]
+            for i in range(len(t_data['ids'])):
+                row = [timestep, int(t_data['ids'][i]), cls, t_data['scores'][i], t_data['im_hs'][i],
+                       t_data['im_ws'][i], t_data['mask_rles'][i]]
+                output_data.append(row)
+
+    # Write results to file
+    out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']),
+                                config['OUTPUT_FOL'].format(split=config['SPLIT']))
+    butils.write_seq(output_data, out_file)
+
+    print('DONE:', seq_todo)
+
+
+if __name__ == '__main__':
+
+    # Required to fix bug in multiprocessing on windows.
+    freeze_support()
+
+    # Obtain list of sequences to run tracker for.
+    if config['Benchmarks']:
+        benchmarks = config['Benchmarks']
+    else:
+        benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao']
+        if config['SPLIT'] != 'train':
+            benchmarks += ['waymo', 'mots_challenge']
+    seqs_todo = []
+    for bench in benchmarks:
+        bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench)
+        seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)]
+
+    # Run in parallel
+    if config['Num_Parallel_Cores']:
+        with Pool(config['Num_Parallel_Cores']) as pool:
+            results = pool.map(do_sequence, seqs_todo)
+
+    # Run in series
+    else:
+        for seq_todo in seqs_todo:
+            do_sequence(seq_todo)
+

yolov7-tracker-example/tracker/trackeval/baselines/vizualize.py

@@ -0,0 +1,94 @@
+"""
+Vizualize: Code which converts .txt rle tracking results into a visual .png format.
+
+Author: Jonathon Luiten
+"""
+
+import os
+import sys
+from multiprocessing.pool import Pool
+from multiprocessing import freeze_support
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from trackeval.baselines import baseline_utils as butils
+from trackeval.utils import get_code_path
+from trackeval.datasets.rob_mots_classmap import cls_id_to_name
+
+code_path = get_code_path()
+config = {
+    # Tracker format:
+    'INPUT_FOL': os.path.join(code_path, 'data/trackers/rob_mots/{split}/STP/data/{bench}'),
+    'OUTPUT_FOL': os.path.join(code_path, 'data/viz/rob_mots/{split}/STP/data/{bench}'),
+    # GT format:
+    # 'INPUT_FOL': os.path.join(code_path, 'data/gt/rob_mots/{split}/{bench}/data/'),
+    # 'OUTPUT_FOL': os.path.join(code_path, 'data/gt_viz/rob_mots/{split}/{bench}/'),
+    'SPLIT': 'train',  # valid: 'train', 'val', 'test'.
+    'Benchmarks': None,  # If None, all benchmarks in SPLIT.
+    'Num_Parallel_Cores': None,  # If None, run without parallel.
+}
+
+
+def do_sequence(seq_file):
+    # Folder to save resulting visualization in
+    out_fol = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT'], bench=bench),
+                               config['OUTPUT_FOL'].format(split=config['SPLIT'], bench=bench)).replace('.txt', '')
+
+    # Load input data from file (e.g. provided detections)
+    # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'}
+    data = butils.load_seq(seq_file)
+
+    # Get frame size for visualizing empty frames
+    im_h, im_w = butils.get_frame_size(data)
+
+    # First run for each class.
+    for cls, cls_data in data.items():
+
+        if cls >= 100:
+            continue
+
+        # Run for each timestep.
+        for timestep, t_data in enumerate(cls_data):
+            # Save out visualization
+            out_file = os.path.join(out_fol, cls_id_to_name[cls], str(timestep).zfill(5) + '.png')
+            butils.save_as_png(t_data, out_file, im_h, im_w)
+
+
+    # Then run for all classes combined
+    # Converts data from a class-separated to a class-combined format.
+    data = butils.combine_classes(data)
+
+    # Run for each timestep.
+    for timestep, t_data in enumerate(data):
+        # Save out visualization
+        out_file = os.path.join(out_fol, 'all_classes', str(timestep).zfill(5) + '.png')
+        butils.save_as_png(t_data, out_file, im_h, im_w)
+
+    print('DONE:', seq_file)
+
+
+if __name__ == '__main__':
+
+    # Required to fix bug in multiprocessing on windows.
+    freeze_support()
+
+    # Obtain list of sequences to run tracker for.
+    if config['Benchmarks']:
+        benchmarks = config['Benchmarks']
+    else:
+        benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao']
+        if config['SPLIT'] != 'train':
+            benchmarks += ['waymo', 'mots_challenge']
+    seqs_todo = []
+    for bench in benchmarks:
+        bench_fol = config['INPUT_FOL'].format(split=config['SPLIT'], bench=bench)
+        seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)]
+
+    # Run in parallel
+    if config['Num_Parallel_Cores']:
+        with Pool(config['Num_Parallel_Cores']) as pool:
+            results = pool.map(do_sequence, seqs_todo)
+
+    # Run in series
+    else:
+        for seq_todo in seqs_todo:
+            do_sequence(seq_todo)

yolov7-tracker-example/tracker/trackeval/datasets/__init__.py

@@ -0,0 +1,15 @@
+from .kitti_2d_box import Kitti2DBox
+from .kitti_mots import KittiMOTS
+from .mot_challenge_2d_box import MotChallenge2DBox
+from .mots_challenge import MOTSChallenge
+from .bdd100k import BDD100K
+from .davis import DAVIS
+from .tao import TAO
+from .tao_ow import TAO_OW
+from .burst import BURST
+from .burst_ow import BURST_OW
+from .youtube_vis import YouTubeVIS
+from .head_tracking_challenge import HeadTrackingChallenge
+from .rob_mots import RobMOTS
+from .person_path_22 import PersonPath22
+from .visdrone import VisDrone2DBox

yolov7-tracker-example/tracker/trackeval/datasets/_base_dataset.py

@@ -0,0 +1,326 @@
+import csv
+import io
+import zipfile
+import os
+import traceback
+import numpy as np
+from copy import deepcopy
+from abc import ABC, abstractmethod
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class _BaseDataset(ABC):
+    @abstractmethod
+    def __init__(self):
+        self.tracker_list = None
+        self.seq_list = None
+        self.class_list = None
+        self.output_fol = None
+        self.output_sub_fol = None
+        self.should_classes_combine = True
+        self.use_super_categories = False
+
+    # Functions to implement:
+
+    @staticmethod
+    @abstractmethod
+    def get_default_dataset_config():
+        ...
+
+    @abstractmethod
+    def _load_raw_file(self, tracker, seq, is_gt):
+        ...
+
+    @_timing.time
+    @abstractmethod
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        ...
+
+    @abstractmethod
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        ...
+
+    # Helper functions for all datasets:
+
+    @classmethod
+    def get_class_name(cls):
+        return cls.__name__
+
+    def get_name(self):
+        return self.get_class_name()
+
+    def get_output_fol(self, tracker):
+        return os.path.join(self.output_fol, tracker, self.output_sub_fol)
+
+    def get_display_name(self, tracker):
+        """ Can be overwritten if the trackers name (in files) is different to how it should be displayed.
+        By default this method just returns the trackers name as is.
+        """
+        return tracker
+
+    def get_eval_info(self):
+        """Return info about the dataset needed for the Evaluator"""
+        return self.tracker_list, self.seq_list, self.class_list
+
+    @_timing.time
+    def get_raw_seq_data(self, tracker, seq):
+        """ Loads raw data (tracker and ground-truth) for a single tracker on a single sequence.
+        Raw data includes all of the information needed for both preprocessing and evaluation, for all classes.
+        A later function (get_processed_seq_data) will perform such preprocessing and extract relevant information for
+        the evaluation of each class.
+
+        This returns a dict which contains the fields:
+        [num_timesteps]: integer
+        [gt_ids, tracker_ids, gt_classes, tracker_classes, tracker_confidences]:
+                                                                list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, tracker_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        [gt_extras]: dict (for each extra) of lists (for each timestep) of 1D NDArrays (for each det).
+
+        gt_extras contains dataset specific information used for preprocessing such as occlusion and truncation levels.
+
+        Note that similarities are extracted as part of the dataset and not the metric, because almost all metrics are
+        independent of the exact method of calculating the similarity. However datasets are not (e.g. segmentation
+        masks vs 2D boxes vs 3D boxes).
+        We calculate the similarity before preprocessing because often both preprocessing and evaluation require it and
+        we don't wish to calculate this twice.
+        We calculate similarity between all gt and tracker classes (not just each class individually) to allow for
+        calculation of metrics such as class confusion matrices. Typically the impact of this on performance is low.
+        """
+        # Load raw data.
+        raw_gt_data = self._load_raw_file(tracker, seq, is_gt=True)
+        raw_tracker_data = self._load_raw_file(tracker, seq, is_gt=False)
+        raw_data = {**raw_tracker_data, **raw_gt_data}  # Merges dictionaries
+
+        # Calculate similarities for each timestep.
+        similarity_scores = []
+        for t, (gt_dets_t, tracker_dets_t) in enumerate(zip(raw_data['gt_dets'], raw_data['tracker_dets'])):
+            ious = self._calculate_similarities(gt_dets_t, tracker_dets_t)
+            similarity_scores.append(ious)
+        raw_data['similarity_scores'] = similarity_scores
+        return raw_data
+
+    @staticmethod
+    def _load_simple_text_file(file, time_col=0, id_col=None, remove_negative_ids=False, valid_filter=None,
+                               crowd_ignore_filter=None, convert_filter=None, is_zipped=False, zip_file=None,
+                               force_delimiters=None):
+        """ Function that loads data which is in a commonly used text file format.
+        Assumes each det is given by one row of a text file.
+        There is no limit to the number or meaning of each column,
+        however one column needs to give the timestep of each det (time_col) which is default col 0.
+
+        The file dialect (deliminator, num cols, etc) is determined automatically.
+        This function automatically separates dets by timestep,
+        and is much faster than alternatives such as np.loadtext or pandas.
+
+        If remove_negative_ids is True and id_col is not None, dets with negative values in id_col are excluded.
+        These are not excluded from ignore data.
+
+        valid_filter can be used to only include certain classes.
+        It is a dict with ints as keys, and lists as values,
+        such that a row is included if "row[key].lower() is in value" for all key/value pairs in the dict.
+        If None, all classes are included.
+
+        crowd_ignore_filter can be used to read crowd_ignore regions separately. It has the same format as valid filter.
+
+        convert_filter can be used to convert value read to another format.
+        This is used most commonly to convert classes given as string to a class id.
+        This is a dict such that the key is the column to convert, and the value is another dict giving the mapping.
+
+        Optionally, input files could be a zip of multiple text files for storage efficiency.
+
+        Returns read_data and ignore_data.
+        Each is a dict (with keys as timesteps as strings) of lists (over dets) of lists (over column values).
+        Note that all data is returned as strings, and must be converted to float/int later if needed.
+        Note that timesteps will not be present in the returned dict keys if there are no dets for them
+        """
+
+        if remove_negative_ids and id_col is None:
+            raise TrackEvalException('remove_negative_ids is True, but id_col is not given.')
+        if crowd_ignore_filter is None:
+            crowd_ignore_filter = {}
+        if convert_filter is None:
+            convert_filter = {}
+        try:
+            if is_zipped:  # Either open file directly or within a zip.
+                if zip_file is None:
+                    raise TrackEvalException('is_zipped set to True, but no zip_file is given.')
+                archive = zipfile.ZipFile(os.path.join(zip_file), 'r')
+                fp = io.TextIOWrapper(archive.open(file, 'r'))
+            else:
+                fp = open(file)
+            read_data = {}
+            crowd_ignore_data = {}
+            fp.seek(0, os.SEEK_END)
+            # check if file is empty
+            if fp.tell():
+                fp.seek(0)
+                dialect = csv.Sniffer().sniff(fp.readline(), delimiters=force_delimiters)  # Auto determine structure.
+                dialect.skipinitialspace = True  # Deal with extra spaces between columns
+                fp.seek(0)
+                reader = csv.reader(fp, dialect)
+                for row in reader:
+                    try:
+                        # Deal with extra trailing spaces at the end of rows
+                        if row[-1] in '':
+                            row = row[:-1]
+                        timestep = str(int(float(row[time_col])))
+                        # Read ignore regions separately.
+                        is_ignored = False
+                        for ignore_key, ignore_value in crowd_ignore_filter.items():
+                            if row[ignore_key].lower() in ignore_value:
+                                # Convert values in one column (e.g. string to id)
+                                for convert_key, convert_value in convert_filter.items():
+                                    row[convert_key] = convert_value[row[convert_key].lower()]
+                                # Save data separated by timestep.
+                                if timestep in crowd_ignore_data.keys():
+                                    crowd_ignore_data[timestep].append(row)
+                                else:
+                                    crowd_ignore_data[timestep] = [row]
+                                is_ignored = True
+                        if is_ignored:  # if det is an ignore region, it cannot be a normal det.
+                            continue
+                        # Exclude some dets if not valid.
+                        if valid_filter is not None:
+                            for key, value in valid_filter.items():
+                                if row[key].lower() not in value:
+                                    continue
+                        if remove_negative_ids:
+                            if int(float(row[id_col])) < 0:
+                                continue
+                        # Convert values in one column (e.g. string to id)
+                        for convert_key, convert_value in convert_filter.items():
+                            row[convert_key] = convert_value[row[convert_key].lower()]
+                        # Save data separated by timestep.
+                        if timestep in read_data.keys():
+                            read_data[timestep].append(row)
+                        else:
+                            read_data[timestep] = [row]
+                    except Exception:
+                        exc_str_init = 'In file %s the following line cannot be read correctly: \n' % os.path.basename(
+                            file)
+                        exc_str = ' '.join([exc_str_init]+row)
+                        raise TrackEvalException(exc_str)
+            fp.close()
+        except Exception:
+            print('Error loading file: %s, printing traceback.' % file)
+            traceback.print_exc()
+            raise TrackEvalException(
+                'File %s cannot be read because it is either not present or invalidly formatted' % os.path.basename(
+                    file))
+        return read_data, crowd_ignore_data
+
+    @staticmethod
+    def _calculate_mask_ious(masks1, masks2, is_encoded=False, do_ioa=False):
+        """ Calculates the IOU (intersection over union) between two arrays of segmentation masks.
+        If is_encoded a run length encoding with pycocotools is assumed as input format, otherwise an input of numpy
+        arrays of the shape (num_masks, height, width) is assumed and the encoding is performed.
+        If do_ioa (intersection over area) , then calculates the intersection over the area of masks1 - this is commonly
+        used to determine if detections are within crowd ignore region.
+        :param masks1:  first set of masks (numpy array of shape (num_masks, height, width) if not encoded,
+                        else pycocotools rle encoded format)
+        :param masks2:  second set of masks (numpy array of shape (num_masks, height, width) if not encoded,
+                        else pycocotools rle encoded format)
+        :param is_encoded: whether the input is in pycocotools rle encoded format
+        :param do_ioa: whether to perform IoA computation
+        :return: the IoU/IoA scores
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        # use pycocotools for run length encoding of masks
+        if not is_encoded:
+            masks1 = mask_utils.encode(np.array(np.transpose(masks1, (1, 2, 0)), order='F'))
+            masks2 = mask_utils.encode(np.array(np.transpose(masks2, (1, 2, 0)), order='F'))
+
+        # use pycocotools for iou computation of rle encoded masks
+        ious = mask_utils.iou(masks1, masks2, [do_ioa]*len(masks2))
+        if len(masks1) == 0 or len(masks2) == 0:
+            ious = np.asarray(ious).reshape(len(masks1), len(masks2))
+        assert (ious >= 0 - np.finfo('float').eps).all()
+        assert (ious <= 1 + np.finfo('float').eps).all()
+
+        return ious
+
+    @staticmethod
+    def _calculate_box_ious(bboxes1, bboxes2, box_format='xywh', do_ioa=False):
+        """ Calculates the IOU (intersection over union) between two arrays of boxes.
+        Allows variable box formats ('xywh' and 'x0y0x1y1').
+        If do_ioa (intersection over area) , then calculates the intersection over the area of boxes1 - this is commonly
+        used to determine if detections are within crowd ignore region.
+        """
+        if box_format in 'xywh':
+            # layout: (x0, y0, w, h)
+            bboxes1 = deepcopy(bboxes1)
+            bboxes2 = deepcopy(bboxes2)
+
+            bboxes1[:, 2] = bboxes1[:, 0] + bboxes1[:, 2]
+            bboxes1[:, 3] = bboxes1[:, 1] + bboxes1[:, 3]
+            bboxes2[:, 2] = bboxes2[:, 0] + bboxes2[:, 2]
+            bboxes2[:, 3] = bboxes2[:, 1] + bboxes2[:, 3]
+        elif box_format not in 'x0y0x1y1':
+            raise (TrackEvalException('box_format %s is not implemented' % box_format))
+
+        # layout: (x0, y0, x1, y1)
+        min_ = np.minimum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
+        max_ = np.maximum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
+        intersection = np.maximum(min_[..., 2] - max_[..., 0], 0) * np.maximum(min_[..., 3] - max_[..., 1], 0)
+        area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1])
+
+        if do_ioa:
+            ioas = np.zeros_like(intersection)
+            valid_mask = area1 > 0 + np.finfo('float').eps
+            ioas[valid_mask, :] = intersection[valid_mask, :] / area1[valid_mask][:, np.newaxis]
+
+            return ioas
+        else:
+            area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1])
+            union = area1[:, np.newaxis] + area2[np.newaxis, :] - intersection
+            intersection[area1 <= 0 + np.finfo('float').eps, :] = 0
+            intersection[:, area2 <= 0 + np.finfo('float').eps] = 0
+            intersection[union <= 0 + np.finfo('float').eps] = 0
+            union[union <= 0 + np.finfo('float').eps] = 1
+            ious = intersection / union
+            return ious
+
+    @staticmethod
+    def _calculate_euclidean_similarity(dets1, dets2, zero_distance=2.0):
+        """ Calculates the euclidean distance between two sets of detections, and then converts this into a similarity
+        measure with values between 0 and 1 using the following formula: sim = max(0, 1 - dist/zero_distance).
+        The default zero_distance of 2.0, corresponds to the default used in MOT15_3D, such that a 0.5 similarity
+        threshold corresponds to a 1m distance threshold for TPs.
+        """
+        dist = np.linalg.norm(dets1[:, np.newaxis]-dets2[np.newaxis, :], axis=2)
+        sim = np.maximum(0, 1 - dist/zero_distance)
+        return sim
+
+    @staticmethod
+    def _check_unique_ids(data, after_preproc=False):
+        """Check the requirement that the tracker_ids and gt_ids are unique per timestep"""
+        gt_ids = data['gt_ids']
+        tracker_ids = data['tracker_ids']
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(gt_ids, tracker_ids)):
+            if len(tracker_ids_t) > 0:
+                unique_ids, counts = np.unique(tracker_ids_t, return_counts=True)
+                if np.max(counts) != 1:
+                    duplicate_ids = unique_ids[counts > 1]
+                    exc_str_init = 'Tracker predicts the same ID more than once in a single timestep ' \
+                                   '(seq: %s, frame: %i, ids:' % (data['seq'], t+1)
+                    exc_str = ' '.join([exc_str_init] + [str(d) for d in duplicate_ids]) + ')'
+                    if after_preproc:
+                        exc_str_init += '\n Note that this error occurred after preprocessing (but not before), ' \
+                                        'so ids may not be as in file, and something seems wrong with preproc.'
+                    raise TrackEvalException(exc_str)
+            if len(gt_ids_t) > 0:
+                unique_ids, counts = np.unique(gt_ids_t, return_counts=True)
+                if np.max(counts) != 1:
+                    duplicate_ids = unique_ids[counts > 1]
+                    exc_str_init = 'Ground-truth has the same ID more than once in a single timestep ' \
+                                   '(seq: %s, frame: %i, ids:' % (data['seq'], t+1)
+                    exc_str = ' '.join([exc_str_init] + [str(d) for d in duplicate_ids]) + ')'
+                    if after_preproc:
+                        exc_str_init += '\n Note that this error occurred after preprocessing (but not before), ' \
+                                        'so ids may not be as in file, and something seems wrong with preproc.'
+                    raise TrackEvalException(exc_str)

yolov7-tracker-example/tracker/trackeval/datasets/bdd100k.py

@@ -0,0 +1,302 @@
+
+import os
+import json
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ..utils import TrackEvalException
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+
+
+class BDD100K(_BaseDataset):
+    """Dataset class for BDD100K tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/bdd100k/bdd100k_val'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/bdd100k/bdd100k_val'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle'],
+            # Valid: ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle']
+            'SPLIT_TO_EVAL': 'val',  # Valid: 'training', 'val',
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = True
+        self.use_super_categories = True
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only classes [pedestrian, rider, car, '
+                                     'bus, truck, train, motorcycle, bicycle] are valid.')
+        self.super_categories = {"HUMAN": [cls for cls in ["pedestrian", "rider"] if cls in self.class_list],
+                                 "VEHICLE": [cls for cls in ["car", "truck", "bus", "train"] if cls in self.class_list],
+                                 "BIKE": [cls for cls in ["motorcycle", "bicycle"] if cls in self.class_list]}
+        self.distractor_classes = ['other person', 'trailer', 'other vehicle']
+        self.class_name_to_class_id = {'pedestrian': 1, 'rider': 2, 'other person': 3, 'car': 4, 'bus': 5, 'truck': 6,
+                                       'train': 7, 'trailer': 8, 'other vehicle': 9, 'motorcycle': 10, 'bicycle': 11}
+
+        # Get sequences to eval
+        self.seq_list = []
+        self.seq_lengths = {}
+
+        self.seq_list = [seq_file.replace('.json', '') for seq_file in os.listdir(self.gt_fol)]
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            for seq in self.seq_list:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.json')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException(
+                        'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                            curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the BDD100K format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if is_gt:
+            file = os.path.join(self.gt_fol, seq + '.json')
+        else:
+            file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.json')
+
+        with open(file) as f:
+            data = json.load(f)
+
+        # sort data by frame index
+        data = sorted(data, key=lambda x: x['index'])
+
+        # check sequence length
+        if is_gt:
+            self.seq_lengths[seq] = len(data)
+            num_timesteps = len(data)
+        else:
+            num_timesteps = self.seq_lengths[seq]
+            if num_timesteps != len(data):
+                raise TrackEvalException('Number of ground truth and tracker timesteps do not match for sequence %s'
+                                         % seq)
+
+        # Convert data to required format
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for t in range(num_timesteps):
+            ig_ids = []
+            keep_ids = []
+            for i in range(len(data[t]['labels'])):
+                ann = data[t]['labels'][i]
+                if is_gt and (ann['category'] in self.distractor_classes or 'attributes' in ann.keys()
+                              and ann['attributes']['Crowd']):
+                    ig_ids.append(i)
+                else:
+                    keep_ids.append(i)
+
+            if keep_ids:
+                raw_data['dets'][t] = np.atleast_2d([[data[t]['labels'][i]['box2d']['x1'],
+                                                      data[t]['labels'][i]['box2d']['y1'],
+                                                      data[t]['labels'][i]['box2d']['x2'],
+                                                      data[t]['labels'][i]['box2d']['y2']
+                                                      ] for i in keep_ids]).astype(float)
+                raw_data['ids'][t] = np.atleast_1d([data[t]['labels'][i]['id'] for i in keep_ids]).astype(int)
+                raw_data['classes'][t] = np.atleast_1d([self.class_name_to_class_id[data[t]['labels'][i]['category']]
+                                                        for i in keep_ids]).astype(int)
+            else:
+                raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+
+            if is_gt:
+                if ig_ids:
+                    raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d([[data[t]['labels'][i]['box2d']['x1'],
+                                                                             data[t]['labels'][i]['box2d']['y1'],
+                                                                             data[t]['labels'][i]['box2d']['x2'],
+                                                                             data[t]['labels'][i]['box2d']['y2']
+                                                                             ] for i in ig_ids]).astype(float)
+                else:
+                    raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)).astype(float)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        BDD100K:
+            In BDD100K, the 4 preproc steps are as follow:
+                1) There are eight classes (pedestrian, rider, car, bus, truck, train, motorcycle, bicycle)
+                    which are evaluated separately.
+                2) For BDD100K there is no removal of matched tracker dets.
+                3) Crowd ignore regions are used to remove unmatched detections.
+                4) No removal of gt dets.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm)
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region.
+            unmatched_tracker_dets = tracker_dets[unmatched_indices, :]
+            crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t]
+            intersection_with_ignore_region = self._calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions,
+                                                                       box_format='x0y0x1y1', do_ioa=True)
+            is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps,
+                                                   axis=1)
+
+            # Apply preprocessing to remove unwanted tracker dets.
+            to_remove_tracker = unmatched_indices[is_within_crowd_ignore_region]
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='x0y0x1y1')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/burst.py

@@ -0,0 +1,49 @@
+import os
+from .burst_helpers.burst_base import BURSTBase
+from .burst_helpers.format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter
+from .. import utils
+
+
+class BURST(BURSTBase):
+    """Dataset class for TAO tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        tao_config = BURSTBase.get_default_dataset_config()
+        code_path = utils.get_code_path()
+
+        # e.g. 'data/gt/tsunami/exemplar_guided/'
+        tao_config['GT_FOLDER'] = os.path.join(
+            code_path, 'data/gt/burst/val/')  # Location of GT data
+        # e.g. 'data/trackers/tsunami/exemplar_guided/mask_guided/validation/'
+        tao_config['TRACKERS_FOLDER'] = os.path.join(
+            code_path, 'data/trackers/burst/class-guided/')  # Trackers location
+        # set to True or False
+        tao_config['EXEMPLAR_GUIDED'] = False
+        return tao_config
+
+    def _iou_type(self):
+        return 'mask'
+
+    def _box_or_mask_from_det(self, det):
+        return det['segmentation']
+
+    def _calculate_area_for_ann(self, ann):
+        import pycocotools.mask as cocomask
+        return cocomask.area(ann["segmentation"])
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores
+
+    def _is_exemplar_guided(self):
+        exemplar_guided = self.config['EXEMPLAR_GUIDED']
+        return exemplar_guided
+
+    def _postproc_ground_truth_data(self, data):
+        return GroundTruthBURSTFormatToTAOFormatConverter(data).convert()
+
+    def _postproc_prediction_data(self, data):
+        return PredictionBURSTFormatToTAOFormatConverter(
+            self.gt_data, data,
+            exemplar_guided=self._is_exemplar_guided()).convert()

yolov7-tracker-example/tracker/trackeval/datasets/burst_helpers/BURST_SPECIFIC_ISSUES.md

@@ -0,0 +1,7 @@
+The track ids in both ground truth and predictions are not globally unique, but
+start from 1 for each video. At the moment when converting from Ali format to
+TAO format, we remap the ids to be globally unique. It would be better to
+directly have this in the data though.
+
+
+Improve setting of EXEMPLAR_GUIDED flag, maybe this can be done automatically.

yolov7-tracker-example/tracker/trackeval/datasets/burst_helpers/burst_base.py

@@ -0,0 +1,591 @@
+import os
+import numpy as np
+import json
+import itertools
+from collections import defaultdict
+from scipy.optimize import linear_sum_assignment
+from trackeval.utils import TrackEvalException
+from trackeval.datasets._base_dataset import _BaseDataset
+from trackeval import utils
+from trackeval import _timing
+
+
+class BURSTBase(_BaseDataset):
+    """Dataset class for TAO tracking"""
+
+    def _postproc_ground_truth_data(self, data):
+        return data
+
+    def _postproc_prediction_data(self, data):
+        return data
+
+    def _iou_type(self):
+        return 'bbox'
+
+    def _box_or_mask_from_det(self, det):
+        return np.atleast_1d(det['bbox'])
+
+    def _calculate_area_for_ann(self, ann):
+        return ann["bbox"][2] * ann["bbox"][3]
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
+            'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val'
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'MAX_DETECTIONS': 300,  # Number of maximal allowed detections per image (0 for unlimited)
+            'EXEMPLAR_GUIDED': False,
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = True
+        self.use_super_categories = False
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')]
+        if len(gt_dir_files) != 1:
+            raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.')
+
+        with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
+            self.gt_data = self._postproc_ground_truth_data(json.load(f))
+
+        # merge categories marked with a merged tag in TAO dataset
+        self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks'])
+
+        # Get sequences to eval and sequence information
+        self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']]
+        self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']}
+        # compute mappings from videos to annotation data
+        self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations'])
+        # compute sequence lengths
+        self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']}
+        for img in self.gt_data['images']:
+            self.seq_lengths[img['video_id']] += 1
+        self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings()
+        self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track
+                                                                in self.videos_to_gt_tracks[vid['id']]}),
+                                           'neg_cat_ids': vid['neg_category_ids'],
+                                           'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']}
+                               for vid in self.gt_data['videos']}
+
+        # Get classes to eval
+        considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list]
+        seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id
+                         in self.seq_to_classes[vid_id]['pos_cat_ids']])
+        # only classes with ground truth are evaluated in TAO, also we don't evaluate distactors.
+        distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567,
+                       569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883,
+                       912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220}
+        self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if (cls['id'] in seen_cats) and (cls['id'] not in distractors)]
+        cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
+
+        if self.config['CLASSES_TO_EVAL']:
+            self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                               for cls in self.config['CLASSES_TO_EVAL']]
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(self.valid_classes) +
+                                         ' are valid (classes present in ground truth data).')
+        else:
+            self.class_list = [cls for cls in self.valid_classes]
+        self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        self.tracker_data = {tracker: dict() for tracker in self.tracker_list}
+
+        for tracker in self.tracker_list:
+            tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol))
+                            if file.endswith('.json')]
+            if len(tr_dir_files) != 1:
+                raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+                                         + ' does not contain exactly one json file.')
+            with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f:
+                curr_data = self._postproc_prediction_data(json.load(f))
+
+            # limit detections if MAX_DETECTIONS > 0
+            if self.config['MAX_DETECTIONS']:
+                curr_data = self._limit_dets_per_image(curr_data)
+
+            # fill missing video ids
+            self._fill_video_ids_inplace(curr_data)
+
+            # make track ids unique over whole evaluation set
+            self._make_track_ids_unique(curr_data)
+
+            # merge categories marked with a merged tag in TAO dataset
+            self._merge_categories(curr_data)
+
+            # get tracker sequence information
+            curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data)
+            self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks
+            self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the TAO format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values
+                                as keys and lists (for each track) as values
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values
+                                                                                           as keys and lists as values
+        [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
+        """
+        seq_id = self.seq_name_to_seq_id[seq]
+        # File location
+        if is_gt:
+            imgs = self.videos_to_gt_images[seq_id]
+        else:
+            imgs = self.tracker_data[tracker]['vids_to_images'][seq_id]
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq_id]
+        img_to_timestep = self.seq_to_images_to_timestep[seq_id]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for img in imgs:
+            # some tracker data contains images without any ground truth information, these are ignored
+            try:
+                t = img_to_timestep[img['id']]
+            except KeyError:
+                continue
+            annotations = img['annotations']
+            raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float)
+            raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int)
+            raw_data['classes'][t] = np.atleast_1d([ann['category_id'] for ann in annotations]).astype(int)
+            if not is_gt:
+                raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float)
+
+        for t, d in enumerate(raw_data['dets']):
+            if d is None:
+                raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if not is_gt:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list]
+        if is_gt:
+            classes_to_consider = all_classes
+            all_tracks = self.videos_to_gt_tracks[seq_id]
+        else:
+            classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \
+                                  + self.seq_to_classes[seq_id]['neg_cat_ids']
+            all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id]
+
+        classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls]
+                             if cls in classes_to_consider else [] for cls in all_classes}
+
+        # mapping from classes to track information
+        raw_data['classes_to_tracks'] = {cls: [{det['image_id']: self._box_or_mask_from_det(det)
+                                                for det in track['annotations']} for track in tracks]
+                                         for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks]
+                                            for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks]
+                                              for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks]
+                                                for cls, tracks in classes_to_tracks.items()}
+
+        if not is_gt:
+            raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score'])
+                                                                              for x in track['annotations']])
+                                                                     for track in tracks])
+                                                      for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            key_map = {'classes_to_tracks': 'classes_to_gt_tracks',
+                       'classes_to_track_ids': 'classes_to_gt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_gt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_gt_track_areas'}
+        else:
+            key_map = {'classes_to_tracks': 'classes_to_dt_tracks',
+                       'classes_to_track_ids': 'classes_to_dt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_dt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_dt_track_areas'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids']
+        raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids']
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+        TAO:
+            In TAO, the 4 preproc steps are as follow:
+                1) All classes present in the ground truth data are evaluated separately.
+                2) No matched tracker detections are removed.
+                3) Unmatched tracker detections are removed if there is not ground truth data and the class does not
+                    belong to the categories marked as negative for this sequence. Additionally, unmatched tracker
+                    detections for classes which are marked as not exhaustively labeled are removed.
+                4) No gt detections are removed.
+            Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
+            and the tracks from the tracker data are sorted according to the tracker confidence.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+        is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls']
+        is_neg_category = cls_id in raw_data['neg_cat_ids']
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            if not self.config['EXEMPLAR_GUIDED']:
+                # Match tracker and gt dets (with hungarian algorithm).
+                unmatched_indices = np.arange(tracker_ids.shape[0])
+                if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                    matching_scores = similarity_scores.copy()
+                    matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                    match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                    actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                    match_cols = match_cols[actually_matched_mask]
+                    unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+                if gt_ids.shape[0] == 0 and not is_neg_category:
+                    to_remove_tracker = unmatched_indices
+                elif is_not_exhaustively_labeled:
+                    to_remove_tracker = unmatched_indices
+                else:
+                    to_remove_tracker = np.array([], dtype=np.int)
+
+                # remove all unwanted unmatched tracker detections
+                data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+                data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+                data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+                similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+            else:
+                data['tracker_ids'][t] = tracker_ids
+                data['tracker_dets'][t] = tracker_dets
+                data['tracker_confidences'][t] = tracker_confidences
+
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # get track representations
+        data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id]
+        data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id]
+        data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id]
+        data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id]
+        data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id]
+        data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id]
+        data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id]
+        data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id]
+        data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id]
+        data['not_exhaustively_labeled'] = is_not_exhaustively_labeled
+        data['iou_type'] = self._iou_type()
+
+        # sort tracker data tracks by tracker confidence scores
+        if data['dt_tracks']:
+            idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort")
+            data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx]
+            data['dt_tracks'] = [data['dt_tracks'][i] for i in idx]
+            data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx]
+            data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx]
+            data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx]
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t)
+        return similarity_scores
+
+    def _merge_categories(self, annotations):
+        """
+        Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset
+        :param annotations: the annotations in which the classes should be merged
+        :return: None
+        """
+        merge_map = {}
+        for category in self.gt_data['categories']:
+            if 'merged' in category:
+                for to_merge in category['merged']:
+                    merge_map[to_merge['id']] = category['id']
+
+        for ann in annotations:
+            ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id'])
+
+    def _compute_vid_mappings(self, annotations):
+        """
+        Computes mappings from Videos to corresponding tracks and images.
+        :param annotations: the annotations for which the mapping should be generated
+        :return: the video-to-track-mapping, the video-to-image-mapping
+        """
+        vids_to_tracks = {}
+        vids_to_imgs = {}
+        vid_ids = [vid['id'] for vid in self.gt_data['videos']]
+
+        # compute an mapping from image IDs to images
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        for ann in annotations:
+            ann["area"] = self._calculate_area_for_ann(ann)
+
+            vid = ann["video_id"]
+            if ann["video_id"] not in vids_to_tracks.keys():
+                vids_to_tracks[ann["video_id"]] = list()
+            if ann["video_id"] not in vids_to_imgs.keys():
+                vids_to_imgs[ann["video_id"]] = list()
+
+            # Fill in vids_to_tracks
+            tid = ann["track_id"]
+            exist_tids = [track["id"] for track in vids_to_tracks[vid]]
+            try:
+                index1 = exist_tids.index(tid)
+            except ValueError:
+                index1 = -1
+            if tid not in exist_tids:
+                curr_track = {"id": tid, "category_id": ann["category_id"],
+                              "video_id": vid, "annotations": [ann]}
+                vids_to_tracks[vid].append(curr_track)
+            else:
+                vids_to_tracks[vid][index1]["annotations"].append(ann)
+
+            # Fill in vids_to_imgs
+            img_id = ann['image_id']
+            exist_img_ids = [img["id"] for img in vids_to_imgs[vid]]
+            try:
+                index2 = exist_img_ids.index(img_id)
+            except ValueError:
+                index2 = -1
+            if index2 == -1:
+                curr_img = {"id": img_id, "annotations": [ann]}
+                vids_to_imgs[vid].append(curr_img)
+            else:
+                vids_to_imgs[vid][index2]["annotations"].append(ann)
+
+        # sort annotations by frame index and compute track area
+        for vid, tracks in vids_to_tracks.items():
+            for track in tracks:
+                track["annotations"] = sorted(
+                    track['annotations'],
+                    key=lambda x: images[x['image_id']]['frame_index'])
+                # Computer average area
+                track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations']))
+
+        # Ensure all videos are present
+        for vid_id in vid_ids:
+            if vid_id not in vids_to_tracks.keys():
+                vids_to_tracks[vid_id] = []
+            if vid_id not in vids_to_imgs.keys():
+                vids_to_imgs[vid_id] = []
+
+        return vids_to_tracks, vids_to_imgs
+
+    def _compute_image_to_timestep_mappings(self):
+        """
+        Computes a mapping from images to the corresponding timestep in the sequence.
+        :return: the image-to-timestep-mapping
+        """
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']}
+        for vid in seq_to_imgs_to_timestep:
+            curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]]
+            curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index'])
+            seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))}
+
+        return seq_to_imgs_to_timestep
+
+    def _limit_dets_per_image(self, annotations):
+        """
+        Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from
+        https://github.com/TAO-Dataset/
+        :param annotations: the annotations in which the detections should be limited
+        :return: the annotations with limited detections
+        """
+        max_dets = self.config['MAX_DETECTIONS']
+        img_ann = defaultdict(list)
+        for ann in annotations:
+            img_ann[ann["image_id"]].append(ann)
+
+        for img_id, _anns in img_ann.items():
+            if len(_anns) <= max_dets:
+                continue
+            _anns = sorted(_anns, key=lambda x: x["score"], reverse=True)
+            img_ann[img_id] = _anns[:max_dets]
+
+        return [ann for anns in img_ann.values() for ann in anns]
+
+    def _fill_video_ids_inplace(self, annotations):
+        """
+        Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotations for which the videos IDs should be filled inplace
+        :return: None
+        """
+        missing_video_id = [x for x in annotations if 'video_id' not in x]
+        if missing_video_id:
+            image_id_to_video_id = {
+                x['id']: x['video_id'] for x in self.gt_data['images']
+            }
+            for x in missing_video_id:
+                x['video_id'] = image_id_to_video_id[x['image_id']]
+
+    @staticmethod
+    def _make_track_ids_unique(annotations):
+        """
+        Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotation set
+        :return: the number of updated IDs
+        """
+        track_id_videos = {}
+        track_ids_to_update = set()
+        max_track_id = 0
+        for ann in annotations:
+            t = ann['track_id']
+            if t not in track_id_videos:
+                track_id_videos[t] = ann['video_id']
+
+            if ann['video_id'] != track_id_videos[t]:
+                # Track id is assigned to multiple videos
+                track_ids_to_update.add(t)
+            max_track_id = max(max_track_id, t)
+
+        if track_ids_to_update:
+            #print('true')
+            next_id = itertools.count(max_track_id + 1)
+            new_track_ids = defaultdict(lambda: next(next_id))
+            for ann in annotations:
+                t = ann['track_id']
+                v = ann['video_id']
+                if t in track_ids_to_update:
+                    ann['track_id'] = new_track_ids[t, v]
+        return len(track_ids_to_update)

yolov7-tracker-example/tracker/trackeval/datasets/burst_helpers/burst_ow_base.py

@@ -0,0 +1,675 @@
+import os
+import numpy as np
+import json
+import itertools
+from collections import defaultdict
+from scipy.optimize import linear_sum_assignment
+from trackeval.utils import TrackEvalException
+from trackeval.datasets._base_dataset import _BaseDataset
+from trackeval import utils
+from trackeval import _timing
+
+
+class BURST_OW_Base(_BaseDataset):
+    """Dataset class for TAO tracking"""
+
+    def _postproc_ground_truth_data(self, data):
+        return data
+
+    def _postproc_prediction_data(self, data):
+        return data
+
+    def _iou_type(self):
+        return 'bbox'
+
+    def _box_or_mask_from_det(self, det):
+        return np.atleast_1d(det['bbox'])
+
+    def _calculate_area_for_ann(self, ann):
+        return ann["bbox"][2] * ann["bbox"][3]
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
+            'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val'
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'MAX_DETECTIONS': 300,  # Number of maximal allowed detections per image (0 for unlimited)
+            'SUBSET': 'all'
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = True
+        self.use_super_categories = False
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')]
+        if len(gt_dir_files) != 1:
+            raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.')
+
+        with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
+            self.gt_data = self._postproc_ground_truth_data(json.load(f))
+
+        self.subset = self.config['SUBSET']
+        if self.subset != 'all':
+            # Split GT data into `known`, `unknown` or `distractor`
+            self._split_known_unknown_distractor()
+            self.gt_data = self._filter_gt_data(self.gt_data)
+
+        # merge categories marked with a merged tag in TAO dataset
+        self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks'])
+
+        # Get sequences to eval and sequence information
+        self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']]
+        self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']}
+        # compute mappings from videos to annotation data
+        self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations'])
+        # compute sequence lengths
+        self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']}
+        for img in self.gt_data['images']:
+            self.seq_lengths[img['video_id']] += 1
+        self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings()
+        self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track
+                                                                in self.videos_to_gt_tracks[vid['id']]}),
+                                           'neg_cat_ids': vid['neg_category_ids'],
+                                           'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']}
+                               for vid in self.gt_data['videos']}
+
+        # Get classes to eval
+        considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list]
+        seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id
+                         in self.seq_to_classes[vid_id]['pos_cat_ids']])
+        # only classes with ground truth are evaluated in TAO
+        self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats]
+        # cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
+
+        if self.config['CLASSES_TO_EVAL']:
+            # self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+            #                    for cls in self.config['CLASSES_TO_EVAL']]
+            self.class_list = ["object"]  # class-agnostic
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(self.valid_classes) +
+                                         ' are valid (classes present in ground truth data).')
+        else:
+            # self.class_list = [cls for cls in self.valid_classes]
+            self.class_list = ["object"]  # class-agnostic
+        # self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
+        self.class_name_to_class_id = {"object": 1}  # class-agnostic
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        self.tracker_data = {tracker: dict() for tracker in self.tracker_list}
+
+        for tracker in self.tracker_list:
+            tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol))
+                            if file.endswith('.json')]
+            if len(tr_dir_files) != 1:
+                raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+                                         + ' does not contain exactly one json file.')
+            with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f:
+                curr_data = self._postproc_prediction_data(json.load(f))
+
+            # limit detections if MAX_DETECTIONS > 0
+            if self.config['MAX_DETECTIONS']:
+                curr_data = self._limit_dets_per_image(curr_data)
+
+            # fill missing video ids
+            self._fill_video_ids_inplace(curr_data)
+
+            # make track ids unique over whole evaluation set
+            self._make_track_ids_unique(curr_data)
+
+            # merge categories marked with a merged tag in TAO dataset
+            self._merge_categories(curr_data)
+
+            # get tracker sequence information
+            curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data)
+            self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks
+            self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the TAO format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values
+                                as keys and lists (for each track) as values
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values
+                                                                                           as keys and lists as values
+        [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
+        """
+        seq_id = self.seq_name_to_seq_id[seq]
+        # File location
+        if is_gt:
+            imgs = self.videos_to_gt_images[seq_id]
+        else:
+            imgs = self.tracker_data[tracker]['vids_to_images'][seq_id]
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq_id]
+        img_to_timestep = self.seq_to_images_to_timestep[seq_id]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for img in imgs:
+            # some tracker data contains images without any ground truth information, these are ignored
+            try:
+                t = img_to_timestep[img['id']]
+            except KeyError:
+                continue
+            annotations = img['annotations']
+            raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float)
+            raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int)
+            raw_data['classes'][t] = np.atleast_1d([1 for _ in annotations]).astype(int)   # class-agnostic
+            if not is_gt:
+                raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float)
+
+        for t, d in enumerate(raw_data['dets']):
+            if d is None:
+                raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if not is_gt:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        # all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list]
+        all_classes = [1]  # class-agnostic
+
+        if is_gt:
+            classes_to_consider = all_classes
+            all_tracks = self.videos_to_gt_tracks[seq_id]
+        else:
+            # classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \
+            #                       + self.seq_to_classes[seq_id]['neg_cat_ids']
+            classes_to_consider = all_classes  # class-agnostic
+            all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id]
+
+        # classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls]
+        #                      if cls in classes_to_consider else [] for cls in all_classes}
+        classes_to_tracks = {cls: [track for track in all_tracks]
+        if cls in classes_to_consider else [] for cls in all_classes}  # class-agnostic
+
+        # mapping from classes to track information
+        raw_data['classes_to_tracks'] = {cls: [{det['image_id']: self._box_or_mask_from_det(det)
+                                                for det in track['annotations']} for track in tracks]
+                                         for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks]
+                                            for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks]
+                                              for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks]
+                                                for cls, tracks in classes_to_tracks.items()}
+
+        if not is_gt:
+            raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score'])
+                                                                              for x in track['annotations']])
+                                                                     for track in tracks])
+                                                      for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            key_map = {'classes_to_tracks': 'classes_to_gt_tracks',
+                       'classes_to_track_ids': 'classes_to_gt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_gt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_gt_track_areas'}
+        else:
+            key_map = {'classes_to_tracks': 'classes_to_dt_tracks',
+                       'classes_to_track_ids': 'classes_to_dt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_dt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_dt_track_areas'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids']
+        raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids']
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+        TAO:
+            In TAO, the 4 preproc steps are as follow:
+                1) All classes present in the ground truth data are evaluated separately.
+                2) No matched tracker detections are removed.
+                3) Unmatched tracker detections are removed if there is not ground truth data and the class does not
+                    belong to the categories marked as negative for this sequence. Additionally, unmatched tracker
+                    detections for classes which are marked as not exhaustively labeled are removed.
+                4) No gt detections are removed.
+            Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
+            and the tracks from the tracker data are sorted according to the tracker confidence.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+        is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls']
+        is_neg_category = cls_id in raw_data['neg_cat_ids']
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm).
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            if gt_ids.shape[0] == 0 and not is_neg_category:
+                to_remove_tracker = unmatched_indices
+            elif is_not_exhaustively_labeled:
+                to_remove_tracker = unmatched_indices
+            else:
+                to_remove_tracker = np.array([], dtype=np.int)
+
+            # remove all unwanted unmatched tracker detections
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # get track representations
+        data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id]
+        data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id]
+        data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id]
+        data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id]
+        data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id]
+        data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id]
+        data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id]
+        data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id]
+        data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id]
+        data['not_exhaustively_labeled'] = is_not_exhaustively_labeled
+        data['iou_type'] = self._iou_type()
+
+        # sort tracker data tracks by tracker confidence scores
+        if data['dt_tracks']:
+            idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort")
+            data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx]
+            data['dt_tracks'] = [data['dt_tracks'][i] for i in idx]
+            data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx]
+            data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx]
+            data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx]
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t)
+        return similarity_scores
+
+    def _merge_categories(self, annotations):
+        """
+        Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset
+        :param annotations: the annotations in which the classes should be merged
+        :return: None
+        """
+        merge_map = {}
+        for category in self.gt_data['categories']:
+            if 'merged' in category:
+                for to_merge in category['merged']:
+                    merge_map[to_merge['id']] = category['id']
+
+        for ann in annotations:
+            ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id'])
+
+    def _compute_vid_mappings(self, annotations):
+        """
+        Computes mappings from Videos to corresponding tracks and images.
+        :param annotations: the annotations for which the mapping should be generated
+        :return: the video-to-track-mapping, the video-to-image-mapping
+        """
+        vids_to_tracks = {}
+        vids_to_imgs = {}
+        vid_ids = [vid['id'] for vid in self.gt_data['videos']]
+
+        # compute an mapping from image IDs to images
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        for ann in annotations:
+            ann["area"] = self._calculate_area_for_ann(ann)
+
+            vid = ann["video_id"]
+            if ann["video_id"] not in vids_to_tracks.keys():
+                vids_to_tracks[ann["video_id"]] = list()
+            if ann["video_id"] not in vids_to_imgs.keys():
+                vids_to_imgs[ann["video_id"]] = list()
+
+            # Fill in vids_to_tracks
+            tid = ann["track_id"]
+            exist_tids = [track["id"] for track in vids_to_tracks[vid]]
+            try:
+                index1 = exist_tids.index(tid)
+            except ValueError:
+                index1 = -1
+            if tid not in exist_tids:
+                curr_track = {"id": tid, "category_id": ann["category_id"],
+                              "video_id": vid, "annotations": [ann]}
+                vids_to_tracks[vid].append(curr_track)
+            else:
+                vids_to_tracks[vid][index1]["annotations"].append(ann)
+
+            # Fill in vids_to_imgs
+            img_id = ann['image_id']
+            exist_img_ids = [img["id"] for img in vids_to_imgs[vid]]
+            try:
+                index2 = exist_img_ids.index(img_id)
+            except ValueError:
+                index2 = -1
+            if index2 == -1:
+                curr_img = {"id": img_id, "annotations": [ann]}
+                vids_to_imgs[vid].append(curr_img)
+            else:
+                vids_to_imgs[vid][index2]["annotations"].append(ann)
+
+        # sort annotations by frame index and compute track area
+        for vid, tracks in vids_to_tracks.items():
+            for track in tracks:
+                track["annotations"] = sorted(
+                    track['annotations'],
+                    key=lambda x: images[x['image_id']]['frame_index'])
+                # Computer average area
+                track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations']))
+
+        # Ensure all videos are present
+        for vid_id in vid_ids:
+            if vid_id not in vids_to_tracks.keys():
+                vids_to_tracks[vid_id] = []
+            if vid_id not in vids_to_imgs.keys():
+                vids_to_imgs[vid_id] = []
+
+        return vids_to_tracks, vids_to_imgs
+
+    def _compute_image_to_timestep_mappings(self):
+        """
+        Computes a mapping from images to the corresponding timestep in the sequence.
+        :return: the image-to-timestep-mapping
+        """
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']}
+        for vid in seq_to_imgs_to_timestep:
+            curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]]
+            curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index'])
+            seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))}
+
+        return seq_to_imgs_to_timestep
+
+    def _limit_dets_per_image(self, annotations):
+        """
+        Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from
+        https://github.com/TAO-Dataset/
+        :param annotations: the annotations in which the detections should be limited
+        :return: the annotations with limited detections
+        """
+        max_dets = self.config['MAX_DETECTIONS']
+        img_ann = defaultdict(list)
+        for ann in annotations:
+            img_ann[ann["image_id"]].append(ann)
+
+        for img_id, _anns in img_ann.items():
+            if len(_anns) <= max_dets:
+                continue
+            _anns = sorted(_anns, key=lambda x: x["score"], reverse=True)
+            img_ann[img_id] = _anns[:max_dets]
+
+        return [ann for anns in img_ann.values() for ann in anns]
+
+    def _fill_video_ids_inplace(self, annotations):
+        """
+        Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotations for which the videos IDs should be filled inplace
+        :return: None
+        """
+        missing_video_id = [x for x in annotations if 'video_id' not in x]
+        if missing_video_id:
+            image_id_to_video_id = {
+                x['id']: x['video_id'] for x in self.gt_data['images']
+            }
+            for x in missing_video_id:
+                x['video_id'] = image_id_to_video_id[x['image_id']]
+
+    @staticmethod
+    def _make_track_ids_unique(annotations):
+        """
+        Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotation set
+        :return: the number of updated IDs
+        """
+        track_id_videos = {}
+        track_ids_to_update = set()
+        max_track_id = 0
+        for ann in annotations:
+            t = ann['track_id']
+            if t not in track_id_videos:
+                track_id_videos[t] = ann['video_id']
+
+            if ann['video_id'] != track_id_videos[t]:
+                # Track id is assigned to multiple videos
+                track_ids_to_update.add(t)
+            max_track_id = max(max_track_id, t)
+
+        if track_ids_to_update:
+            #print('true')
+            next_id = itertools.count(max_track_id + 1)
+            new_track_ids = defaultdict(lambda: next(next_id))
+            for ann in annotations:
+                t = ann['track_id']
+                v = ann['video_id']
+                if t in track_ids_to_update:
+                    ann['track_id'] = new_track_ids[t, v]
+        return len(track_ids_to_update)
+
+    def _split_known_unknown_distractor(self):
+        all_ids = set([i for i in range(1, 2000)])  # 2000 is larger than the max category id in TAO-OW.
+        # `knowns` includes 78 TAO_category_ids that corresponds to 78 COCO classes.
+        # (The other 2 COCO classes do not have corresponding classes in TAO).
+        self.knowns = {4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115,
+                     1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185,
+                     699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347,
+                     371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018}
+        # `distractors` is defined as in the paper "Opening up Open-World Tracking"
+        self.distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567,
+                            569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883,
+                            912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220}
+        self.unknowns = all_ids.difference(self.knowns.union(self.distractors))
+
+    def _filter_gt_data(self, raw_gt_data):
+        """
+        Filter out irrelevant data in the raw_gt_data
+        Args:
+            raw_gt_data: directly loaded from json.
+
+        Returns:
+            filtered gt_data
+        """
+        valid_cat_ids = list()
+        if self.subset == "known":
+            valid_cat_ids = self.knowns
+        elif self.subset == "distractor":
+            valid_cat_ids = self.distractors
+        elif self.subset == "unknown":
+            valid_cat_ids = self.unknowns
+        # elif self.subset == "test_only_unknowns":
+        #     valid_cat_ids = test_only_unknowns
+        else:
+            raise Exception("The parameter `SUBSET` is incorrect")
+
+        filtered = dict()
+        filtered["videos"] = raw_gt_data["videos"]
+        # filtered["videos"] = list()
+        unwanted_vid = set()
+        # for video in raw_gt_data["videos"]:
+        #     datasrc = video["name"].split('/')[1]
+        #     if datasrc in data_srcs:
+        #         filtered["videos"].append(video)
+        #     else:
+        #         unwanted_vid.add(video["id"])
+
+        filtered["annotations"] = list()
+        for ann in raw_gt_data["annotations"]:
+            if (ann["video_id"] not in unwanted_vid) and (ann["category_id"] in valid_cat_ids):
+                filtered["annotations"].append(ann)
+
+        filtered["tracks"] = list()
+        for track in raw_gt_data["tracks"]:
+            if (track["video_id"] not in unwanted_vid) and (track["category_id"] in valid_cat_ids):
+                filtered["tracks"].append(track)
+
+        filtered["images"] = list()
+        for image in raw_gt_data["images"]:
+            if image["video_id"] not in unwanted_vid:
+                filtered["images"].append(image)
+
+        filtered["categories"] = list()
+        for cat in raw_gt_data["categories"]:
+            if cat["id"] in valid_cat_ids:
+                filtered["categories"].append(cat)
+
+        if "info" in raw_gt_data:
+            filtered["info"] = raw_gt_data["info"]
+        if "licenses" in raw_gt_data:
+            filtered["licenses"] = raw_gt_data["licenses"]
+
+        if "track_id_offsets" in raw_gt_data:
+            filtered["track_id_offsets"] = raw_gt_data["track_id_offsets"]
+
+        if "split" in raw_gt_data:
+            filtered["split"] = raw_gt_data["split"]
+
+        return filtered

yolov7-tracker-example/tracker/trackeval/datasets/burst_helpers/convert_burst_format_to_tao_format.py

@@ -0,0 +1,39 @@
+import json
+import argparse
+from .format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter
+
+
+def main(args):
+    with open(args.gt_input_file) as f:
+        ali_format_gt = json.load(f)
+    tao_format_gt = GroundTruthBURSTFormatToTAOFormatConverter(
+        ali_format_gt, args.split).convert()
+    with open(args.gt_output_file, 'w') as f:
+        json.dump(tao_format_gt, f)
+
+    if args.pred_input_file is None:
+        return
+    with open(args.pred_input_file) as f:
+        ali_format_pred = json.load(f)
+    tao_format_pred = PredictionBURSTFormatToTAOFormatConverter(
+        tao_format_gt, ali_format_pred, args.split,
+        args.exemplar_guided).convert()
+    with open(args.pred_output_file, 'w') as f:
+        json.dump(tao_format_pred, f)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--gt_input_file', type=str,
+        default='../data/gt/tsunami/exemplar_guided/validation_all_annotations.json')
+    parser.add_argument('--gt_output_file', type=str,
+                        default='/tmp/val_gt.json')
+    parser.add_argument('--pred_input_file', type=str,
+                        default='../data/trackers/tsunami/exemplar_guided/STCN_off_the_shelf/data/results.json')
+    parser.add_argument('--pred_output_file', type=str,
+                        default='/tmp/pred.json')
+    parser.add_argument('--split', type=str, default='validation')
+    parser.add_argument('--exemplar_guided', type=bool, default=True)
+    args_ = parser.parse_args()
+    main(args_)

yolov7-tracker-example/tracker/trackeval/datasets/burst_helpers/format_converter.py

@@ -0,0 +1,259 @@
+import os
+import json
+import pycocotools.mask as cocomask
+from tabulate import tabulate
+from typing import Union
+
+
+def _global_track_id(*, local_track_id: Union[str, int],
+                     video_id: Union[str, int],
+                     track_id_mapping) -> int:
+    # remap local track ids into globally unique ids
+    return track_id_mapping[str(video_id)][str(local_track_id)]
+
+
+class GroundTruthBURSTFormatToTAOFormatConverter:
+    def __init__(self, ali_format):
+        self._ali_format = ali_format
+        self._split = ali_format['split']
+        self._categories = self._make_categories()
+        self._videos = []
+        self._annotations = []
+        self._tracks = {}
+        self._images = []
+        self._next_img_id = 0
+        self._next_ann_id = 0
+
+        self._track_id_mapping = self._load_track_id_mapping()
+
+        for seq in ali_format['sequences']:
+            self._visit_seq(seq)
+
+    def _load_track_id_mapping(self):
+        id_map = {}
+        next_global_track_id = 1
+        for seq in self._ali_format['sequences']:
+            seq_id = seq['id']
+            seq_id_map = {}
+            id_map[str(seq_id)] = seq_id_map
+            for local_track_id in seq['track_category_ids']:
+                seq_id_map[str(local_track_id)] = next_global_track_id
+                next_global_track_id += 1
+        return id_map
+
+    def global_track_id(self, *, local_track_id: Union[str, int],
+                        video_id: Union[str, int]) -> int:
+        return _global_track_id(local_track_id=local_track_id,
+                                video_id=video_id,
+                                track_id_mapping=self._track_id_mapping)
+
+    def _visit_seq(self, seq):
+        self._make_video(seq)
+        imgs = self._make_images(seq)
+        self._make_annotations_and_tracks(seq, imgs)
+
+    def _make_images(self, seq):
+        imgs = []
+        for img_path in seq['annotated_image_paths']:
+            video = self._split + '/' + seq['dataset'] + '/' + seq['seq_name']
+            file_name = video + '/' + img_path
+
+            # TODO: once python 3.9 is more common, we can use this nicer and safer code
+            #stripped = img_path.removesuffix('.jpg').removesuffix('.png').removeprefix('frame')
+            stripped = img_path.replace('.jpg', '').replace('.png', '').replace('frame', '')
+
+            last = stripped.split('_')[-1]
+            frame_idx = int(last)
+
+            img = {'id': self._next_img_id, 'video': video,
+                   'width': seq['width'], 'height': seq['height'],
+                   'file_name': file_name,
+                   'frame_index': frame_idx,
+                   'video_id': seq['id']}
+            self._next_img_id += 1
+            self._images.append(img)
+            imgs.append(img)
+        return imgs
+
+    def _make_video(self, seq):
+        video_id = seq['id']
+        dataset = seq['dataset']
+        seq_name = seq['seq_name']
+        name = f'{self._split}/' + dataset + '/' + seq_name
+        video = {
+            'id': video_id, 'width': seq['width'], 'height': seq['height'],
+            'neg_category_ids': seq['neg_category_ids'],
+            'not_exhaustive_category_ids': seq['not_exhaustive_category_ids'],
+            'name': name, 'metadata': {'dataset': dataset}}
+        self._videos.append(video)
+
+    def _make_annotations_and_tracks(self, seq, imgs):
+        video_id = seq['id']
+        segs = seq['segmentations']
+        assert len(segs) == len(imgs), (len(segs), len(imgs))
+        for frame_segs, img in zip(segs, imgs):
+            for local_track_id, seg in frame_segs.items():
+                distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567,
+                               569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883,
+                               912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220}
+                global_track_id = self.global_track_id(
+                    local_track_id=local_track_id, video_id=seq['id'])
+                rle = seg['rle']
+                segmentation = {'counts': rle,
+                                'size': [img['height'], img['width']]}
+                image_id = img['id']
+                category_id = int(seq['track_category_ids'][local_track_id])
+                if category_id in distractors:
+                    continue
+                coco_bbox = cocomask.toBbox(segmentation)
+                bbox = [int(x) for x in coco_bbox]
+                ann = {'segmentation': segmentation, 'id': self._next_ann_id,
+                       'image_id': image_id, 'category_id': category_id,
+                       'track_id': global_track_id, 'video_id': video_id,
+                       'bbox': bbox}
+                self._next_ann_id += 1
+                self._annotations.append(ann)
+
+                if global_track_id not in self._tracks:
+                    track = {'id': global_track_id, 'category_id': category_id,
+                             'video_id': video_id}
+                    self._tracks[global_track_id] = track
+
+    def convert(self):
+        tracks = sorted(self._tracks.values(), key=lambda t: t['id'])
+        return {'videos': self._videos, 'annotations': self._annotations,
+                'tracks': tracks, 'images': self._images,
+                'categories': self._categories,
+                'track_id_mapping': self._track_id_mapping,
+                'split': self._split}
+
+    def _make_categories(self):
+        tao_categories_path = os.path.join(os.path.dirname(__file__), 'tao_categories.json')
+        with open(tao_categories_path) as f:
+            return json.load(f)
+
+
+class PredictionBURSTFormatToTAOFormatConverter:
+    def __init__(self, gt, ali_format, exemplar_guided):
+        self._gt = gt
+        self._ali_format = ali_format
+        if 'split' in ali_format:
+            self._split = ali_format['split']
+            gt_split = self._gt['split']
+            assert self._split == gt_split, (self._split, gt_split)
+        else:
+            self._split = self._gt['split']
+        self._exemplar_guided = exemplar_guided
+        self._result = []
+        self._next_det_id = 0
+
+        self._img_by_filename = {}
+        for img in self._gt['images']:
+            file_name = img['file_name']
+            assert file_name not in self._img_by_filename
+            self._img_by_filename[file_name] = img
+
+        self._gt_track_by_track_id = {}
+        for track in self._gt['tracks']:
+            self._gt_track_by_track_id[int(track['id'])] = track
+
+        self._filtered_out_track_ids = set()
+
+        for seq in ali_format['sequences']:
+            self._visit_seq(seq)
+
+        if exemplar_guided and len(self._filtered_out_track_ids) > 0:
+            self.print_filter_out_debug_info(ali_format)
+
+    def print_filter_out_debug_info(self, ali_format):
+        track_ids_in_pred = set()
+        a_dict_for_debugging = {}
+        for seq in ali_format['sequences']:
+            for local_track_id in seq['track_category_ids']:
+                global_track_id = _global_track_id(
+                    local_track_id=local_track_id, video_id=seq['id'],
+                    track_id_mapping=self._gt['track_id_mapping'])
+                track_ids_in_pred.add(global_track_id)
+                a_dict_for_debugging[global_track_id] = {'seq': seq,
+                                                         'local_track_id': local_track_id}
+        print('Number of Track ids in pred:', len(track_ids_in_pred))
+        print('Exemplar Guided: Filtered out',
+              len(self._filtered_out_track_ids),
+              'tracks which were not found in the ground truth.')
+        track_ids_after_filtering = set(d['track_id'] for d in self._result)
+        print('Number of tracks after filtering:',
+              len(track_ids_after_filtering))
+        problem_tracks = list(
+            track_ids_in_pred - track_ids_after_filtering - self._filtered_out_track_ids)
+        if len(problem_tracks) > 0:
+            print("\nWARNING:", len(problem_tracks),
+                  "object tracks are not present. There could be a number of reasons for this:\n"
+                  "(1) If you are running evaluation for the box/point exemplar-guided task then this is to be expected"
+                  " because your tracker probably didn't predict masks for every ground-truth object instance.\n"
+                  "(2) If you are running evaluation for the mask exemplar-guided task, then this could indicate a "
+                  "problem. Assume that you copied the given first-frame object mask to your predicted result, this "
+                  "should not happen. It could be that your predictions are at the wrong frame-rate i.e. you have no "
+                  "predicted masks for video frames which will be evaluated.\n")
+
+            rows = []
+            for xx in problem_tracks:
+                rows.append([a_dict_for_debugging[xx]['seq']['dataset'],
+                             a_dict_for_debugging[xx]['seq']['seq_name'],
+                             a_dict_for_debugging[xx]['local_track_id']])
+
+            print("For your reference, the sequence name and track IDs for these missing tracks are:")
+            print(tabulate(rows, ["Dataset", "Sequence Name", "Track ID"]))
+
+    def _visit_seq(self, seq):
+        dataset = seq['dataset']
+        seq_name = seq['seq_name']
+        assert len(seq['segmentations']) == len(seq['annotated_image_paths'])
+        for frame_segs, img_path in zip(seq['segmentations'],
+                                        seq['annotated_image_paths']):
+            for local_track_id_str, track_det in frame_segs.items():
+                rle = track_det['rle']
+
+                file_name = self._split + '/' + dataset + '/' + seq_name + '/' + img_path
+                # the result might have a higher frame rate than the ground truth
+                if file_name not in self._img_by_filename:
+                    continue
+
+                img = self._img_by_filename[file_name]
+                img_id = img['id']
+                height = img['height']
+                width = img['width']
+                segmentation = {'counts': rle, 'size': [height, width]}
+
+                local_track_id = int(local_track_id_str)
+                if self._exemplar_guided:
+                    global_track_id = _global_track_id(
+                        local_track_id=local_track_id, video_id=seq['id'],
+                        track_id_mapping=self._gt['track_id_mapping'])
+                else:
+                    global_track_id = local_track_id
+                coco_bbox = cocomask.toBbox(segmentation)
+                bbox = [int(x) for x in coco_bbox]
+                det = {'id': self._next_det_id, 'image_id': img_id,
+                       'track_id': global_track_id, 'bbox': bbox,
+                       'segmentation': segmentation}
+                if self._exemplar_guided:
+                    if global_track_id not in self._gt_track_by_track_id:
+                        self._filtered_out_track_ids.add(global_track_id)
+                        continue
+                    gt_track = self._gt_track_by_track_id[global_track_id]
+                    category_id = gt_track['category_id']
+                    det['category_id'] = category_id
+                elif 'category_id' in track_det:
+                    det['category_id'] = track_det['category_id']
+                else:
+                    category_id = seq['track_category_ids'][local_track_id_str]
+                    det['category_id'] = category_id
+                self._next_det_id += 1
+                if 'score' in track_det:
+                    det['score'] = track_det['score']
+                else:
+                    det['score'] = 1.0
+                self._result.append(det)
+
+    def convert(self):
+        return self._result

yolov7-tracker-example/tracker/trackeval/datasets/burst_ow.py

@@ -0,0 +1,91 @@
+import json
+import os
+from .burst_helpers.burst_ow_base import BURST_OW_Base
+from .burst_helpers.format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter
+from .. import utils
+
+
+class BURST_OW(BURST_OW_Base):
+    """Dataset class for TAO tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        tao_config = BURST_OW_Base.get_default_dataset_config()
+        code_path = utils.get_code_path()
+        tao_config['GT_FOLDER'] = os.path.join(
+            code_path, 'data/gt/burst/all_classes/val/')  # Location of GT data
+        tao_config['TRACKERS_FOLDER'] = os.path.join(
+            code_path, 'data/trackers/burst/open-world/val/')  # Trackers location
+        return tao_config
+
+    def _iou_type(self):
+        return 'mask'
+
+    def _box_or_mask_from_det(self, det):
+        if "segmentation" in det:
+            return det["segmentation"]
+        else:
+            return det["mask"]
+
+    def _calculate_area_for_ann(self, ann):
+        import pycocotools.mask as cocomask
+        seg = self._box_or_mask_from_det(ann)
+        return cocomask.area(seg)
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores
+
+    def _postproc_ground_truth_data(self, data):
+        return GroundTruthBURSTFormatToTAOFormatConverter(data).convert()
+
+    def _postproc_prediction_data(self, data):
+        # if it's a list, it's already in TAO format and not in Ali format
+        # however the image ids do not match and need to be remapped
+        if isinstance(data, list):
+            _remap_image_ids(data, self.gt_data)
+            return data
+
+        return PredictionBURSTFormatToTAOFormatConverter(
+            self.gt_data, data,
+            exemplar_guided=False).convert()
+
+
+def _remap_image_ids(pred_data, ali_gt_data):
+    code_path = utils.get_code_path()
+    if 'split' in ali_gt_data:
+        split = ali_gt_data['split']
+    else:
+        split = 'val'
+
+    if split in ('val', 'validation'):
+        tao_gt_path = os.path.join(
+            code_path, 'data/gt/tao/tao_validation/gt.json')
+    else:
+        tao_gt_path = os.path.join(
+            code_path, 'data/gt/tao/tao_test/test_without_annotations.json')
+
+    with open(tao_gt_path) as f:
+        tao_gt = json.load(f)
+
+    tao_img_by_id = {}
+    for img in tao_gt['images']:
+        img_id = img['id']
+        tao_img_by_id[img_id] = img
+
+    ali_img_id_by_filename = {}
+    for ali_img in ali_gt_data['images']:
+        ali_img_id = ali_img['id']
+        file_name = ali_img['file_name'].replace("validation", "val")
+        ali_img_id_by_filename[file_name] = ali_img_id
+
+    ali_img_id_by_tao_img_id = {}
+    for tao_img_id, tao_img in tao_img_by_id.items():
+        file_name = tao_img['file_name']
+        ali_img_id = ali_img_id_by_filename[file_name]
+        ali_img_id_by_tao_img_id[tao_img_id] = ali_img_id
+
+    for det in pred_data:
+        tao_img_id = det['image_id']
+        ali_img_id = ali_img_id_by_tao_img_id[tao_img_id]
+        det['image_id'] = ali_img_id

yolov7-tracker-example/tracker/trackeval/datasets/davis.py

@@ -0,0 +1,276 @@
+import os
+import csv
+import numpy as np
+from ._base_dataset import _BaseDataset
+from ..utils import TrackEvalException
+from .. import utils
+from .. import _timing
+
+
+class DAVIS(_BaseDataset):
+    """Dataset class for DAVIS tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/davis/davis_unsupervised_val/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/davis/davis_unsupervised_val/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'SPLIT_TO_EVAL': 'val',  # Valid: 'val', 'train'
+            'CLASSES_TO_EVAL': ['general'],
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FILE': None,  # Specify seqmap file
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            # '{gt_folder}/Annotations_unsupervised/480p/{seq}'
+            'MAX_DETECTIONS': 0  # Maximum number of allowed detections per sequence (0 for no threshold)
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        # defining a default class since there are no classes in DAVIS
+        self.should_classes_combine = False
+        self.use_super_categories = False
+
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.config['TRACKERS_FOLDER']
+
+        self.max_det = self.config['MAX_DETECTIONS']
+
+        # Get classes to eval
+        self.valid_classes = ['general']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only general class is valid.')
+
+        # Get sequences to eval
+        if self.config["SEQ_INFO"]:
+            self.seq_list = list(self.config["SEQ_INFO"].keys())
+            self.seq_lengths = self.config["SEQ_INFO"]
+        elif self.config["SEQMAP_FILE"]:
+            self.seq_list = []
+            seqmap_file = self.config["SEQMAP_FILE"]
+            if not os.path.isfile(seqmap_file):
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if row[0] == '':
+                        continue
+                    seq = row[0]
+                    self.seq_list.append(seq)
+        else:
+            self.seq_list = os.listdir(self.gt_fol)
+
+        self.seq_lengths = {seq: len(os.listdir(os.path.join(self.gt_fol, seq))) for seq in self.seq_list}
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+        for tracker in self.tracker_list:
+            for seq in self.seq_list:
+                curr_dir = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq)
+                if not os.path.isdir(curr_dir):
+                    print('Tracker directory not found: ' + curr_dir)
+                    raise TrackEvalException('Tracker directory not found: ' +
+                                             os.path.join(tracker, self.tracker_sub_fol, seq))
+                tr_timesteps = len(os.listdir(curr_dir))
+                if self.seq_lengths[seq] != tr_timesteps:
+                    raise TrackEvalException('GT folder and tracker folder have a different number'
+                                             'timesteps for tracker %s and sequence %s' % (tracker, seq))
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the DAVIS format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [masks_void]: list of masks with void pixels (pixels to be ignored during evaluation)
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+        from PIL import Image
+
+        # File location
+        if is_gt:
+            seq_dir = os.path.join(self.gt_fol, seq)
+        else:
+            seq_dir = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq)
+
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'dets', 'masks_void']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # read frames
+        frames = [os.path.join(seq_dir, im_name) for im_name in sorted(os.listdir(seq_dir))]
+
+        id_list = []
+        for t in range(num_timesteps):
+            frame = np.array(Image.open(frames[t]))
+            if is_gt:
+                void = frame == 255
+                frame[void] = 0
+                raw_data['masks_void'][t] = mask_utils.encode(np.asfortranarray(void.astype(np.uint8)))
+            id_values = np.unique(frame)
+            id_values = id_values[id_values != 0]
+            id_list += list(id_values)
+            tmp = np.ones((len(id_values), *frame.shape))
+            tmp = tmp * id_values[:, None, None]
+            masks = np.array(tmp == frame[None, ...]).astype(np.uint8)
+            raw_data['dets'][t] = mask_utils.encode(np.array(np.transpose(masks, (1, 2, 0)), order='F'))
+            raw_data['ids'][t] = id_values.astype(int)
+        num_objects = len(np.unique(id_list))
+
+        if not is_gt and num_objects > self.max_det > 0:
+            raise Exception('Number of proposals (%i) for sequence %s exceeds number of maximum allowed proposals (%i).'
+                            % (num_objects, seq, self.max_det))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data["num_timesteps"] = num_timesteps
+        raw_data['mask_shape'] = np.array(Image.open(frames[0])).shape
+        if is_gt:
+            raw_data['num_gt_ids'] = num_objects
+        else:
+            raw_data['num_tracker_ids'] = num_objects
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        DAVIS:
+            In DAVIS, the 4 preproc steps are as follow:
+                1) There are no classes, all detections are evaluated jointly
+                2) No matched tracker detections are removed.
+                3) No unmatched tracker detections are removed.
+                4) There are no ground truth detections (e.g. those of distractor classes) to be removed.
+            Preprocessing special to DAVIS: Pixels which are marked as void in the ground truth are set to zero in the
+                tracker detections since they are not considered during evaluation.
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_timesteps = raw_data['num_timesteps']
+
+        # count detections
+        for t in range(num_timesteps):
+            num_gt_dets += len(raw_data['gt_dets'][t])
+            num_tracker_dets += len(raw_data['tracker_dets'][t])
+            unique_gt_ids += list(np.unique(raw_data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(raw_data['tracker_ids'][t]))
+
+        data['gt_ids'] = raw_data['gt_ids']
+        data['gt_dets'] = raw_data['gt_dets']
+        data['similarity_scores'] = raw_data['similarity_scores']
+        data['tracker_ids'] = raw_data['tracker_ids']
+
+        # set void pixels in tracker detections to zero
+        for t in range(num_timesteps):
+            void_mask = raw_data['masks_void'][t]
+            if mask_utils.area(void_mask) > 0:
+                void_mask_ious = np.atleast_1d(mask_utils.iou(raw_data['tracker_dets'][t], [void_mask], [False]))
+                if void_mask_ious.any():
+                    rows, columns = np.where(void_mask_ious > 0)
+                    for r in rows:
+                        det = mask_utils.decode(raw_data['tracker_dets'][t][r])
+                        void = mask_utils.decode(void_mask).astype(np.bool)
+                        det[void] = 0
+                        det = mask_utils.encode(np.array(det, order='F').astype(np.uint8))
+                        raw_data['tracker_dets'][t][r] = det
+        data['tracker_dets'] = raw_data['tracker_dets']
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = raw_data['num_tracker_ids']
+        data['num_gt_ids'] = raw_data['num_gt_ids']
+        data['mask_shape'] = raw_data['mask_shape']
+        data['num_timesteps'] = num_timesteps
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/head_tracking_challenge.py

@@ -0,0 +1,459 @@
+import os
+import csv
+import configparser
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class HeadTrackingChallenge(_BaseDataset):
+    """Dataset class for Head Tracking Challenge - 2D bounding box tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrian'],  # Valid: ['pedestrian']
+            'BENCHMARK': 'HT',  # Valid: 'HT'. Refers to "Head Tracking or the dataset CroHD"
+            'SPLIT_TO_EVAL': 'train',  # Valid: 'train', 'test', 'all'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'DO_PREPROC': True,  # Whether to perform preprocessing (never done for MOT15)
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt',  # '{gt_folder}/{seq}/gt/gt.txt'
+            'SKIP_SPLIT_FOL': False,  # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in
+                                      # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/
+                                      # If True, then the middle 'benchmark-split' folder is skipped for both.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+
+        self.benchmark = self.config['BENCHMARK']
+        gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL']
+        self.gt_set = gt_set
+        if not self.config['SKIP_SPLIT_FOL']:
+            split_fol = gt_set
+        else:
+            split_fol = ''
+        self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol)
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol)
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+        self.do_preproc = self.config['DO_PREPROC']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.')
+        self.class_name_to_class_id = {'pedestrian': 1, 'static': 2, 'ignore': 3, 'person_on_vehicle': 4}
+        self.valid_class_numbers = list(self.class_name_to_class_id.values())
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                print('GT file not found ' + curr_file)
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+
+            # If sequence length is 'None' tries to read sequence length from .ini files.
+            for seq, seq_length in seq_lengths.items():
+                if seq_length is None:
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt')
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt')
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if i == 0 or row[0] == '':
+                        continue
+                    seq = row[0]
+                    seq_list.append(seq)
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the MOT Challenge 2D box format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det).
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file)
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions', 'gt_extras']
+        else:
+            data_keys += ['tracker_confidences']
+
+        if self.benchmark == 'HT':
+            data_keys += ['visibility']
+            data_keys += ['gt_conf']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str( t+ 1) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t+1)
+            if time_key in read_data.keys():
+                try:
+                    time_data = np.asarray(read_data[time_key], dtype=np.float)
+                except ValueError:
+                    if is_gt:
+                        raise TrackEvalException(
+                            'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq)
+                    else:
+                        raise TrackEvalException(
+                            'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % (
+                                tracker, seq))
+                try:
+                    raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6])
+                    raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int)
+                except IndexError:
+                    if is_gt:
+                        err = 'Cannot load gt data from sequence %s, because there is not enough ' \
+                              'columns in the data.' % seq
+                        raise TrackEvalException(err)
+                    else:
+                        err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \
+                              'columns in the data.' % (tracker, seq)
+                        raise TrackEvalException(err)
+                if time_data.shape[1] >= 8:
+                    raw_data['gt_conf'][t] = np.atleast_1d(time_data[:, 6]).astype(float)
+                    raw_data['visibility'][t] = np.atleast_1d(time_data[:, 8]).astype(float)
+                    raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int)
+                else:
+                    if not is_gt:
+                        raw_data['classes'][t] = np.ones_like(raw_data['ids'][t])
+                    else:
+                        raise TrackEvalException(
+                            'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % (
+                                seq, t))
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6])
+            else:
+                raw_data['dets'][t] = np.empty((0, 4))
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.empty(0)}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+            if is_gt:
+                raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        MOT Challenge:
+            In MOT Challenge, the 4 preproc steps are as follow:
+                1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc.
+                2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor
+                    objects are removed.
+                3) There is no crowd ignore regions.
+                4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        # 'static': 2, 'ignore': 3, 'person_on_vehicle':
+
+        distractor_class_names = ['static', 'ignore', 'person_on_vehicle']
+
+        distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names]
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences',
+                     'similarity_scores', 'gt_visibility']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Get all data
+            gt_ids = raw_data['gt_ids'][t]
+            gt_dets = raw_data['gt_dets'][t]
+            gt_classes = raw_data['gt_classes'][t]
+            gt_visibility = raw_data['visibility'][t]
+            gt_conf = raw_data['gt_conf'][t]
+
+            gt_zero_marked = raw_data['gt_extras'][t]['zero_marked']
+
+            tracker_ids = raw_data['tracker_ids'][t]
+            tracker_dets = raw_data['tracker_dets'][t]
+            tracker_classes = raw_data['tracker_classes'][t]
+            tracker_confidences = raw_data['tracker_confidences'][t]
+            similarity_scores = raw_data['similarity_scores'][t]
+
+            # Evaluation is ONLY valid for pedestrian class
+            if len(tracker_classes) > 0 and np.max(tracker_classes) > 1:
+                raise TrackEvalException(
+                    'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at '
+                    'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t))
+
+            # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets
+            # which are labeled as belonging to a distractor class.
+            to_remove_tracker = np.array([], np.int)
+            if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+
+                # Check all classes are valid:
+                invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers)
+                if len(invalid_classes) > 0:
+                    print(' '.join([str(x) for x in invalid_classes]))
+                    raise(TrackEvalException('Attempting to evaluate using invalid gt classes. '
+                                             'This warning only triggers if preprocessing is performed, '
+                                             'e.g. not for MOT15 or where prepropressing is explicitly disabled. '
+                                             'Please either check your gt data, or disable preprocessing. '
+                                             'The following invalid classes were found in timestep ' + str(t) + ': ' +
+                                             ' '.join([str(x) for x in invalid_classes])))
+
+                matching_scores = similarity_scores.copy()
+
+                matching_scores[matching_scores < 0.4 - np.finfo('float').eps] = 0
+
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_rows = match_rows[actually_matched_mask]
+                match_cols = match_cols[actually_matched_mask]
+
+                is_distractor_class = np.logical_not(np.isin(gt_classes[match_rows], cls_id))
+                if self.benchmark == 'HT':
+                    is_invisible_class = gt_visibility[match_rows] < np.finfo('float').eps
+                    low_conf_class = gt_conf[match_rows] < np.finfo('float').eps
+                    are_distractors = np.logical_or(is_invisible_class, is_distractor_class, low_conf_class)
+                    to_remove_tracker = match_cols[are_distractors]
+                else:
+                    to_remove_tracker = match_cols[is_distractor_class]
+
+            # Apply preprocessing to remove all unwanted tracker dets.
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian
+            if self.do_preproc and self.benchmark == 'HT':
+                gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \
+                                  (np.equal(gt_classes, cls_id)) & \
+                                  (gt_visibility > 0.) & \
+                                  (gt_conf > 0.)
+
+            else:
+                # There are no classes for MOT15
+                gt_to_keep_mask = np.not_equal(gt_zero_marked, 0)
+            data['gt_ids'][t] = gt_ids[gt_to_keep_mask]
+            data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :]
+            data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask]
+            data['gt_visibility'][t] = gt_visibility # No mask!
+
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/kitti_2d_box.py

@@ -0,0 +1,389 @@
+
+import os
+import csv
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from ..utils import TrackEvalException
+from .. import _timing
+
+
+class Kitti2DBox(_BaseDataset):
+    """Dataset class for KITTI 2D bounding box tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/kitti/kitti_2d_box_train'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/kitti/kitti_2d_box_train/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['car', 'pedestrian'],  # Valid: ['car', 'pedestrian']
+            'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val', 'training_minus_val', 'test'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        self.max_occlusion = 2
+        self.max_truncation = 0
+        self.min_height = 25
+
+        # Get classes to eval
+        self.valid_classes = ['car', 'pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only classes [car, pedestrian] are valid.')
+        self.class_name_to_class_id = {'car': 1, 'van': 2, 'truck': 3, 'pedestrian': 4, 'person': 5,  # person sitting
+                                       'cyclist': 6, 'tram': 7, 'misc': 8, 'dontcare': 9, 'car_2': 1}
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list = []
+        self.seq_lengths = {}
+        seqmap_name = 'evaluate_tracking.seqmap.' + self.config['SPLIT_TO_EVAL']
+        seqmap_file = os.path.join(self.gt_fol, seqmap_name)
+        if not os.path.isfile(seqmap_file):
+            raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+        with open(seqmap_file) as fp:
+            dialect = csv.Sniffer().sniff(fp.read(1024))
+            fp.seek(0)
+            reader = csv.reader(fp, dialect)
+            for row in reader:
+                if len(row) >= 4:
+                    seq = row[0]
+                    self.seq_list.append(seq)
+                    self.seq_lengths[seq] = int(row[3])
+                    if not self.data_is_zipped:
+                        curr_file = os.path.join(self.gt_fol, 'label_02', seq + '.txt')
+                        if not os.path.isfile(curr_file):
+                            raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.gt_fol, 'data.zip')
+                if not os.path.isfile(curr_file):
+                    raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the kitti 2D box format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det).
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = os.path.join(self.gt_fol, 'label_02', seq + '.txt')
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Ignore regions
+        if is_gt:
+            crowd_ignore_filter = {2: ['dontcare']}
+        else:
+            crowd_ignore_filter = None
+
+        # Valid classes
+        valid_filter = {2: [x for x in self.class_list]}
+        if is_gt:
+            if 'car' in self.class_list:
+                valid_filter[2].append('van')
+            if 'pedestrian' in self.class_list:
+                valid_filter[2] += ['person']
+
+        # Convert kitti class strings to class ids
+        convert_filter = {2: self.class_name_to_class_id}
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, time_col=0, id_col=1, remove_negative_ids=True,
+                                                             valid_filter=valid_filter,
+                                                             crowd_ignore_filter=crowd_ignore_filter,
+                                                             convert_filter=convert_filter,
+                                                             is_zipped=self.data_is_zipped, zip_file=zip_file)
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions', 'gt_extras']
+        else:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str(t) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t)
+            if time_key in read_data.keys():
+                time_data = np.asarray(read_data[time_key], dtype=np.float)
+                raw_data['dets'][t] = np.atleast_2d(time_data[:, 6:10])
+                raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int)
+                raw_data['classes'][t] = np.atleast_1d(time_data[:, 2]).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'truncation': np.atleast_1d(time_data[:, 3].astype(int)),
+                                      'occlusion': np.atleast_1d(time_data[:, 4].astype(int))}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    if time_data.shape[1] > 17:
+                        raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 17])
+                    else:
+                        raw_data['tracker_confidences'][t] = np.ones(time_data.shape[0])
+            else:
+                raw_data['dets'][t] = np.empty((0, 4))
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'truncation': np.empty(0),
+                                      'occlusion': np.empty(0)}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+            if is_gt:
+                if time_key in ignore_data.keys():
+                    time_ignore = np.asarray(ignore_data[time_key], dtype=np.float)
+                    raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d(time_ignore[:, 6:10])
+                else:
+                    raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        KITTI:
+            In KITTI, the 4 preproc steps are as follow:
+                1) There are two classes (pedestrian and car) which are evaluated separately.
+                2) For the pedestrian class, the 'person' class is distractor objects (people sitting).
+                    For the car class, the 'van' class are distractor objects.
+                    GT boxes marked as having occlusion level > 2 or truncation level > 0 are also treated as
+                        distractors.
+                3) Crowd ignore regions are used to remove unmatched detections. Also unmatched detections with
+                    height <= 25 pixels are removed.
+                4) Distractor gt dets (including truncated and occluded) are removed.
+        """
+        if cls == 'pedestrian':
+            distractor_classes = [self.class_name_to_class_id['person']]
+        elif cls == 'car':
+            distractor_classes = [self.class_name_to_class_id['van']]
+        else:
+            raise (TrackEvalException('Class %s is not evaluatable' % cls))
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls + distractor classes)
+            gt_class_mask = np.sum([raw_data['gt_classes'][t] == c for c in [cls_id] + distractor_classes], axis=0)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+            gt_classes = raw_data['gt_classes'][t][gt_class_mask]
+            gt_occlusion = raw_data['gt_extras'][t]['occlusion'][gt_class_mask]
+            gt_truncation = raw_data['gt_extras'][t]['truncation'][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets
+            # which are labeled as truncated, occluded, or belonging to a distractor class.
+            to_remove_matched = np.array([], np.int)
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_rows = match_rows[actually_matched_mask]
+                match_cols = match_cols[actually_matched_mask]
+
+                is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes)
+                is_occluded_or_truncated = np.logical_or(
+                    gt_occlusion[match_rows] > self.max_occlusion + np.finfo('float').eps,
+                    gt_truncation[match_rows] > self.max_truncation + np.finfo('float').eps)
+                to_remove_matched = np.logical_or(is_distractor_class, is_occluded_or_truncated)
+                to_remove_matched = match_cols[to_remove_matched]
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            # For unmatched tracker dets, also remove those smaller than a minimum height.
+            unmatched_tracker_dets = tracker_dets[unmatched_indices, :]
+            unmatched_heights = unmatched_tracker_dets[:, 3] - unmatched_tracker_dets[:, 1]
+            is_too_small = unmatched_heights <= self.min_height + np.finfo('float').eps
+
+            # For unmatched tracker dets, also remove those that are greater than 50% within a crowd ignore region.
+            crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t]
+            intersection_with_ignore_region = self._calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions,
+                                                                       box_format='x0y0x1y1', do_ioa=True)
+            is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1)
+
+            # Apply preprocessing to remove all unwanted tracker dets.
+            to_remove_unmatched = unmatched_indices[np.logical_or(is_too_small, is_within_crowd_ignore_region)]
+            to_remove_tracker = np.concatenate((to_remove_matched, to_remove_unmatched), axis=0)
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Also remove gt dets that were only useful for preprocessing and are not needed for evaluation.
+            # These are those that are occluded, truncated and from distractor objects.
+            gt_to_keep_mask = (np.less_equal(gt_occlusion, self.max_occlusion)) & \
+                              (np.less_equal(gt_truncation, self.max_truncation)) & \
+                              (np.equal(gt_classes, cls_id))
+            data['gt_ids'][t] = gt_ids[gt_to_keep_mask]
+            data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :]
+            data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask]
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='x0y0x1y1')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/kitti_mots.py

@@ -0,0 +1,426 @@
+import os
+import csv
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class KittiMOTS(_BaseDataset):
+    """Dataset class for KITTI MOTS tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/kitti/kitti_mots_val'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/kitti/kitti_mots_val'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['car', 'pedestrian'],  # Valid: ['car', 'pedestrian']
+            'SPLIT_TO_EVAL': 'val',  # Valid: 'training', 'val'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER)
+            'SEQMAP_FILE': None,    # Directly specify seqmap file (if none use seqmap_folder/split_to_eval.seqmap)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/label_02/{seq}.txt',  # format of gt localization
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.split_to_eval = self.config['SPLIT_TO_EVAL']
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['car', 'pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. '
+                                     'Only classes [car, pedestrian] are valid.')
+        self.class_name_to_class_id = {'car': '1', 'pedestrian': '2', 'ignore': '10'}
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        seqmap_name = 'evaluate_mots.seqmap.' + self.config['SPLIT_TO_EVAL']
+
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], seqmap_name)
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], seqmap_name)
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, _ in enumerate(reader):
+                    dialect = csv.Sniffer().sniff(fp.read(1024))
+                    fp.seek(0)
+                    reader = csv.reader(fp, dialect)
+                    for row in reader:
+                        if len(row) >= 4:
+                            seq = "%04d" % int(row[0])
+                            seq_list.append(seq)
+                            seq_lengths[seq] = int(row[3]) + 1
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the KITTI MOTS format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [gt_ignore_region]: list (for each timestep) of masks for the ignore regions
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Ignore regions
+        if is_gt:
+            crowd_ignore_filter = {2: ['10']}
+        else:
+            crowd_ignore_filter = None
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, crowd_ignore_filter=crowd_ignore_filter,
+                                                             is_zipped=self.data_is_zipped, zip_file=zip_file,
+                                                             force_delimiters=' ')
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_ignore_region']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str(t) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t)
+            # list to collect all masks of a timestep to check for overlapping areas
+            all_masks = []
+            if time_key in read_data.keys():
+                try:
+                    raw_data['dets'][t] = [{'size': [int(region[3]), int(region[4])],
+                                            'counts': region[5].encode(encoding='UTF-8')}
+                                           for region in read_data[time_key]]
+                    raw_data['ids'][t] = np.atleast_1d([region[1] for region in read_data[time_key]]).astype(int)
+                    raw_data['classes'][t] = np.atleast_1d([region[2] for region in read_data[time_key]]).astype(int)
+                    all_masks += raw_data['dets'][t]
+                except IndexError:
+                    self._raise_index_error(is_gt, tracker, seq)
+                except ValueError:
+                    self._raise_value_error(is_gt, tracker, seq)
+            else:
+                raw_data['dets'][t] = []
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+            if is_gt:
+                if time_key in ignore_data.keys():
+                    try:
+                        time_ignore = [{'size': [int(region[3]), int(region[4])],
+                                        'counts': region[5].encode(encoding='UTF-8')}
+                                       for region in ignore_data[time_key]]
+                        raw_data['gt_ignore_region'][t] = mask_utils.merge([mask for mask in time_ignore],
+                                                                           intersect=False)
+                        all_masks += [raw_data['gt_ignore_region'][t]]
+                    except IndexError:
+                        self._raise_index_error(is_gt, tracker, seq)
+                    except ValueError:
+                        self._raise_value_error(is_gt, tracker, seq)
+                else:
+                    raw_data['gt_ignore_region'][t] = mask_utils.merge([], intersect=False)
+
+            # check for overlapping masks
+            if all_masks:
+                masks_merged = all_masks[0]
+                for mask in all_masks[1:]:
+                    if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0:
+                        raise TrackEvalException(
+                            'Tracker has overlapping masks. Tracker: ' + tracker + ' Seq: ' + seq + ' Timestep: ' + str(
+                                t))
+                    masks_merged = mask_utils.merge([masks_merged, mask], intersect=False)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data["num_timesteps"] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        KITTI MOTS:
+            In KITTI MOTS, the 4 preproc steps are as follow:
+                1) There are two classes (car and pedestrian) which are evaluated separately.
+                2) There are no ground truth detections marked as to be removed/distractor classes.
+                    Therefore also no matched tracker detections are removed.
+                3) Ignore regions are used to remove unmatched detections (at least 50% overlap with ignore region).
+                4) There are no ground truth detections (e.g. those of distractor classes) to be removed.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        cls_id = int(self.class_name_to_class_id[cls])
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if
+                            tracker_class_mask[ind]]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm)
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = -10000
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region.
+            unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if i in unmatched_indices]
+            ignore_region = raw_data['gt_ignore_region'][t]
+            intersection_with_ignore_region = self._calculate_mask_ious(unmatched_tracker_dets, [ignore_region],
+                                                                        is_encoded=True, do_ioa=True)
+            is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1)
+
+            # Apply preprocessing to remove unwanted tracker dets.
+            to_remove_tracker = unmatched_indices[is_within_ignore_region]
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Keep all ground truth detections
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+        data['cls'] = cls
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores
+
+    @staticmethod
+    def _raise_index_error(is_gt, tracker, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an index error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            err = 'Cannot load gt data from sequence %s, because there are not enough ' \
+                  'columns in the data.' % seq
+            raise TrackEvalException(err)
+        else:
+            err = 'Cannot load tracker data from tracker %s, sequence %s, because there are not enough ' \
+                  'columns in the data.' % (tracker, seq)
+            raise TrackEvalException(err)
+
+    @staticmethod
+    def _raise_value_error(is_gt, tracker, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an value error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            raise TrackEvalException(
+                'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq)
+        else:
+            raise TrackEvalException(
+                'Tracking data from tracker %s, sequence %s cannot be converted to the right format. '
+                'Is data corrupted?' % (tracker, seq))

yolov7-tracker-example/tracker/trackeval/datasets/mot_challenge_2d_box.py

@@ -0,0 +1,437 @@
+import os
+import csv
+import configparser
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class MotChallenge2DBox(_BaseDataset):
+    """Dataset class for MOT Challenge 2D bounding box tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrian'],  # Valid: ['pedestrian']
+            'BENCHMARK': 'MOT17',  # Valid: 'MOT17', 'MOT16', 'MOT20', 'MOT15'
+            'SPLIT_TO_EVAL': 'train',  # Valid: 'train', 'test', 'all'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'DO_PREPROC': True,  # Whether to perform preprocessing (never done for MOT15)
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt',  # '{gt_folder}/{seq}/gt/gt.txt'
+            'SKIP_SPLIT_FOL': False,  # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in
+                                      # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/
+                                      # If True, then the middle 'benchmark-split' folder is skipped for both.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+
+        self.benchmark = self.config['BENCHMARK']
+        gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL']
+        self.gt_set = gt_set
+        if not self.config['SKIP_SPLIT_FOL']:
+            split_fol = gt_set
+        else:
+            split_fol = ''
+        self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol)
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol)
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+        self.do_preproc = self.config['DO_PREPROC']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.')
+        self.class_name_to_class_id = {'pedestrian': 1, 'person_on_vehicle': 2, 'car': 3, 'bicycle': 4, 'motorbike': 5,
+                                       'non_mot_vehicle': 6, 'static_person': 7, 'distractor': 8, 'occluder': 9,
+                                       'occluder_on_ground': 10, 'occluder_full': 11, 'reflection': 12, 'crowd': 13}
+        self.valid_class_numbers = list(self.class_name_to_class_id.values())
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                print('GT file not found ' + curr_file)
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+
+            # If sequence length is 'None' tries to read sequence length from .ini files.
+            for seq, seq_length in seq_lengths.items():
+                if seq_length is None:
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt')
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt')
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if i == 0 or row[0] == '':
+                        continue
+                    seq = row[0]
+                    seq_list.append(seq)
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the MOT Challenge 2D box format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det).
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file)
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions', 'gt_extras']
+        else:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str( t+ 1) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t+1)
+            if time_key in read_data.keys():
+                try:
+                    time_data = np.asarray(read_data[time_key], dtype=np.float)
+                except ValueError:
+                    if is_gt:
+                        raise TrackEvalException(
+                            'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq)
+                    else:
+                        raise TrackEvalException(
+                            'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % (
+                                tracker, seq))
+                try:
+                    raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6])
+                    raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int)
+                except IndexError:
+                    if is_gt:
+                        err = 'Cannot load gt data from sequence %s, because there is not enough ' \
+                              'columns in the data.' % seq
+                        raise TrackEvalException(err)
+                    else:
+                        err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \
+                              'columns in the data.' % (tracker, seq)
+                        raise TrackEvalException(err)
+                if time_data.shape[1] >= 8:
+                    raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int)
+                else:
+                    if not is_gt:
+                        raw_data['classes'][t] = np.ones_like(raw_data['ids'][t])
+                    else:
+                        raise TrackEvalException(
+                            'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % (
+                                seq, t))
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6])
+            else:
+                raw_data['dets'][t] = np.empty((0, 4))
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.empty(0)}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+            if is_gt:
+                raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        MOT Challenge:
+            In MOT Challenge, the 4 preproc steps are as follow:
+                1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc.
+                2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor
+                    objects are removed.
+                3) There is no crowd ignore regions.
+                4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection']
+        if self.benchmark == 'MOT20':
+            distractor_class_names.append('non_mot_vehicle')
+        distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names]
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Get all data
+            gt_ids = raw_data['gt_ids'][t]
+            gt_dets = raw_data['gt_dets'][t]
+            gt_classes = raw_data['gt_classes'][t]
+            gt_zero_marked = raw_data['gt_extras'][t]['zero_marked']
+
+            tracker_ids = raw_data['tracker_ids'][t]
+            tracker_dets = raw_data['tracker_dets'][t]
+            tracker_classes = raw_data['tracker_classes'][t]
+            tracker_confidences = raw_data['tracker_confidences'][t]
+            similarity_scores = raw_data['similarity_scores'][t]
+
+            # Evaluation is ONLY valid for pedestrian class
+            if len(tracker_classes) > 0 and np.max(tracker_classes) > 1:
+                raise TrackEvalException(
+                    'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at '
+                    'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t))
+
+            # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets
+            # which are labeled as belonging to a distractor class.
+            to_remove_tracker = np.array([], np.int)
+            if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+
+                # Check all classes are valid:
+                invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers)
+                if len(invalid_classes) > 0:
+                    print(' '.join([str(x) for x in invalid_classes]))
+                    raise(TrackEvalException('Attempting to evaluate using invalid gt classes. '
+                                             'This warning only triggers if preprocessing is performed, '
+                                             'e.g. not for MOT15 or where prepropressing is explicitly disabled. '
+                                             'Please either check your gt data, or disable preprocessing. '
+                                             'The following invalid classes were found in timestep ' + str(t) + ': ' +
+                                             ' '.join([str(x) for x in invalid_classes])))
+
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_rows = match_rows[actually_matched_mask]
+                match_cols = match_cols[actually_matched_mask]
+
+                is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes)
+                to_remove_tracker = match_cols[is_distractor_class]
+
+            # Apply preprocessing to remove all unwanted tracker dets.
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian
+            # class (not applicable for MOT15)
+            if self.do_preproc and self.benchmark != 'MOT15':
+                gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \
+                                  (np.equal(gt_classes, cls_id))
+            else:
+                # There are no classes for MOT15
+                gt_to_keep_mask = np.not_equal(gt_zero_marked, 0)
+            data['gt_ids'][t] = gt_ids[gt_to_keep_mask]
+            data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :]
+            data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask]
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/mots_challenge.py

@@ -0,0 +1,446 @@
+import os
+import csv
+import configparser
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class MOTSChallenge(_BaseDataset):
+    """Dataset class for MOTS Challenge tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrian'],  # Valid: ['pedestrian']
+            'SPLIT_TO_EVAL': 'train',  # Valid: 'train', 'test'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use seqmap_folder/MOTS-split_to_eval)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt',  # '{gt_folder}/{seq}/gt/gt.txt'
+            'SKIP_SPLIT_FOL': False,  # If False, data is in GT_FOLDER/MOTS-SPLIT_TO_EVAL/ and in
+                                      # TRACKERS_FOLDER/MOTS-SPLIT_TO_EVAL/tracker/
+                                      # If True, then the middle 'MOTS-split' folder is skipped for both.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+
+        self.benchmark = 'MOTS'
+        self.gt_set = self.benchmark + '-' + self.config['SPLIT_TO_EVAL']
+        if not self.config['SKIP_SPLIT_FOL']:
+            split_fol = self.gt_set
+        else:
+            split_fol = ''
+        self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol)
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol)
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.')
+        self.class_name_to_class_id = {'pedestrian': '2', 'ignore': '10'}
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                print('GT file not found ' + curr_file)
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+
+            # If sequence length is 'None' tries to read sequence length from .ini files.
+            for seq, seq_length in seq_lengths.items():
+                if seq_length is None:
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt')
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt')
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if i == 0 or row[0] == '':
+                        continue
+                    seq = row[0]
+                    seq_list.append(seq)
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the MOTS Challenge format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [gt_ignore_region]: list (for each timestep) of masks for the ignore regions
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Ignore regions
+        if is_gt:
+            crowd_ignore_filter = {2: ['10']}
+        else:
+            crowd_ignore_filter = None
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, crowd_ignore_filter=crowd_ignore_filter,
+                                                             is_zipped=self.data_is_zipped, zip_file=zip_file,
+                                                             force_delimiters=' ')
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_ignore_region']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str(t + 1) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t+1)
+            # list to collect all masks of a timestep to check for overlapping areas
+            all_masks = []
+            if time_key in read_data.keys():
+                try:
+                    raw_data['dets'][t] = [{'size': [int(region[3]), int(region[4])],
+                                            'counts': region[5].encode(encoding='UTF-8')}
+                                           for region in read_data[time_key]]
+                    raw_data['ids'][t] = np.atleast_1d([region[1] for region in read_data[time_key]]).astype(int)
+                    raw_data['classes'][t] = np.atleast_1d([region[2] for region in read_data[time_key]]).astype(int)
+                    all_masks += raw_data['dets'][t]
+                except IndexError:
+                    self._raise_index_error(is_gt, tracker, seq)
+                except ValueError:
+                    self._raise_value_error(is_gt, tracker, seq)
+            else:
+                raw_data['dets'][t] = []
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+            if is_gt:
+                if time_key in ignore_data.keys():
+                    try:
+                        time_ignore = [{'size': [int(region[3]), int(region[4])],
+                                        'counts': region[5].encode(encoding='UTF-8')}
+                                       for region in ignore_data[time_key]]
+                        raw_data['gt_ignore_region'][t] = mask_utils.merge([mask for mask in time_ignore],
+                                                                           intersect=False)
+                        all_masks += [raw_data['gt_ignore_region'][t]]
+                    except IndexError:
+                        self._raise_index_error(is_gt, tracker, seq)
+                    except ValueError:
+                        self._raise_value_error(is_gt, tracker, seq)
+                else:
+                    raw_data['gt_ignore_region'][t] = mask_utils.merge([], intersect=False)
+
+            # check for overlapping masks
+            if all_masks:
+                masks_merged = all_masks[0]
+                for mask in all_masks[1:]:
+                    if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0:
+                        raise TrackEvalException(
+                            'Tracker has overlapping masks. Tracker: ' + tracker + ' Seq: ' + seq + ' Timestep: ' + str(
+                                t))
+                    masks_merged = mask_utils.merge([masks_merged, mask], intersect=False)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        MOTS Challenge:
+            In MOTS Challenge, the 4 preproc steps are as follow:
+                1) There is only one class (pedestrians) to be evaluated.
+                2) There are no ground truth detections marked as to be removed/distractor classes.
+                    Therefore also no matched tracker detections are removed.
+                3) Ignore regions are used to remove unmatched detections (at least 50% overlap with ignore region).
+                4) There are no ground truth detections (e.g. those of distractor classes) to be removed.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        cls_id = int(self.class_name_to_class_id[cls])
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if
+                            tracker_class_mask[ind]]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm)
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = -10000
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region.
+            unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if i in unmatched_indices]
+            ignore_region = raw_data['gt_ignore_region'][t]
+            intersection_with_ignore_region = self._calculate_mask_ious(unmatched_tracker_dets, [ignore_region],
+                                                                        is_encoded=True, do_ioa=True)
+            is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1)
+
+            # Apply preprocessing to remove unwanted tracker dets.
+            to_remove_tracker = unmatched_indices[is_within_ignore_region]
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Keep all ground truth detections
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores
+
+    @staticmethod
+    def _raise_index_error(is_gt, tracker, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an index error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            err = 'Cannot load gt data from sequence %s, because there are not enough ' \
+                  'columns in the data.' % seq
+            raise TrackEvalException(err)
+        else:
+            err = 'Cannot load tracker data from tracker %s, sequence %s, because there are not enough ' \
+                  'columns in the data.' % (tracker, seq)
+            raise TrackEvalException(err)
+
+    @staticmethod
+    def _raise_value_error(is_gt, tracker, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an value error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            raise TrackEvalException(
+                'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq)
+        else:
+            raise TrackEvalException(
+                'Tracking data from tracker %s, sequence %s cannot be converted to the right format. '
+                'Is data corrupted?' % (tracker, seq))

yolov7-tracker-example/tracker/trackeval/datasets/person_path_22.py

@@ -0,0 +1,452 @@
+import os
+import csv
+import configparser
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+class PersonPath22(_BaseDataset):
+    """Dataset class for MOT Challenge 2D bounding box tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/person_path_22/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/person_path_22/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrian'],  # Valid: ['pedestrian']
+            'BENCHMARK': 'person_path_22',  # Valid: 'person_path_22'
+            'SPLIT_TO_EVAL': 'test',  # Valid: 'train', 'test', 'all'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'DO_PREPROC': True,  # Whether to perform preprocessing
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt',  # '{gt_folder}/{seq}/gt/gt.txt'
+            'SKIP_SPLIT_FOL': False,  # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in
+                                      # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/
+                                      # If True, then the middle 'benchmark-split' folder is skipped for both.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+
+        self.benchmark = self.config['BENCHMARK']
+        gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL']
+        self.gt_set = gt_set
+        if not self.config['SKIP_SPLIT_FOL']:
+            split_fol = gt_set
+        else:
+            split_fol = ''
+        self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol)
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol)
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+        self.do_preproc = self.config['DO_PREPROC']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.')
+        self.class_name_to_class_id = {'pedestrian': 1, 'person_on_vehicle': 2, 'car': 3, 'bicycle': 4, 'motorbike': 5,
+                                       'non_mot_vehicle': 6, 'static_person': 7, 'distractor': 8, 'occluder': 9,
+                                       'occluder_on_ground': 10, 'occluder_full': 11, 'reflection': 12, 'crowd': 13}
+        self.valid_class_numbers = list(self.class_name_to_class_id.values())
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                print('GT file not found ' + curr_file)
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+
+            # If sequence length is 'None' tries to read sequence length from .ini files.
+            for seq, seq_length in seq_lengths.items():
+                if seq_length is None:
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt')
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt')
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if i == 0 or row[0] == '':
+                        continue
+                    seq = row[0]
+                    seq_list.append(seq)
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the MOT Challenge 2D box format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det).
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Ignore regions
+        if is_gt:
+            crowd_ignore_filter = {7: ['13']}
+        else:
+            crowd_ignore_filter = None
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file, crowd_ignore_filter=crowd_ignore_filter)
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions', 'gt_extras']
+        else:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str( t+ 1) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t+1)
+            if time_key in read_data.keys():
+                try:
+                    time_data = np.asarray(read_data[time_key], dtype=np.float)
+                except ValueError:
+                    if is_gt:
+                        raise TrackEvalException(
+                            'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq)
+                    else:
+                        raise TrackEvalException(
+                            'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % (
+                                tracker, seq))
+                try:
+                    raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6])
+                    raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int)
+                except IndexError:
+                    if is_gt:
+                        err = 'Cannot load gt data from sequence %s, because there is not enough ' \
+                              'columns in the data.' % seq
+                        raise TrackEvalException(err)
+                    else:
+                        err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \
+                              'columns in the data.' % (tracker, seq)
+                        raise TrackEvalException(err)
+                if time_data.shape[1] >= 8:
+                    raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int)
+                else:
+                    if not is_gt:
+                        raw_data['classes'][t] = np.ones_like(raw_data['ids'][t])
+                    else:
+                        raise TrackEvalException(
+                            'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % (
+                                seq, t))
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6])
+            else:
+                raw_data['dets'][t] = np.empty((0, 4))
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.empty(0)}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+            if is_gt:
+                if time_key in ignore_data.keys():
+                    time_ignore = np.asarray(ignore_data[time_key], dtype=np.float)
+                    raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d(time_ignore[:, 2:6])
+                else:
+                    raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        MOT Challenge:
+            In MOT Challenge, the 4 preproc steps are as follow:
+                1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc.
+                2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor
+                    objects are removed.
+                3) There is no crowd ignore regions.
+                4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection']
+        if self.benchmark == 'MOT20':
+            distractor_class_names.append('non_mot_vehicle')
+        distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names]
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Get all data
+            gt_ids = raw_data['gt_ids'][t]
+            gt_dets = raw_data['gt_dets'][t]
+            gt_classes = raw_data['gt_classes'][t]
+            gt_zero_marked = raw_data['gt_extras'][t]['zero_marked']
+
+            tracker_ids = raw_data['tracker_ids'][t]
+            tracker_dets = raw_data['tracker_dets'][t]
+            tracker_classes = raw_data['tracker_classes'][t]
+            tracker_confidences = raw_data['tracker_confidences'][t]
+            similarity_scores = raw_data['similarity_scores'][t]
+            crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t]
+
+            # Evaluation is ONLY valid for pedestrian class
+            if len(tracker_classes) > 0 and np.max(tracker_classes) > 1:
+                raise TrackEvalException(
+                    'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at '
+                    'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t))
+
+            # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets
+            # which are labeled as belonging to a distractor class.
+            to_remove_tracker = np.array([], np.int)
+            if self.do_preproc and self.benchmark != 'MOT15' and (gt_ids.shape[0] > 0 or len(crowd_ignore_regions) > 0) and tracker_ids.shape[0] > 0:
+
+                # Check all classes are valid:
+                invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers)
+                if len(invalid_classes) > 0:
+                    print(' '.join([str(x) for x in invalid_classes]))
+                    raise(TrackEvalException('Attempting to evaluate using invalid gt classes. '
+                                             'This warning only triggers if preprocessing is performed, '
+                                             'e.g. not for MOT15 or where prepropressing is explicitly disabled. '
+                                             'Please either check your gt data, or disable preprocessing. '
+                                             'The following invalid classes were found in timestep ' + str(t) + ': ' +
+                                             ' '.join([str(x) for x in invalid_classes])))
+
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_rows = match_rows[actually_matched_mask]
+                match_cols = match_cols[actually_matched_mask]
+
+                is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes)
+                to_remove_tracker = match_cols[is_distractor_class]
+
+                # remove bounding boxes that overlap with crowd ignore region.
+                intersection_with_ignore_region = self._calculate_box_ious(tracker_dets, crowd_ignore_regions, box_format='xywh', do_ioa=True)
+                is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.95 + np.finfo('float').eps, axis=1)
+                to_remove_tracker = np.unique(np.concatenate([to_remove_tracker, np.where(is_within_crowd_ignore_region)[0]]))
+
+            # Apply preprocessing to remove all unwanted tracker dets.
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian
+            # class (not applicable for MOT15)
+            if self.do_preproc and self.benchmark != 'MOT15':
+                gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \
+                                  (np.equal(gt_classes, cls_id))
+            else:
+                # There are no classes for MOT15
+                gt_to_keep_mask = np.not_equal(gt_zero_marked, 0)
+            data['gt_ids'][t] = gt_ids[gt_to_keep_mask]
+            data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :]
+            data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask]
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/rob_mots.py

@@ -0,0 +1,508 @@
+
+import os
+import csv
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from ..utils import TrackEvalException
+from .. import _timing
+from ..datasets.rob_mots_classmap import cls_id_to_name
+
+
+class RobMOTS(_BaseDataset):
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/rob_mots'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/rob_mots'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'SUB_BENCHMARK': None,  # REQUIRED. Sub-benchmark to eval. If None, then error.
+            # ['mots_challenge', 'kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'waymo', 'tao']
+            'CLASSES_TO_EVAL': None,  # List of classes to eval. If None, then it does all COCO classes.
+            'SPLIT_TO_EVAL': 'train',  # valid: ['train', 'val', 'test']
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'OUTPUT_SUB_FOLDER': 'results',  # Output files are saved in OUTPUT_FOLDER/DATA_LOC_FORMAT/OUTPUT_SUB_FOLDER
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/DATA_LOC_FORMAT/TRACKER_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/dataset_subfolder/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use SEQMAP_FOLDER/BENCHMARK_SPLIT_TO_EVAL)
+            'CLSMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/dataset_subfolder/clsmaps)
+            'CLSMAP_FILE': None,  # Directly specify seqmap file (if none use CLSMAP_FOLDER/BENCHMARK_SPLIT_TO_EVAL)
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config())
+
+        self.split = self.config['SPLIT_TO_EVAL']
+        valid_benchmarks = ['mots_challenge', 'kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'waymo', 'tao']
+        self.box_gt_benchmarks = ['waymo', 'tao']
+
+        self.sub_benchmark = self.config['SUB_BENCHMARK']
+        if not self.sub_benchmark:
+            raise TrackEvalException('SUB_BENCHMARK config input is required (there is no default value)' +
+                                     ', '.join(valid_benchmarks) + ' are valid.')
+        if self.sub_benchmark not in valid_benchmarks:
+            raise TrackEvalException('Attempted to evaluate an invalid benchmark: ' + self.sub_benchmark + '. Only benchmarks ' +
+                                     ', '.join(valid_benchmarks) + ' are valid.')
+
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], self.config['SPLIT_TO_EVAL'])
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = os.path.join(self.config['OUTPUT_SUB_FOLDER'], self.sub_benchmark)
+
+        # Loops through all sub-benchmarks, and reads in seqmaps to info on all sequences to eval.
+        self._get_seq_info()
+
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        valid_class_ids = np.atleast_1d(np.genfromtxt(os.path.join(self.gt_fol, self.split, self.sub_benchmark,
+                                                                   'clsmap.txt')))
+        valid_classes = [cls_id_to_name[int(x)] for x in valid_class_ids] + ['all']
+        self.valid_class_ids = valid_class_ids
+        self.class_name_to_class_id = {cls_name: cls_id for cls_id, cls_name in cls_id_to_name.items()}
+        self.class_name_to_class_id['all'] = -1
+        if not self.config['CLASSES_TO_EVAL']:
+            self.class_list = valid_classes
+        else:
+            self.class_list = [cls if cls in valid_classes else None
+                               for cls in self.config['CLASSES_TO_EVAL']]
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(valid_classes) + ' are valid.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data', seq + '.txt')
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data.zip')
+            if not os.path.isfile(curr_file):
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, 'data.zip')
+                if not os.path.isfile(curr_file):
+                    raise TrackEvalException('Tracker file not found: ' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, self.sub_benchmark, seq
+                                             + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + self.sub_benchmark + '/' + os.path.basename(curr_file))
+
+    def get_name(self):
+        return self.get_class_name() + '.' + self.sub_benchmark
+
+    def _get_seq_info(self):
+        self.seq_list = []
+        self.seq_lengths = {}
+        self.seq_sizes = {}
+        self.seq_ignore_class_ids = {}
+        if self.config["SEQMAP_FILE"]:
+            seqmap_file = self.config["SEQMAP_FILE"]
+        else:
+            if self.config["SEQMAP_FOLDER"] is None:
+                seqmap_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'seqmap.txt')
+            else:
+                seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.split + '.seqmap')
+        if not os.path.isfile(seqmap_file):
+            print('no seqmap found: ' + seqmap_file)
+            raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+        with open(seqmap_file) as fp:
+            dialect = csv.Sniffer().sniff(fp.readline(), delimiters=' ')
+            fp.seek(0)
+            reader = csv.reader(fp, dialect)
+            for i, row in enumerate(reader):
+                if len(row) >= 4:
+                    # first col: sequence, second col: sequence length, third and fourth col: sequence height/width
+                    # The rest of the columns list the 'sequence ignore class ids' which are classes not penalized as
+                    # FPs for this sequence.
+                    seq = row[0]
+                    self.seq_list.append(seq)
+                    self.seq_lengths[seq] = int(row[1])
+                    self.seq_sizes[seq] = (int(row[2]), int(row[3]))
+                    self.seq_ignore_class_ids[seq] = [int(row[x]) for x in range(4, len(row))]
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the unified RobMOTS format.
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # import to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, 'data.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data', seq + '.txt')
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, self.sub_benchmark, seq + '.txt')
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file,
+                                                             force_delimiters=' ')
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for t in range(num_timesteps):
+            time_key = str(t)
+            # list to collect all masks of a timestep to check for overlapping areas (for segmentation datasets)
+            all_valid_masks = []
+            if time_key in read_data.keys():
+                try:
+                    raw_data['ids'][t] = np.atleast_1d([det[1] for det in read_data[time_key]]).astype(int)
+                    raw_data['classes'][t] = np.atleast_1d([det[2] for det in read_data[time_key]]).astype(int)
+                    if (not is_gt) or (self.sub_benchmark not in self.box_gt_benchmarks):
+                        raw_data['dets'][t] = [{'size': [int(region[4]), int(region[5])],
+                                                'counts': region[6].encode(encoding='UTF-8')}
+                                               for region in read_data[time_key]]
+                        all_valid_masks += [mask for mask, cls in zip(raw_data['dets'][t], raw_data['classes'][t]) if
+                                      cls < 100]
+                    else:
+                        raw_data['dets'][t] = np.atleast_2d([det[4:8] for det in read_data[time_key]]).astype(float)
+
+                    if not is_gt:
+                        raw_data['tracker_confidences'][t] = np.atleast_1d([det[3] for det
+                                                                            in read_data[time_key]]).astype(float)
+                except IndexError:
+                    self._raise_index_error(is_gt, self.sub_benchmark, seq)
+                except ValueError:
+                    self._raise_value_error(is_gt, self.sub_benchmark, seq)
+            # no detection in this timestep
+            else:
+                if (not is_gt) or (self.sub_benchmark not in self.box_gt_benchmarks):
+                    raw_data['dets'][t] = []
+                else:
+                    raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if not is_gt:
+                    raw_data['tracker_confidences'][t] = np.empty(0).astype(float)
+
+            # check for overlapping masks
+            if all_valid_masks:
+                masks_merged = all_valid_masks[0]
+                for mask in all_valid_masks[1:]:
+                    if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0:
+                        err = 'Overlapping masks in frame %d' % t
+                        raise TrackEvalException(err)
+                    masks_merged = mask_utils.merge([masks_merged, mask], intersect=False)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['frame_size'] = self.seq_sizes[seq]
+        raw_data['seq'] = seq
+        return raw_data
+
+    @staticmethod
+    def _raise_index_error(is_gt, sub_benchmark, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an index error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            err = 'Cannot load gt data from sequence %s, because there are not enough ' \
+                  'columns in the data.' % seq
+            raise TrackEvalException(err)
+        else:
+            err = 'Cannot load tracker data from benchmark %s, sequence %s, because there are not enough ' \
+                  'columns in the data.' % (sub_benchmark, seq)
+            raise TrackEvalException(err)
+
+    @staticmethod
+    def _raise_value_error(is_gt, sub_benchmark, seq):
+        """
+        Auxiliary method to raise an evaluation error in case of an value error while reading files.
+        :param is_gt: whether gt or tracker data is read
+        :param tracker: the name of the tracker
+        :param seq: the name of the seq
+        :return: None
+        """
+        if is_gt:
+            raise TrackEvalException(
+                'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq)
+        else:
+            raise TrackEvalException(
+                'Tracking data from benchmark %s, sequence %s cannot be converted to the right format. '
+                'Is data corrupted?' % (sub_benchmark, seq))
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            Preprocessing (preproc) occurs in 3 steps.
+                1) Extract only detections relevant for the class to be evaluated.
+                2) Match gt dets and tracker dets. Tracker dets that are to a gt det (TPs) are marked as not to be
+                    removed.
+                3) Remove unmatched tracker dets if they fall within an ignore region or are too small, or if that class
+                    is marked as an ignore class for that sequence.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+            Note that there is a special 'all' class, which evaluates all of the COCO classes together in a
+                'class agnostic' fashion.
+        """
+        # import to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        cls_id = self.class_name_to_class_id[cls]
+        ignore_class_id = cls_id+100
+        seq = raw_data['seq']
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class
+            if cls == 'all':
+                gt_class_mask = raw_data['gt_classes'][t] < 100
+            # For waymo, combine predictions for [car, truck, bus, motorcycle] into car, because they are all annotated
+            # together as one 'vehicle' class.
+            elif self.sub_benchmark == 'waymo' and cls == 'car':
+                waymo_vehicle_classes = np.array([3, 4, 6, 8])
+                gt_class_mask = np.isin(raw_data['gt_classes'][t], waymo_vehicle_classes)
+            else:
+                gt_class_mask = raw_data['gt_classes'][t] == cls_id
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            if cls == 'all':
+                ignore_regions_mask = raw_data['gt_classes'][t] >= 100
+            else:
+                ignore_regions_mask = raw_data['gt_classes'][t] == ignore_class_id
+                ignore_regions_mask = np.logical_or(ignore_regions_mask, raw_data['gt_classes'][t] == 100)
+            if self.sub_benchmark in self.box_gt_benchmarks:
+                gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+                ignore_regions_box = raw_data['gt_dets'][t][ignore_regions_mask]
+                if len(ignore_regions_box) > 0:
+                    ignore_regions_box[:, 2] = ignore_regions_box[:, 2] - ignore_regions_box[:, 0]
+                    ignore_regions_box[:, 3] = ignore_regions_box[:, 3] - ignore_regions_box[:, 1]
+                    ignore_regions = mask_utils.frPyObjects(ignore_regions_box, self.seq_sizes[seq][0], self.seq_sizes[seq][1])
+                else:
+                    ignore_regions = []
+            else:
+                gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]]
+                ignore_regions = [raw_data['gt_dets'][t][ind] for ind in range(len(ignore_regions_mask)) if
+                                  ignore_regions_mask[ind]]
+
+            if cls == 'all':
+                tracker_class_mask = np.ones_like(raw_data['tracker_classes'][t])
+            else:
+                tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if
+                            tracker_class_mask[ind]]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+            tracker_classes = raw_data['tracker_classes'][t][tracker_class_mask]
+
+            # Only do preproc if there are ignore regions defined to remove
+            if tracker_ids.shape[0] > 0:
+
+                # Match tracker and gt dets (with hungarian algorithm)
+                unmatched_indices = np.arange(tracker_ids.shape[0])
+                if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                    matching_scores = similarity_scores.copy()
+                    matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                    match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                    actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                    # match_rows = match_rows[actually_matched_mask]
+                    match_cols = match_cols[actually_matched_mask]
+                    unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+                # For unmatched tracker dets remove those that are greater than 50% within an ignore region.
+                # unmatched_tracker_dets = tracker_dets[unmatched_indices, :]
+                # crowd_ignore_regions = raw_data['gt_ignore_regions'][t]
+                # intersection_with_ignore_region = self. \
+                #     _calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions, box_format='x0y0x1y1',
+                #                         do_ioa=True)
+
+
+                if cls_id in self.seq_ignore_class_ids[seq]:
+                    # Remove unmatched detections for classes that are marked as 'ignore' for the whole sequence.
+                    to_remove_tracker = unmatched_indices
+                else:
+                    unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if
+                                              i in unmatched_indices]
+
+                    # For unmatched tracker dets remove those that are too small.
+                    tracker_boxes_t = mask_utils.toBbox(unmatched_tracker_dets)
+                    unmatched_widths = tracker_boxes_t[:, 2]
+                    unmatched_heights = tracker_boxes_t[:, 3]
+                    unmatched_size = np.maximum(unmatched_heights, unmatched_widths)
+                    min_size = np.min(self.seq_sizes[seq])/8
+                    is_too_small = unmatched_size <= min_size + np.finfo('float').eps
+
+                    # For unmatched tracker dets remove those that are greater than 50% within an ignore region.
+                    if ignore_regions:
+                        ignore_region_merged = ignore_regions[0]
+                        for mask in ignore_regions[1:]:
+                            ignore_region_merged = mask_utils.merge([ignore_region_merged, mask], intersect=False)
+                        intersection_with_ignore_region = self. \
+                            _calculate_mask_ious(unmatched_tracker_dets, [ignore_region_merged], is_encoded=True, do_ioa=True)
+                        is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1)
+                        to_remove_tracker = unmatched_indices[np.logical_or(is_too_small, is_within_ignore_region)]
+                    else:
+                        to_remove_tracker = unmatched_indices[is_too_small]
+
+                # For the special 'all' class, you need to remove unmatched detections from all ignore classes and
+                #   non-evaluated classes.
+                if cls == 'all':
+                    unmatched_tracker_classes = [tracker_classes[i] for i in range(len(tracker_classes)) if
+                                              i in unmatched_indices]
+                    is_ignore_class = np.isin(unmatched_tracker_classes, self.seq_ignore_class_ids[seq])
+                    is_not_evaled_class = np.logical_not(np.isin(unmatched_tracker_classes, self.valid_class_ids))
+                    to_remove_all = unmatched_indices[np.logical_or(is_ignore_class, is_not_evaled_class)]
+                    to_remove_tracker = np.concatenate([to_remove_tracker, to_remove_all], axis=0)
+            else:
+                to_remove_tracker = np.array([], dtype=np.int)
+
+            # remove all unwanted tracker detections
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # keep all ground truth detections
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+        data['frame_size'] = raw_data['frame_size']
+
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        if self.sub_benchmark in self.box_gt_benchmarks:
+            # Convert tracker masks to bboxes (for benchmarks with only bbox ground-truth),
+            # and then convert to x0y0x1y1 format.
+            tracker_boxes_t = mask_utils.toBbox(tracker_dets_t)
+            tracker_boxes_t[:, 2] = tracker_boxes_t[:, 0] + tracker_boxes_t[:, 2]
+            tracker_boxes_t[:, 3] = tracker_boxes_t[:, 1] + tracker_boxes_t[:, 3]
+            similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_boxes_t, box_format='x0y0x1y1')
+        else:
+            similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/rob_mots_classmap.py

@@ -0,0 +1,81 @@
+cls_id_to_name = {
+ 1: 'person',
+ 2: 'bicycle',
+ 3: 'car',
+ 4: 'motorcycle',
+ 5: 'airplane',
+ 6: 'bus',
+ 7: 'train',
+ 8: 'truck',
+ 9: 'boat',
+ 10: 'traffic light',
+ 11: 'fire hydrant',
+ 12: 'stop sign',
+ 13: 'parking meter',
+ 14: 'bench',
+ 15: 'bird',
+ 16: 'cat',
+ 17: 'dog',
+ 18: 'horse',
+ 19: 'sheep',
+ 20: 'cow',
+ 21: 'elephant',
+ 22: 'bear',
+ 23: 'zebra',
+ 24: 'giraffe',
+ 25: 'backpack',
+ 26: 'umbrella',
+ 27: 'handbag',
+ 28: 'tie',
+ 29: 'suitcase',
+ 30: 'frisbee',
+ 31: 'skis',
+ 32: 'snowboard',
+ 33: 'sports ball',
+ 34: 'kite',
+ 35: 'baseball bat',
+ 36: 'baseball glove',
+ 37: 'skateboard',
+ 38: 'surfboard',
+ 39: 'tennis racket',
+ 40: 'bottle',
+ 41: 'wine glass',
+ 42: 'cup',
+ 43: 'fork',
+ 44: 'knife',
+ 45: 'spoon',
+ 46: 'bowl',
+ 47: 'banana',
+ 48: 'apple',
+ 49: 'sandwich',
+ 50: 'orange',
+ 51: 'broccoli',
+ 52: 'carrot',
+ 53: 'hot dog',
+ 54: 'pizza',
+ 55: 'donut',
+ 56: 'cake',
+ 57: 'chair',
+ 58: 'couch',
+ 59: 'potted plant',
+ 60: 'bed',
+ 61: 'dining table',
+ 62: 'toilet',
+ 63: 'tv',
+ 64: 'laptop',
+ 65: 'mouse',
+ 66: 'remote',
+ 67: 'keyboard',
+ 68: 'cell phone',
+ 69: 'microwave',
+ 70: 'oven',
+ 71: 'toaster',
+ 72: 'sink',
+ 73: 'refrigerator',
+ 74: 'book',
+ 75: 'clock',
+ 76: 'vase',
+ 77: 'scissors',
+ 78: 'teddy bear',
+ 79: 'hair drier',
+ 80: 'toothbrush'}

yolov7-tracker-example/tracker/trackeval/datasets/run_rob_mots.py

@@ -0,0 +1,113 @@
+
+# python3 scripts\run_rob_mots.py --ROBMOTS_SPLIT val --TRACKERS_TO_EVAL tracker_name (e.g. STP) --USE_PARALLEL True --NUM_PARALLEL_CORES 4
+
+import sys
+import os
+import csv
+import numpy as np
+from multiprocessing import freeze_support
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+import trackeval  # noqa: E402
+from trackeval import utils
+code_path = utils.get_code_path()
+
+if __name__ == '__main__':
+    freeze_support()
+
+    script_config = {
+        'ROBMOTS_SPLIT': 'train',  # 'train',  # valid: 'train', 'val', 'test', 'test_live', 'test_post', 'test_all'
+        'BENCHMARKS': ['kitti_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao'], # 'bdd_mots' coming soon
+        'GT_FOLDER': os.path.join(code_path, 'data/gt/rob_mots'),
+        'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/rob_mots'),
+    }
+
+    default_eval_config = trackeval.Evaluator.get_default_eval_config()
+    default_eval_config['PRINT_ONLY_COMBINED'] = True
+    default_eval_config['DISPLAY_LESS_PROGRESS'] = True
+    default_dataset_config = trackeval.datasets.RobMOTS.get_default_dataset_config()
+    config = {**default_eval_config, **default_dataset_config, **script_config}
+
+    # Command line interface:
+    config = utils.update_config(config)
+
+    if config['ROBMOTS_SPLIT'] == 'val':
+        config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis',
+                                       'tao', 'mots_challenge']
+        config['SPLIT_TO_EVAL'] = 'val'
+    elif config['ROBMOTS_SPLIT'] == 'test' or config['SPLIT_TO_EVAL'] == 'test_live':
+        config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao']
+        config['SPLIT_TO_EVAL'] = 'test'
+    elif config['ROBMOTS_SPLIT'] == 'test_post':
+        config['BENCHMARKS'] = ['mots_challenge', 'waymo']
+        config['SPLIT_TO_EVAL'] = 'test'
+    elif config['ROBMOTS_SPLIT'] == 'test_all':
+        config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis',
+                                       'tao', 'mots_challenge', 'waymo']
+        config['SPLIT_TO_EVAL'] = 'test'
+    elif config['ROBMOTS_SPLIT'] == 'train':
+        config['BENCHMARKS'] = ['kitti_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao']  # 'bdd_mots' coming soon
+        config['SPLIT_TO_EVAL'] = 'train'
+
+    metrics_config = {'METRICS': ['HOTA']}
+    # metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity']}
+    eval_config = {k: v for k, v in config.items() if k in config.keys()}
+    dataset_config = {k: v for k, v in config.items() if k in config.keys()}
+
+    # Run code
+    dataset_list = []
+    for bench in config['BENCHMARKS']:
+        dataset_config['SUB_BENCHMARK'] = bench
+        dataset_list.append(trackeval.datasets.RobMOTS(dataset_config))
+    evaluator = trackeval.Evaluator(eval_config)
+    metrics_list = []
+    for metric in [trackeval.metrics.HOTA, trackeval.metrics.CLEAR, trackeval.metrics.Identity]:
+        if metric.get_name() in metrics_config['METRICS']:
+            metrics_list.append(metric())
+    if len(metrics_list) == 0:
+        raise Exception('No metrics selected for evaluation')
+    output_res, output_msg = evaluator.evaluate(dataset_list, metrics_list)
+
+
+    # For each benchmark, combine the 'all' score with the 'cls_averaged' using geometric mean.
+    metrics_to_calc = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA']
+    trackers = list(output_res['RobMOTS.' + config['BENCHMARKS'][0]].keys())
+    for tracker in trackers:
+        # final_results[benchmark][result_type][metric]
+        final_results = {}
+        res = {bench: output_res['RobMOTS.' + bench][tracker]['COMBINED_SEQ'] for bench in config['BENCHMARKS']}
+        for bench in config['BENCHMARKS']:
+            final_results[bench] = {'cls_av': {}, 'det_av': {}, 'final': {}}
+            for metric in metrics_to_calc:
+                final_results[bench]['cls_av'][metric] = np.mean(res[bench]['cls_comb_cls_av']['HOTA'][metric])
+                final_results[bench]['det_av'][metric] = np.mean(res[bench]['all']['HOTA'][metric])
+                final_results[bench]['final'][metric] = \
+                    np.sqrt(final_results[bench]['cls_av'][metric] * final_results[bench]['det_av'][metric])
+
+        # Take the arithmetic mean over all the benchmarks
+        final_results['overall'] = {'cls_av': {}, 'det_av': {}, 'final': {}}
+        for metric in metrics_to_calc:
+            final_results['overall']['cls_av'][metric] = \
+                np.mean([final_results[bench]['cls_av'][metric] for bench in config['BENCHMARKS']])
+            final_results['overall']['det_av'][metric] = \
+                np.mean([final_results[bench]['det_av'][metric] for bench in config['BENCHMARKS']])
+            final_results['overall']['final'][metric] = \
+                np.mean([final_results[bench]['final'][metric] for bench in config['BENCHMARKS']])
+
+        # Save out result
+        headers = [config['SPLIT_TO_EVAL']] + [x + '___' + metric for x in ['f', 'c', 'd'] for metric in metrics_to_calc]
+
+        def rowify(d):
+            return [d[x][metric] for x in ['final', 'cls_av', 'det_av'] for metric in metrics_to_calc]
+
+        out_file = os.path.join(script_config['TRACKERS_FOLDER'], script_config['ROBMOTS_SPLIT'], tracker,
+                                'final_results.csv')
+
+        with open(out_file, 'w', newline='') as f:
+            writer = csv.writer(f, delimiter=',')
+            writer.writerow(headers)
+            writer.writerow(['overall'] + rowify(final_results['overall']))
+            for bench in config['BENCHMARKS']:
+                if bench == 'overall':
+                    continue
+                writer.writerow([bench] + rowify(final_results[bench]))

yolov7-tracker-example/tracker/trackeval/datasets/tao.py

@@ -0,0 +1,566 @@
+import os
+import numpy as np
+import json
+import itertools
+from collections import defaultdict
+from scipy.optimize import linear_sum_assignment
+from ..utils import TrackEvalException
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+
+
+class TAO(_BaseDataset):
+    """Dataset class for TAO tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
+            'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val'
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'MAX_DETECTIONS': 300,  # Number of maximal allowed detections per image (0 for unlimited)
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = True
+        self.use_super_categories = False
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')]
+        if len(gt_dir_files) != 1:
+            raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.')
+
+        with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
+            self.gt_data = json.load(f)
+
+        # merge categories marked with a merged tag in TAO dataset
+        self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks'])
+
+        # Get sequences to eval and sequence information
+        self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']]
+        self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']}
+        # compute mappings from videos to annotation data
+        self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations'])
+        # compute sequence lengths
+        self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']}
+        for img in self.gt_data['images']:
+            self.seq_lengths[img['video_id']] += 1
+        self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings()
+        self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track
+                                                                in self.videos_to_gt_tracks[vid['id']]}),
+                                           'neg_cat_ids': vid['neg_category_ids'],
+                                           'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']}
+                               for vid in self.gt_data['videos']}
+
+        # Get classes to eval
+        considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list]
+        seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id
+                         in self.seq_to_classes[vid_id]['pos_cat_ids']])
+        # only classes with ground truth are evaluated in TAO
+        self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats]
+        cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
+
+        if self.config['CLASSES_TO_EVAL']:
+            self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                               for cls in self.config['CLASSES_TO_EVAL']]
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(self.valid_classes) +
+                                         ' are valid (classes present in ground truth data).')
+        else:
+            self.class_list = [cls for cls in self.valid_classes]
+        self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        self.tracker_data = {tracker: dict() for tracker in self.tracker_list}
+
+        for tracker in self.tracker_list:
+            tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol))
+                            if file.endswith('.json')]
+            if len(tr_dir_files) != 1:
+                raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+                                         + ' does not contain exactly one json file.')
+            with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f:
+                curr_data = json.load(f)
+
+            # limit detections if MAX_DETECTIONS > 0
+            if self.config['MAX_DETECTIONS']:
+                curr_data = self._limit_dets_per_image(curr_data)
+
+            # fill missing video ids
+            self._fill_video_ids_inplace(curr_data)
+
+            # make track ids unique over whole evaluation set
+            self._make_track_ids_unique(curr_data)
+
+            # merge categories marked with a merged tag in TAO dataset
+            self._merge_categories(curr_data)
+
+            # get tracker sequence information
+            curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data)
+            self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks
+            self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the TAO format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values
+                                as keys and lists (for each track) as values
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values
+                                                                                           as keys and lists as values
+        [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
+        """
+        seq_id = self.seq_name_to_seq_id[seq]
+        # File location
+        if is_gt:
+            imgs = self.videos_to_gt_images[seq_id]
+        else:
+            imgs = self.tracker_data[tracker]['vids_to_images'][seq_id]
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq_id]
+        img_to_timestep = self.seq_to_images_to_timestep[seq_id]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for img in imgs:
+            # some tracker data contains images without any ground truth information, these are ignored
+            try:
+                t = img_to_timestep[img['id']]
+            except KeyError:
+                continue
+            annotations = img['annotations']
+            raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float)
+            raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int)
+            raw_data['classes'][t] = np.atleast_1d([ann['category_id'] for ann in annotations]).astype(int)
+            if not is_gt:
+                raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float)
+
+        for t, d in enumerate(raw_data['dets']):
+            if d is None:
+                raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if not is_gt:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list]
+        if is_gt:
+            classes_to_consider = all_classes
+            all_tracks = self.videos_to_gt_tracks[seq_id]
+        else:
+            classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \
+                                  + self.seq_to_classes[seq_id]['neg_cat_ids']
+            all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id]
+
+        classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls]
+                             if cls in classes_to_consider else [] for cls in all_classes}
+
+        # mapping from classes to track information
+        raw_data['classes_to_tracks'] = {cls: [{det['image_id']: np.atleast_1d(det['bbox'])
+                                                for det in track['annotations']} for track in tracks]
+                                         for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks]
+                                            for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks]
+                                              for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks]
+                                                for cls, tracks in classes_to_tracks.items()}
+
+        if not is_gt:
+            raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score'])
+                                                                              for x in track['annotations']])
+                                                                     for track in tracks])
+                                                      for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            key_map = {'classes_to_tracks': 'classes_to_gt_tracks',
+                       'classes_to_track_ids': 'classes_to_gt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_gt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_gt_track_areas'}
+        else:
+            key_map = {'classes_to_tracks': 'classes_to_dt_tracks',
+                       'classes_to_track_ids': 'classes_to_dt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_dt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_dt_track_areas'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids']
+        raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids']
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+        TAO:
+            In TAO, the 4 preproc steps are as follow:
+                1) All classes present in the ground truth data are evaluated separately.
+                2) No matched tracker detections are removed.
+                3) Unmatched tracker detections are removed if there is not ground truth data and the class does not
+                    belong to the categories marked as negative for this sequence. Additionally, unmatched tracker
+                    detections for classes which are marked as not exhaustively labeled are removed.
+                4) No gt detections are removed.
+            Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
+            and the tracks from the tracker data are sorted according to the tracker confidence.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+        is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls']
+        is_neg_category = cls_id in raw_data['neg_cat_ids']
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm).
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            if gt_ids.shape[0] == 0 and not is_neg_category:
+                to_remove_tracker = unmatched_indices
+            elif is_not_exhaustively_labeled:
+                to_remove_tracker = unmatched_indices
+            else:
+                to_remove_tracker = np.array([], dtype=np.int)
+
+            # remove all unwanted unmatched tracker detections
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # get track representations
+        data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id]
+        data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id]
+        data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id]
+        data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id]
+        data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id]
+        data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id]
+        data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id]
+        data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id]
+        data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id]
+        data['not_exhaustively_labeled'] = is_not_exhaustively_labeled
+        data['iou_type'] = 'bbox'
+
+        # sort tracker data tracks by tracker confidence scores
+        if data['dt_tracks']:
+            idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort")
+            data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx]
+            data['dt_tracks'] = [data['dt_tracks'][i] for i in idx]
+            data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx]
+            data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx]
+            data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx]
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t)
+        return similarity_scores
+
+    def _merge_categories(self, annotations):
+        """
+        Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset
+        :param annotations: the annotations in which the classes should be merged
+        :return: None
+        """
+        merge_map = {}
+        for category in self.gt_data['categories']:
+            if 'merged' in category:
+                for to_merge in category['merged']:
+                    merge_map[to_merge['id']] = category['id']
+
+        for ann in annotations:
+            ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id'])
+
+    def _compute_vid_mappings(self, annotations):
+        """
+        Computes mappings from Videos to corresponding tracks and images.
+        :param annotations: the annotations for which the mapping should be generated
+        :return: the video-to-track-mapping, the video-to-image-mapping
+        """
+        vids_to_tracks = {}
+        vids_to_imgs = {}
+        vid_ids = [vid['id'] for vid in self.gt_data['videos']]
+
+        # compute an mapping from image IDs to images
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        for ann in annotations:
+            ann["area"] = ann["bbox"][2] * ann["bbox"][3]
+
+            vid = ann["video_id"]
+            if ann["video_id"] not in vids_to_tracks.keys():
+                vids_to_tracks[ann["video_id"]] = list()
+            if ann["video_id"] not in vids_to_imgs.keys():
+                vids_to_imgs[ann["video_id"]] = list()
+
+            # Fill in vids_to_tracks
+            tid = ann["track_id"]
+            exist_tids = [track["id"] for track in vids_to_tracks[vid]]
+            try:
+                index1 = exist_tids.index(tid)
+            except ValueError:
+                index1 = -1
+            if tid not in exist_tids:
+                curr_track = {"id": tid, "category_id": ann['category_id'],
+                              "video_id": vid, "annotations": [ann]}
+                vids_to_tracks[vid].append(curr_track)
+            else:
+                vids_to_tracks[vid][index1]["annotations"].append(ann)
+
+            # Fill in vids_to_imgs
+            img_id = ann['image_id']
+            exist_img_ids = [img["id"] for img in vids_to_imgs[vid]]
+            try:
+                index2 = exist_img_ids.index(img_id)
+            except ValueError:
+                index2 = -1
+            if index2 == -1:
+                curr_img = {"id": img_id, "annotations": [ann]}
+                vids_to_imgs[vid].append(curr_img)
+            else:
+                vids_to_imgs[vid][index2]["annotations"].append(ann)
+
+        # sort annotations by frame index and compute track area
+        for vid, tracks in vids_to_tracks.items():
+            for track in tracks:
+                track["annotations"] = sorted(
+                    track['annotations'],
+                    key=lambda x: images[x['image_id']]['frame_index'])
+                # Computer average area
+                track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations']))
+
+        # Ensure all videos are present
+        for vid_id in vid_ids:
+            if vid_id not in vids_to_tracks.keys():
+                vids_to_tracks[vid_id] = []
+            if vid_id not in vids_to_imgs.keys():
+                vids_to_imgs[vid_id] = []
+
+        return vids_to_tracks, vids_to_imgs
+
+    def _compute_image_to_timestep_mappings(self):
+        """
+        Computes a mapping from images to the corresponding timestep in the sequence.
+        :return: the image-to-timestep-mapping
+        """
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']}
+        for vid in seq_to_imgs_to_timestep:
+            curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]]
+            curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index'])
+            seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))}
+
+        return seq_to_imgs_to_timestep
+
+    def _limit_dets_per_image(self, annotations):
+        """
+        Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from
+        https://github.com/TAO-Dataset/
+        :param annotations: the annotations in which the detections should be limited
+        :return: the annotations with limited detections
+        """
+        max_dets = self.config['MAX_DETECTIONS']
+        img_ann = defaultdict(list)
+        for ann in annotations:
+            img_ann[ann["image_id"]].append(ann)
+
+        for img_id, _anns in img_ann.items():
+            if len(_anns) <= max_dets:
+                continue
+            _anns = sorted(_anns, key=lambda x: x["score"], reverse=True)
+            img_ann[img_id] = _anns[:max_dets]
+
+        return [ann for anns in img_ann.values() for ann in anns]
+
+    def _fill_video_ids_inplace(self, annotations):
+        """
+        Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotations for which the videos IDs should be filled inplace
+        :return: None
+        """
+        missing_video_id = [x for x in annotations if 'video_id' not in x]
+        if missing_video_id:
+            image_id_to_video_id = {
+                x['id']: x['video_id'] for x in self.gt_data['images']
+            }
+            for x in missing_video_id:
+                x['video_id'] = image_id_to_video_id[x['image_id']]
+
+    @staticmethod
+    def _make_track_ids_unique(annotations):
+        """
+        Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotation set
+        :return: the number of updated IDs
+        """
+        track_id_videos = {}
+        track_ids_to_update = set()
+        max_track_id = 0
+        for ann in annotations:
+            t = ann['track_id']
+            if t not in track_id_videos:
+                track_id_videos[t] = ann['video_id']
+
+            if ann['video_id'] != track_id_videos[t]:
+                # Track id is assigned to multiple videos
+                track_ids_to_update.add(t)
+            max_track_id = max(max_track_id, t)
+
+        if track_ids_to_update:
+            print('true')
+            next_id = itertools.count(max_track_id + 1)
+            new_track_ids = defaultdict(lambda: next(next_id))
+            for ann in annotations:
+                t = ann['track_id']
+                v = ann['video_id']
+                if t in track_ids_to_update:
+                    ann['track_id'] = new_track_ids[t, v]
+        return len(track_ids_to_update)

yolov7-tracker-example/tracker/trackeval/datasets/tao_ow.py

@@ -0,0 +1,652 @@
+import os
+import numpy as np
+import json
+import itertools
+from collections import defaultdict
+from scipy.optimize import linear_sum_assignment
+from ..utils import TrackEvalException
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+
+
+class TAO_OW(_BaseDataset):
+    """Dataset class for TAO tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
+            'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val'
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'MAX_DETECTIONS': 300,  # Number of maximal allowed detections per image (0 for unlimited)
+            'SUBSET': 'all'
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER']
+        self.tracker_fol = self.config['TRACKERS_FOLDER']
+        self.should_classes_combine = True
+        self.use_super_categories = False
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')]
+        if len(gt_dir_files) != 1:
+            raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.')
+
+        with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
+            self.gt_data = json.load(f)
+
+        self.subset = self.config['SUBSET']
+        if self.subset != 'all':
+            # Split GT data into `known`, `unknown` or `distractor`
+            self._split_known_unknown_distractor()
+            self.gt_data = self._filter_gt_data(self.gt_data)
+
+        # merge categories marked with a merged tag in TAO dataset
+        self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks'])
+
+        # Get sequences to eval and sequence information
+        self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']]
+        self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']}
+        # compute mappings from videos to annotation data
+        self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations'])
+        # compute sequence lengths
+        self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']}
+        for img in self.gt_data['images']:
+            self.seq_lengths[img['video_id']] += 1
+        self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings()
+        self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track
+                                                                in self.videos_to_gt_tracks[vid['id']]}),
+                                           'neg_cat_ids': vid['neg_category_ids'],
+                                           'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']}
+                               for vid in self.gt_data['videos']}
+
+        # Get classes to eval
+        considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list]
+        seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id
+                         in self.seq_to_classes[vid_id]['pos_cat_ids']])
+        # only classes with ground truth are evaluated in TAO
+        self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats]
+        # cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
+
+        if self.config['CLASSES_TO_EVAL']:
+            # self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+            #                    for cls in self.config['CLASSES_TO_EVAL']]
+            self.class_list = ["object"]  # class-agnostic
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(self.valid_classes) +
+                                         ' are valid (classes present in ground truth data).')
+        else:
+            # self.class_list = [cls for cls in self.valid_classes]
+            self.class_list = ["object"]  # class-agnostic
+        # self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
+        self.class_name_to_class_id = {"object": 1}  # class-agnostic
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        self.tracker_data = {tracker: dict() for tracker in self.tracker_list}
+
+        for tracker in self.tracker_list:
+            tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol))
+                            if file.endswith('.json')]
+            if len(tr_dir_files) != 1:
+                raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+                                         + ' does not contain exactly one json file.')
+            with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f:
+                curr_data = json.load(f)
+
+            # limit detections if MAX_DETECTIONS > 0
+            if self.config['MAX_DETECTIONS']:
+                curr_data = self._limit_dets_per_image(curr_data)
+
+            # fill missing video ids
+            self._fill_video_ids_inplace(curr_data)
+
+            # make track ids unique over whole evaluation set
+            self._make_track_ids_unique(curr_data)
+
+            # merge categories marked with a merged tag in TAO dataset
+            self._merge_categories(curr_data)
+
+            # get tracker sequence information
+            curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data)
+            self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks
+            self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the TAO format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values
+                                as keys and lists (for each track) as values
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values
+                                                                                           as keys and lists as values
+        [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
+        """
+        seq_id = self.seq_name_to_seq_id[seq]
+        # File location
+        if is_gt:
+            imgs = self.videos_to_gt_images[seq_id]
+        else:
+            imgs = self.tracker_data[tracker]['vids_to_images'][seq_id]
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq_id]
+        img_to_timestep = self.seq_to_images_to_timestep[seq_id]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for img in imgs:
+            # some tracker data contains images without any ground truth information, these are ignored
+            try:
+                t = img_to_timestep[img['id']]
+            except KeyError:
+                continue
+            annotations = img['annotations']
+            raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float)
+            raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int)
+            raw_data['classes'][t] = np.atleast_1d([1 for _ in annotations]).astype(int)   # class-agnostic
+            if not is_gt:
+                raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float)
+
+        for t, d in enumerate(raw_data['dets']):
+            if d is None:
+                raw_data['dets'][t] = np.empty((0, 4)).astype(float)
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if not is_gt:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        # all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list]
+        all_classes = [1]  # class-agnostic
+
+        if is_gt:
+            classes_to_consider = all_classes
+            all_tracks = self.videos_to_gt_tracks[seq_id]
+        else:
+            # classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \
+            #                       + self.seq_to_classes[seq_id]['neg_cat_ids']
+            classes_to_consider = all_classes  # class-agnostic
+            all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id]
+
+        # classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls]
+        #                      if cls in classes_to_consider else [] for cls in all_classes}
+        classes_to_tracks = {cls: [track for track in all_tracks]
+        if cls in classes_to_consider else [] for cls in all_classes}  # class-agnostic
+
+        # mapping from classes to track information
+        raw_data['classes_to_tracks'] = {cls: [{det['image_id']: np.atleast_1d(det['bbox'])
+                                                for det in track['annotations']} for track in tracks]
+                                         for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks]
+                                            for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks]
+                                              for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks]
+                                                for cls, tracks in classes_to_tracks.items()}
+
+        if not is_gt:
+            raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score'])
+                                                                              for x in track['annotations']])
+                                                                     for track in tracks])
+                                                      for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            key_map = {'classes_to_tracks': 'classes_to_gt_tracks',
+                       'classes_to_track_ids': 'classes_to_gt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_gt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_gt_track_areas'}
+        else:
+            key_map = {'classes_to_tracks': 'classes_to_dt_tracks',
+                       'classes_to_track_ids': 'classes_to_dt_track_ids',
+                       'classes_to_track_lengths': 'classes_to_dt_track_lengths',
+                       'classes_to_track_areas': 'classes_to_dt_track_areas'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids']
+        raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids']
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+        TAO:
+            In TAO, the 4 preproc steps are as follow:
+                1) All classes present in the ground truth data are evaluated separately.
+                2) No matched tracker detections are removed.
+                3) Unmatched tracker detections are removed if there is not ground truth data and the class does not
+                    belong to the categories marked as negative for this sequence. Additionally, unmatched tracker
+                    detections for classes which are marked as not exhaustively labeled are removed.
+                4) No gt detections are removed.
+            Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
+            and the tracks from the tracker data are sorted according to the tracker confidence.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+        is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls']
+        is_neg_category = cls_id in raw_data['neg_cat_ids']
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for preproc and eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = raw_data['gt_dets'][t][gt_class_mask]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask]
+            tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            # Match tracker and gt dets (with hungarian algorithm).
+            unmatched_indices = np.arange(tracker_ids.shape[0])
+            if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_cols = match_cols[actually_matched_mask]
+                unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
+
+            if gt_ids.shape[0] == 0 and not is_neg_category:
+                to_remove_tracker = unmatched_indices
+            elif is_not_exhaustively_labeled:
+                to_remove_tracker = unmatched_indices
+            else:
+                to_remove_tracker = np.array([], dtype=np.int)
+
+            # remove all unwanted unmatched tracker detections
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # get track representations
+        data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id]
+        data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id]
+        data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id]
+        data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id]
+        data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id]
+        data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id]
+        data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id]
+        data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id]
+        data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id]
+        data['not_exhaustively_labeled'] = is_not_exhaustively_labeled
+        data['iou_type'] = 'bbox'
+
+        # sort tracker data tracks by tracker confidence scores
+        if data['dt_tracks']:
+            idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort")
+            data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx]
+            data['dt_tracks'] = [data['dt_tracks'][i] for i in idx]
+            data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx]
+            data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx]
+            data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx]
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t)
+        return similarity_scores
+
+    def _merge_categories(self, annotations):
+        """
+        Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset
+        :param annotations: the annotations in which the classes should be merged
+        :return: None
+        """
+        merge_map = {}
+        for category in self.gt_data['categories']:
+            if 'merged' in category:
+                for to_merge in category['merged']:
+                    merge_map[to_merge['id']] = category['id']
+
+        for ann in annotations:
+            ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id'])
+
+    def _compute_vid_mappings(self, annotations):
+        """
+        Computes mappings from Videos to corresponding tracks and images.
+        :param annotations: the annotations for which the mapping should be generated
+        :return: the video-to-track-mapping, the video-to-image-mapping
+        """
+        vids_to_tracks = {}
+        vids_to_imgs = {}
+        vid_ids = [vid['id'] for vid in self.gt_data['videos']]
+
+        # compute an mapping from image IDs to images
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        for ann in annotations:
+            ann["area"] = ann["bbox"][2] * ann["bbox"][3]
+
+            vid = ann["video_id"]
+            if ann["video_id"] not in vids_to_tracks.keys():
+                vids_to_tracks[ann["video_id"]] = list()
+            if ann["video_id"] not in vids_to_imgs.keys():
+                vids_to_imgs[ann["video_id"]] = list()
+
+            # Fill in vids_to_tracks
+            tid = ann["track_id"]
+            exist_tids = [track["id"] for track in vids_to_tracks[vid]]
+            try:
+                index1 = exist_tids.index(tid)
+            except ValueError:
+                index1 = -1
+            if tid not in exist_tids:
+                curr_track = {"id": tid, "category_id": ann['category_id'],
+                              "video_id": vid, "annotations": [ann]}
+                vids_to_tracks[vid].append(curr_track)
+            else:
+                vids_to_tracks[vid][index1]["annotations"].append(ann)
+
+            # Fill in vids_to_imgs
+            img_id = ann['image_id']
+            exist_img_ids = [img["id"] for img in vids_to_imgs[vid]]
+            try:
+                index2 = exist_img_ids.index(img_id)
+            except ValueError:
+                index2 = -1
+            if index2 == -1:
+                curr_img = {"id": img_id, "annotations": [ann]}
+                vids_to_imgs[vid].append(curr_img)
+            else:
+                vids_to_imgs[vid][index2]["annotations"].append(ann)
+
+        # sort annotations by frame index and compute track area
+        for vid, tracks in vids_to_tracks.items():
+            for track in tracks:
+                track["annotations"] = sorted(
+                    track['annotations'],
+                    key=lambda x: images[x['image_id']]['frame_index'])
+                # Computer average area
+                track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations']))
+
+        # Ensure all videos are present
+        for vid_id in vid_ids:
+            if vid_id not in vids_to_tracks.keys():
+                vids_to_tracks[vid_id] = []
+            if vid_id not in vids_to_imgs.keys():
+                vids_to_imgs[vid_id] = []
+
+        return vids_to_tracks, vids_to_imgs
+
+    def _compute_image_to_timestep_mappings(self):
+        """
+        Computes a mapping from images to the corresponding timestep in the sequence.
+        :return: the image-to-timestep-mapping
+        """
+        images = {}
+        for image in self.gt_data['images']:
+            images[image['id']] = image
+
+        seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']}
+        for vid in seq_to_imgs_to_timestep:
+            curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]]
+            curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index'])
+            seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))}
+
+        return seq_to_imgs_to_timestep
+
+    def _limit_dets_per_image(self, annotations):
+        """
+        Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from
+        https://github.com/TAO-Dataset/
+        :param annotations: the annotations in which the detections should be limited
+        :return: the annotations with limited detections
+        """
+        max_dets = self.config['MAX_DETECTIONS']
+        img_ann = defaultdict(list)
+        for ann in annotations:
+            img_ann[ann["image_id"]].append(ann)
+
+        for img_id, _anns in img_ann.items():
+            if len(_anns) <= max_dets:
+                continue
+            _anns = sorted(_anns, key=lambda x: x["score"], reverse=True)
+            img_ann[img_id] = _anns[:max_dets]
+
+        return [ann for anns in img_ann.values() for ann in anns]
+
+    def _fill_video_ids_inplace(self, annotations):
+        """
+        Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotations for which the videos IDs should be filled inplace
+        :return: None
+        """
+        missing_video_id = [x for x in annotations if 'video_id' not in x]
+        if missing_video_id:
+            image_id_to_video_id = {
+                x['id']: x['video_id'] for x in self.gt_data['images']
+            }
+            for x in missing_video_id:
+                x['video_id'] = image_id_to_video_id[x['image_id']]
+
+    @staticmethod
+    def _make_track_ids_unique(annotations):
+        """
+        Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/
+        :param annotations: the annotation set
+        :return: the number of updated IDs
+        """
+        track_id_videos = {}
+        track_ids_to_update = set()
+        max_track_id = 0
+        for ann in annotations:
+            t = ann['track_id']
+            if t not in track_id_videos:
+                track_id_videos[t] = ann['video_id']
+
+            if ann['video_id'] != track_id_videos[t]:
+                # Track id is assigned to multiple videos
+                track_ids_to_update.add(t)
+            max_track_id = max(max_track_id, t)
+
+        if track_ids_to_update:
+            print('true')
+            next_id = itertools.count(max_track_id + 1)
+            new_track_ids = defaultdict(lambda: next(next_id))
+            for ann in annotations:
+                t = ann['track_id']
+                v = ann['video_id']
+                if t in track_ids_to_update:
+                    ann['track_id'] = new_track_ids[t, v]
+        return len(track_ids_to_update)
+
+    def _split_known_unknown_distractor(self):
+        all_ids = set([i for i in range(1, 2000)])  # 2000 is larger than the max category id in TAO-OW.
+        # `knowns` includes 78 TAO_category_ids that corresponds to 78 COCO classes.
+        # (The other 2 COCO classes do not have corresponding classes in TAO).
+        self.knowns = {4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115,
+                     1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185,
+                     699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347,
+                     371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018}
+        # `distractors` is defined as in the paper "Opening up Open-World Tracking"
+        self.distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567,
+                            569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883,
+                            912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220}
+        self.unknowns = all_ids.difference(self.knowns.union(self.distractors))
+
+    def _filter_gt_data(self, raw_gt_data):
+        """
+        Filter out irrelevant data in the raw_gt_data
+        Args:
+            raw_gt_data: directly loaded from json.
+
+        Returns:
+            filtered gt_data
+        """
+        valid_cat_ids = list()
+        if self.subset == "known":
+            valid_cat_ids = self.knowns
+        elif self.subset == "distractor":
+            valid_cat_ids = self.distractors
+        elif self.subset == "unknown":
+            valid_cat_ids = self.unknowns
+        # elif self.subset == "test_only_unknowns":
+        #     valid_cat_ids = test_only_unknowns
+        else:
+            raise Exception("The parameter `SUBSET` is incorrect")
+
+        filtered = dict()
+        filtered["videos"] = raw_gt_data["videos"]
+        # filtered["videos"] = list()
+        unwanted_vid = set()
+        # for video in raw_gt_data["videos"]:
+        #     datasrc = video["name"].split('/')[1]
+        #     if datasrc in data_srcs:
+        #         filtered["videos"].append(video)
+        #     else:
+        #         unwanted_vid.add(video["id"])
+
+        filtered["annotations"] = list()
+        for ann in raw_gt_data["annotations"]:
+            if (ann["video_id"] not in unwanted_vid) and (ann["category_id"] in valid_cat_ids):
+                filtered["annotations"].append(ann)
+
+        filtered["tracks"] = list()
+        for track in raw_gt_data["tracks"]:
+            if (track["video_id"] not in unwanted_vid) and (track["category_id"] in valid_cat_ids):
+                filtered["tracks"].append(track)
+
+        filtered["images"] = list()
+        for image in raw_gt_data["images"]:
+            if image["video_id"] not in unwanted_vid:
+                filtered["images"].append(image)
+
+        filtered["categories"] = list()
+        for cat in raw_gt_data["categories"]:
+            if cat["id"] in valid_cat_ids:
+                filtered["categories"].append(cat)
+
+        filtered["info"] = raw_gt_data["info"]
+        filtered["licenses"] = raw_gt_data["licenses"]
+
+        return filtered

yolov7-tracker-example/tracker/trackeval/datasets/visdrone.py

@@ -0,0 +1,438 @@
+import os
+import csv
+import configparser
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_dataset import _BaseDataset
+from .. import utils
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class VisDrone2DBox(_BaseDataset):
+    """Dataset class for MOT Challenge 2D bounding box tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'),  # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': ['pedestrain', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor'],  # Valid: ['pedestrian']
+            'BENCHMARK': 'MOT17',  # Valid: 'MOT17', 'MOT16', 'MOT20', 'MOT15'
+            'SPLIT_TO_EVAL': 'train',  # Valid: 'train', 'test', 'all'
+            'INPUT_AS_ZIP': False,  # Whether tracker input files are zipped
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'DO_PREPROC': True,  # Whether to perform preprocessing (never done for MOT15)
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+            'SEQMAP_FOLDER': None,  # Where seqmaps are found (if None, GT_FOLDER/seqmaps)
+            'SEQMAP_FILE': None,  # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval)
+            'SEQ_INFO': None,  # If not None, directly specify sequences to eval and their number of timesteps
+            'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt',  # '{gt_folder}/{seq}/gt/gt.txt'
+            'SKIP_SPLIT_FOL': False,  # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in
+                                      # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/
+                                      # If True, then the middle 'benchmark-split' folder is skipped for both.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+
+        self.benchmark = self.config['BENCHMARK']
+        gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL']
+        self.gt_set = gt_set
+        if not self.config['SKIP_SPLIT_FOL']:
+            split_fol = gt_set
+        else:
+            split_fol = ''
+        self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol)
+        self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol)
+        self.should_classes_combine = False
+        self.use_super_categories = False
+        self.data_is_zipped = self.config['INPUT_AS_ZIP']
+        self.do_preproc = self.config['DO_PREPROC']
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+
+        # Get classes to eval
+        self.valid_classes = ['pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor']
+        self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                           for cls in self.config['CLASSES_TO_EVAL']]
+        if not all(self.class_list):
+            raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.')
+        self.class_name_to_class_id = {'ignored': 0, 'pedestrian': 1, 'people': 2, 'bicycle': 3, 'car': 4, 'van': 5,
+                                       'truck': 6, 'tricycle': 7, 'awning-tricycle': 8, 'bus': 9,
+                                       'motor': 10, 'other': 11}
+        self.valid_class_numbers = list(self.class_name_to_class_id.values())
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list, self.seq_lengths = self._get_seq_info()
+        if len(self.seq_list) < 1:
+            raise TrackEvalException('No sequences are selected to be evaluated.')
+
+        # Check gt files exist
+        for seq in self.seq_list:
+            if not self.data_is_zipped:
+                curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+                if not os.path.isfile(curr_file):
+                    print('GT file not found ' + curr_file)
+                    raise TrackEvalException('GT file not found for sequence: ' + seq)
+        if self.data_is_zipped:
+            curr_file = os.path.join(self.gt_fol, 'data.zip')
+            if not os.path.isfile(curr_file):
+                print('GT file not found ' + curr_file)
+                raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file))
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        for tracker in self.tracker_list:
+            if self.data_is_zipped:
+                curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+                if not os.path.isfile(curr_file):
+                    print('Tracker file not found: ' + curr_file)
+                    raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file))
+            else:
+                for seq in self.seq_list:
+                    curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+                    if not os.path.isfile(curr_file):
+                        print('Tracker file not found: ' + curr_file)
+                        raise TrackEvalException(
+                            'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename(
+                                curr_file))
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _get_seq_info(self):
+        seq_list = []
+        seq_lengths = {}
+        if self.config["SEQ_INFO"]:
+            seq_list = list(self.config["SEQ_INFO"].keys())
+            seq_lengths = self.config["SEQ_INFO"]
+
+            # If sequence length is 'None' tries to read sequence length from .ini files.
+            for seq, seq_length in seq_lengths.items():
+                if seq_length is None:
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+
+        else:
+            if self.config["SEQMAP_FILE"]:
+                seqmap_file = self.config["SEQMAP_FILE"]
+            else:
+                if self.config["SEQMAP_FOLDER"] is None:
+                    seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt')
+                else:
+                    seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt')
+            if not os.path.isfile(seqmap_file):
+                print('no seqmap found: ' + seqmap_file)
+                raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file))
+            with open(seqmap_file) as fp:
+                reader = csv.reader(fp)
+                for i, row in enumerate(reader):
+                    if i == 0 or row[0] == '':
+                        continue
+                    seq = row[0]
+                    seq_list.append(seq)
+                    ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini')
+                    if not os.path.isfile(ini_file):
+                        raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file))
+                    ini_data = configparser.ConfigParser()
+                    ini_data.read(ini_file)
+                    seq_lengths[seq] = int(ini_data['Sequence']['seqLength'])
+        return seq_list, seq_lengths
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the MOT Challenge 2D box format
+
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
+        [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det).
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        """
+        # File location
+        if self.data_is_zipped:
+            if is_gt:
+                zip_file = os.path.join(self.gt_fol, 'data.zip')
+            else:
+                zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip')
+            file = seq + '.txt'
+        else:
+            zip_file = None
+            if is_gt:
+                file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq)
+            else:
+                file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt')
+
+        # Load raw data from text file
+        read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file)
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq]
+        data_keys = ['ids', 'classes', 'dets']
+        if is_gt:
+            data_keys += ['gt_crowd_ignore_regions', 'gt_extras']
+        else:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+
+        # Check for any extra time keys
+        current_time_keys = [str( t+ 1) for t in range(num_timesteps)]
+        extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys]
+        if len(extra_time_keys) > 0:
+            if is_gt:
+                text = 'Ground-truth'
+            else:
+                text = 'Tracking'
+            raise TrackEvalException(
+                text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join(
+                    [str(x) + ', ' for x in extra_time_keys]))
+
+        for t in range(num_timesteps):
+            time_key = str(t+1)
+            if time_key in read_data.keys():
+                try:
+                    time_data = np.asarray(read_data[time_key], dtype=np.float)
+                except ValueError:
+                    if is_gt:
+                        raise TrackEvalException(
+                            'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq)
+                    else:
+                        raise TrackEvalException(
+                            'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % (
+                                tracker, seq))
+                try:
+                    raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6])
+                    raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int)
+                except IndexError:
+                    if is_gt:
+                        err = 'Cannot load gt data from sequence %s, because there is not enough ' \
+                              'columns in the data.' % seq
+                        raise TrackEvalException(err)
+                    else:
+                        err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \
+                              'columns in the data.' % (tracker, seq)
+                        raise TrackEvalException(err)
+                if time_data.shape[1] >= 8:
+                    raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int)
+                else:
+                    if not is_gt:
+                        raw_data['classes'][t] = np.ones_like(raw_data['ids'][t])
+                    else:
+                        raise TrackEvalException(
+                            'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % (
+                                seq, t))
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6])
+            else:
+                raw_data['dets'][t] = np.empty((0, 4))
+                raw_data['ids'][t] = np.empty(0).astype(int)
+                raw_data['classes'][t] = np.empty(0).astype(int)
+                if is_gt:
+                    gt_extras_dict = {'zero_marked': np.empty(0)}
+                    raw_data['gt_extras'][t] = gt_extras_dict
+                else:
+                    raw_data['tracker_confidences'][t] = np.empty(0)
+            if is_gt:
+                raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4))
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+
+        MOT Challenge:
+            In MOT Challenge, the 4 preproc steps are as follow:
+                1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc.
+                2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor
+                    objects are removed.
+                3) There is no crowd ignore regions.
+                4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked.
+        """
+        # Check that input data has unique ids
+        self._check_unique_ids(raw_data)
+
+        # distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection']
+        distractor_class_names = ['ignored', 'other']
+        if self.benchmark == 'MOT20':
+            distractor_class_names.append('non_mot_vehicle')
+        distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names]
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+        for t in range(raw_data['num_timesteps']):
+
+            # Get all data
+            gt_ids = raw_data['gt_ids'][t]
+            gt_dets = raw_data['gt_dets'][t]
+            gt_classes = raw_data['gt_classes'][t]
+            gt_zero_marked = raw_data['gt_extras'][t]['zero_marked']
+
+            tracker_ids = raw_data['tracker_ids'][t]
+            tracker_dets = raw_data['tracker_dets'][t]
+            tracker_classes = raw_data['tracker_classes'][t]
+            tracker_confidences = raw_data['tracker_confidences'][t]
+            similarity_scores = raw_data['similarity_scores'][t]
+
+            # Evaluation is ONLY valid for pedestrian class
+            if len(tracker_classes) > 0 and np.max(tracker_classes) > 1:
+                raise TrackEvalException(
+                    'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at '
+                    'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t))
+
+            # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets
+            # which are labeled as belonging to a distractor class.
+            to_remove_tracker = np.array([], np.int)
+            if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
+
+                # Check all classes are valid:
+                invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers)
+                if len(invalid_classes) > 0:
+                    print(' '.join([str(x) for x in invalid_classes]))
+                    raise(TrackEvalException('Attempting to evaluate using invalid gt classes. '
+                                             'This warning only triggers if preprocessing is performed, '
+                                             'e.g. not for MOT15 or where prepropressing is explicitly disabled. '
+                                             'Please either check your gt data, or disable preprocessing. '
+                                             'The following invalid classes were found in timestep ' + str(t) + ': ' +
+                                             ' '.join([str(x) for x in invalid_classes])))
+
+                matching_scores = similarity_scores.copy()
+                matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0
+                match_rows, match_cols = linear_sum_assignment(-matching_scores)
+                actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps
+                match_rows = match_rows[actually_matched_mask]
+                match_cols = match_cols[actually_matched_mask]
+
+                is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes)
+                to_remove_tracker = match_cols[is_distractor_class]
+
+            # Apply preprocessing to remove all unwanted tracker dets.
+            data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
+            data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
+            data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0)
+            similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
+
+            # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian
+            # class (not applicable for MOT15)
+            if self.do_preproc and self.benchmark != 'MOT15':
+                gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \
+                                  (np.equal(gt_classes, cls_id))
+            else:
+                # There are no classes for MOT15
+                gt_to_keep_mask = np.not_equal(gt_zero_marked, 0)
+            data['gt_ids'][t] = gt_ids[gt_to_keep_mask]
+            data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :]
+            data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask]
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # Ensure again that ids are unique per timestep after preproc.
+        self._check_unique_ids(data, after_preproc=True)
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh')
+        return similarity_scores

yolov7-tracker-example/tracker/trackeval/datasets/youtube_vis.py

@@ -0,0 +1,364 @@
+import os
+import numpy as np
+import json
+from ._base_dataset import _BaseDataset
+from ..utils import TrackEvalException
+from .. import utils
+from .. import _timing
+
+
+class YouTubeVIS(_BaseDataset):
+    """Dataset class for YouTubeVIS tracking"""
+
+    @staticmethod
+    def get_default_dataset_config():
+        """Default class config values"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'GT_FOLDER': os.path.join(code_path, 'data/gt/youtube_vis/'),  # Location of GT data
+            'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/youtube_vis/'),
+            # Trackers location
+            'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
+            'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
+            'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
+            'SPLIT_TO_EVAL': 'train_sub_split',  # Valid: 'train', 'val', 'train_sub_split'
+            'PRINT_CONFIG': True,  # Whether to print current config
+            'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
+            'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
+            'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise dataset, checking that all required files are present"""
+        super().__init__()
+        # Fill non-given config values with defaults
+        self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name())
+        self.gt_fol = self.config['GT_FOLDER'] + 'youtube_vis_' + self.config['SPLIT_TO_EVAL']
+        self.tracker_fol = self.config['TRACKERS_FOLDER'] + 'youtube_vis_' + self.config['SPLIT_TO_EVAL']
+        self.use_super_categories = False
+        self.should_classes_combine = True
+
+        self.output_fol = self.config['OUTPUT_FOLDER']
+        if self.output_fol is None:
+            self.output_fol = self.tracker_fol
+        self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER']
+        self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER']
+
+        if not os.path.exists(self.gt_fol):
+            print("GT folder not found: " + self.gt_fol)
+            raise TrackEvalException("GT folder not found: " + os.path.basename(self.gt_fol))
+        gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')]
+        if len(gt_dir_files) != 1:
+            raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.')
+
+        with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
+            self.gt_data = json.load(f)
+
+        # Get classes to eval
+        self.valid_classes = [cls['name'] for cls in self.gt_data['categories']]
+        cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
+
+        if self.config['CLASSES_TO_EVAL']:
+            self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
+                               for cls in self.config['CLASSES_TO_EVAL']]
+            if not all(self.class_list):
+                raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' +
+                                         ', '.join(self.valid_classes) + ' are valid.')
+        else:
+            self.class_list = [cls['name'] for cls in self.gt_data['categories']]
+        self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
+
+        # Get sequences to eval and check gt files exist
+        self.seq_list = [vid['file_names'][0].split('/')[0] for vid in self.gt_data['videos']]
+        self.seq_name_to_seq_id = {vid['file_names'][0].split('/')[0]: vid['id'] for vid in self.gt_data['videos']}
+        self.seq_lengths = {vid['id']: len(vid['file_names']) for vid in self.gt_data['videos']}
+
+        # encode masks and compute track areas
+        self._prepare_gt_annotations()
+
+        # Get trackers to eval
+        if self.config['TRACKERS_TO_EVAL'] is None:
+            self.tracker_list = os.listdir(self.tracker_fol)
+        else:
+            self.tracker_list = self.config['TRACKERS_TO_EVAL']
+
+        if self.config['TRACKER_DISPLAY_NAMES'] is None:
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
+        elif (self.config['TRACKERS_TO_EVAL'] is not None) and (
+                len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)):
+            self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES']))
+        else:
+            raise TrackEvalException('List of tracker files and tracker display names do not match.')
+
+        # counter for globally unique track IDs
+        self.global_tid_counter = 0
+
+        self.tracker_data = dict()
+        for tracker in self.tracker_list:
+            tracker_dir_path = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+            tr_dir_files = [file for file in os.listdir(tracker_dir_path) if file.endswith('.json')]
+            if len(tr_dir_files) != 1:
+                raise TrackEvalException(tracker_dir_path + ' does not contain exactly one json file.')
+
+            with open(os.path.join(tracker_dir_path, tr_dir_files[0])) as f:
+                curr_data = json.load(f)
+
+            self.tracker_data[tracker] = curr_data
+
+    def get_display_name(self, tracker):
+        return self.tracker_to_disp[tracker]
+
+    def _load_raw_file(self, tracker, seq, is_gt):
+        """Load a file (gt or tracker) in the YouTubeVIS format
+        If is_gt, this returns a dict which contains the fields:
+        [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
+        [gt_dets]: list (for each timestep) of lists of detections.
+        [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_iscrowd]: dictionary with class values
+                                as keys and lists (for each track) as values
+
+        if not is_gt, this returns a dict which contains the fields:
+        [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
+        [tracker_dets]: list (for each timestep) of lists of detections.
+        [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
+                                keys and corresponding segmentations as values) for each track
+        [classes_to_dt_track_ids, classes_to_dt_track_areas]: dictionary with class values as keys and lists as values
+        [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
+        """
+        # select sequence tracks
+        seq_id = self.seq_name_to_seq_id[seq]
+        if is_gt:
+            tracks = [ann for ann in self.gt_data['annotations'] if ann['video_id'] == seq_id]
+        else:
+            tracks = self._get_tracker_seq_tracks(tracker, seq_id)
+
+        # Convert data to required format
+        num_timesteps = self.seq_lengths[seq_id]
+        data_keys = ['ids', 'classes', 'dets']
+        if not is_gt:
+            data_keys += ['tracker_confidences']
+        raw_data = {key: [None] * num_timesteps for key in data_keys}
+        for t in range(num_timesteps):
+            raw_data['dets'][t] = [track['segmentations'][t] for track in tracks if track['segmentations'][t]]
+            raw_data['ids'][t] = np.atleast_1d([track['id'] for track in tracks
+                                                if track['segmentations'][t]]).astype(int)
+            raw_data['classes'][t] = np.atleast_1d([track['category_id'] for track in tracks
+                                                    if track['segmentations'][t]]).astype(int)
+            if not is_gt:
+                raw_data['tracker_confidences'][t] = np.atleast_1d([track['score'] for track in tracks
+                                                                    if track['segmentations'][t]]).astype(float)
+
+        if is_gt:
+            key_map = {'ids': 'gt_ids',
+                       'classes': 'gt_classes',
+                       'dets': 'gt_dets'}
+        else:
+            key_map = {'ids': 'tracker_ids',
+                       'classes': 'tracker_classes',
+                       'dets': 'tracker_dets'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        all_cls_ids = {self.class_name_to_class_id[cls] for cls in self.class_list}
+        classes_to_tracks = {cls: [track for track in tracks if track['category_id'] == cls] for cls in all_cls_ids}
+
+        # mapping from classes to track representations and track information
+        raw_data['classes_to_tracks'] = {cls: [{i: track['segmentations'][i]
+                                                for i in range(len(track['segmentations']))} for track in tracks]
+                                         for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks]
+                                            for cls, tracks in classes_to_tracks.items()}
+        raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks]
+                                              for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            raw_data['classes_to_gt_track_iscrowd'] = {cls: [track['iscrowd'] for track in tracks]
+                                                       for cls, tracks in classes_to_tracks.items()}
+        else:
+            raw_data['classes_to_dt_track_scores'] = {cls: np.array([track['score'] for track in tracks])
+                                                      for cls, tracks in classes_to_tracks.items()}
+
+        if is_gt:
+            key_map = {'classes_to_tracks': 'classes_to_gt_tracks',
+                       'classes_to_track_ids': 'classes_to_gt_track_ids',
+                       'classes_to_track_areas': 'classes_to_gt_track_areas'}
+        else:
+            key_map = {'classes_to_tracks': 'classes_to_dt_tracks',
+                       'classes_to_track_ids': 'classes_to_dt_track_ids',
+                       'classes_to_track_areas': 'classes_to_dt_track_areas'}
+        for k, v in key_map.items():
+            raw_data[v] = raw_data.pop(k)
+
+        raw_data['num_timesteps'] = num_timesteps
+        raw_data['seq'] = seq
+        return raw_data
+
+    @_timing.time
+    def get_preprocessed_seq_data(self, raw_data, cls):
+        """ Preprocess data for a single sequence for a single class ready for evaluation.
+        Inputs:
+             - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
+             - cls is the class to be evaluated.
+        Outputs:
+             - data is a dict containing all of the information that metrics need to perform evaluation.
+                It contains the following fields:
+                    [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
+                    [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
+                    [gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
+                    [similarity_scores]: list (for each timestep) of 2D NDArrays.
+        Notes:
+            General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
+                1) Extract only detections relevant for the class to be evaluated (including distractor detections).
+                2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
+                    distractor class, or otherwise marked as to be removed.
+                3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
+                    other criteria (e.g. are too small).
+                4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
+            After the above preprocessing steps, this function also calculates the number of gt and tracker detections
+                and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
+                unique within each timestep.
+        YouTubeVIS:
+            In YouTubeVIS, the 4 preproc steps are as follow:
+                1) There are 40 classes which are evaluated separately.
+                2) No matched tracker dets are removed.
+                3) No unmatched tracker dets are removed.
+                4) No gt dets are removed.
+            Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
+            and the tracks from the tracker data are sorted according to the tracker confidence.
+        """
+        cls_id = self.class_name_to_class_id[cls]
+
+        data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores']
+        data = {key: [None] * raw_data['num_timesteps'] for key in data_keys}
+        unique_gt_ids = []
+        unique_tracker_ids = []
+        num_gt_dets = 0
+        num_tracker_dets = 0
+
+        for t in range(raw_data['num_timesteps']):
+
+            # Only extract relevant dets for this class for eval (cls)
+            gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id)
+            gt_class_mask = gt_class_mask.astype(np.bool)
+            gt_ids = raw_data['gt_ids'][t][gt_class_mask]
+            gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]]
+
+            tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id)
+            tracker_class_mask = tracker_class_mask.astype(np.bool)
+            tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask]
+            tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if
+                            tracker_class_mask[ind]]
+            similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask]
+
+            data['tracker_ids'][t] = tracker_ids
+            data['tracker_dets'][t] = tracker_dets
+            data['gt_ids'][t] = gt_ids
+            data['gt_dets'][t] = gt_dets
+            data['similarity_scores'][t] = similarity_scores
+
+            unique_gt_ids += list(np.unique(data['gt_ids'][t]))
+            unique_tracker_ids += list(np.unique(data['tracker_ids'][t]))
+            num_tracker_dets += len(data['tracker_ids'][t])
+            num_gt_dets += len(data['gt_ids'][t])
+
+        # Re-label IDs such that there are no empty IDs
+        if len(unique_gt_ids) > 0:
+            unique_gt_ids = np.unique(unique_gt_ids)
+            gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
+            gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['gt_ids'][t]) > 0:
+                    data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int)
+        if len(unique_tracker_ids) > 0:
+            unique_tracker_ids = np.unique(unique_tracker_ids)
+            tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
+            tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
+            for t in range(raw_data['num_timesteps']):
+                if len(data['tracker_ids'][t]) > 0:
+                    data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int)
+
+        # Ensure that ids are unique per timestep.
+        self._check_unique_ids(data)
+
+        # Record overview statistics.
+        data['num_tracker_dets'] = num_tracker_dets
+        data['num_gt_dets'] = num_gt_dets
+        data['num_tracker_ids'] = len(unique_tracker_ids)
+        data['num_gt_ids'] = len(unique_gt_ids)
+        data['num_timesteps'] = raw_data['num_timesteps']
+        data['seq'] = raw_data['seq']
+
+        # get track representations
+        data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id]
+        data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id]
+        data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id]
+        data['gt_track_iscrowd'] = raw_data['classes_to_gt_track_iscrowd'][cls_id]
+        data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id]
+        data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id]
+        data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id]
+        data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id]
+        data['iou_type'] = 'mask'
+
+        # sort tracker data tracks by tracker confidence scores
+        if data['dt_tracks']:
+            idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort")
+            data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx]
+            data['dt_tracks'] = [data['dt_tracks'][i] for i in idx]
+            data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx]
+            data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx]
+
+        return data
+
+    def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
+        similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False)
+        return similarity_scores
+
+    def _prepare_gt_annotations(self):
+        """
+        Prepares GT data by rle encoding segmentations and computing the average track area.
+        :return: None
+        """
+        # only loaded when needed to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        for track in self.gt_data['annotations']:
+            h = track['height']
+            w = track['width']
+            for i, seg in enumerate(track['segmentations']):
+                if seg:
+                    track['segmentations'][i] = mask_utils.frPyObjects(seg, h, w)
+            areas = [a for a in track['areas'] if a]
+            if len(areas) == 0:
+                track['area'] = 0
+            else:
+                track['area'] = np.array(areas).mean()
+
+    def _get_tracker_seq_tracks(self, tracker, seq_id):
+        """
+        Prepares tracker data for a given sequence. Extracts all annotations for given sequence ID, computes
+        average track area and assigns a track ID.
+        :param tracker: the given tracker
+        :param seq_id: the sequence ID
+        :return: the extracted tracks
+        """
+        # only loaded when needed to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        tracks = [ann for ann in self.tracker_data[tracker] if ann['video_id'] == seq_id]
+        for track in tracks:
+            track['areas'] = []
+            for seg in track['segmentations']:
+                if seg:
+                    track['areas'].append(mask_utils.area(seg))
+                else:
+                    track['areas'].append(None)
+            areas = [a for a in track['areas'] if a]
+            if len(areas) == 0:
+                track['area'] = 0
+            else:
+                track['area'] = np.array(areas).mean()
+            track['id'] = self.global_tid_counter
+            self.global_tid_counter += 1
+        return tracks

yolov7-tracker-example/tracker/trackeval/eval.py

@@ -0,0 +1,225 @@
+import time
+import traceback
+from multiprocessing.pool import Pool
+from functools import partial
+import os
+from . import utils
+from .utils import TrackEvalException
+from . import _timing
+from .metrics import Count
+
+try:
+    import tqdm
+    TQDM_IMPORTED = True
+except ImportError as _:
+    TQDM_IMPORTED = False
+
+
+class Evaluator:
+    """Evaluator class for evaluating different metrics for different datasets"""
+
+    @staticmethod
+    def get_default_eval_config():
+        """Returns the default config values for evaluation"""
+        code_path = utils.get_code_path()
+        default_config = {
+            'USE_PARALLEL': False,
+            'NUM_PARALLEL_CORES': 8,
+            'BREAK_ON_ERROR': True,  # Raises exception and exits with error
+            'RETURN_ON_ERROR': False,  # if not BREAK_ON_ERROR, then returns from function on error
+            'LOG_ON_ERROR': os.path.join(code_path, 'error_log.txt'),  # if not None, save any errors into a log file.
+
+            'PRINT_RESULTS': True,
+            'PRINT_ONLY_COMBINED': False,
+            'PRINT_CONFIG': True,
+            'TIME_PROGRESS': True,
+            'DISPLAY_LESS_PROGRESS': True,
+
+            'OUTPUT_SUMMARY': True,
+            'OUTPUT_EMPTY_CLASSES': True,  # If False, summary files are not output for classes with no detections
+            'OUTPUT_DETAILED': True,
+            'PLOT_CURVES': True,
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        """Initialise the evaluator with a config file"""
+        self.config = utils.init_config(config, self.get_default_eval_config(), 'Eval')
+        # Only run timing analysis if not run in parallel.
+        if self.config['TIME_PROGRESS'] and not self.config['USE_PARALLEL']:
+            _timing.DO_TIMING = True
+            if self.config['DISPLAY_LESS_PROGRESS']:
+                _timing.DISPLAY_LESS_PROGRESS = True
+
+    @_timing.time
+    def evaluate(self, dataset_list, metrics_list, show_progressbar=False):
+        """Evaluate a set of metrics on a set of datasets"""
+        config = self.config
+        metrics_list = metrics_list + [Count()]  # Count metrics are always run
+        metric_names = utils.validate_metrics_list(metrics_list)
+        dataset_names = [dataset.get_name() for dataset in dataset_list]
+        output_res = {}
+        output_msg = {}
+
+        for dataset, dataset_name in zip(dataset_list, dataset_names):
+            # Get dataset info about what to evaluate
+            output_res[dataset_name] = {}
+            output_msg[dataset_name] = {}
+            tracker_list, seq_list, class_list = dataset.get_eval_info()
+            print('\nEvaluating %i tracker(s) on %i sequence(s) for %i class(es) on %s dataset using the following '
+                  'metrics: %s\n' % (len(tracker_list), len(seq_list), len(class_list), dataset_name,
+                                     ', '.join(metric_names)))
+
+            # Evaluate each tracker
+            for tracker in tracker_list:
+                # if not config['BREAK_ON_ERROR'] then go to next tracker without breaking
+                try:
+                    # Evaluate each sequence in parallel or in series.
+                    # returns a nested dict (res), indexed like: res[seq][class][metric_name][sub_metric field]
+                    # e.g. res[seq_0001][pedestrian][hota][DetA]
+                    print('\nEvaluating %s\n' % tracker)
+                    time_start = time.time()
+                    if config['USE_PARALLEL']:
+                        if show_progressbar and TQDM_IMPORTED:
+                            seq_list_sorted = sorted(seq_list)
+
+                            with Pool(config['NUM_PARALLEL_CORES']) as pool, tqdm.tqdm(total=len(seq_list)) as pbar:
+                                _eval_sequence = partial(eval_sequence, dataset=dataset, tracker=tracker,
+                                                         class_list=class_list, metrics_list=metrics_list,
+                                                         metric_names=metric_names)
+                                results = []
+                                for r in pool.imap(_eval_sequence, seq_list_sorted,
+                                                   chunksize=20):
+                                    results.append(r)
+                                    pbar.update()
+                                res = dict(zip(seq_list_sorted, results))
+
+                        else:
+                            with Pool(config['NUM_PARALLEL_CORES']) as pool:
+                                _eval_sequence = partial(eval_sequence, dataset=dataset, tracker=tracker,
+                                                         class_list=class_list, metrics_list=metrics_list,
+                                                         metric_names=metric_names)
+                                results = pool.map(_eval_sequence, seq_list)
+                                res = dict(zip(seq_list, results))
+                    else:
+                        res = {}
+                        if show_progressbar and TQDM_IMPORTED:
+                            seq_list_sorted = sorted(seq_list)
+                            for curr_seq in tqdm.tqdm(seq_list_sorted):
+                                res[curr_seq] = eval_sequence(curr_seq, dataset, tracker, class_list, metrics_list,
+                                                              metric_names)
+                        else:
+                            for curr_seq in sorted(seq_list):
+                                res[curr_seq] = eval_sequence(curr_seq, dataset, tracker, class_list, metrics_list,
+                                                              metric_names)
+
+                    # Combine results over all sequences and then over all classes
+
+                    # collecting combined cls keys (cls averaged, det averaged, super classes)
+                    combined_cls_keys = []
+                    res['COMBINED_SEQ'] = {}
+                    # combine sequences for each class
+                    for c_cls in class_list:
+                        res['COMBINED_SEQ'][c_cls] = {}
+                        for metric, metric_name in zip(metrics_list, metric_names):
+                            curr_res = {seq_key: seq_value[c_cls][metric_name] for seq_key, seq_value in res.items() if
+                                        seq_key != 'COMBINED_SEQ'}
+                            res['COMBINED_SEQ'][c_cls][metric_name] = metric.combine_sequences(curr_res)
+                    # combine classes
+                    if dataset.should_classes_combine:
+                        combined_cls_keys += ['cls_comb_cls_av', 'cls_comb_det_av', 'all']
+                        res['COMBINED_SEQ']['cls_comb_cls_av'] = {}
+                        res['COMBINED_SEQ']['cls_comb_det_av'] = {}
+                        for metric, metric_name in zip(metrics_list, metric_names):
+                            cls_res = {cls_key: cls_value[metric_name] for cls_key, cls_value in
+                                       res['COMBINED_SEQ'].items() if cls_key not in combined_cls_keys}
+                            res['COMBINED_SEQ']['cls_comb_cls_av'][metric_name] = \
+                                metric.combine_classes_class_averaged(cls_res)
+                            res['COMBINED_SEQ']['cls_comb_det_av'][metric_name] = \
+                                metric.combine_classes_det_averaged(cls_res)
+                    # combine classes to super classes
+                    if dataset.use_super_categories:
+                        for cat, sub_cats in dataset.super_categories.items():
+                            combined_cls_keys.append(cat)
+                            res['COMBINED_SEQ'][cat] = {}
+                            for metric, metric_name in zip(metrics_list, metric_names):
+                                cat_res = {cls_key: cls_value[metric_name] for cls_key, cls_value in
+                                           res['COMBINED_SEQ'].items() if cls_key in sub_cats}
+                                res['COMBINED_SEQ'][cat][metric_name] = metric.combine_classes_det_averaged(cat_res)
+
+                    # Print and output results in various formats
+                    if config['TIME_PROGRESS']:
+                        print('\nAll sequences for %s finished in %.2f seconds' % (tracker, time.time() - time_start))
+                    output_fol = dataset.get_output_fol(tracker)
+                    tracker_display_name = dataset.get_display_name(tracker)
+                    for c_cls in res['COMBINED_SEQ'].keys():  # class_list + combined classes if calculated
+                        summaries = []
+                        details = []
+                        num_dets = res['COMBINED_SEQ'][c_cls]['Count']['Dets']
+                        if config['OUTPUT_EMPTY_CLASSES'] or num_dets > 0:
+                            for metric, metric_name in zip(metrics_list, metric_names):
+                                # for combined classes there is no per sequence evaluation
+                                if c_cls in combined_cls_keys:
+                                    table_res = {'COMBINED_SEQ': res['COMBINED_SEQ'][c_cls][metric_name]}
+                                else:
+                                    table_res = {seq_key: seq_value[c_cls][metric_name] for seq_key, seq_value
+                                                 in res.items()}
+
+                                if config['PRINT_RESULTS'] and config['PRINT_ONLY_COMBINED']:
+                                    dont_print = dataset.should_classes_combine and c_cls not in combined_cls_keys
+                                    if not dont_print:
+                                        metric.print_table({'COMBINED_SEQ': table_res['COMBINED_SEQ']},
+                                                           tracker_display_name, c_cls)
+                                elif config['PRINT_RESULTS']:
+                                    metric.print_table(table_res, tracker_display_name, c_cls)
+                                if config['OUTPUT_SUMMARY']:
+                                    summaries.append(metric.summary_results(table_res))
+                                if config['OUTPUT_DETAILED']:
+                                    details.append(metric.detailed_results(table_res))
+                                if config['PLOT_CURVES']:
+                                    metric.plot_single_tracker_results(table_res, tracker_display_name, c_cls,
+                                                                       output_fol)
+                            if config['OUTPUT_SUMMARY']:
+                                utils.write_summary_results(summaries, c_cls, output_fol)
+                            if config['OUTPUT_DETAILED']:
+                                utils.write_detailed_results(details, c_cls, output_fol)
+
+                    # Output for returning from function
+                    output_res[dataset_name][tracker] = res
+                    output_msg[dataset_name][tracker] = 'Success'
+
+                except Exception as err:
+                    output_res[dataset_name][tracker] = None
+                    if type(err) == TrackEvalException:
+                        output_msg[dataset_name][tracker] = str(err)
+                    else:
+                        output_msg[dataset_name][tracker] = 'Unknown error occurred.'
+                    print('Tracker %s was unable to be evaluated.' % tracker)
+                    print(err)
+                    traceback.print_exc()
+                    if config['LOG_ON_ERROR'] is not None:
+                        with open(config['LOG_ON_ERROR'], 'a') as f:
+                            print(dataset_name, file=f)
+                            print(tracker, file=f)
+                            print(traceback.format_exc(), file=f)
+                            print('\n\n\n', file=f)
+                    if config['BREAK_ON_ERROR']:
+                        raise err
+                    elif config['RETURN_ON_ERROR']:
+                        return output_res, output_msg
+
+        return output_res, output_msg
+
+
+@_timing.time
+def eval_sequence(seq, dataset, tracker, class_list, metrics_list, metric_names):
+    """Function for evaluating a single sequence"""
+
+    raw_data = dataset.get_raw_seq_data(tracker, seq)
+    seq_res = {}
+    for cls in class_list:
+        seq_res[cls] = {}
+        data = dataset.get_preprocessed_seq_data(raw_data, cls)
+        for metric, met_name in zip(metrics_list, metric_names):
+            seq_res[cls][met_name] = metric.eval_sequence(data)
+    return seq_res

yolov7-tracker-example/tracker/trackeval/metrics/__init__.py

@@ -0,0 +1,8 @@
+from .hota import HOTA
+from .clear import CLEAR
+from .identity import Identity
+from .count import Count
+from .j_and_f import JAndF
+from .track_map import TrackMAP
+from .vace import VACE
+from .ideucl import IDEucl

yolov7-tracker-example/tracker/trackeval/metrics/_base_metric.py

@@ -0,0 +1,133 @@
+
+import numpy as np
+from abc import ABC, abstractmethod
+from .. import _timing
+from ..utils import TrackEvalException
+
+
+class _BaseMetric(ABC):
+    @abstractmethod
+    def __init__(self):
+        self.plottable = False
+        self.integer_fields = []
+        self.float_fields = []
+        self.array_labels = []
+        self.integer_array_fields = []
+        self.float_array_fields = []
+        self.fields = []
+        self.summary_fields = []
+        self.registered = False
+
+    #####################################################################
+    # Abstract functions for subclasses to implement
+
+    @_timing.time
+    @abstractmethod
+    def eval_sequence(self, data):
+        ...
+
+    @abstractmethod
+    def combine_sequences(self, all_res):
+        ...
+
+    @abstractmethod
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        ...
+
+    @ abstractmethod
+    def combine_classes_det_averaged(self, all_res):
+        ...
+
+    def plot_single_tracker_results(self, all_res, tracker, output_folder, cls):
+        """Plot results of metrics, only valid for metrics with self.plottable"""
+        if self.plottable:
+            raise NotImplementedError('plot_results is not implemented for metric %s' % self.get_name())
+        else:
+            pass
+
+    #####################################################################
+    # Helper functions which are useful for all metrics:
+
+    @classmethod
+    def get_name(cls):
+        return cls.__name__
+
+    @staticmethod
+    def _combine_sum(all_res, field):
+        """Combine sequence results via sum"""
+        return sum([all_res[k][field] for k in all_res.keys()])
+
+    @staticmethod
+    def _combine_weighted_av(all_res, field, comb_res, weight_field):
+        """Combine sequence results via weighted average"""
+        return sum([all_res[k][field] * all_res[k][weight_field] for k in all_res.keys()]) / np.maximum(1.0, comb_res[
+            weight_field])
+
+    def print_table(self, table_res, tracker, cls):
+        """Prints table of results for all sequences"""
+        print('')
+        metric_name = self.get_name()
+        self._row_print([metric_name + ': ' + tracker + '-' + cls] + self.summary_fields)
+        for seq, results in sorted(table_res.items()):
+            if seq == 'COMBINED_SEQ':
+                continue
+            summary_res = self._summary_row(results)
+            self._row_print([seq] + summary_res)
+        summary_res = self._summary_row(table_res['COMBINED_SEQ'])
+        self._row_print(['COMBINED'] + summary_res)
+
+    def _summary_row(self, results_):
+        vals = []
+        for h in self.summary_fields:
+            if h in self.float_array_fields:
+                vals.append("{0:1.5g}".format(100 * np.mean(results_[h])))
+            elif h in self.float_fields:
+                vals.append("{0:1.5g}".format(100 * float(results_[h])))
+            elif h in self.integer_fields:
+                vals.append("{0:d}".format(int(results_[h])))
+            else:
+                raise NotImplementedError("Summary function not implemented for this field type.")
+        return vals
+
+    @staticmethod
+    def _row_print(*argv):
+        """Prints results in an evenly spaced rows, with more space in first row"""
+        if len(argv) == 1:
+            argv = argv[0]
+        to_print = '%-35s' % argv[0]
+        for v in argv[1:]:
+            to_print += '%-10s' % str(v)
+        print(to_print)
+
+    def summary_results(self, table_res):
+        """Returns a simple summary of final results for a tracker"""
+        return dict(zip(self.summary_fields, self._summary_row(table_res['COMBINED_SEQ'])))
+
+    def detailed_results(self, table_res):
+        """Returns detailed final results for a tracker"""
+        # Get detailed field information
+        detailed_fields = self.float_fields + self.integer_fields
+        for h in self.float_array_fields + self.integer_array_fields:
+            for alpha in [int(100*x) for x in self.array_labels]:
+                detailed_fields.append(h + '___' + str(alpha))
+            detailed_fields.append(h + '___AUC')
+
+        # Get detailed results
+        detailed_results = {}
+        for seq, res in table_res.items():
+            detailed_row = self._detailed_row(res)
+            if len(detailed_row) != len(detailed_fields):
+                raise TrackEvalException(
+                    'Field names and data have different sizes (%i and %i)' % (len(detailed_row), len(detailed_fields)))
+            detailed_results[seq] = dict(zip(detailed_fields, detailed_row))
+        return detailed_results
+
+    def _detailed_row(self, res):
+        detailed_row = []
+        for h in self.float_fields + self.integer_fields:
+            detailed_row.append(res[h])
+        for h in self.float_array_fields + self.integer_array_fields:
+            for i, alpha in enumerate([int(100 * x) for x in self.array_labels]):
+                detailed_row.append(res[h][i])
+            detailed_row.append(np.mean(res[h]))
+        return detailed_row

yolov7-tracker-example/tracker/trackeval/metrics/clear.py

@@ -0,0 +1,186 @@
+
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_metric import _BaseMetric
+from .. import _timing
+from .. import utils
+
+class CLEAR(_BaseMetric):
+    """Class which implements the CLEAR metrics"""
+
+    @staticmethod
+    def get_default_config():
+        """Default class config values"""
+        default_config = {
+            'THRESHOLD': 0.5,  # Similarity score threshold required for a TP match. Default 0.5.
+            'PRINT_CONFIG': True,  # Whether to print the config information on init. Default: False.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        super().__init__()
+        main_integer_fields = ['CLR_TP', 'CLR_FN', 'CLR_FP', 'IDSW', 'MT', 'PT', 'ML', 'Frag']
+        extra_integer_fields = ['CLR_Frames']
+        self.integer_fields = main_integer_fields + extra_integer_fields
+        main_float_fields = ['MOTA', 'MOTP', 'MODA', 'CLR_Re', 'CLR_Pr', 'MTR', 'PTR', 'MLR', 'sMOTA']
+        extra_float_fields = ['CLR_F1', 'FP_per_frame', 'MOTAL', 'MOTP_sum']
+        self.float_fields = main_float_fields + extra_float_fields
+        self.fields = self.float_fields + self.integer_fields
+        self.summed_fields = self.integer_fields + ['MOTP_sum']
+        self.summary_fields = main_float_fields + main_integer_fields
+
+        # Configuration options:
+        self.config = utils.init_config(config, self.get_default_config(), self.get_name())
+        self.threshold = float(self.config['THRESHOLD'])
+
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates CLEAR metrics for one sequence"""
+        # Initialise results
+        res = {}
+        for field in self.fields:
+            res[field] = 0
+
+        # Return result quickly if tracker or gt sequence is empty
+        if data['num_tracker_dets'] == 0:
+            res['CLR_FN'] = data['num_gt_dets']
+            res['ML'] = data['num_gt_ids']
+            res['MLR'] = 1.0
+            return res
+        if data['num_gt_dets'] == 0:
+            res['CLR_FP'] = data['num_tracker_dets']
+            res['MLR'] = 1.0
+            return res
+
+        # Variables counting global association
+        num_gt_ids = data['num_gt_ids']
+        gt_id_count = np.zeros(num_gt_ids)  # For MT/ML/PT
+        gt_matched_count = np.zeros(num_gt_ids)  # For MT/ML/PT
+        gt_frag_count = np.zeros(num_gt_ids)  # For Frag
+
+        # Note that IDSWs are counted based on the last time each gt_id was present (any number of frames previously),
+        # but are only used in matching to continue current tracks based on the gt_id in the single previous timestep.
+        prev_tracker_id = np.nan * np.zeros(num_gt_ids)  # For scoring IDSW
+        prev_timestep_tracker_id = np.nan * np.zeros(num_gt_ids)  # For matching IDSW
+
+        # Calculate scores for each timestep
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            # Deal with the case that there are no gt_det/tracker_det in a timestep.
+            if len(gt_ids_t) == 0:
+                res['CLR_FP'] += len(tracker_ids_t)
+                continue
+            if len(tracker_ids_t) == 0:
+                res['CLR_FN'] += len(gt_ids_t)
+                gt_id_count[gt_ids_t] += 1
+                continue
+
+            # Calc score matrix to first minimise IDSWs from previous frame, and then maximise MOTP secondarily
+            similarity = data['similarity_scores'][t]
+            score_mat = (tracker_ids_t[np.newaxis, :] == prev_timestep_tracker_id[gt_ids_t[:, np.newaxis]])
+            score_mat = 1000 * score_mat + similarity
+            score_mat[similarity < self.threshold - np.finfo('float').eps] = 0
+
+            # Hungarian algorithm to find best matches
+            match_rows, match_cols = linear_sum_assignment(-score_mat)
+            actually_matched_mask = score_mat[match_rows, match_cols] > 0 + np.finfo('float').eps
+            match_rows = match_rows[actually_matched_mask]
+            match_cols = match_cols[actually_matched_mask]
+
+            matched_gt_ids = gt_ids_t[match_rows]
+            matched_tracker_ids = tracker_ids_t[match_cols]
+
+            # Calc IDSW for MOTA
+            prev_matched_tracker_ids = prev_tracker_id[matched_gt_ids]
+            is_idsw = (np.logical_not(np.isnan(prev_matched_tracker_ids))) & (
+                np.not_equal(matched_tracker_ids, prev_matched_tracker_ids))
+            res['IDSW'] += np.sum(is_idsw)
+
+            # Update counters for MT/ML/PT/Frag and record for IDSW/Frag for next timestep
+            gt_id_count[gt_ids_t] += 1
+            gt_matched_count[matched_gt_ids] += 1
+            not_previously_tracked = np.isnan(prev_timestep_tracker_id)
+            prev_tracker_id[matched_gt_ids] = matched_tracker_ids
+            prev_timestep_tracker_id[:] = np.nan
+            prev_timestep_tracker_id[matched_gt_ids] = matched_tracker_ids
+            currently_tracked = np.logical_not(np.isnan(prev_timestep_tracker_id))
+            gt_frag_count += np.logical_and(not_previously_tracked, currently_tracked)
+
+            # Calculate and accumulate basic statistics
+            num_matches = len(matched_gt_ids)
+            res['CLR_TP'] += num_matches
+            res['CLR_FN'] += len(gt_ids_t) - num_matches
+            res['CLR_FP'] += len(tracker_ids_t) - num_matches
+            if num_matches > 0:
+                res['MOTP_sum'] += sum(similarity[match_rows, match_cols])
+
+        # Calculate MT/ML/PT/Frag/MOTP
+        tracked_ratio = gt_matched_count[gt_id_count > 0] / gt_id_count[gt_id_count > 0]
+        res['MT'] = np.sum(np.greater(tracked_ratio, 0.8))
+        res['PT'] = np.sum(np.greater_equal(tracked_ratio, 0.2)) - res['MT']
+        res['ML'] = num_gt_ids - res['MT'] - res['PT']
+        res['Frag'] = np.sum(np.subtract(gt_frag_count[gt_frag_count > 0], 1))
+        res['MOTP'] = res['MOTP_sum'] / np.maximum(1.0, res['CLR_TP'])
+
+        res['CLR_Frames'] = data['num_timesteps']
+
+        # Calculate final CLEAR scores
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for field in self.summed_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self.summed_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        """Combines metrics across all classes by averaging over the class values.
+        If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
+        """
+        res = {}
+        for field in self.integer_fields:
+            if ignore_empty_classes:
+                res[field] = self._combine_sum(
+                    {k: v for k, v in all_res.items() if v['CLR_TP'] + v['CLR_FN'] + v['CLR_FP'] > 0}, field)
+            else:
+                res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field)
+        for field in self.float_fields:
+            if ignore_empty_classes:
+                res[field] = np.mean(
+                    [v[field] for v in all_res.values() if v['CLR_TP'] + v['CLR_FN'] + v['CLR_FP'] > 0], axis=0)
+            else:
+                res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
+        return res
+
+    @staticmethod
+    def _compute_final_fields(res):
+        """Calculate sub-metric ('field') values which only depend on other sub-metric values.
+        This function is used both for both per-sequence calculation, and in combining values across sequences.
+        """
+        num_gt_ids = res['MT'] + res['ML'] + res['PT']
+        res['MTR'] = res['MT'] / np.maximum(1.0, num_gt_ids)
+        res['MLR'] = res['ML'] / np.maximum(1.0, num_gt_ids)
+        res['PTR'] = res['PT'] / np.maximum(1.0, num_gt_ids)
+        res['CLR_Re'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN'])
+        res['CLR_Pr'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + res['CLR_FP'])
+        res['MODA'] = (res['CLR_TP'] - res['CLR_FP']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN'])
+        res['MOTA'] = (res['CLR_TP'] - res['CLR_FP'] - res['IDSW']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN'])
+        res['MOTP'] = res['MOTP_sum'] / np.maximum(1.0, res['CLR_TP'])
+        res['sMOTA'] = (res['MOTP_sum'] - res['CLR_FP'] - res['IDSW']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN'])
+
+        res['CLR_F1'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + 0.5*res['CLR_FN'] + 0.5*res['CLR_FP'])
+        res['FP_per_frame'] = res['CLR_FP'] / np.maximum(1.0, res['CLR_Frames'])
+        safe_log_idsw = np.log10(res['IDSW']) if res['IDSW'] > 0 else res['IDSW']
+        res['MOTAL'] = (res['CLR_TP'] - res['CLR_FP'] - safe_log_idsw) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN'])
+        return res

yolov7-tracker-example/tracker/trackeval/metrics/count.py

@@ -0,0 +1,44 @@
+
+from ._base_metric import _BaseMetric
+from .. import _timing
+
+
+class Count(_BaseMetric):
+    """Class which simply counts the number of tracker and gt detections and ids."""
+    def __init__(self, config=None):
+        super().__init__()
+        self.integer_fields = ['Dets', 'GT_Dets', 'IDs', 'GT_IDs']
+        self.fields = self.integer_fields
+        self.summary_fields = self.fields
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Returns counts for one sequence"""
+        # Get results
+        res = {'Dets': data['num_tracker_dets'],
+               'GT_Dets': data['num_gt_dets'],
+               'IDs': data['num_tracker_ids'],
+               'GT_IDs': data['num_gt_ids'],
+               'Frames': data['num_timesteps']}
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for field in self.integer_fields:
+            res[field] = self._combine_sum(all_res, field)
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=None):
+        """Combines metrics across all classes by averaging over the class values"""
+        res = {}
+        for field in self.integer_fields:
+            res[field] = self._combine_sum(all_res, field)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self.integer_fields:
+            res[field] = self._combine_sum(all_res, field)
+        return res

yolov7-tracker-example/tracker/trackeval/metrics/hota.py

@@ -0,0 +1,203 @@
+
+import os
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_metric import _BaseMetric
+from .. import _timing
+
+
+class HOTA(_BaseMetric):
+    """Class which implements the HOTA metrics.
+    See: https://link.springer.com/article/10.1007/s11263-020-01375-2
+    """
+
+    def __init__(self, config=None):
+        super().__init__()
+        self.plottable = True
+        self.array_labels = np.arange(0.05, 0.99, 0.05)
+        self.integer_array_fields = ['HOTA_TP', 'HOTA_FN', 'HOTA_FP']
+        self.float_array_fields = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA', 'OWTA']
+        self.float_fields = ['HOTA(0)', 'LocA(0)', 'HOTALocA(0)']
+        self.fields = self.float_array_fields + self.integer_array_fields + self.float_fields
+        self.summary_fields = self.float_array_fields + self.float_fields
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates the HOTA metrics for one sequence"""
+
+        # Initialise results
+        res = {}
+        for field in self.float_array_fields + self.integer_array_fields:
+            res[field] = np.zeros((len(self.array_labels)), dtype=np.float)
+        for field in self.float_fields:
+            res[field] = 0
+
+        # Return result quickly if tracker or gt sequence is empty
+        if data['num_tracker_dets'] == 0:
+            res['HOTA_FN'] = data['num_gt_dets'] * np.ones((len(self.array_labels)), dtype=np.float)
+            res['LocA'] = np.ones((len(self.array_labels)), dtype=np.float)
+            res['LocA(0)'] = 1.0
+            return res
+        if data['num_gt_dets'] == 0:
+            res['HOTA_FP'] = data['num_tracker_dets'] * np.ones((len(self.array_labels)), dtype=np.float)
+            res['LocA'] = np.ones((len(self.array_labels)), dtype=np.float)
+            res['LocA(0)'] = 1.0
+            return res
+
+        # Variables counting global association
+        potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids']))
+        gt_id_count = np.zeros((data['num_gt_ids'], 1))
+        tracker_id_count = np.zeros((1, data['num_tracker_ids']))
+
+        # First loop through each timestep and accumulate global track information.
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            # Count the potential matches between ids in each timestep
+            # These are normalised, weighted by the match similarity.
+            similarity = data['similarity_scores'][t]
+            sim_iou_denom = similarity.sum(0)[np.newaxis, :] + similarity.sum(1)[:, np.newaxis] - similarity
+            sim_iou = np.zeros_like(similarity)
+            sim_iou_mask = sim_iou_denom > 0 + np.finfo('float').eps
+            sim_iou[sim_iou_mask] = similarity[sim_iou_mask] / sim_iou_denom[sim_iou_mask]
+            potential_matches_count[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] += sim_iou
+
+            # Calculate the total number of dets for each gt_id and tracker_id.
+            gt_id_count[gt_ids_t] += 1
+            tracker_id_count[0, tracker_ids_t] += 1
+
+        # Calculate overall jaccard alignment score (before unique matching) between IDs
+        global_alignment_score = potential_matches_count / (gt_id_count + tracker_id_count - potential_matches_count)
+        matches_counts = [np.zeros_like(potential_matches_count) for _ in self.array_labels]
+
+        # Calculate scores for each timestep
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            # Deal with the case that there are no gt_det/tracker_det in a timestep.
+            if len(gt_ids_t) == 0:
+                for a, alpha in enumerate(self.array_labels):
+                    res['HOTA_FP'][a] += len(tracker_ids_t)
+                continue
+            if len(tracker_ids_t) == 0:
+                for a, alpha in enumerate(self.array_labels):
+                    res['HOTA_FN'][a] += len(gt_ids_t)
+                continue
+
+            # Get matching scores between pairs of dets for optimizing HOTA
+            similarity = data['similarity_scores'][t]
+            score_mat = global_alignment_score[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] * similarity
+
+            # Hungarian algorithm to find best matches
+            match_rows, match_cols = linear_sum_assignment(-score_mat)
+
+            # Calculate and accumulate basic statistics
+            for a, alpha in enumerate(self.array_labels):
+                actually_matched_mask = similarity[match_rows, match_cols] >= alpha - np.finfo('float').eps
+                alpha_match_rows = match_rows[actually_matched_mask]
+                alpha_match_cols = match_cols[actually_matched_mask]
+                num_matches = len(alpha_match_rows)
+                res['HOTA_TP'][a] += num_matches
+                res['HOTA_FN'][a] += len(gt_ids_t) - num_matches
+                res['HOTA_FP'][a] += len(tracker_ids_t) - num_matches
+                if num_matches > 0:
+                    res['LocA'][a] += sum(similarity[alpha_match_rows, alpha_match_cols])
+                    matches_counts[a][gt_ids_t[alpha_match_rows], tracker_ids_t[alpha_match_cols]] += 1
+
+        # Calculate association scores (AssA, AssRe, AssPr) for the alpha value.
+        # First calculate scores per gt_id/tracker_id combo and then average over the number of detections.
+        for a, alpha in enumerate(self.array_labels):
+            matches_count = matches_counts[a]
+            ass_a = matches_count / np.maximum(1, gt_id_count + tracker_id_count - matches_count)
+            res['AssA'][a] = np.sum(matches_count * ass_a) / np.maximum(1, res['HOTA_TP'][a])
+            ass_re = matches_count / np.maximum(1, gt_id_count)
+            res['AssRe'][a] = np.sum(matches_count * ass_re) / np.maximum(1, res['HOTA_TP'][a])
+            ass_pr = matches_count / np.maximum(1, tracker_id_count)
+            res['AssPr'][a] = np.sum(matches_count * ass_pr) / np.maximum(1, res['HOTA_TP'][a])
+
+        # Calculate final scores
+        res['LocA'] = np.maximum(1e-10, res['LocA']) / np.maximum(1e-10, res['HOTA_TP'])
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for field in self.integer_array_fields:
+            res[field] = self._combine_sum(all_res, field)
+        for field in ['AssRe', 'AssPr', 'AssA']:
+            res[field] = self._combine_weighted_av(all_res, field, res, weight_field='HOTA_TP')
+        loca_weighted_sum = sum([all_res[k]['LocA'] * all_res[k]['HOTA_TP'] for k in all_res.keys()])
+        res['LocA'] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(1e-10, res['HOTA_TP'])
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        """Combines metrics across all classes by averaging over the class values.
+        If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
+        """
+        res = {}
+        for field in self.integer_array_fields:
+            if ignore_empty_classes:
+                res[field] = self._combine_sum(
+                    {k: v for k, v in all_res.items()
+                     if (v['HOTA_TP'] + v['HOTA_FN'] + v['HOTA_FP'] > 0 + np.finfo('float').eps).any()}, field)
+            else:
+                res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field)
+
+        for field in self.float_fields + self.float_array_fields:
+            if ignore_empty_classes:
+                res[field] = np.mean([v[field] for v in all_res.values() if
+                                      (v['HOTA_TP'] + v['HOTA_FN'] + v['HOTA_FP'] > 0 + np.finfo('float').eps).any()],
+                                     axis=0)
+            else:
+                res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self.integer_array_fields:
+            res[field] = self._combine_sum(all_res, field)
+        for field in ['AssRe', 'AssPr', 'AssA']:
+            res[field] = self._combine_weighted_av(all_res, field, res, weight_field='HOTA_TP')
+        loca_weighted_sum = sum([all_res[k]['LocA'] * all_res[k]['HOTA_TP'] for k in all_res.keys()])
+        res['LocA'] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(1e-10, res['HOTA_TP'])
+        res = self._compute_final_fields(res)
+        return res
+
+    @staticmethod
+    def _compute_final_fields(res):
+        """Calculate sub-metric ('field') values which only depend on other sub-metric values.
+        This function is used both for both per-sequence calculation, and in combining values across sequences.
+        """
+        res['DetRe'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FN'])
+        res['DetPr'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FP'])
+        res['DetA'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FN'] + res['HOTA_FP'])
+        res['HOTA'] = np.sqrt(res['DetA'] * res['AssA'])
+        res['OWTA'] = np.sqrt(res['DetRe'] * res['AssA'])
+
+        res['HOTA(0)'] = res['HOTA'][0]
+        res['LocA(0)'] = res['LocA'][0]
+        res['HOTALocA(0)'] = res['HOTA(0)']*res['LocA(0)']
+        return res
+
+    def plot_single_tracker_results(self, table_res, tracker, cls, output_folder):
+        """Create plot of results"""
+
+        # Only loaded when run to reduce minimum requirements
+        from matplotlib import pyplot as plt
+
+        res = table_res['COMBINED_SEQ']
+        styles_to_plot = ['r', 'b', 'g', 'b--', 'b:', 'g--', 'g:', 'm']
+        for name, style in zip(self.float_array_fields, styles_to_plot):
+            plt.plot(self.array_labels, res[name], style)
+        plt.xlabel('alpha')
+        plt.ylabel('score')
+        plt.title(tracker + ' - ' + cls)
+        plt.axis([0, 1, 0, 1])
+        legend = []
+        for name in self.float_array_fields:
+            legend += [name + ' (' + str(np.round(np.mean(res[name]), 2)) + ')']
+        plt.legend(legend, loc='lower left')
+        out_file = os.path.join(output_folder, cls + '_plot.pdf')
+        os.makedirs(os.path.dirname(out_file), exist_ok=True)
+        plt.savefig(out_file)
+        plt.savefig(out_file.replace('.pdf', '.png'))
+        plt.clf()

yolov7-tracker-example/tracker/trackeval/metrics/identity.py

@@ -0,0 +1,135 @@
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_metric import _BaseMetric
+from .. import _timing
+from .. import utils
+
+
+class Identity(_BaseMetric):
+    """Class which implements the ID metrics"""
+
+    @staticmethod
+    def get_default_config():
+        """Default class config values"""
+        default_config = {
+            'THRESHOLD': 0.5,  # Similarity score threshold required for a IDTP match. Default 0.5.
+            'PRINT_CONFIG': True,  # Whether to print the config information on init. Default: False.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        super().__init__()
+        self.integer_fields = ['IDTP', 'IDFN', 'IDFP']
+        self.float_fields = ['IDF1', 'IDR', 'IDP']
+        self.fields = self.float_fields + self.integer_fields
+        self.summary_fields = self.fields
+
+        # Configuration options:
+        self.config = utils.init_config(config, self.get_default_config(), self.get_name())
+        self.threshold = float(self.config['THRESHOLD'])
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates ID metrics for one sequence"""
+        # Initialise results
+        res = {}
+        for field in self.fields:
+            res[field] = 0
+
+        # Return result quickly if tracker or gt sequence is empty
+        if data['num_tracker_dets'] == 0:
+            res['IDFN'] = data['num_gt_dets']
+            return res
+        if data['num_gt_dets'] == 0:
+            res['IDFP'] = data['num_tracker_dets']
+            return res
+
+        # Variables counting global association
+        potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids']))
+        gt_id_count = np.zeros(data['num_gt_ids'])
+        tracker_id_count = np.zeros(data['num_tracker_ids'])
+
+        # First loop through each timestep and accumulate global track information.
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            # Count the potential matches between ids in each timestep
+            matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold)
+            match_idx_gt, match_idx_tracker = np.nonzero(matches_mask)
+            potential_matches_count[gt_ids_t[match_idx_gt], tracker_ids_t[match_idx_tracker]] += 1
+
+            # Calculate the total number of dets for each gt_id and tracker_id.
+            gt_id_count[gt_ids_t] += 1
+            tracker_id_count[tracker_ids_t] += 1
+
+        # Calculate optimal assignment cost matrix for ID metrics
+        num_gt_ids = data['num_gt_ids']
+        num_tracker_ids = data['num_tracker_ids']
+        fp_mat = np.zeros((num_gt_ids + num_tracker_ids, num_gt_ids + num_tracker_ids))
+        fn_mat = np.zeros((num_gt_ids + num_tracker_ids, num_gt_ids + num_tracker_ids))
+        fp_mat[num_gt_ids:, :num_tracker_ids] = 1e10
+        fn_mat[:num_gt_ids, num_tracker_ids:] = 1e10
+        for gt_id in range(num_gt_ids):
+            fn_mat[gt_id, :num_tracker_ids] = gt_id_count[gt_id]
+            fn_mat[gt_id, num_tracker_ids + gt_id] = gt_id_count[gt_id]
+        for tracker_id in range(num_tracker_ids):
+            fp_mat[:num_gt_ids, tracker_id] = tracker_id_count[tracker_id]
+            fp_mat[tracker_id + num_gt_ids, tracker_id] = tracker_id_count[tracker_id]
+        fn_mat[:num_gt_ids, :num_tracker_ids] -= potential_matches_count
+        fp_mat[:num_gt_ids, :num_tracker_ids] -= potential_matches_count
+
+        # Hungarian algorithm
+        match_rows, match_cols = linear_sum_assignment(fn_mat + fp_mat)
+
+        # Accumulate basic statistics
+        res['IDFN'] = fn_mat[match_rows, match_cols].sum().astype(np.int)
+        res['IDFP'] = fp_mat[match_rows, match_cols].sum().astype(np.int)
+        res['IDTP'] = (gt_id_count.sum() - res['IDFN']).astype(np.int)
+
+        # Calculate final ID scores
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        """Combines metrics across all classes by averaging over the class values.
+        If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
+        """
+        res = {}
+        for field in self.integer_fields:
+            if ignore_empty_classes:
+                res[field] = self._combine_sum({k: v for k, v in all_res.items()
+                                                if v['IDTP'] + v['IDFN'] + v['IDFP'] > 0 + np.finfo('float').eps},
+                                               field)
+            else:
+                res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field)
+        for field in self.float_fields:
+            if ignore_empty_classes:
+                res[field] = np.mean([v[field] for v in all_res.values()
+                                      if v['IDTP'] + v['IDFN'] + v['IDFP'] > 0 + np.finfo('float').eps], axis=0)
+            else:
+                res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self.integer_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for field in self.integer_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res)
+        return res
+
+    @staticmethod
+    def _compute_final_fields(res):
+        """Calculate sub-metric ('field') values which only depend on other sub-metric values.
+        This function is used both for both per-sequence calculation, and in combining values across sequences.
+        """
+        res['IDR'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + res['IDFN'])
+        res['IDP'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + res['IDFP'])
+        res['IDF1'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + 0.5 * res['IDFP'] + 0.5 * res['IDFN'])
+        return res

yolov7-tracker-example/tracker/trackeval/metrics/ideucl.py

@@ -0,0 +1,135 @@
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_metric import _BaseMetric
+from .. import _timing
+from collections import defaultdict
+from .. import utils
+
+
+class IDEucl(_BaseMetric):
+    """Class which implements the ID metrics"""
+
+    @staticmethod
+    def get_default_config():
+        """Default class config values"""
+        default_config = {
+            'THRESHOLD': 0.4,  # Similarity score threshold required for a IDTP match. 0.4 for IDEucl.
+            'PRINT_CONFIG': True,  # Whether to print the config information on init. Default: False.
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        super().__init__()
+        self.fields = ['IDEucl']
+        self.float_fields = self.fields
+        self.summary_fields = self.fields
+
+        # Configuration options:
+        self.config = utils.init_config(config, self.get_default_config(), self.get_name())
+        self.threshold = float(self.config['THRESHOLD'])
+
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates IDEucl metrics for all frames"""
+        # Initialise results
+        res = {'IDEucl' : 0}
+
+        # Return result quickly if tracker or gt sequence is empty
+        if data['num_tracker_dets'] == 0 or data['num_gt_dets'] == 0.:
+            return res
+
+        data['centroid'] = []
+        for t, gt_det in enumerate(data['gt_dets']):
+            # import pdb;pdb.set_trace()
+            data['centroid'].append(self._compute_centroid(gt_det))
+
+        oid_hid_cent = defaultdict(list)
+        oid_cent = defaultdict(list)
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold)
+
+            # I hope the orders of ids and boxes are maintained in `data`
+            for ind, gid in enumerate(gt_ids_t):
+                oid_cent[gid].append(data['centroid'][t][ind])
+
+            match_idx_gt, match_idx_tracker = np.nonzero(matches_mask)
+            for m_gid, m_tid in zip(match_idx_gt, match_idx_tracker):
+                oid_hid_cent[gt_ids_t[m_gid], tracker_ids_t[m_tid]].append(data['centroid'][t][m_gid])
+
+        oid_hid_dist = {k : np.sum(np.linalg.norm(np.diff(np.array(v), axis=0), axis=1)) for k, v in oid_hid_cent.items()}
+        oid_dist = {int(k) : np.sum(np.linalg.norm(np.diff(np.array(v), axis=0), axis=1)) for k, v in oid_cent.items()}
+
+        unique_oid = np.unique([i[0] for i in oid_hid_dist.keys()]).tolist()
+        unique_hid = np.unique([i[1] for i in oid_hid_dist.keys()]).tolist()
+        o_len = len(unique_oid)
+        h_len = len(unique_hid)
+        dist_matrix = np.zeros((o_len, h_len))
+        for ((oid, hid), dist) in oid_hid_dist.items():
+            oid_ind = unique_oid.index(oid)
+            hid_ind = unique_hid.index(hid)
+            dist_matrix[oid_ind, hid_ind] = dist
+
+        # opt_hyp_dist contains GT ID : max dist covered by track
+        opt_hyp_dist = dict.fromkeys(oid_dist.keys(), 0.)
+        cost_matrix = np.max(dist_matrix) - dist_matrix
+        rows, cols = linear_sum_assignment(cost_matrix)
+        for (row, col) in zip(rows, cols):
+            value = dist_matrix[row, col]
+            opt_hyp_dist[int(unique_oid[row])] = value
+
+        assert len(opt_hyp_dist.keys()) == len(oid_dist.keys())
+        hyp_length = np.sum(list(opt_hyp_dist.values()))
+        gt_length = np.sum(list(oid_dist.values()))
+        id_eucl =np.mean([np.divide(a, b, out=np.zeros_like(a), where=b!=0) for a, b in zip(opt_hyp_dist.values(), oid_dist.values())])
+        res['IDEucl'] = np.divide(hyp_length, gt_length, out=np.zeros_like(hyp_length), where=gt_length!=0)
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        """Combines metrics across all classes by averaging over the class values.
+        If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
+        """
+        res = {}
+
+        for field in self.float_fields:
+            if ignore_empty_classes:
+                res[field] = np.mean([v[field] for v in all_res.values()
+                                      if v['IDEucl'] > 0 + np.finfo('float').eps], axis=0)
+            else:
+                res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self.float_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res, len(all_res))
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for field in self.float_fields:
+            res[field] = self._combine_sum(all_res, field)
+        res = self._compute_final_fields(res, len(all_res))
+        return res
+
+
+    @staticmethod
+    def _compute_centroid(box):
+        box = np.array(box)
+        if len(box.shape) == 1:
+            centroid = (box[0:2] + box[2:4])/2
+        else:
+            centroid = (box[:, 0:2] + box[:, 2:4])/2
+        return  np.flip(centroid, axis=1)
+
+
+    @staticmethod
+    def _compute_final_fields(res, res_len):
+        """
+        Exists only to match signature with the original Identiy class.
+
+        """
+        return {k:v/res_len for k,v in res.items()}

yolov7-tracker-example/tracker/trackeval/metrics/j_and_f.py

@@ -0,0 +1,310 @@
+
+import numpy as np
+import math
+from scipy.optimize import linear_sum_assignment
+from ..utils import TrackEvalException
+from ._base_metric import _BaseMetric
+from .. import _timing
+
+
+class JAndF(_BaseMetric):
+    """Class which implements the J&F metrics"""
+    def __init__(self, config=None):
+        super().__init__()
+        self.integer_fields = ['num_gt_tracks']
+        self.float_fields = ['J-Mean', 'J-Recall', 'J-Decay', 'F-Mean', 'F-Recall', 'F-Decay', 'J&F']
+        self.fields = self.float_fields + self.integer_fields
+        self.summary_fields = self.float_fields
+        self.optim_type = 'J'  # possible values J, J&F
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Returns J&F metrics for one sequence"""
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        num_timesteps = data['num_timesteps']
+        num_tracker_ids = data['num_tracker_ids']
+        num_gt_ids = data['num_gt_ids']
+        gt_dets = data['gt_dets']
+        tracker_dets = data['tracker_dets']
+        gt_ids = data['gt_ids']
+        tracker_ids = data['tracker_ids']
+
+        # get shape of frames
+        frame_shape = None
+        if num_gt_ids > 0:
+            for t in range(num_timesteps):
+                if len(gt_ids[t]) > 0:
+                    frame_shape = gt_dets[t][0]['size']
+                    break
+        elif num_tracker_ids > 0:
+            for t in range(num_timesteps):
+                if len(tracker_ids[t]) > 0:
+                    frame_shape = tracker_dets[t][0]['size']
+                    break
+
+        if frame_shape:
+            # append all zero masks for timesteps in which tracks do not have a detection
+            zero_padding = np.zeros((frame_shape), order= 'F').astype(np.uint8)
+            padding_mask = mask_utils.encode(zero_padding)
+            for t in range(num_timesteps):
+                gt_id_det_mapping = {gt_ids[t][i]: gt_dets[t][i] for i in range(len(gt_ids[t]))}
+                gt_dets[t] = [gt_id_det_mapping[index] if index in gt_ids[t] else padding_mask for index
+                              in range(num_gt_ids)]
+                tracker_id_det_mapping = {tracker_ids[t][i]: tracker_dets[t][i] for i in range(len(tracker_ids[t]))}
+                tracker_dets[t] = [tracker_id_det_mapping[index] if index in tracker_ids[t] else padding_mask for index
+                                   in range(num_tracker_ids)]
+            # also perform zero padding if number of tracker IDs < number of ground truth IDs
+            if num_tracker_ids < num_gt_ids:
+                diff = num_gt_ids - num_tracker_ids
+                for t in range(num_timesteps):
+                    tracker_dets[t] = tracker_dets[t] + [padding_mask for _ in range(diff)]
+                num_tracker_ids += diff
+
+        j = self._compute_j(gt_dets, tracker_dets, num_gt_ids, num_tracker_ids, num_timesteps)
+
+        # boundary threshold for F computation
+        bound_th = 0.008
+
+        # perform matching
+        if self.optim_type == 'J&F':
+            f = np.zeros_like(j)
+            for k in range(num_tracker_ids):
+                for i in range(num_gt_ids):
+                    f[k, i, :] = self._compute_f(gt_dets, tracker_dets, k, i, bound_th)
+            optim_metrics = (np.mean(j, axis=2) + np.mean(f, axis=2)) / 2
+            row_ind, col_ind = linear_sum_assignment(- optim_metrics)
+            j_m = j[row_ind, col_ind, :]
+            f_m = f[row_ind, col_ind, :]
+        elif self.optim_type == 'J':
+            optim_metrics = np.mean(j, axis=2)
+            row_ind, col_ind = linear_sum_assignment(- optim_metrics)
+            j_m = j[row_ind, col_ind, :]
+            f_m = np.zeros_like(j_m)
+            for i, (tr_ind, gt_ind) in enumerate(zip(row_ind, col_ind)):
+                f_m[i] = self._compute_f(gt_dets, tracker_dets, tr_ind, gt_ind, bound_th)
+        else:
+            raise TrackEvalException('Unsupported optimization type %s for J&F metric.' % self.optim_type)
+
+        # append zeros for false negatives
+        if j_m.shape[0] < data['num_gt_ids']:
+            diff = data['num_gt_ids'] - j_m.shape[0]
+            j_m = np.concatenate((j_m, np.zeros((diff, j_m.shape[1]))), axis=0)
+            f_m = np.concatenate((f_m, np.zeros((diff, f_m.shape[1]))), axis=0)
+
+        # compute the metrics for each ground truth track
+        res = {
+            'J-Mean': [np.nanmean(j_m[i, :]) for i in range(j_m.shape[0])],
+            'J-Recall': [np.nanmean(j_m[i, :] > 0.5 + np.finfo('float').eps) for i in range(j_m.shape[0])],
+            'F-Mean': [np.nanmean(f_m[i, :]) for i in range(f_m.shape[0])],
+            'F-Recall': [np.nanmean(f_m[i, :] > 0.5 + np.finfo('float').eps) for i in range(f_m.shape[0])],
+            'J-Decay': [],
+            'F-Decay': []
+        }
+        n_bins = 4
+        ids = np.round(np.linspace(1, data['num_timesteps'], n_bins + 1) + 1e-10) - 1
+        ids = ids.astype(np.uint8)
+
+        for k in range(j_m.shape[0]):
+            d_bins_j = [j_m[k][ids[i]:ids[i + 1] + 1] for i in range(0, n_bins)]
+            res['J-Decay'].append(np.nanmean(d_bins_j[0]) - np.nanmean(d_bins_j[3]))
+        for k in range(f_m.shape[0]):
+            d_bins_f = [f_m[k][ids[i]:ids[i + 1] + 1] for i in range(0, n_bins)]
+            res['F-Decay'].append(np.nanmean(d_bins_f[0]) - np.nanmean(d_bins_f[3]))
+
+        # count number of tracks for weighting of the result
+        res['num_gt_tracks'] = len(res['J-Mean'])
+        for field in ['J-Mean', 'J-Recall', 'J-Decay', 'F-Mean', 'F-Recall', 'F-Decay']:
+            res[field] = np.mean(res[field])
+        res['J&F'] = (res['J-Mean'] + res['F-Mean']) / 2
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')}
+        for field in self.summary_fields:
+            res[field] = self._combine_weighted_av(all_res, field, res, weight_field='num_gt_tracks')
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
+        """Combines metrics across all classes by averaging over the class values
+        'ignore empty classes' is not yet implemented here.
+        """
+        res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')}
+        for field in self.float_fields:
+            res[field] = np.mean([v[field] for v in all_res.values()])
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')}
+        for field in self.float_fields:
+            res[field] = np.mean([v[field] for v in all_res.values()])
+        return res
+
+    @staticmethod
+    def _seg2bmap(seg, width=None, height=None):
+        """
+        From a segmentation, compute a binary boundary map with 1 pixel wide
+        boundaries.  The boundary pixels are offset by 1/2 pixel towards the
+        origin from the actual segment boundary.
+        Arguments:
+            seg     : Segments labeled from 1..k.
+            width	  :	Width of desired bmap  <= seg.shape[1]
+            height  :	Height of desired bmap <= seg.shape[0]
+        Returns:
+            bmap (ndarray):	Binary boundary map.
+         David Martin <dmartin@eecs.berkeley.edu>
+         January 2003
+        """
+
+        seg = seg.astype(np.bool)
+        seg[seg > 0] = 1
+
+        assert np.atleast_3d(seg).shape[2] == 1
+
+        width = seg.shape[1] if width is None else width
+        height = seg.shape[0] if height is None else height
+
+        h, w = seg.shape[:2]
+
+        ar1 = float(width) / float(height)
+        ar2 = float(w) / float(h)
+
+        assert not (
+                width > w | height > h | abs(ar1 - ar2) > 0.01
+        ), "Can" "t convert %dx%d seg to %dx%d bmap." % (w, h, width, height)
+
+        e = np.zeros_like(seg)
+        s = np.zeros_like(seg)
+        se = np.zeros_like(seg)
+
+        e[:, :-1] = seg[:, 1:]
+        s[:-1, :] = seg[1:, :]
+        se[:-1, :-1] = seg[1:, 1:]
+
+        b = seg ^ e | seg ^ s | seg ^ se
+        b[-1, :] = seg[-1, :] ^ e[-1, :]
+        b[:, -1] = seg[:, -1] ^ s[:, -1]
+        b[-1, -1] = 0
+
+        if w == width and h == height:
+            bmap = b
+        else:
+            bmap = np.zeros((height, width))
+            for x in range(w):
+                for y in range(h):
+                    if b[y, x]:
+                        j = 1 + math.floor((y - 1) + height / h)
+                        i = 1 + math.floor((x - 1) + width / h)
+                        bmap[j, i] = 1
+
+        return bmap
+
+    @staticmethod
+    def _compute_f(gt_data, tracker_data, tracker_data_id, gt_id, bound_th):
+        """
+        Perform F computation for a given gt and a given tracker ID. Adapted from
+        https://github.com/davisvideochallenge/davis2017-evaluation
+        :param gt_data: the encoded gt masks
+        :param tracker_data: the encoded tracker masks
+        :param tracker_data_id: the tracker ID
+        :param gt_id: the ground truth ID
+        :param bound_th: boundary threshold parameter
+        :return: the F value for the given tracker and gt ID
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+        from skimage.morphology import disk
+        import cv2
+
+        f = np.zeros(len(gt_data))
+
+        for t, (gt_masks, tracker_masks) in enumerate(zip(gt_data, tracker_data)):
+            curr_tracker_mask = mask_utils.decode(tracker_masks[tracker_data_id])
+            curr_gt_mask = mask_utils.decode(gt_masks[gt_id])
+            
+            bound_pix = bound_th if bound_th >= 1 - np.finfo('float').eps else \
+                np.ceil(bound_th * np.linalg.norm(curr_tracker_mask.shape))
+
+            # Get the pixel boundaries of both masks
+            fg_boundary = JAndF._seg2bmap(curr_tracker_mask)
+            gt_boundary = JAndF._seg2bmap(curr_gt_mask)
+
+            # fg_dil = binary_dilation(fg_boundary, disk(bound_pix))
+            fg_dil = cv2.dilate(fg_boundary.astype(np.uint8), disk(bound_pix).astype(np.uint8))
+            # gt_dil = binary_dilation(gt_boundary, disk(bound_pix))
+            gt_dil = cv2.dilate(gt_boundary.astype(np.uint8), disk(bound_pix).astype(np.uint8))
+
+            # Get the intersection
+            gt_match = gt_boundary * fg_dil
+            fg_match = fg_boundary * gt_dil
+
+            # Area of the intersection
+            n_fg = np.sum(fg_boundary)
+            n_gt = np.sum(gt_boundary)
+
+            # % Compute precision and recall
+            if n_fg == 0 and n_gt > 0:
+                precision = 1
+                recall = 0
+            elif n_fg > 0 and n_gt == 0:
+                precision = 0
+                recall = 1
+            elif n_fg == 0 and n_gt == 0:
+                precision = 1
+                recall = 1
+            else:
+                precision = np.sum(fg_match) / float(n_fg)
+                recall = np.sum(gt_match) / float(n_gt)
+
+            # Compute F measure
+            if precision + recall == 0:
+                f_val = 0
+            else:
+                f_val = 2 * precision * recall / (precision + recall)
+
+            f[t] = f_val
+
+        return f
+
+    @staticmethod
+    def _compute_j(gt_data, tracker_data, num_gt_ids, num_tracker_ids, num_timesteps):
+        """
+        Computation of J value for all ground truth IDs and all tracker IDs in the given sequence. Adapted from
+        https://github.com/davisvideochallenge/davis2017-evaluation
+        :param gt_data: the ground truth masks
+        :param tracker_data: the tracker masks
+        :param num_gt_ids: the number of ground truth IDs
+        :param num_tracker_ids: the number of tracker IDs
+        :param num_timesteps: the number of timesteps
+        :return: the J values
+        """
+
+        # Only loaded when run to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        j = np.zeros((num_tracker_ids, num_gt_ids, num_timesteps))
+
+        for t, (time_gt, time_data) in enumerate(zip(gt_data, tracker_data)):
+            # run length encoded masks with pycocotools
+            area_gt = mask_utils.area(time_gt)
+            time_data = list(time_data)
+            area_tr = mask_utils.area(time_data)
+
+            area_tr = np.repeat(area_tr[:, np.newaxis], len(area_gt), axis=1)
+            area_gt = np.repeat(area_gt[np.newaxis, :], len(area_tr), axis=0)
+
+            # mask iou computation with pycocotools
+            ious = np.atleast_2d(mask_utils.iou(time_data, time_gt, [0]*len(time_gt)))
+            # set iou to 1 if both masks are close to 0 (no ground truth and no predicted mask in timestep)
+            ious[np.isclose(area_tr, 0) & np.isclose(area_gt, 0)] = 1
+            assert (ious >= 0 - np.finfo('float').eps).all()
+            assert (ious <= 1 + np.finfo('float').eps).all()
+
+            j[..., t] = ious
+
+        return j

yolov7-tracker-example/tracker/trackeval/metrics/track_map.py

@@ -0,0 +1,462 @@
+import numpy as np
+from ._base_metric import _BaseMetric
+from .. import _timing
+from functools import partial
+from .. import utils
+from ..utils import TrackEvalException
+
+
+class TrackMAP(_BaseMetric):
+    """Class which implements the TrackMAP metrics"""
+
+    @staticmethod
+    def get_default_metric_config():
+        """Default class config values"""
+        default_config = {
+            'USE_AREA_RANGES': True,  # whether to evaluate for certain area ranges
+            'AREA_RANGES': [[0 ** 2, 32 ** 2],  # additional area range sets for which TrackMAP is evaluated
+                            [32 ** 2, 96 ** 2],  # (all area range always included), default values for TAO
+                            [96 ** 2, 1e5 ** 2]],  # evaluation
+            'AREA_RANGE_LABELS': ["area_s", "area_m", "area_l"],  # the labels for the area ranges
+            'USE_TIME_RANGES': True,  # whether to evaluate for certain time ranges (length of tracks)
+            'TIME_RANGES': [[0, 3], [3, 10], [10, 1e5]],  # additional time range sets for which TrackMAP is evaluated
+            # (all time range always included) , default values for TAO evaluation
+            'TIME_RANGE_LABELS': ["time_s", "time_m", "time_l"],  # the labels for the time ranges
+            'IOU_THRESHOLDS': np.arange(0.5, 0.96, 0.05),  # the IoU thresholds
+            'RECALL_THRESHOLDS': np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01) + 1), endpoint=True),
+            # recall thresholds at which precision is evaluated
+            'MAX_DETECTIONS': 0,  # limit the maximum number of considered tracks per sequence (0 for unlimited)
+            'PRINT_CONFIG': True
+        }
+        return default_config
+
+    def __init__(self, config=None):
+        super().__init__()
+        self.config = utils.init_config(config, self.get_default_metric_config(), self.get_name())
+
+        self.num_ig_masks = 1
+        self.lbls = ['all']
+        self.use_area_rngs = self.config['USE_AREA_RANGES']
+        if self.use_area_rngs:
+            self.area_rngs = self.config['AREA_RANGES']
+            self.area_rng_lbls = self.config['AREA_RANGE_LABELS']
+            self.num_ig_masks += len(self.area_rng_lbls)
+            self.lbls += self.area_rng_lbls
+
+        self.use_time_rngs = self.config['USE_TIME_RANGES']
+        if self.use_time_rngs:
+            self.time_rngs = self.config['TIME_RANGES']
+            self.time_rng_lbls = self.config['TIME_RANGE_LABELS']
+            self.num_ig_masks += len(self.time_rng_lbls)
+            self.lbls += self.time_rng_lbls
+
+        self.array_labels = self.config['IOU_THRESHOLDS']
+        self.rec_thrs = self.config['RECALL_THRESHOLDS']
+
+        self.maxDet = self.config['MAX_DETECTIONS']
+        self.float_array_fields = ['AP_' + lbl for lbl in self.lbls] + ['AR_' + lbl for lbl in self.lbls]
+        self.fields = self.float_array_fields
+        self.summary_fields = self.float_array_fields
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates GT and Tracker matches for one sequence for TrackMAP metrics. Adapted from
+        https://github.com/TAO-Dataset/"""
+
+        # Initialise results to zero for each sequence as the fields are only defined over the set of all sequences
+        res = {}
+        for field in self.fields:
+            res[field] = [0 for _ in self.array_labels]
+
+        gt_ids, dt_ids = data['gt_track_ids'], data['dt_track_ids']
+
+        if len(gt_ids) == 0 and len(dt_ids) == 0:
+            for idx in range(self.num_ig_masks):
+                res[idx] = None
+            return res
+
+        # get track data
+        gt_tr_areas = data.get('gt_track_areas', None) if self.use_area_rngs else None
+        gt_tr_lengths = data.get('gt_track_lengths', None) if self.use_time_rngs else None
+        gt_tr_iscrowd = data.get('gt_track_iscrowd', None)
+        dt_tr_areas = data.get('dt_track_areas', None) if self.use_area_rngs else None
+        dt_tr_lengths = data.get('dt_track_lengths', None) if self.use_time_rngs else None
+        is_nel = data.get('not_exhaustively_labeled', False)
+
+        # compute ignore masks for different track sets to eval
+        gt_ig_masks = self._compute_track_ig_masks(len(gt_ids), track_lengths=gt_tr_lengths, track_areas=gt_tr_areas,
+                                                   iscrowd=gt_tr_iscrowd)
+        dt_ig_masks = self._compute_track_ig_masks(len(dt_ids), track_lengths=dt_tr_lengths, track_areas=dt_tr_areas,
+                                                   is_not_exhaustively_labeled=is_nel, is_gt=False)
+
+        boxformat = data.get('boxformat', 'xywh')
+        ious = self._compute_track_ious(data['dt_tracks'], data['gt_tracks'], iou_function=data['iou_type'],
+                                        boxformat=boxformat)
+
+        for mask_idx in range(self.num_ig_masks):
+            gt_ig_mask = gt_ig_masks[mask_idx]
+
+            # Sort gt ignore last
+            gt_idx = np.argsort([g for g in gt_ig_mask], kind="mergesort")
+            gt_ids = [gt_ids[i] for i in gt_idx]
+
+            ious_sorted = ious[:, gt_idx] if len(ious) > 0 else ious
+
+            num_thrs = len(self.array_labels)
+            num_gt = len(gt_ids)
+            num_dt = len(dt_ids)
+
+            # Array to store the "id" of the matched dt/gt
+            gt_m = np.zeros((num_thrs, num_gt)) - 1
+            dt_m = np.zeros((num_thrs, num_dt)) - 1
+
+            gt_ig = np.array([gt_ig_mask[idx] for idx in gt_idx])
+            dt_ig = np.zeros((num_thrs, num_dt))
+
+            for iou_thr_idx, iou_thr in enumerate(self.array_labels):
+                if len(ious_sorted) == 0:
+                    break
+
+                for dt_idx, _dt in enumerate(dt_ids):
+                    iou = min([iou_thr, 1 - 1e-10])
+                    # information about best match so far (m=-1 -> unmatched)
+                    # store the gt_idx which matched for _dt
+                    m = -1
+                    for gt_idx, _ in enumerate(gt_ids):
+                        # if this gt already matched continue
+                        if gt_m[iou_thr_idx, gt_idx] > 0:
+                            continue
+                        # if _dt matched to reg gt, and on ignore gt, stop
+                        if m > -1 and gt_ig[m] == 0 and gt_ig[gt_idx] == 1:
+                            break
+                        # continue to next gt unless better match made
+                        if ious_sorted[dt_idx, gt_idx] < iou - np.finfo('float').eps:
+                            continue
+                        # if match successful and best so far, store appropriately
+                        iou = ious_sorted[dt_idx, gt_idx]
+                        m = gt_idx
+
+                    # No match found for _dt, go to next _dt
+                    if m == -1:
+                        continue
+
+                    # if gt to ignore for some reason update dt_ig.
+                    # Should not be used in evaluation.
+                    dt_ig[iou_thr_idx, dt_idx] = gt_ig[m]
+                    # _dt match found, update gt_m, and dt_m with "id"
+                    dt_m[iou_thr_idx, dt_idx] = gt_ids[m]
+                    gt_m[iou_thr_idx, m] = _dt
+
+            dt_ig_mask = dt_ig_masks[mask_idx]
+
+            dt_ig_mask = np.array(dt_ig_mask).reshape((1, num_dt))  # 1 X num_dt
+            dt_ig_mask = np.repeat(dt_ig_mask, num_thrs, 0)  # num_thrs X num_dt
+
+            # Based on dt_ig_mask ignore any unmatched detection by updating dt_ig
+            dt_ig = np.logical_or(dt_ig, np.logical_and(dt_m == -1, dt_ig_mask))
+            # store results for given video and category
+            res[mask_idx] = {
+                "dt_ids": dt_ids,
+                "gt_ids": gt_ids,
+                "dt_matches": dt_m,
+                "gt_matches": gt_m,
+                "dt_scores": data['dt_track_scores'],
+                "gt_ignore": gt_ig,
+                "dt_ignore": dt_ig,
+            }
+
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences. Computes precision and recall values based on track matches.
+        Adapted from https://github.com/TAO-Dataset/
+        """
+        num_thrs = len(self.array_labels)
+        num_recalls = len(self.rec_thrs)
+
+        # -1 for absent categories
+        precision = -np.ones(
+            (num_thrs, num_recalls, self.num_ig_masks)
+        )
+        recall = -np.ones((num_thrs, self.num_ig_masks))
+
+        for ig_idx in range(self.num_ig_masks):
+            ig_idx_results = [res[ig_idx] for res in all_res.values() if res[ig_idx] is not None]
+
+            # Remove elements which are None
+            if len(ig_idx_results) == 0:
+                continue
+
+            # Append all scores: shape (N,)
+            # limit considered tracks for each sequence if maxDet > 0
+            if self.maxDet == 0:
+                dt_scores = np.concatenate([res["dt_scores"] for res in ig_idx_results], axis=0)
+
+                dt_idx = np.argsort(-dt_scores, kind="mergesort")
+
+                dt_m = np.concatenate([e["dt_matches"] for e in ig_idx_results],
+                                      axis=1)[:, dt_idx]
+                dt_ig = np.concatenate([e["dt_ignore"] for e in ig_idx_results],
+                                       axis=1)[:, dt_idx]
+            elif self.maxDet > 0:
+                dt_scores = np.concatenate([res["dt_scores"][0:self.maxDet] for res in ig_idx_results], axis=0)
+
+                dt_idx = np.argsort(-dt_scores, kind="mergesort")
+
+                dt_m = np.concatenate([e["dt_matches"][:, 0:self.maxDet] for e in ig_idx_results],
+                                      axis=1)[:, dt_idx]
+                dt_ig = np.concatenate([e["dt_ignore"][:, 0:self.maxDet] for e in ig_idx_results],
+                                       axis=1)[:, dt_idx]
+            else:
+                raise Exception("Number of maximum detections must be >= 0, but is set to %i" % self.maxDet)
+
+            gt_ig = np.concatenate([res["gt_ignore"] for res in ig_idx_results])
+            # num gt anns to consider
+            num_gt = np.count_nonzero(gt_ig == 0)
+
+            if num_gt == 0:
+                continue
+
+            tps = np.logical_and(dt_m != -1, np.logical_not(dt_ig))
+            fps = np.logical_and(dt_m == -1, np.logical_not(dt_ig))
+
+            tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)
+            fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float)
+
+            for iou_thr_idx, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
+                tp = np.array(tp)
+                fp = np.array(fp)
+                num_tp = len(tp)
+                rc = tp / num_gt
+                if num_tp:
+                    recall[iou_thr_idx, ig_idx] = rc[-1]
+                else:
+                    recall[iou_thr_idx, ig_idx] = 0
+
+                # np.spacing(1) ~= eps
+                pr = tp / (fp + tp + np.spacing(1))
+                pr = pr.tolist()
+
+                # Ensure precision values are monotonically decreasing
+                for i in range(num_tp - 1, 0, -1):
+                    if pr[i] > pr[i - 1]:
+                        pr[i - 1] = pr[i]
+
+                # find indices at the predefined recall values
+                rec_thrs_insert_idx = np.searchsorted(rc, self.rec_thrs, side="left")
+
+                pr_at_recall = [0.0] * num_recalls
+
+                try:
+                    for _idx, pr_idx in enumerate(rec_thrs_insert_idx):
+                        pr_at_recall[_idx] = pr[pr_idx]
+                except IndexError:
+                    pass
+
+                precision[iou_thr_idx, :, ig_idx] = (np.array(pr_at_recall))
+
+        res = {'precision': precision, 'recall': recall}
+
+        # compute the precision and recall averages for the respective alpha thresholds and ignore masks
+        for lbl in self.lbls:
+            res['AP_' + lbl] = np.zeros((len(self.array_labels)), dtype=np.float)
+            res['AR_' + lbl] = np.zeros((len(self.array_labels)), dtype=np.float)
+
+        for a_id, alpha in enumerate(self.array_labels):
+            for lbl_idx, lbl in enumerate(self.lbls):
+                p = precision[a_id, :, lbl_idx]
+                if len(p[p > -1]) == 0:
+                    mean_p = -1
+                else:
+                    mean_p = np.mean(p[p > -1])
+                res['AP_' + lbl][a_id] = mean_p
+                res['AR_' + lbl][a_id] = recall[a_id, lbl_idx]
+
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=True):
+        """Combines metrics across all classes by averaging over the class values
+        Note mAP is not well defined for 'empty classes' so 'ignore empty classes' is always true here.
+        """
+        res = {}
+        for field in self.fields:
+            res[field] = np.zeros((len(self.array_labels)), dtype=np.float)
+            field_stacked = np.array([res[field] for res in all_res.values()])
+
+            for a_id, alpha in enumerate(self.array_labels):
+                values = field_stacked[:, a_id]
+                if len(values[values > -1]) == 0:
+                    mean = -1
+                else:
+                    mean = np.mean(values[values > -1])
+                res[field][a_id] = mean
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+
+        res = {}
+        for field in self.fields:
+            res[field] = np.zeros((len(self.array_labels)), dtype=np.float)
+            field_stacked = np.array([res[field] for res in all_res.values()])
+
+            for a_id, alpha in enumerate(self.array_labels):
+                values = field_stacked[:, a_id]
+                if len(values[values > -1]) == 0:
+                    mean = -1
+                else:
+                    mean = np.mean(values[values > -1])
+                res[field][a_id] = mean
+        return res
+
+    def _compute_track_ig_masks(self, num_ids, track_lengths=None, track_areas=None, iscrowd=None,
+                                is_not_exhaustively_labeled=False, is_gt=True):
+        """
+        Computes ignore masks for different track sets to evaluate
+        :param num_ids: the number of track IDs
+        :param track_lengths: the lengths of the tracks (number of timesteps)
+        :param track_areas: the average area of a track
+        :param iscrowd: whether a track is marked as crowd
+        :param is_not_exhaustively_labeled: whether the track category is not exhaustively labeled
+        :param is_gt: whether it is gt
+        :return: the track ignore masks
+        """
+        # for TAO tracks for classes which are not exhaustively labeled are not evaluated
+        if not is_gt and is_not_exhaustively_labeled:
+            track_ig_masks = [[1 for _ in range(num_ids)] for i in range(self.num_ig_masks)]
+        else:
+            # consider all tracks
+            track_ig_masks = [[0 for _ in range(num_ids)]]
+
+            # consider tracks with certain area
+            if self.use_area_rngs:
+                for rng in self.area_rngs:
+                    track_ig_masks.append([0 if rng[0] - np.finfo('float').eps <= area <= rng[1] + np.finfo('float').eps
+                                           else 1 for area in track_areas])
+
+            # consider tracks with certain duration
+            if self.use_time_rngs:
+                for rng in self.time_rngs:
+                    track_ig_masks.append([0 if rng[0] - np.finfo('float').eps <= length
+                                                <= rng[1] + np.finfo('float').eps else 1 for length in track_lengths])
+
+        # for YouTubeVIS evaluation tracks with crowd tag are not evaluated
+        if is_gt and iscrowd:
+            track_ig_masks = [np.logical_or(mask, iscrowd) for mask in track_ig_masks]
+
+        return track_ig_masks
+
+    @staticmethod
+    def _compute_bb_track_iou(dt_track, gt_track, boxformat='xywh'):
+        """
+        Calculates the track IoU for one detected track and one ground truth track for bounding boxes
+        :param dt_track: the detected track (format: dictionary with frame index as keys and
+                            numpy arrays as values)
+        :param gt_track: the ground truth track (format: dictionary with frame index as keys and
+                        numpy array as values)
+        :param boxformat: the format of the boxes
+        :return: the track IoU
+        """
+        intersect = 0
+        union = 0
+        image_ids = set(gt_track.keys()) | set(dt_track.keys())
+        for image in image_ids:
+            g = gt_track.get(image, None)
+            d = dt_track.get(image, None)
+            if boxformat == 'xywh':
+                if d is not None and g is not None:
+                    dx, dy, dw, dh = d
+                    gx, gy, gw, gh = g
+                    w = max(min(dx + dw, gx + gw) - max(dx, gx), 0)
+                    h = max(min(dy + dh, gy + gh) - max(dy, gy), 0)
+                    i = w * h
+                    u = dw * dh + gw * gh - i
+                    intersect += i
+                    union += u
+                elif d is None and g is not None:
+                    union += g[2] * g[3]
+                elif d is not None and g is None:
+                    union += d[2] * d[3]
+            elif boxformat == 'x0y0x1y1':
+                if d is not None and g is not None:
+                    dx0, dy0, dx1, dy1 = d
+                    gx0, gy0, gx1, gy1 = g
+                    w = max(min(dx1, gx1) - max(dx0, gx0), 0)
+                    h = max(min(dy1, gy1) - max(dy0, gy0), 0)
+                    i = w * h
+                    u = (dx1 - dx0) * (dy1 - dy0) + (gx1 - gx0) * (gy1 - gy0) - i
+                    intersect += i
+                    union += u
+                elif d is None and g is not None:
+                    union += (g[2] - g[0]) * (g[3] - g[1])
+                elif d is not None and g is None:
+                    union += (d[2] - d[0]) * (d[3] - d[1])
+            else:
+                raise TrackEvalException('BoxFormat not implemented')
+        if intersect > union:
+            raise TrackEvalException("Intersection value > union value. Are the box values corrupted?")
+        return intersect / union if union > 0 else 0
+
+    @staticmethod
+    def _compute_mask_track_iou(dt_track, gt_track):
+        """
+        Calculates the track IoU for one detected track and one ground truth track for segmentation masks
+        :param dt_track: the detected track (format: dictionary with frame index as keys and
+                            pycocotools rle encoded masks as values)
+        :param gt_track: the ground truth track (format: dictionary with frame index as keys and
+                            pycocotools rle encoded masks as values)
+        :return: the track IoU
+        """
+        # only loaded when needed to reduce minimum requirements
+        from pycocotools import mask as mask_utils
+
+        intersect = .0
+        union = .0
+        image_ids = set(gt_track.keys()) | set(dt_track.keys())
+        for image in image_ids:
+            g = gt_track.get(image, None)
+            d = dt_track.get(image, None)
+            if d and g:
+                intersect += mask_utils.area(mask_utils.merge([d, g], True))
+                union += mask_utils.area(mask_utils.merge([d, g], False))
+            elif not d and g:
+                union += mask_utils.area(g)
+            elif d and not g:
+                union += mask_utils.area(d)
+        if union < 0.0 - np.finfo('float').eps:
+            raise TrackEvalException("Union value < 0. Are the segmentaions corrupted?")
+        if intersect > union:
+            raise TrackEvalException("Intersection value > union value. Are the segmentations corrupted?")
+        iou = intersect / union if union > 0.0 + np.finfo('float').eps else 0.0
+        return iou
+
+    @staticmethod
+    def _compute_track_ious(dt, gt, iou_function='bbox', boxformat='xywh'):
+        """
+        Calculate track IoUs for a set of ground truth tracks and a set of detected tracks
+        """
+
+        if len(gt) == 0 and len(dt) == 0:
+            return []
+
+        if iou_function == 'bbox':
+            track_iou_function = partial(TrackMAP._compute_bb_track_iou, boxformat=boxformat)
+        elif iou_function == 'mask':
+            track_iou_function = partial(TrackMAP._compute_mask_track_iou)
+        else:
+            raise Exception('IoU function not implemented')
+
+        ious = np.zeros([len(dt), len(gt)])
+        for i, j in np.ndindex(ious.shape):
+            ious[i, j] = track_iou_function(dt[i], gt[j])
+        return ious
+
+    @staticmethod
+    def _row_print(*argv):
+        """Prints results in an evenly spaced rows, with more space in first row"""
+        if len(argv) == 1:
+            argv = argv[0]
+        to_print = '%-40s' % argv[0]
+        for v in argv[1:]:
+            to_print += '%-12s' % str(v)
+        print(to_print)

yolov7-tracker-example/tracker/trackeval/metrics/vace.py

@@ -0,0 +1,131 @@
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ._base_metric import _BaseMetric
+from .. import _timing
+
+
+class VACE(_BaseMetric):
+    """Class which implements the VACE metrics.
+
+    The metrics are described in:
+    Manohar et al. (2006) "Performance Evaluation of Object Detection and Tracking in Video"
+    https://link.springer.com/chapter/10.1007/11612704_16
+
+    This implementation uses the "relaxed" variant of the metrics,
+    where an overlap threshold is applied in each frame.
+    """
+
+    def __init__(self, config=None):
+        super().__init__()
+        self.integer_fields = ['VACE_IDs', 'VACE_GT_IDs', 'num_non_empty_timesteps']
+        self.float_fields = ['STDA', 'ATA', 'FDA', 'SFDA']
+        self.fields = self.integer_fields + self.float_fields
+        self.summary_fields = ['SFDA', 'ATA']
+
+        # Fields that are accumulated over multiple videos.
+        self._additive_fields = self.integer_fields + ['STDA', 'FDA']
+
+        self.threshold = 0.5
+
+    @_timing.time
+    def eval_sequence(self, data):
+        """Calculates VACE metrics for one sequence.
+
+        Depends on the fields:
+            data['num_gt_ids']
+            data['num_tracker_ids']
+            data['gt_ids']
+            data['tracker_ids']
+            data['similarity_scores']
+        """
+        res = {}
+
+        # Obtain Average Tracking Accuracy (ATA) using track correspondence.
+        # Obtain counts necessary to compute temporal IOU.
+        # Assume that integer counts can be represented exactly as floats.
+        potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids']))
+        gt_id_count = np.zeros(data['num_gt_ids'])
+        tracker_id_count = np.zeros(data['num_tracker_ids'])
+        both_present_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids']))
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            # Count the number of frames in which two tracks satisfy the overlap criterion.
+            matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold)
+            match_idx_gt, match_idx_tracker = np.nonzero(matches_mask)
+            potential_matches_count[gt_ids_t[match_idx_gt], tracker_ids_t[match_idx_tracker]] += 1
+            # Count the number of frames in which the tracks are present.
+            gt_id_count[gt_ids_t] += 1
+            tracker_id_count[tracker_ids_t] += 1
+            both_present_count[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] += 1
+        # Number of frames in which either track is present (union of the two sets of frames).
+        union_count = (gt_id_count[:, np.newaxis]
+                       + tracker_id_count[np.newaxis, :]
+                       - both_present_count)
+        # The denominator should always be non-zero if all tracks are non-empty.
+        with np.errstate(divide='raise', invalid='raise'):
+            temporal_iou = potential_matches_count / union_count
+        # Find assignment that maximizes temporal IOU.
+        match_rows, match_cols = linear_sum_assignment(-temporal_iou)
+        res['STDA'] = temporal_iou[match_rows, match_cols].sum()
+        res['VACE_IDs'] = data['num_tracker_ids']
+        res['VACE_GT_IDs'] = data['num_gt_ids']
+
+        # Obtain Frame Detection Accuracy (FDA) using per-frame correspondence.
+        non_empty_count = 0
+        fda = 0
+        for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])):
+            n_g = len(gt_ids_t)
+            n_d = len(tracker_ids_t)
+            if not (n_g or n_d):
+                continue
+            # n_g > 0 or n_d > 0
+            non_empty_count += 1
+            if not (n_g and n_d):
+                continue
+            # n_g > 0 and n_d > 0
+            spatial_overlap = data['similarity_scores'][t]
+            match_rows, match_cols = linear_sum_assignment(-spatial_overlap)
+            overlap_ratio = spatial_overlap[match_rows, match_cols].sum()
+            fda += overlap_ratio / (0.5 * (n_g + n_d))
+        res['FDA'] = fda
+        res['num_non_empty_timesteps'] = non_empty_count
+
+        res.update(self._compute_final_fields(res))
+        return res
+
+    def combine_classes_class_averaged(self, all_res, ignore_empty_classes=True):
+        """Combines metrics across all classes by averaging over the class values.
+        If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
+        """
+        res = {}
+        for field in self.fields:
+            if ignore_empty_classes:
+                res[field] = np.mean([v[field] for v in all_res.values()
+                                  if v['VACE_GT_IDs'] > 0 or v['VACE_IDs'] > 0], axis=0)
+            else:
+                res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
+        return res
+
+    def combine_classes_det_averaged(self, all_res):
+        """Combines metrics across all classes by averaging over the detection values"""
+        res = {}
+        for field in self._additive_fields:
+            res[field] = _BaseMetric._combine_sum(all_res, field)
+        res = self._compute_final_fields(res)
+        return res
+
+    def combine_sequences(self, all_res):
+        """Combines metrics across all sequences"""
+        res = {}
+        for header in self._additive_fields:
+            res[header] = _BaseMetric._combine_sum(all_res, header)
+        res.update(self._compute_final_fields(res))
+        return res
+
+    @staticmethod
+    def _compute_final_fields(additive):
+        final = {}
+        with np.errstate(invalid='ignore'):  # Permit nan results.
+            final['ATA'] = (additive['STDA'] /
+                            (0.5 * (additive['VACE_IDs'] + additive['VACE_GT_IDs'])))
+            final['SFDA'] = additive['FDA'] / additive['num_non_empty_timesteps']
+        return final

yolov7-tracker-example/tracker/trackeval/plotting.py

@@ -0,0 +1,230 @@
+
+import os
+import numpy as np
+from .utils import TrackEvalException
+
+
+def plot_compare_trackers(tracker_folder, tracker_list, cls, output_folder, plots_list=None):
+    """Create plots which compare metrics across different trackers."""
+    # Define what to plot
+    if plots_list is None:
+        plots_list = get_default_plots_list()
+
+    # Load data
+    data = load_multiple_tracker_summaries(tracker_folder, tracker_list, cls)
+    out_loc = os.path.join(output_folder, cls)
+
+    # Plot
+    for args in plots_list:
+        create_comparison_plot(data, out_loc, *args)
+
+
+def get_default_plots_list():
+    # y_label, x_label, sort_label, bg_label, bg_function
+    plots_list = [
+        ['AssA', 'DetA', 'HOTA', 'HOTA', 'geometric_mean'],
+        ['AssPr', 'AssRe', 'HOTA', 'AssA', 'jaccard'],
+        ['DetPr', 'DetRe', 'HOTA', 'DetA', 'jaccard'],
+        ['HOTA(0)', 'LocA(0)', 'HOTA', 'HOTALocA(0)', 'multiplication'],
+        ['HOTA', 'LocA', 'HOTA', None, None],
+
+        ['HOTA', 'MOTA', 'HOTA', None, None],
+        ['HOTA', 'IDF1', 'HOTA', None, None],
+        ['IDF1', 'MOTA', 'HOTA', None, None],
+    ]
+    return plots_list
+
+
+def load_multiple_tracker_summaries(tracker_folder, tracker_list, cls):
+    """Loads summary data for multiple trackers."""
+    data = {}
+    for tracker in tracker_list:
+        with open(os.path.join(tracker_folder, tracker, cls + '_summary.txt')) as f:
+            keys = next(f).split(' ')
+            done = False
+            while not done:
+                values = next(f).split(' ')
+                if len(values) == len(keys):
+                    done = True
+            data[tracker] = dict(zip(keys, map(float, values)))
+    return data
+
+
+def create_comparison_plot(data, out_loc, y_label, x_label, sort_label, bg_label=None, bg_function=None, settings=None):
+    """ Creates a scatter plot comparing multiple trackers between two metric fields, with one on the x-axis and the
+    other on the y axis. Adds pareto optical lines and (optionally) a background contour.
+
+    Inputs:
+        data: dict of dicts such that data[tracker_name][metric_field_name] = float
+        y_label: the metric_field_name to be plotted on the y-axis
+        x_label: the metric_field_name to be plotted on the x-axis
+        sort_label: the metric_field_name by which trackers are ordered and ranked
+        bg_label: the metric_field_name by which (optional) background contours are plotted
+        bg_function: the (optional) function bg_function(x,y) which converts the x_label / y_label values into bg_label.
+        settings: dict of plot settings with keys:
+            'gap_val': gap between axis ticks and bg curves.
+            'num_to_plot': maximum number of trackers to plot
+    """
+
+    # Only loaded when run to reduce minimum requirements
+    from matplotlib import pyplot as plt
+
+    # Get plot settings
+    if settings is None:
+        gap_val = 2
+        num_to_plot = 20
+    else:
+        gap_val = settings['gap_val']
+        num_to_plot = settings['num_to_plot']
+
+    if (bg_label is None) != (bg_function is None):
+        raise TrackEvalException('bg_function and bg_label must either be both given or neither given.')
+
+    # Extract data
+    tracker_names = np.array(list(data.keys()))
+    sort_index = np.array([data[t][sort_label] for t in tracker_names]).argsort()[::-1]
+    x_values = np.array([data[t][x_label] for t in tracker_names])[sort_index][:num_to_plot]
+    y_values = np.array([data[t][y_label] for t in tracker_names])[sort_index][:num_to_plot]
+
+    # Print info on what is being plotted
+    tracker_names = tracker_names[sort_index][:num_to_plot]
+    print('\nPlotting %s vs %s, for the following (ordered) trackers:' % (y_label, x_label))
+    for i, name in enumerate(tracker_names):
+        print('%i: %s' % (i+1, name))
+
+    # Find best fitting boundaries for data
+    boundaries = _get_boundaries(x_values, y_values, round_val=gap_val/2)
+
+    fig = plt.figure()
+
+    # Plot background contour
+    if bg_function is not None:
+        _plot_bg_contour(bg_function, boundaries, gap_val)
+
+    # Plot pareto optimal lines
+    _plot_pareto_optimal_lines(x_values, y_values)
+
+    # Plot data points with number labels
+    labels = np.arange(len(y_values)) + 1
+    plt.plot(x_values, y_values, 'b.', markersize=15)
+    for xx, yy, l in zip(x_values, y_values, labels):
+        plt.text(xx, yy, str(l), color="red", fontsize=15)
+
+    # Add extra explanatory text to plots
+    plt.text(0, -0.11, 'label order:\nHOTA', horizontalalignment='left', verticalalignment='center',
+             transform=fig.axes[0].transAxes, color="red", fontsize=12)
+    if bg_label is not None:
+        plt.text(1, -0.11, 'curve values:\n' + bg_label, horizontalalignment='right', verticalalignment='center',
+                 transform=fig.axes[0].transAxes, color="grey", fontsize=12)
+
+    plt.xlabel(x_label, fontsize=15)
+    plt.ylabel(y_label, fontsize=15)
+    title = y_label + ' vs ' + x_label
+    if bg_label is not None:
+        title += ' (' + bg_label + ')'
+    plt.title(title, fontsize=17)
+    plt.xticks(np.arange(0, 100, gap_val))
+    plt.yticks(np.arange(0, 100, gap_val))
+    min_x, max_x, min_y, max_y = boundaries
+    plt.xlim(min_x, max_x)
+    plt.ylim(min_y, max_y)
+    plt.gca().set_aspect('equal', adjustable='box')
+    plt.tight_layout()
+
+    os.makedirs(out_loc, exist_ok=True)
+    filename = os.path.join(out_loc, title.replace(' ', '_'))
+    plt.savefig(filename + '.pdf', bbox_inches='tight', pad_inches=0.05)
+    plt.savefig(filename + '.png', bbox_inches='tight', pad_inches=0.05)
+
+
+def _get_boundaries(x_values, y_values, round_val):
+    x1 = np.min(np.floor((x_values - 0.5) / round_val) * round_val)
+    x2 = np.max(np.ceil((x_values + 0.5) / round_val) * round_val)
+    y1 = np.min(np.floor((y_values - 0.5) / round_val) * round_val)
+    y2 = np.max(np.ceil((y_values + 0.5) / round_val) * round_val)
+    x_range = x2 - x1
+    y_range = y2 - y1
+    max_range = max(x_range, y_range)
+    x_center = (x1 + x2) / 2
+    y_center = (y1 + y2) / 2
+    min_x = max(x_center - max_range / 2, 0)
+    max_x = min(x_center + max_range / 2, 100)
+    min_y = max(y_center - max_range / 2, 0)
+    max_y = min(y_center + max_range / 2, 100)
+    return min_x, max_x, min_y, max_y
+
+
+def geometric_mean(x, y):
+    return np.sqrt(x * y)
+
+
+def jaccard(x, y):
+    x = x / 100
+    y = y / 100
+    return 100 * (x * y) / (x + y - x * y)
+
+
+def multiplication(x, y):
+    return x * y / 100
+
+
+bg_function_dict = {
+    "geometric_mean": geometric_mean,
+    "jaccard": jaccard,
+    "multiplication": multiplication,
+    }
+
+
+def _plot_bg_contour(bg_function, plot_boundaries, gap_val):
+    """ Plot background contour. """
+
+    # Only loaded when run to reduce minimum requirements
+    from matplotlib import pyplot as plt
+
+    # Plot background contour
+    min_x, max_x, min_y, max_y = plot_boundaries
+    x = np.arange(min_x, max_x, 0.1)
+    y = np.arange(min_y, max_y, 0.1)
+    x_grid, y_grid = np.meshgrid(x, y)
+    if bg_function in bg_function_dict.keys():
+        z_grid = bg_function_dict[bg_function](x_grid, y_grid)
+    else:
+        raise TrackEvalException("background plotting function '%s' is not defined." % bg_function)
+    levels = np.arange(0, 100, gap_val)
+    con = plt.contour(x_grid, y_grid, z_grid, levels, colors='grey')
+
+    def bg_format(val):
+        s = '{:1f}'.format(val)
+        return '{:.0f}'.format(val) if s[-1] == '0' else s
+
+    con.levels = [bg_format(val) for val in con.levels]
+    plt.clabel(con, con.levels, inline=True, fmt='%r', fontsize=8)
+
+
+def _plot_pareto_optimal_lines(x_values, y_values):
+    """ Plot pareto optimal lines """
+
+    # Only loaded when run to reduce minimum requirements
+    from matplotlib import pyplot as plt
+
+    # Plot pareto optimal lines
+    cxs = x_values
+    cys = y_values
+    best_y = np.argmax(cys)
+    x_pareto = [0, cxs[best_y]]
+    y_pareto = [cys[best_y], cys[best_y]]
+    t = 2
+    remaining = cxs > x_pareto[t - 1]
+    cys = cys[remaining]
+    cxs = cxs[remaining]
+    while len(cxs) > 0 and len(cys) > 0:
+        best_y = np.argmax(cys)
+        x_pareto += [x_pareto[t - 1], cxs[best_y]]
+        y_pareto += [cys[best_y], cys[best_y]]
+        t += 2
+        remaining = cxs > x_pareto[t - 1]
+        cys = cys[remaining]
+        cxs = cxs[remaining]
+    x_pareto.append(x_pareto[t - 1])
+    y_pareto.append(0)
+    plt.plot(np.array(x_pareto), np.array(y_pareto), '--r')

yolov7-tracker-example/tracker/trackeval/utils.py

@@ -0,0 +1,146 @@
+
+import os
+import csv
+import argparse
+from collections import OrderedDict
+
+
+def init_config(config, default_config, name=None):
+    """Initialise non-given config values with defaults"""
+    if config is None:
+        config = default_config
+    else:
+        for k in default_config.keys():
+            if k not in config.keys():
+                config[k] = default_config[k]
+    if name and config['PRINT_CONFIG']:
+        print('\n%s Config:' % name)
+        for c in config.keys():
+            print('%-20s : %-30s' % (c, config[c]))
+    return config
+
+
+def update_config(config):
+    """
+    Parse the arguments of a script and updates the config values for a given value if specified in the arguments.
+    :param config: the config to update
+    :return: the updated config
+    """
+    parser = argparse.ArgumentParser()
+    for setting in config.keys():
+        if type(config[setting]) == list or type(config[setting]) == type(None):
+            parser.add_argument("--" + setting, nargs='+')
+        else:
+            parser.add_argument("--" + setting)
+    args = parser.parse_args().__dict__
+    for setting in args.keys():
+        if args[setting] is not None:
+            if type(config[setting]) == type(True):
+                if args[setting] == 'True':
+                    x = True
+                elif args[setting] == 'False':
+                    x = False
+                else:
+                    raise Exception('Command line parameter ' + setting + 'must be True or False')
+            elif type(config[setting]) == type(1):
+                x = int(args[setting])
+            elif type(args[setting]) == type(None):
+                x = None
+            else:
+                x = args[setting]
+            config[setting] = x
+    return config
+
+
+def get_code_path():
+    """Get base path where code is"""
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+
+def validate_metrics_list(metrics_list):
+    """Get names of metric class and ensures they are unique, further checks that the fields within each metric class
+    do not have overlapping names.
+    """
+    metric_names = [metric.get_name() for metric in metrics_list]
+    # check metric names are unique
+    if len(metric_names) != len(set(metric_names)):
+        raise TrackEvalException('Code being run with multiple metrics of the same name')
+    fields = []
+    for m in metrics_list:
+        fields += m.fields
+    # check metric fields are unique
+    if len(fields) != len(set(fields)):
+        raise TrackEvalException('Code being run with multiple metrics with fields of the same name')
+    return metric_names
+
+
+def write_summary_results(summaries, cls, output_folder):
+    """Write summary results to file"""
+
+    fields = sum([list(s.keys()) for s in summaries], [])
+    values = sum([list(s.values()) for s in summaries], [])
+
+    # In order to remain consistent upon new fields being adding, for each of the following fields if they are present
+    # they will be output in the summary first in the order below. Any further fields will be output in the order each
+    # metric family is called, and within each family either in the order they were added to the dict (python >= 3.6) or
+    # randomly (python < 3.6).
+    default_order = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA', 'OWTA', 'HOTA(0)', 'LocA(0)',
+                     'HOTALocA(0)', 'MOTA', 'MOTP', 'MODA', 'CLR_Re', 'CLR_Pr', 'MTR', 'PTR', 'MLR', 'CLR_TP', 'CLR_FN',
+                     'CLR_FP', 'IDSW', 'MT', 'PT', 'ML', 'Frag', 'sMOTA', 'IDF1', 'IDR', 'IDP', 'IDTP', 'IDFN', 'IDFP',
+                     'Dets', 'GT_Dets', 'IDs', 'GT_IDs']
+    default_ordered_dict = OrderedDict(zip(default_order, [None for _ in default_order]))
+    for f, v in zip(fields, values):
+        default_ordered_dict[f] = v
+    for df in default_order:
+        if default_ordered_dict[df] is None:
+            del default_ordered_dict[df]
+    fields = list(default_ordered_dict.keys())
+    values = list(default_ordered_dict.values())
+
+    out_file = os.path.join(output_folder, cls + '_summary.txt')
+    os.makedirs(os.path.dirname(out_file), exist_ok=True)
+    with open(out_file, 'w', newline='') as f:
+        writer = csv.writer(f, delimiter=' ')
+        writer.writerow(fields)
+        writer.writerow(values)
+
+
+def write_detailed_results(details, cls, output_folder):
+    """Write detailed results to file"""
+    sequences = details[0].keys()
+    fields = ['seq'] + sum([list(s['COMBINED_SEQ'].keys()) for s in details], [])
+    out_file = os.path.join(output_folder, cls + '_detailed.csv')
+    os.makedirs(os.path.dirname(out_file), exist_ok=True)
+    with open(out_file, 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerow(fields)
+        for seq in sorted(sequences):
+            if seq == 'COMBINED_SEQ':
+                continue
+            writer.writerow([seq] + sum([list(s[seq].values()) for s in details], []))
+        writer.writerow(['COMBINED'] + sum([list(s['COMBINED_SEQ'].values()) for s in details], []))
+
+
+def load_detail(file):
+    """Loads detailed data for a tracker."""
+    data = {}
+    with open(file) as f:
+        for i, row_text in enumerate(f):
+            row = row_text.replace('\r', '').replace('\n', '').split(',')
+            if i == 0:
+                keys = row[1:]
+                continue
+            current_values = row[1:]
+            seq = row[0]
+            if seq == 'COMBINED':
+                seq = 'COMBINED_SEQ'
+            if (len(current_values) == len(keys)) and seq != '':
+                data[seq] = {}
+                for key, value in zip(keys, current_values):
+                    data[seq][key] = float(value)
+    return data
+
+
+class TrackEvalException(Exception):
+    """Custom exception for catching expected errors."""
+    ...