Unclear if detection tutorials cocoeval supports custom coco datasets

emcp: <p>im debugging my dataloader implementation… in order to use the coco_eval.py … but seeing an error around the image IDs… is it expected that you only test this tutorial with the baseline COCO datasets?</p>
<pre><code class="lang-auto">import os

import torch
from torchvision.transforms import functional as F
from torchvision.datasets import CocoDetection
# https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html


class CustomCocoDataset(CocoDetection):

    def __init__(self, data_conf, model_conf, testing_mode_on=False):

        if testing_mode_on is False:
            target_data_set = data_conf["image_data_training_id"]
            coco_filename = data_conf["coco_annotations_training_id"]
        else:
            # This turns off returning the data in a tensor, so COCOEval can work
            target_data_set = data_conf["image_data_testing_id"]
            coco_filename = data_conf["coco_annotations_testing_id"]

        self.coco_data_root = os.path.join(data_conf["image_pool_path"],
                                           target_data_set + "/" +
                                           data_conf["image_data_sub_dir"])

        self.coco_annotations_file = os.path.join(data_conf["image_pool_path"],
                                                  target_data_set,
                                                  "annotations",
                                                  coco_filename + ".json")

        super(CustomCocoDataset, self).__init__(root=self.coco_data_root,
                                                annFile=self.coco_annotations_file)

        self.categories = data_conf["classes_available"]

        print("found " + str(self.get_num_categories()) + " categories in data at: " + str(self.coco_data_root))

    def get_num_categories(self):
        return len(self.categories)

    def __getitem__(self, item):

        (pil_image, targets) = super(CustomCocoDataset, self).__getitem__(item)

        # get bounding box coordinates for each mask
        num_targets = len(targets)
        boxes = []
        for i in range(num_targets):
            box = targets[i]["bbox"]
            xmin = box[0]
            xmax = box[0] + box[2]
            ymin = box[1]
            ymax = box[1] + box[3]
            boxes.append([xmin, ymin, xmax, ymax])

        # convert everything into a torch.Tensor, unless you're performing testing (ie COCOEval)

        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        labels = torch.ones((num_targets,), dtype=torch.int64)
        image_id = torch.tensor([item])
        areas = []
        for i in range(num_targets):
            areas.append(targets[i]["area"])
        areas = torch.as_tensor(areas, dtype=torch.float32)
        iscrowd = torch.zeros((num_targets,), dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["image_id"] = image_id
        target["area"] = areas
        target["iscrowd"] = iscrowd

        image = F.to_tensor(pil_image)

        return image, target


if __name__ == "__main__":
    import json

    with open("../../dataset_template.json") as f:
        config_json = json.load(f)

    with open("../../config.json") as fp:
        model_conf = json.load(fp)

    loader = CustomCocoDataset(data_conf=config_json, model_conf=model_conf)
    print(loader.__len__())
    print(loader.__getitem__(4))

    print("Done")

</code></pre>
<p>when calling the .evaluate()</p>
<pre><code class="lang-auto">annsImgIds = [50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50]
seld.getImgIds()[2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368]
Traceback (most recent call last):
  File "/bootstrap-pytorch-torchvision-fasterrcnn/training.py", line 141, in &lt;module&gt;
    train(data_conf=config_json, model_conf=model_conf)
  File "/bootstrap-pytorch-torchvision-fasterrcnn/training.py", line 80, in train
    evaluate(model, training_data_loader, device=device)
  File "faster-rcnn-torchvision/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 15, in decorate_context
    return func(*args, **kwargs)
  File "/bootstrap-pytorch-torchvision-fasterrcnn/references/detection/engine.py", line 107, in evaluate
    coco_evaluator.update(res)
  File "/bootstrap-pytorch-torchvision-fasterrcnn/references/detection/coco_eval.py", line 36, in update
    coco_dt = loadRes(self.coco_gt, results) if results else COCO()
  File "/bootstrap-pytorch-torchvision-fasterrcnn/references/detection/coco_eval.py", line 256, in loadRes
    assert set(annsImgIds) == (set(annsImgIds) &amp; set(self.getImgIds())), \
AssertionError: Results do not correspond to current coco set

</code></pre>

emcp: <p>well, after a while I gave up and went back and rescued my prior models … bumped them up to pytorch 1.9.x and cuda 11.1+ … but I have to ask , is anyone else still working with faster-r-cnn and custom coco datasets or has the community moved onto something fresher and I am just out of the loop… I would think there should be plenty of tutorials by now, showing how to pour your custom COCO based data into pytorch example code… but again it’s these academic-only example datasets that I cannot really use beyond the tutorial…</p>
<p>am I just out of the loop and everyones working in Transformers now, or?</p>