Why is my rt-detr model giving map zero while train loss and valid loss is being decreased?

saint: <p>I am just a beginner trying to implement a transformer model for object detection on a custom dataset.</p>
<p>i’ve been using a hugging face rt-detr model to fine tune it and train it on my custom dataset i visualised the dataset,checked the data being sent into the model and all other possibilities and none have any issue all of the data loading part is correct but during training the model start with a high loss value like in ranges of 10-100 and the valid loss is low compared to train loss like in ranges of 10-50 however when i calculate the map setting iou threshold as 0.3 or above the map returned is zero for like 30 epochs why is this happening<br>
i followed this tutorial to implement this model : <a href="https://youtu.be/sBARif4DnL4?si=bYdAugeLz2eIS2Dn" rel="noopener nofollow ugc">https://youtu.be/sBARif4DnL4?si=bYdAugeLz2eIS2Dn</a></p>
<p>since the map is very low when i go to do predictions on my test data it gives out nothing for confidence scores above 0.1 also the dataset i use is of images with small objects and images in night/darker environment so is the model struggling with this kind of datasets or is my code wrong ?.</p>
<p>!pip install -q git+https://github.com/huggingface/transformers.git<br>
!pip install -q git+https://github.com/roboflow/supervision.git<br>
!pip install -q accelerate<br>
!pip install -q roboflow<br>
!pip install -q torchmetrics<br>
!pip install -q “albumentations&gt;=1.4.5”</p>
<p>import torch<br>
import requests</p>
<p>import numpy as np<br>
import supervision as sv<br>
import albumentations as A</p>
<p>from PIL import Image<br>
from pprint import pprint<br>
from roboflow import Roboflow<br>
from dataclasses import dataclass, replace<br>
from google.colab import userdata<br>
from torch.utils.data import Dataset<br>
from transformers import (<br>
AutoImageProcessor,<br>
AutoModelForObjectDetection,<br>
TrainingArguments,<br>
Trainer<br>
)</p>
<p>from torchvision import transforms</p>
<p>!pip install -q torchinfo</p>
<p>from torchinfo import summary</p>
<p><strong>Loading the Model</strong></p>
<p>CHECKPOINT = “PekingU/rtdetr_v2_r101vd”<br>
DEVICE = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)</p>
<p>NUM_LABELS = 2<br>
model = AutoModelForObjectDetection.from_pretrained(CHECKPOINT).to(DEVICE)<br>
processor = AutoImageProcessor.from_pretrained(CHECKPOINT)<br>
model</p>
<p><span class="hashtag-raw">#loading</span> images and json file via detectiondataset</p>
<p>ds_train = sv.DetectionDataset.from_coco(<br>
images_directory_path=“dataset link”,<br>
annotations_path=“annotations.coco.json”,<br>
)</p>
<p>ds_val = sv.DetectionDataset.from_coco(<br>
images_directory_path=“dataset”,<br>
annotations_path=“_annotations.coco.json”,<br>
)</p>
<p>ds_test = sv.DetectionDataset.from_coco(<br>
images_directory_path = “dataset”,<br>
annotations_path = “_annotations.coco.json”<br>
)</p>
<p>print(len(ds_train))<br>
len(ds_val),len(ds_test)</p>
<p><span class="hashtag-raw">#visualizing</span> loaded dataset</p>
<p>GRID_SIZE = 3</p>
<p>def annotate(image, annotations, classes):<br>
labels = [<br>
classes[class_id]<br>
for class_id<br>
in annotations.class_id<br>
]</p>
<pre><code class="lang-auto">bounding_box_annotator = sv.BoxAnnotator()
label_annotator = sv.LabelAnnotator(text_scale=1, text_thickness=2)

annotated_image = image.copy()
annotated_image = bounding_box_annotator.annotate(annotated_image, annotations)
annotated_image = label_annotator.annotate(annotated_image, annotations, labels=labels)
return annotated_image
</code></pre>
<p>annotated_images = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for i in range(GRID_SIZE * GRID_SIZE):<br>
_, image, annotations = ds_train[i]<br>
annotated_image = annotate(image, annotations, ds_train.classes)<br>
print(annotations)<br>
annotated_images.append(annotated_image)</p>
<p>sv.plot_images_grid(<br>
annotated_images,<br>
grid_size=(GRID_SIZE, GRID_SIZE),<br>
size=(20, 20)<br>
)</p>
<p><span class="hashtag-raw">#initialising</span> the imageprocessor to convert dataset to the one required by the model</p>
<p>IMAGE_SIZE = 640</p>
<p>processor = AutoImageProcessor.from_pretrained(<br>
“PekingU/rtdetr_v2_r101vd”,<br>
do_convert_annotations=False,<br>
do_resize=True,<br>
do_rescale=False,<br>
do_normalize=True,<br>
size={“width”: IMAGE_SIZE, “height”: IMAGE_SIZE}<br>
)</p>
<h1><a name="p-476329-h-1" class="anchor" href="#p-476329-h-1"></a></h1>
<p><span class="hashtag-raw">#since</span> detections dataset converts coco to pascal_voc transforms are performed on format pascal_voc before the processor converts it back to coco</p>
<p>train_augmentation_and_transform = A.Compose(<br>
[A.NoOp()],<br>
bbox_params = A.BboxParams(<br>
format=“pascal_voc”,<br>
label_fields=[“category”],<br>
clip=True</p>
<pre><code class="lang-auto">),
</code></pre>
<p>)</p>
<p>valid_transform = A.Compose(<br>
[A.NoOp()],<br>
bbox_params = A.BboxParams(<br>
format=“pascal_voc”,<br>
label_fields=[“category”],<br>
clip=True<br>
),<br>
)</p>
<p><span class="hashtag-raw">#converting</span> the loaded dataset to a dataset object to feed it to model</p>
<p>import torch<br>
from torch.utils.data import Dataset<br>
import albumentations as A<br>
import numpy as np</p>
<p>class PyTorchDetectionDataset(Dataset):<br>
def <strong>init</strong>(self, dataset, processor, transform=None):<br>
self.dataset = dataset<br>
self.processor = processor<br>
self.transform = transform</p>
<pre><code class="lang-auto">@staticmethod
def annotations_as_coco(image_id, categories, boxes, img_shape):
    """
    Converts boxes to COCO format ([x, y, w, h] in pixels)
    img_shape = (H, W, C)
    """
    h, w = img_shape[:2]
    annotations = []
    box=[]
    box.append(boxes)
    for category, bbox in zip(categories, box):
        x_min,y_min,x_max,y_max=bbox
        cx = (x_min + x_max)/2
        cy = (y_min + y_max)/2
        w = x_max - x_min
        h = y_max - y_min
        bbox =[cx,cy,w,h]
        # print("manual",bbox)
        cx_norm = cx / 640.0
        cy_norm = cy / 640.0
        w_norm = w / 640.0

        h_norm = h / 640.0

        bbox = [cx_norm, cy_norm, w_norm, h_norm]
        # print("normalised",bbox)
        # bbox = np.array(bbox)
        bbox = torch.tensor(bbox, dtype=torch.float32)

        annotations.append({
            "image_id": image_id,
            "category_id": int(category),
            "bbox": bbox,
            "area": bbox[2] * bbox[3],
            "iscrowd": 0
        })
        # print("anno",annotations)
    return {"image_id": image_id, "annotations": annotations}



def __len__(self):
    return len(self.dataset)

def __getitem__(self, idx):
    _, image, annotations = self.dataset[idx]
    boxes = annotations.xyxy
    categories = annotations.class_id
    if len(boxes) == 0:
      print(f"[Warning] No boxes for image {idx} — skipping")
      print("Categories:", categories)
      print("Annotations:", annotations)

    # Apply Albumentations transform
    if self.transform:
        # Albumentations expects [x_min, y_min, x_max, y_max] for 'pascal_voc'
        transformed = self.transform(
            image=image,
            bboxes=boxes,
            category=categories
        )
        image = transformed["image"]
        boxes = transformed["bboxes"]
        # print(boxes)
        categories = transformed["category"]

    # Convert to COCO-style dict for processor
    formatted_annotations = self.annotations_as_coco(
        image_id=idx, categories=categories, boxes=boxes[0], img_shape=image.shape
    )
    # print("formatted",formatted_annotations["annotations"][0]["bbox"])
    # Feed processor
    result = self.processor(images=image, annotations=formatted_annotations, return_tensors="pt")
    # Remove batch dim
    result = {k: v[0] for k, v in result.items()}
    # print("results",result)
    # print("boxes old",result["labels"]["boxes"])
    boxe=formatted_annotations["annotations"][0]["bbox"]
    boxe=boxe.unsqueeze(0)
    result["labels"]["boxes"]=boxe
    # print("new results",result)
    # print("boxes new",result["labels"]["boxes"])
    return result
</code></pre>
<p>from torchvision.utils import draw_bounding_boxes<br>
import torch<br>
from torch import Tensor<br>
from typing import Optional<br>
from torchvision.ops import box_convert</p>
<p><span class="hashtag-raw">#creating</span> the pytorch dataset object and visualising it without de-normalising the image</p>
<p>import matplotlib.pyplot as plt</p>
<p>pytorch_dataset_train = PyTorchDetectionDataset(<br>
ds_train, processor, transform=train_augmentation_and_transform)<br>
pytorch_dataset_valid = PyTorchDetectionDataset(<br>
ds_val, processor, transform=train_augmentation_and_transform)<br>
pytorch_dataset_test = PyTorchDetectionDataset(<br>
ds_test, processor, transform=train_augmentation_and_transform)</p>
<p>print(pytorch_dataset_train[2])<br>
sample = pytorch_dataset_train[2]<br>
img = sample[“pixel_values”].squeeze(0)<br>
boxes = sample[“labels”][“boxes”]<br>
labels = sample[“labels”][“class_labels”]</p>
<p>_, H, W = img.shape  # get image size<br>
print(img.shape)<br>
boxes[:, [0, 2]] *= W  # scale x, width<br>
boxes[:, [1, 3]] *= H  # scale y, height<br>
print(H,W)</p>
<p>print(boxes)<br>
boxes = box_convert(boxes, in_fmt=“cxcywh”, out_fmt=“xyxy”)</p>
<p>id2label = {0: “debris”, 1: “satellite”}<br>
label_names = [id2label[int(l)] for l in labels]<br>
print(“bi”,boxes)<br>
mean = torch.tensor([0.485, 0.456, 0.406]).view(3,1,1)<br>
std = torch.tensor([0.229, 0.224, 0.225]).view(3,1,1)<br>
img = img * std + mean<br>
img = img.clamp(0,1)</p>
<p>img_with_boxes = draw_bounding_boxes(<br>
image=img,<br>
boxes=boxes,<br>
labels=label_names,<br>
colors=“red”,<br>
width=2,<br>
font_size=16<br>
)</p>
<p>plt.figure(figsize=(8, 8))<br>
plt.imshow(img_with_boxes.permute(1, 2, 0))<br>
plt.axis(“off”)<br>
plt.show()</p>
<p>from torchmetrics.detection.mean_ap import MeanAveragePrecision<br>
from torchmetrics.classification import MulticlassPrecision,MulticlassRecall</p>
<p>def collate_fn(batch):<br>
data = {}<br>
data[“pixel_values”] = torch.stack([x[“pixel_values”] for x in batch])<br>
data[“labels”] = [x[“labels”] for x in batch]<br>
return data</p>
<p>import numpy as np<br>
from sklearn.metrics import precision_score<br>
from torch import tensor</p>
<p><span class="hashtag-raw">#map</span> evaluation</p>
<p>id2label = {id: label for id, label in enumerate(ds_train.classes)}<br>
label2id = {label: id for id, label in enumerate(ds_train.classes)}</p>
<p><span class="mention">@dataclass</span><br>
class ModelOutput:<br>
logits: torch.Tensor<br>
pred_boxes: torch.Tensor</p>
<p>class MAPEvaluator:</p>
<pre><code class="lang-auto">def __init__(self, image_processor, threshold=0.00, id2label=None):
    self.image_processor = image_processor
    self.threshold = threshold
    self.id2label = id2label

def collect_image_sizes(self, targets):
    """Collect image sizes across the dataset as list of tensors with shape [batch_size, 2]."""
    image_sizes = []
    for batch in targets:
        batch_image_sizes = torch.tensor(np.array([x["size"] for x in batch]))
        image_sizes.append(batch_image_sizes)
    return image_sizes

def collect_targets(self, targets, image_sizes):
    post_processed_targets = []
    for target_batch, image_size_batch in zip(targets, image_sizes):
        for target, (height, width) in zip(target_batch, image_size_batch):
            boxes = target["boxes"]
            boxes = sv.xcycwh_to_xyxy(boxes)
            boxes = boxes * np.array([width, height, width, height])
            boxes = torch.tensor(boxes)
            labels = torch.tensor(target["class_labels"])
            post_processed_targets.append({"boxes": boxes, "labels": labels})
    return post_processed_targets

def collect_predictions(self, predictions, image_sizes):
    post_processed_predictions = []
    for batch, target_sizes in zip(predictions, image_sizes):
        batch_logits, batch_boxes = batch[1], batch[2]
        output = ModelOutput(logits=torch.tensor(batch_logits), pred_boxes=torch.tensor(batch_boxes))
        post_processed_output = self.image_processor.post_process_object_detection(
            output, threshold=self.threshold, target_sizes=target_sizes
        )
        post_processed_predictions.extend(post_processed_output)
    return post_processed_predictions

@torch.no_grad()
def __call__(self, evaluation_results):

    predictions, targets = evaluation_results.predictions, evaluation_results.label_ids

    image_sizes = self.collect_image_sizes(targets)
    post_processed_targets = self.collect_targets(targets, image_sizes)
    post_processed_predictions = self.collect_predictions(predictions, image_sizes)
    # print("preds",post_processed_predictions)
    # print("targets",post_processed_targets)
    evaluator = MeanAveragePrecision(iou_type="bbox",box_format="xyxy", class_metrics=True)
    evaluator.warn_on_many_detections = False
    evaluator.update(post_processed_predictions, post_processed_targets)

    metrics = evaluator.compute()

    # Replace list of per class metrics with separate metric for each class
    classes = metrics.pop("classes")
    map_per_class = metrics.pop("map_per_class")
    mar_100_per_class = metrics.pop("mar_100_per_class")
    for class_id, class_map, class_mar in zip(classes, map_per_class, mar_100_per_class):
        class_name = id2label[class_id.item()] if id2label is not None else class_id.item()
        metrics[f"map_{class_name}"] = class_map
        metrics[f"mar_100_{class_name}"] = class_mar

    metrics = {k: round(v.item(), 4) for k, v in metrics.items()}
    # metrico = MulticlassPrecision(num_classes=2,multidim_average='samplewise')
    # print("precision at epoch 1: ",metrico(tensor(post_processed_predictions), tensor(post_processed_targets)))

    return metrics
</code></pre>
<p>eval_compute_metrics_fn = MAPEvaluator(image_processor=processor, threshold=0.3, id2label=id2label)</p>
<p><span class="hashtag-raw">#loading</span> the model but redifining classes and out_features</p>
<p>model = AutoModelForObjectDetection.from_pretrained(<br>
“PekingU/rtdetr_v2_r101vd”,<br>
id2label=id2label,<br>
label2id=label2id,<br>
anchor_image_size=None,<br>
ignore_mismatched_sizes=True,<br>
)<br>
summary(model=model,input=[1,3,64,64],col_names=[“trainable”])<br>
print(model.config)</p>
<p><span class="hashtag-raw">#freezing</span> the backbone of the model</p>
<p>for p in model.model.backbone.parameters():<br>
p.requires_grad=False<br>
summary(model=model,input=[1,3,64,64],col_names=[“trainable”])</p>
<p>model.config.num_labels</p>
<p>backbone_params = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
transformer_params = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>for name, param in model.named_parameters():<br>
if “backbone” in name:<br>
backbone_params.append(param)<br>
else:<br>
transformer_params.append(param)</p>
<p><span class="hashtag-raw">#setting</span> lr and optimiser with regularisation</p>
<p>optimizer = torch.optim.AdamW([<br>
{‘params’: backbone_params, ‘lr’: 1e-5}, # Lower learning rate for the backbone<br>
{‘params’: transformer_params, ‘lr’: 5e-5} # Higher learning rate for the transformer<br>
])</p>
<p><span class="hashtag-raw">#setting</span> train args to pass to trainer api</p>
<p>training_args = TrainingArguments(<br>
output_dir=f"{dataset.name.replace(’ ', ‘-’)}-finetune",<br>
num_train_epochs=10,<br>
max_grad_norm=0.1,<br>
per_device_train_batch_size=2,<br>
gradient_accumulation_steps=4,<br>
dataloader_num_workers=2,<br>
metric_for_best_model=“eval_map”,<br>
greater_is_better=True,<br>
load_best_model_at_end=True,<br>
eval_strategy=“epoch”,<br>
save_strategy=“epoch”,<br>
save_total_limit=2,<br>
remove_unused_columns=False,<br>
eval_do_concat_batches=False,<br>
)</p>
<p>from torch import seed<br>
torch.manual_seed(42)</p>
<p>trainer = Trainer(<br>
model=model,<br>
args=training_args,<br>
train_dataset=pytorch_dataset_train,<br>
eval_dataset=pytorch_dataset_valid,<br>
processing_class=processor,<br>
data_collator=collate_fn,<br>
optimizers=(optimizer, None),<br>
compute_metrics=eval_compute_metrics_fn<br>
)</p>
<p>trainer.train()</p>
<p>Please check this code and let me know what are the mistakes i made .</p>