Repository: huggingface/transformers
Pull request: Add DINOv3Backbone for ConvNext variant

I've taken a stab as it was a convolutional model it's very straightforward. The tests I borrowed from Dinov2, I hope it's ok @qubvel can you review if you have bandwidth? 

If it's good I will try with AIMv2, PE and some other image backbones as well! 

Changed files:
filename: docs/source/en/model_doc/dinov3.md
patch: @@ -178,3 +178,8 @@ print("Pooled output shape:", pooled_output.shape)
 
 [[autodoc]] DINOv3ViTImageProcessorFast
     - preprocess
+
+## DINOv3ConvNextBackbone
+
+[[autodoc]] DINOv3ConvNextBackbone
+    - forward
\ No newline at end of file

filename: src/transformers/models/auto/modeling_auto.py
patch: @@ -1699,6 +1699,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("dinat", "DinatBackbone"),
         ("dinov2", "Dinov2Backbone"),
         ("dinov2_with_registers", "Dinov2WithRegistersBackbone"),
+        ("dinov3_convnext", "DINOv3ConvNextBackbone"),
         ("focalnet", "FocalNetBackbone"),
         ("hgnet_v2", "HGNetV2Backbone"),
         ("hiera", "HieraBackbone"),

filename: src/transformers/models/dinov3_convnext/configuration_dinov3_convnext.py
patch: @@ -18,12 +18,13 @@
 
 from ...configuration_utils import PreTrainedConfig
 from ...utils import logging
+from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices
 
 
 logger = logging.get_logger(__name__)
 
 
-class DINOv3ConvNextConfig(PreTrainedConfig):
+class DINOv3ConvNextConfig(BackboneConfigMixin, PreTrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`DINOv3ConvNextModel`]. It is used to instantiate an
     DINOv3ConvNext model according to the specified arguments, defining the model architecture. Instantiating a configuration
@@ -53,6 +54,16 @@ class DINOv3ConvNextConfig(PreTrainedConfig):
             The drop rate for stochastic depth.
         image_size (`int`, *optional*, defaults to 224):
             The size (resolution) of input images.
+        out_features (`list[str]`, *optional*):
+            If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc.
+            (depending on how many stages the model has). If unset and `out_indices` is set, will default to the
+            corresponding stages. If unset and `out_indices` is unset, will default to the last stage. Must be in the
+            same order as defined in the `stage_names` attribute.
+        out_indices (`list[int]`, *optional*):
+            If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how
+            many stages the model has). If unset and `out_features` is set, will default to the corresponding stages.
+            If unset and `out_features` is unset, will default to the last stage. Must be in the
+            same order as defined in the `stage_names` attribute.
 
     Example:
     ```python
@@ -81,6 +92,8 @@ def __init__(
         layer_scale_init_value: float = 1e-6,
         drop_path_rate: float = 0.0,
         image_size: int = 224,
+        out_features: Optional[list[str]] = None,
+        out_indices: Optional[list[int]] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -94,6 +107,10 @@ def __init__(
         self.layer_scale_init_value = layer_scale_init_value
         self.drop_path_rate = drop_path_rate
         self.image_size = image_size
+        self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(self.depths) + 1)]
+        self._out_features, self._out_indices = get_aligned_output_features_output_indices(
+            out_features=out_features, out_indices=out_indices, stage_names=self.stage_names
+        )
 
     @property
     def num_stages(self) -> int:

filename: src/transformers/models/dinov3_convnext/modeling_dinov3_convnext.py
patch: @@ -21,11 +21,10 @@
 from torch import nn
 
 from ...activations import ACT2FN
-from ...modeling_outputs import (
-    BaseModelOutputWithPoolingAndNoAttention,
-)
+from ...modeling_outputs import BackboneOutput, BaseModelOutputWithPoolingAndNoAttention
 from ...modeling_utils import PreTrainedModel
 from ...utils import auto_docstring, logging
+from ...utils.backbone_utils import BackboneMixin
 from ...utils.generic import can_return_tuple
 from .configuration_dinov3_convnext import DINOv3ConvNextConfig
 
@@ -250,4 +249,51 @@ def forward(
         )
 
 
-__all__ = ["DINOv3ConvNextModel", "DINOv3ConvNextPreTrainedModel"]
+@auto_docstring
+class DINOv3ConvNextBackbone(DINOv3ConvNextPreTrainedModel, BackboneMixin):
+    config: DINOv3ConvNextConfig
+
+    def __init__(self, config: DINOv3ConvNextConfig):
+        super().__init__(config)
+        super()._init_backbone(config)
+
+        self.num_features = [config.num_channels] + list(config.hidden_sizes)
+
+        self.stages = nn.ModuleList([DINOv3ConvNextStage(config, s) for s in range(config.num_stages)])
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return None
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        output_hidden_states: Optional[bool] = None,
+        **kwargs,
+    ) -> BackboneOutput:
+        if output_hidden_states is None:
+            output_hidden_states = self.config.output_hidden_states
+
+        hidden_states = pixel_values
+        all_hidden_states: list[torch.Tensor] = [hidden_states]
+
+        for stage in self.stages:
+            hidden_states = stage(hidden_states)
+            all_hidden_states.append(hidden_states)
+
+        # hidden_states are already in NCHW (batch_size, channels, height, width) format
+        feature_maps: list[torch.Tensor] = []
+        for stage, hidden_states in zip(self.stage_names, all_hidden_states):
+            if stage in self.out_features:
+                feature_maps.append(hidden_states)
+
+        return BackboneOutput(
+            feature_maps=tuple(feature_maps),
+            hidden_states=tuple(all_hidden_states) if output_hidden_states else None,
+        )
+
+
+__all__ = ["DINOv3ConvNextModel", "DINOv3ConvNextPreTrainedModel", "DINOv3ConvNextBackbone"]

filename: tests/models/dinov3_convnext/test_modeling_dinov3_convnext.py
patch: @@ -20,6 +20,7 @@
 from transformers.testing_utils import require_torch, require_vision, slow, torch_device
 from transformers.utils import is_torch_available, is_vision_available
 
+from ...test_backbone_common import BackboneTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
 from ...test_pipeline_mixin import PipelineTesterMixin
@@ -28,7 +29,7 @@
 if is_torch_available():
     import torch
 
-    from transformers import DINOv3ConvNextModel
+    from transformers import DINOv3ConvNextBackbone, DINOv3ConvNextModel
 
 
 if is_vision_available():
@@ -104,6 +105,49 @@ def create_and_check_model(self, config, pixel_values, labels):
             ),
         )
 
+    def create_and_check_backbone(self, config, pixel_values, labels):
+        model = DINOv3ConvNextBackbone(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(pixel_values)
+
+        # verify hidden states
+        self.parent.assertEqual(len(result.feature_maps), len(config.out_features))
+        expected_size = self.image_size // (4 * (2 ** (len(config.depths) - 1)))
+        self.parent.assertListEqual(
+            list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], expected_size, expected_size]
+        )
+
+        # verify channels
+        self.parent.assertEqual(len(model.channels), len(config.out_features))
+
+        # verify backbone works with out_features=None
+        config.out_features = None
+        model = DINOv3ConvNextBackbone(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(pixel_values)
+
+        # verify feature maps
+        self.parent.assertEqual(len(result.feature_maps), 1)
+        self.parent.assertListEqual(
+            list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], expected_size, expected_size]
+        )
+
+        # verify channels
+        self.parent.assertEqual(len(model.channels), 1)
+
+        model = DINOv3ConvNextBackbone(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(pixel_values)
+
+        # verify feature maps
+        self.parent.assertEqual(len(result.feature_maps), 1)
+        self.parent.assertListEqual(
+            list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], expected_size, expected_size]
+        )
+
     def prepare_config_and_inputs_for_common(self):
         config_and_inputs = self.prepare_config_and_inputs()
         config, pixel_values, labels = config_and_inputs
@@ -156,6 +200,10 @@ def test_model(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*config_and_inputs)
 
+    def test_backbone(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_backbone(*config_and_inputs)
+
     def test_hidden_states_output(self):
         def check_hidden_states_output(inputs_dict, config, model_class):
             model = model_class(config)
@@ -240,3 +288,14 @@ def test_inference_no_head(self):
         last_layer_patch_tokens = outputs.last_hidden_state[:, 1:]
         expected_slice = torch.tensor([0.4905, -3.7135, 1.8485, -1.0403, -1.0908], device=torch_device)
         torch.testing.assert_close(last_layer_patch_tokens[0, 0, :5], expected_slice, rtol=1e-4, atol=1e-4)
+
+
+@require_torch
+class DINOv3ConvNextBackboneTest(unittest.TestCase, BackboneTesterMixin):
+    all_model_classes = (DINOv3ConvNextBackbone,) if is_torch_available() else ()
+    config_class = DINOv3ConvNextConfig
+
+    has_attentions = False
+
+    def setUp(self):
+        self.model_tester = DINOv3ConvNextModelTester(self)