Repository: huggingface/transformers
Pull request: Add support for Florence-2 training

# What does this PR do?

This PR adds support for Florence-2 training.
Without shifting tokens to the right, the model cannot compute the forward loss correctly because the decoder input IDs are not generated from the labels.
In the PR https://github.com/huggingface/transformers/pull/38188, I thought it was handled by the language model (bart) but it was not 

## Who can review?
@zucchini-nlp @Cyrilvallez @SunMarc 

Changed files:
filename: docs/source/en/model_doc/florence2.md
patch: @@ -44,7 +44,7 @@ from transformers import pipeline
 
 pipeline = pipeline(
     "image-text-to-text",
-    model="ducviet00/Florence-2-base-hf",
+    model="florence-community/Florence-2-base",
     device=0,
     dtype=torch.bfloat16
 )

filename: src/transformers/models/florence2/modeling_florence2.py
patch: @@ -33,6 +33,7 @@
     auto_docstring,
     can_return_tuple,
     is_torch_available,
+    logging,
 )
 from ..auto import AutoModel
 from .configuration_florence2 import Florence2Config, Florence2VisionConfig
@@ -44,6 +45,9 @@
     import torch.nn.functional as F
 
 
+logger = logging.get_logger(__name__)
+
+
 def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor:
     """
     Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
@@ -793,6 +797,22 @@ def get_encoder(self):
         return self.language_model.get_encoder()
 
 
+def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+
 @auto_docstring(
     custom_intro="""
     Florence-2 is a vision model for captioning, detection, and segmentation.
@@ -901,6 +921,15 @@ def forward(
         )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
+        if labels is not None:
+            if use_cache:
+                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
+            use_cache = False
+            if decoder_input_ids is None and decoder_inputs_embeds is None:
+                decoder_input_ids = shift_tokens_right(
+                    labels, self.config.text_config.pad_token_id, self.config.text_config.decoder_start_token_id
+                )
+
         outputs = self.model(
             input_ids=input_ids,
             pixel_values=pixel_values,

filename: src/transformers/models/florence2/modular_florence2.py
patch: @@ -36,7 +36,7 @@
     logging,
 )
 from ..auto import CONFIG_MAPPING, AutoConfig
-from ..bart.modeling_bart import eager_attention_forward
+from ..bart.modeling_bart import eager_attention_forward, shift_tokens_right
 from ..beit.modeling_beit import BeitDropPath
 from ..llama4.modeling_llama4 import Llama4VisionMLP
 from ..llava.modeling_llava import LlavaForConditionalGeneration, LlavaModel, LlavaPreTrainedModel
@@ -1710,6 +1710,15 @@ def forward(
         )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
+        if labels is not None:
+            if use_cache:
+                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
+            use_cache = False
+            if decoder_input_ids is None and decoder_inputs_embeds is None:
+                decoder_input_ids = shift_tokens_right(
+                    labels, self.config.text_config.pad_token_id, self.config.text_config.decoder_start_token_id
+                )
+
         outputs = self.model(
             input_ids=input_ids,
             pixel_values=pixel_values,

filename: tests/models/florence2/test_modeling_florence2.py
patch: @@ -58,11 +58,11 @@ def __init__(
         vocab_size=99,
         max_position_embeddings=64,
         encoder_layers=1,
-        encoder_ffn_dim=8,
+        encoder_ffn_dim=16,
         decoder_layers=1,
-        decoder_ffn_dim=8,
+        decoder_ffn_dim=16,
         num_attention_heads=1,
-        d_model=8,
+        d_model=16,
         activation_function="gelu",
         dropout=0.1,
         eos_token_id=2,
@@ -74,12 +74,12 @@ def __init__(
         patch_stride=[4],
         patch_padding=[3],
         patch_prenorm=[False],
-        embed_dim=[8],
+        embed_dim=[16],
         num_heads=[1],
         num_groups=[1],
         window_size=12,
         drop_path_rate=0.1,
-        projection_dim=8,
+        projection_dim=16,
     ):
         self.parent = parent
         self.batch_size = batch_size
@@ -215,6 +215,10 @@ def create_and_check_florence2_model_fp16_forward(self, config, input_ids, pixel
     def test_load_save_without_tied_weights(self):
         pass
 
+    @unittest.skip(reason="SDPA can't dispatch on flash due to unsupported qkv stride")
+    def test_sdpa_can_dispatch_on_flash(self):
+        pass
+
 
 @require_torch
 class Florence2ForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
@@ -271,7 +275,7 @@ def tearDown(self):
         cleanup(torch_device, gc_collect=True)
 
     def test_base_model_inference_eager(self):
-        model_name = "ducviet00/Florence-2-base-hf"
+        model_name = "florence-community/Florence-2-base"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="eager").to(
             torch_device
@@ -295,7 +299,7 @@ def test_base_model_inference_eager(self):
         self.assertEqual(generated_text, EXPECTED_GENERATED_TEXT)
 
     def test_base_model_batching_inference_eager(self):
-        model_name = "ducviet00/Florence-2-base-hf"
+        model_name = "florence-community/Florence-2-base"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="eager").to(
             torch_device
@@ -343,7 +347,7 @@ def test_base_model_batching_inference_eager(self):
         self.assertEqual(parsed_answer_1, EXPECTED_PARSED_ANSWER_1)
 
     def test_base_model_inference_sdpa(self):
-        model_name = "ducviet00/Florence-2-base-hf"
+        model_name = "florence-community/Florence-2-base"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="sdpa").to(
             torch_device
@@ -375,7 +379,7 @@ def test_base_model_inference_sdpa(self):
         self.assertEqual(parsed_answer, EXPECTED_PARSED_ANSWER)
 
     def test_base_model_batching_inference_sdpa(self):
-        model_name = "ducviet00/Florence-2-base-hf"
+        model_name = "florence-community/Florence-2-base"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="sdpa").to(
             torch_device
@@ -415,7 +419,7 @@ def test_base_model_batching_inference_sdpa(self):
         self.assertEqual(parsed_answer, EXPECTED_PARSED_ANSWER)
 
     def test_large_model_inference_eager(self):
-        model_name = "ducviet00/Florence-2-large-hf"
+        model_name = "florence-community/Florence-2-large"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="eager").to(
             torch_device
@@ -439,7 +443,7 @@ def test_large_model_inference_eager(self):
         self.assertEqual(generated_text, EXPECTED_GENERATED_TEXT)
 
     def test_large_model_batching_inference_eager(self):
-        model_name = "ducviet00/Florence-2-large-hf"
+        model_name = "florence-community/Florence-2-large"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="eager").to(
             torch_device
@@ -485,7 +489,7 @@ def test_large_model_batching_inference_eager(self):
         self.assertEqual(parsed_answer_1, EXPECTED_PARSED_ANSWER_1)
 
     def test_large_model_inference_sdpa(self):
-        model_name = "ducviet00/Florence-2-large-hf"
+        model_name = "florence-community/Florence-2-large"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="sdpa").to(
             torch_device
@@ -517,7 +521,7 @@ def test_large_model_inference_sdpa(self):
         self.assertEqual(parsed_answer, EXPECTED_PARSED_ANSWER)
 
     def test_large_model_batching_inference_sdpa(self):
-        model_name = "ducviet00/Florence-2-large-hf"
+        model_name = "florence-community/Florence-2-large"
         processor = AutoProcessor.from_pretrained(model_name)
         model = Florence2ForConditionalGeneration.from_pretrained(model_name, attn_implementation="sdpa").to(
             torch_device

filename: tests/models/florence2/test_processing_florence2.py
patch: @@ -38,9 +38,9 @@ class Florence2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
     def setUpClass(cls):
         cls.tmpdirname = tempfile.mkdtemp()
 
-        image_processor = CLIPImageProcessor.from_pretrained("ducviet00/Florence-2-base-hf")
+        image_processor = CLIPImageProcessor.from_pretrained("florence-community/Florence-2-base")
         image_processor.image_seq_length = 0
-        tokenizer = BartTokenizerFast.from_pretrained("ducviet00/Florence-2-base-hf")
+        tokenizer = BartTokenizerFast.from_pretrained("florence-community/Florence-2-base")
         tokenizer.image_token = "<image>"
         tokenizer.image_token_id = tokenizer.encode(tokenizer.image_token, add_special_tokens=False)[0]
         tokenizer.extra_special_tokens = {"image_token": "<image>"}