Repository: huggingface/transformers
File: src/transformers/models/kosmos2_5/modeling_kosmos2_5.py (Python)

Code before:
        # Overwritten -- in specific circumstances we don't want to forward image inputs to the model

        model_inputs = super().prepare_inputs_for_generation(
            input_ids,
            inputs_embeds=inputs_embeds,
            image_embeds=image_embeds,
            image_embeds_position_mask=image_embeds_position_mask,
            past_key_values=past_key_values,
            attention_mask=attention_mask,
            use_cache=use_cache,
            cache_position=cache_position,
            position_ids=position_ids,
            is_first_iteration=is_first_iteration,
            **model_kwargs,
        )

        # Pixel values are used only in the first iteration if available
        # In subsequent iterations, they are already cached
        if past_key_values is not None and past_key_values.get_seq_length() > 0:
            model_inputs["image_embeds"] = None
            model_inputs["image_embeds_position_mask"] = None

            # Kosmos2.5 starts position_ids at `pad_token_id`...
            if model_inputs.get("position_ids") is not None:
                model_inputs["position_ids"] = model_inputs["position_ids"] + 1 + self.config.pad_token_id

        # appending `False` to `image_embeds_position_mask` (because `input_ids` grows during generation)
        elif image_embeds_position_mask is not None:
            batch_size, seq_len = inputs_embeds.size()[:-1] if inputs_embeds is not None else input_ids.size()
            mask_len = image_embeds_position_mask.size()[-1]
            model_inputs["image_embeds_position_mask"] = torch.cat(
                (
                    image_embeds_position_mask,
                    torch.zeros(size=(batch_size, seq_len - mask_len), dtype=torch.bool, device=input_ids.device),
                ),
                dim=1,
            )
            # Kosmos2.5 has offset for position ids, so we need to create them correctly in PositionEmbedding layer
            model_inputs.pop("position_ids", None)

        return model_inputs


@add_start_docstrings(
    """
    KOSMOS-2.5 Model for generating text and bounding boxes given an image. The model consists of a vision encoder and a
    language model.
    """,
    KOSMOS2_5_START_DOCSTRING,
)
class Kosmos2_5ForConditionalGeneration(Kosmos2_5PreTrainedModel, GenerationMixin):

Review comment (suggestion):
oh, maybe then we can just override `self._prepare_position_ids` for this model to account for offset

Code after:
        # Overwritten -- in specific circumstances we don't want to forward image inputs to the model

        model_inputs = super().prepare_inputs_for_generation(
            input_ids,
            inputs_embeds=inputs_embeds,
            image_embeds=image_embeds,
            image_embeds_position_mask=image_embeds_position_mask,
            past_key_values=past_key_values,
            attention_mask=attention_mask,
            use_cache=use_cache,
            cache_position=cache_position,
            position_ids=position_ids,
            is_first_iteration=is_first_iteration,
            **model_kwargs,
        )

        # Pixel values are used only in the first iteration if available
        # In subsequent iterations, they are already cached
        if past_key_values is not None and past_key_values.get_seq_length() > 0:
            model_inputs["image_embeds"] = None
            model_inputs["image_embeds_position_mask"] = None

            # Kosmos2.5 starts position_ids at `pad_token_id`...
            if model_inputs.get("position_ids") is not None:
                # NOTE: we need this op out-of-place, otherwise it modifies the `model_kwargs` dict used in `generate` in-place!
                model_inputs["position_ids"] = model_inputs["position_ids"] + 1 + self.config.pad_token_id

        # appending `False` to `image_embeds_position_mask` (because `input_ids` grows during generation)
        elif image_embeds_position_mask is not None:
            batch_size, seq_len = inputs_embeds.size()[:-1] if inputs_embeds is not None else input_ids.size()
            mask_len = image_embeds_position_mask.size()[-1]
            model_inputs["image_embeds_position_mask"] = torch.cat(
                (
                    image_embeds_position_mask,
                    torch.zeros(size=(batch_size, seq_len - mask_len), dtype=torch.bool, device=input_ids.device),
                ),
                dim=1,
            )
            # Kosmos2.5 has offset for position ids, so we need to create them correctly in PositionEmbedding layer
            model_inputs.pop("position_ids", None)

        return model_inputs


@add_start_docstrings(
    """
    KOSMOS-2.5 Model for generating text and bounding boxes given an image. The model consists of a vision encoder and a
    language model.
    """,
    KOSMOS2_5_START_DOCSTRING,
)