Repository: hiyouga/LLaMA-Factory
File: src/llamafactory/data/mm_plugin.py (Python)

Code before:
                audio_lengths = (input_lengths - 2) // 2 + 1
        else:
            mm_inputs = {}
            image_grid_thw = [None] * len(images)
            video_grid_thw = [None] * len(videos)
            audio_lengths = [None] * len(audios)

        if self.expand_mm_tokens and use_audio_in_video and audios:
            if "feature_attention_mask" not in mm_inputs:
                raise ValueError("audio_lengths should exist when use_audio_in_video is `True`.")

            if "video_grid_thw" not in mm_inputs and videos:
                raise ValueError("video_grid_thw should exist when use_audio_in_video is `True`.")

        for message in messages:
            content = message["content"]
            # separate with audio-video
            while IMAGE_PLACEHOLDER in content:
                if num_image_tokens >= len(images):
                    raise ValueError(f"`len(images)` is less than the number of {IMAGE_PLACEHOLDER} tokens.")

                image_seqlen = image_grid_thw[num_image_tokens].prod() // merge_length if self.expand_mm_tokens else 1
                # remove <*_bos> and <*_eos> tokens
                # ref to qwen2_5_omni/processing_qwen2_5_omni.py#L235-L238
                content = content.replace(
                    IMAGE_PLACEHOLDER, f"{self.image_token * image_seqlen}", 1
                )
                num_image_tokens += 1

            if not use_audio_in_video or (audios and not videos):  # for fake audio inputs when use_audio=True
                while AUDIO_PLACEHOLDER in content:
                    if num_audio_tokens >= len(audios):
                        raise ValueError(f"`len(audios)` is less than the number of {AUDIO_PLACEHOLDER} tokens.")

                    audio_seqlen = audio_lengths[num_audio_tokens] if self.expand_mm_tokens else 1
                    content = content.replace(
                        AUDIO_PLACEHOLDER, f"{self.audio_token * audio_seqlen}", 1
                    )
                    num_audio_tokens += 1

                # TODO handle video_input and use_audio_in_video
                while VIDEO_PLACEHOLDER in content:
                    if num_video_tokens >= len(videos):
                        raise ValueError(f"`len(videos)` is less than the number of {VIDEO_PLACEHOLDER} tokens.")

                    video_seqlen = (
                        video_grid_thw[num_video_tokens].prod() // merge_length if self.expand_mm_tokens else 1
                    )
                    content = content.replace(
                        VIDEO_PLACEHOLDER, f"{self.video_token * video_seqlen}", 1
                    )

Review comment (suggestion):
We should add `
` and 
` tokens according to https://huggingface.co/Qwen/Qwen2.5-Omni-7B/blob/main/chat_template.json#L2

Code after:
                audio_lengths = (input_lengths - 2) // 2 + 1
        else:
            mm_inputs = {}
            image_grid_thw = [None] * len(images)
            video_grid_thw = [None] * len(videos)
            audio_lengths = [None] * len(audios)

        for message in messages:
            content = message["content"]
            while IMAGE_PLACEHOLDER in content:
                if num_image_tokens >= len(images):
                    raise ValueError(f"`len(images)` is less than the number of {IMAGE_PLACEHOLDER} tokens.")

                image_seqlen = image_grid_thw[num_image_tokens].prod() // merge_length if self.expand_mm_tokens else 1
                content = content.replace(
                    IMAGE_PLACEHOLDER, f"
{self.image_token * image_seqlen}
", 1
                )
                num_image_tokens += 1

            if (
                use_audio_in_video and len(audios) and len(videos)
            ):  # if use the audio of video # deal video token and audio token togather
                if len(videos) != len(audios):
                    raise ValueError(
                        f"Number of videos ({len(videos)}) must match number of audios ({len(audios)}) when using audio in video."
                    )

                while VIDEO_PLACEHOLDER in content:
                    if num_video_tokens >= len(videos):
                        raise ValueError(f"`len(videos)` is less than the number of {VIDEO_PLACEHOLDER} tokens.")
                    if num_audio_tokens >= len(audios):
                        raise ValueError(f"`len(audios)` is less than the number of {AUDIO_PLACEHOLDER} tokens.")

                    video_pos = content.find(VIDEO_PLACEHOLDER)
                    audio_pos = content.find(AUDIO_PLACEHOLDER, video_pos)
                    if audio_pos == -1 or audio_pos < video_pos:
                        raise ValueError(
                            f"Each {VIDEO_PLACEHOLDER} must be followed by an {AUDIO_PLACEHOLDER} when using audio in video."
                        )

                    audio_t_index = torch.arange(audio_lengths[num_audio_tokens])
                    video_t_index = (
                        torch.arange(video_grid_thw[num_video_tokens][0])
                        .view(-1, 1, 1)
                        .expand(
                            -1,
                            video_grid_thw[num_video_tokens][1] // image_processor.merge_size,
                            video_grid_thw[num_video_tokens][2] // image_processor.merge_size,
                        )
                        .flatten()
                        * mm_inputs["video_second_per_grid"][num_video_tokens]