Repository: huggingface/transformers
Pull request: Convenient default behavior for pipeline TTS usage.

# What does this PR do?

Related to offline discussion with @eustlb and @Deep-unlearning, let's change default pipeline TTS behavior to make it easier to users.

I pinned `output_audio=True` for CSM but also did manual insertion of speaker IDs (for CSM and Dia) to make usage more intuitive for simple TTS usage.

See below some CSM and Dia examples.
```python
import soundfile as sf
import torch
from datasets import Audio, load_dataset

from transformers import pipeline


device = "cuda" if torch.cuda.is_available() else "cpu"


"""CSM"""
pipe = pipeline("text-to-audio", model="sesame/csm-1b", device=device)

# -- minimal TTS example
torch.manual_seed(0)
outputs = pipe("Hello from Sesame.")     # instead of pipe("[0]Hello from Sesame.")
fn = "csm_pipeline_tts.wav"
sf.write(fn, outputs["audio"], outputs["sampling_rate"])
print(f"Audio saved to {fn}")

# -- minimal TTS example with voice cloning
torch.manual_seed(0)
ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train")
ds = ds.cast_column("audio", Audio(sampling_rate=24000))
conversation = [
    # audio/text pair(s) for voice cloning
    {
        "role": "0",
        "content": [
            {"type": "text", "text": "What are you working on?"},
            {"type": "audio", "path": ds[0]["audio"]["array"]},
        ],
    },
    # desired audio response for voice cloning
    {"role": "0", "content": [{"type": "text", "text": "How much money can you spend?"}]},
]
outputs = pipe(conversation)
fn = "csm_pipeline_voice_cloning.wav"
sf.write(fn, outputs["audio"], outputs["sampling_rate"])
print(f"Audio saved to {fn}")


"""Dia"""
pipe = pipeline("text-to-audio", model="nari-labs/Dia-1.6B-0626", device=device)

# -- minimal TTS example
torch.manual_seed(42)
outputs = pipe(
    "Dia is an open weights text to dialogue model.",      # instead of pipe("[S1] Dia is an open weights text to dialogue model..")
    generate_kwargs={"max_new_tokens": 256},
)
fn = "dia_pipeline_tts.wav"
sf.write(fn, outputs["audio"], outputs["sampling_rate"])
print(f"Audio saved to {fn}")

# -- minimal conversation example
# note: Dia doesn't support chat template for voice cloning
# explicit model loading should be used instead: https://huggingface.co/nari-labs/Dia-1.6B-0626#generation-with-text-and-audio-voice-cloning
torch.manual_seed(0)
outputs = pipe(
    "[S1] Dia is an open weights text to dialogue model. [S2] That's cool, tell me how it works.",
)
fn = "dia_pipeline_conversation.wav"
sf.write(fn, outputs["audio"], outputs["sampling_rate"])
print(f"Audio saved to {fn}")
```

@Deep-unlearning what do you think about adding such examples to the [TTS page](https://huggingface.co/docs/transformers/en/tasks/text-to-speech) (while pruning the verbose comments). 

At least the CSM voice cloning example (and pointing to [this dataset](https://huggingface.co/datasets/hf-internal-testing/dailytalk-dummy) so they know what the original voice sounds like).

```python
import soundfile as sf
import torch
from datasets import Audio, load_dataset
from transformers import pipeline


device = "cuda" if torch.cuda.is_available() else "cpu"
pipe = pipeline("text-to-audio", model="sesame/csm-1b", device=device)

# prepare input
ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train")
ds = ds.cast_column("audio", Audio(sampling_rate=24000))
conversation = [
    # -- audio/text pair(s) for voice cloning
    {
        "role": "0",
        "content": [
            {"type": "text", "text": "What are you working on?"}, 
            {"type": "audio", "path": ds[0]["audio"]["array"]},
        ],
    },
    # -- desired audio response for voice cloning
    {"role": "0", "content": [{"type": "text", "text": "How much money can you spend?"}]},
]
outputs = pipe(conversation)
fn = "csm_pipeline_voice_cloning.wav"
sf.write(fn, outputs["audio"], outputs["sampling_rate"])
print(f"Audio saved to {fn}")
```




Changed files:
filename: src/transformers/pipelines/text_to_audio.py
patch: @@ -172,6 +172,11 @@ def preprocess(self, text, **kwargs):
                 **kwargs,
             )
         else:
+            # Add speaker ID if needed and user didn't insert at start of text
+            if self.model.config.model_type == "csm":
+                text = [f"[0]{t}" if not t.startswith("[") else t for t in text]
+            if self.model.config.model_type == "dia":
+                text = [f"[S1] {t}" if not t.startswith("[") else t for t in text]
             output = preprocessor(text, **kwargs, return_tensors="pt")
 
         return output
@@ -196,6 +201,12 @@ def _forward(self, model_inputs, **kwargs):
             # ensure dict output to facilitate postprocessing
             forward_params.update({"return_dict_in_generate": True})
 
+            if self.model.config.model_type in ["csm"]:
+                # NOTE (ebezzam): CSM does not have the audio tokenizer in the processor therefore `output_audio=True`
+                # needed for decoding to audio
+                if "output_audio" not in forward_params:
+                    forward_params["output_audio"] = True
+
             output = self.model.generate(**model_inputs, **forward_params)
         else:
             if len(generate_kwargs):

filename: tests/pipelines/test_pipelines_text_to_audio.py
patch: @@ -251,26 +251,26 @@ def test_generative_model_kwargs(self):
     @require_torch
     def test_csm_model_pt(self):
         speech_generator = pipeline(task="text-to-audio", model="sesame/csm-1b", device=torch_device)
-        generate_kwargs = {"max_new_tokens": 10, "output_audio": True}
+        generate_kwargs = {"max_new_tokens": 10}
         num_channels = 1  # model generates mono audio
 
-        outputs = speech_generator("[0]This is a test", generate_kwargs=generate_kwargs)
+        outputs = speech_generator("This is a test", generate_kwargs=generate_kwargs)
         self.assertEqual(outputs["sampling_rate"], 24000)
         audio = outputs["audio"]
         self.assertEqual(ANY(np.ndarray), audio)
         # ensure audio and not discrete codes
         self.assertEqual(len(audio.shape), num_channels)
 
         # test two examples side-by-side
-        outputs = speech_generator(["[0]This is a test", "[0]This is a second test"], generate_kwargs=generate_kwargs)
+        outputs = speech_generator(["This is a test", "This is a second test"], generate_kwargs=generate_kwargs)
         audio = [output["audio"] for output in outputs]
         self.assertEqual([ANY(np.ndarray), ANY(np.ndarray)], audio)
         self.assertEqual(len(audio[0].shape), num_channels)
 
         # test batching
         batch_size = 2
         outputs = speech_generator(
-            ["[0]This is a test", "[0]This is a second test"], generate_kwargs=generate_kwargs, batch_size=batch_size
+            ["This is a test", "This is a second test"], generate_kwargs=generate_kwargs, batch_size=batch_size
         )
         self.assertEqual(len(outputs), batch_size)
         audio = [output["audio"] for output in outputs]
@@ -284,9 +284,7 @@ def test_dia_model(self):
         generate_kwargs = {"max_new_tokens": 20}
         num_channels = 1  # model generates mono audio
 
-        outputs = speech_generator(
-            "[S1] Dia is an open weights text to dialogue model.", generate_kwargs=generate_kwargs
-        )
+        outputs = speech_generator("Dia is an open weights text to dialogue model.", generate_kwargs=generate_kwargs)
         self.assertEqual(outputs["sampling_rate"], 44100)
         audio = outputs["audio"]
         self.assertEqual(ANY(np.ndarray), audio)
@@ -295,7 +293,7 @@ def test_dia_model(self):
 
         # test two examples side-by-side
         outputs = speech_generator(
-            ["[S1] Dia is an open weights text to dialogue model.", "[S2] This is a second example."],
+            ["Dia is an open weights text to dialogue model.", "This is a second example."],
             generate_kwargs=generate_kwargs,
         )
         audio = [output["audio"] for output in outputs]
@@ -305,7 +303,7 @@ def test_dia_model(self):
         # test batching
         batch_size = 2
         outputs = speech_generator(
-            ["[S1] Dia is an open weights text to dialogue model.", "[S2] This is a second example."],
+            ["Dia is an open weights text to dialogue model.", "This is a second example."],
             generate_kwargs=generate_kwargs,
             batch_size=2,
         )