Repository: deepset-ai/haystack

Issue:
Add GenerationConfig option to PromptNode's HuggingFace invocation layer 
**Is your feature request related to a problem? Please describe.**
As a PromptNode and Agent user, I can't control all the parameters of text2text-generation and text-generation pipelines.  Transformers library recently introduced [GenerationConfig](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig) class that simplifies setting generation parameters for the underlying model used in a pipeline. 

**Describe the solution you'd like**
I would like to have the possibility to pass the GenerationConfig both at the time of PromptNode instantiation and also override these GenerationConfig values (if possible) with another runtime parameter passed to both PromptNode itself prompt method and run method of a pipeline. 

**Describe alternatives you've considered**
There are no alternatives. Currently, there is no support for passing these generation parameters. 

**Additional context**
See HuggingFace [documentation](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig) for more details


Discussion:

Resolving patch:
diff --git a/haystack/nodes/prompt/invocation_layer/hugging_face.py b/haystack/nodes/prompt/invocation_layer/hugging_face.py
--- a/haystack/nodes/prompt/invocation_layer/hugging_face.py
+++ b/haystack/nodes/prompt/invocation_layer/hugging_face.py
@@ -2,7 +2,14 @@
 import logging
 
 import torch
-from transformers import pipeline, StoppingCriteriaList, StoppingCriteria, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers import (
+    pipeline,
+    StoppingCriteriaList,
+    StoppingCriteria,
+    PreTrainedTokenizer,
+    PreTrainedTokenizerFast,
+    GenerationConfig,
+)
 from transformers.pipelines import get_task
 
 from haystack.modeling.utils import initialize_device_settings
@@ -71,6 +78,7 @@ def __init__(
                 "use_fast",
                 "torch_dtype",
                 "device_map",
+                "generation_kwargs",
             ]
             if key in kwargs
         }
@@ -79,6 +87,9 @@ def __init__(
             mkwargs = model_input_kwargs.pop("model_kwargs")
             model_input_kwargs.update(mkwargs)
 
+        # save generation_kwargs for pipeline invocation
+        self.generation_kwargs = model_input_kwargs.pop("generation_kwargs", {})
+
         torch_dtype = model_input_kwargs.get("torch_dtype")
         if torch_dtype is not None:
             if isinstance(torch_dtype, str):
@@ -136,9 +147,18 @@ def invoke(self, *args, **kwargs):
                     "return_full_text",
                     "clean_up_tokenization_spaces",
                     "truncation",
+                    "generation_kwargs",
                 ]
                 if key in kwargs
             }
+            generation_kwargs = model_input_kwargs.pop("generation_kwargs", self.generation_kwargs)
+            if isinstance(generation_kwargs, dict):
+                model_input_kwargs.update(generation_kwargs)
+            elif isinstance(generation_kwargs, GenerationConfig):
+                gen_dict = generation_kwargs.to_diff_dict()
+                gen_dict.pop("transformers_version", None)
+                model_input_kwargs.update(gen_dict)
+
             is_text_generation = "text-generation" == self.task_name
             # Prefer return_full_text is False for text-generation (unless explicitly set)
             # Thus only generated text is returned (excluding prompt)


Test patch:
diff --git a/test/prompt/test_prompt_node.py b/test/prompt/test_prompt_node.py
--- a/test/prompt/test_prompt_node.py
+++ b/test/prompt/test_prompt_node.py
@@ -1,9 +1,10 @@
 import os
 import logging
 from typing import Optional, Union, List, Dict, Any, Tuple
-from unittest.mock import patch, Mock
+from unittest.mock import patch, Mock, MagicMock
 
 import pytest
+from transformers import AutoTokenizer, GenerationConfig
 
 from haystack import Document, Pipeline, BaseComponent, MultiLabel
 from haystack.nodes.prompt import PromptTemplate, PromptNode, PromptModel
@@ -201,6 +202,54 @@ def test_invalid_template_params():
         node.prompt("question-answering-per-document", {"some_crazy_key": "Berlin is the capital of Germany."})
 
 
+@pytest.mark.integration
+def test_generation_kwargs_from_prompt_node_init():
+    the_question = "What does 42 mean?"
+    # test that generation_kwargs are passed to the underlying HF model
+    node = PromptNode(model_kwargs={"generation_kwargs": {"do_sample": True}})
+    with patch.object(node.prompt_model.model_invocation_layer.pipe, "run_single", MagicMock()) as mock_call:
+        node(the_question)
+        mock_call.assert_called_with(
+            the_question, {}, {"do_sample": True, "num_return_sequences": 1, "num_beams": 1, "max_length": 100}, {}
+        )
+
+    # test that generation_kwargs in the form of GenerationConfig are passed to the underlying HF model
+    node = PromptNode(model_kwargs={"generation_kwargs": GenerationConfig(do_sample=True, top_p=0.9)})
+    with patch.object(node.prompt_model.model_invocation_layer.pipe, "run_single", MagicMock()) as mock_call:
+        node(the_question)
+        mock_call.assert_called_with(
+            the_question,
+            {},
+            {"do_sample": True, "top_p": 0.9, "num_return_sequences": 1, "num_beams": 1, "max_length": 100},
+            {},
+        )
+
+
+@pytest.mark.integration
+def test_generation_kwargs_from_prompt_node_call():
+    the_question = "What does 42 mean?"
+    # default node with local HF model
+    node = PromptNode()
+    with patch.object(node.prompt_model.model_invocation_layer.pipe, "run_single", MagicMock()) as mock_call:
+        # test that generation_kwargs are passed to the underlying HF model
+        node(the_question, generation_kwargs={"do_sample": True})
+        mock_call.assert_called_with(
+            the_question, {}, {"do_sample": True, "num_return_sequences": 1, "num_beams": 1, "max_length": 100}, {}
+        )
+
+    # default node with local HF model
+    node = PromptNode()
+    with patch.object(node.prompt_model.model_invocation_layer.pipe, "run_single", MagicMock()) as mock_call:
+        # test that generation_kwargs in the form of GenerationConfig are passed to the underlying HF model
+        node(the_question, generation_kwargs=GenerationConfig(do_sample=True, top_p=0.9))
+        mock_call.assert_called_with(
+            the_question,
+            {},
+            {"do_sample": True, "top_p": 0.9, "num_return_sequences": 1, "num_beams": 1, "max_length": 100},
+            {},
+        )
+
+
 @pytest.mark.integration
 @pytest.mark.parametrize("prompt_model", ["hf", "openai", "azure"], indirect=True)
 def test_stop_words(prompt_model):