Repository: deepset-ai/haystack

Issue:
Agent's answer contains the prompt text for HF models 
**Describe the bug**
When running [this Agent Tutorial](https://haystack.deepset.ai/tutorials/25_customizing_agent), I tried to replace the openai model with the open source model vicuna and found that there is a discrepancy in the behavior of openai model output and the HF model output. 

**Error message**
Here is the agent output of the OpenAI model:
![image](https://github.com/deepset-ai/haystack/assets/24477841/c9dceb13-bec0-4e68-89fa-f3cc60f2d6ee)

Here is a screenshot of the agent output using HF Model. You can reproduce it with my script provided below: 
 
![image](https://github.com/deepset-ai/haystack/assets/24477841/41705b9d-1911-4a19-810c-8206074342ac)

**Expected behavior**
The Agent should answer without the prompt text.

**Additional context**
Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.

**To Reproduce**
```python
from datasets import load_dataset

from prompt_templates import agent_prompt, qa_prompt_template

from haystack.document_stores import InMemoryDocumentStore
from haystack.nodes import PromptNode, PromptTemplate, AnswerParser, BM25Retriever
from haystack.pipelines import Pipeline
from haystack.agents import Tool
from haystack.nodes import PromptNode
from haystack.agents.memory import ConversationSummaryMemory
from haystack.agents import AgentStep, Agent
from haystack.agents.base import Agent, ToolsManager
from haystack.utils import print_answers

def launch_document_store():

    dataset = load_dataset("bilgeyucel/seven-wonders", split="train")
    document_store = InMemoryDocumentStore(use_bm25=True)
    document_store.write_documents(dataset)

    return document_store 

def create_memory():

    memory_prompt_node = PromptNode(
        "philschmid/bart-large-cnn-samsum", max_length=256, model_kwargs={"task_name": "text2text-generation"}
    )
    memory = ConversationSummaryMemory(memory_prompt_node, prompt_template="{chat_transcript}")

    return memory


def resolver_function(query, agent, agent_step):
    
    return {
        "query": query,
        "tool_names_with_descriptions": agent.tm.get_tool_names_with_descriptions(),
        "transcript": agent_step.transcript,
        "memory": agent.memory.load(),
    }

def create_agent(agent_prompt, memory, tools_manager):

    agent_prompt_node = PromptNode(
        'lmsys/vicuna-13b-v1.3',
        max_length=256,
        stop_words=["Observation:"],
        model_kwargs={"temperature": 0.5},
    )

    conversational_agent = Agent(
        agent_prompt_node,
        prompt_template=agent_prompt,
        prompt_parameters_resolver=resolver_function,
        memory=memory,
        tools_manager=tools_manager,
    )

    return conversational_agent


if __name__ == "__main__":
    document_store = launch_document_store()   
    retriever = BM25Retriever(document_store=document_store, top_k=3)
    prompt_node = PromptNode(
         model_name_or_path="lmsys/vicuna-13b-v1.3", default_prompt_template=qa_prompt_template
    )
    
    generative_pipeline = Pipeline()
    generative_pipeline.add_node(component=retriever, name="retriever", inputs=["Query"])
    generative_pipeline.add_node(component=prompt_node, name="prompt_node", inputs=["retriever"])
    search_tool = Tool(
        name="seven_wonders_search",
        pipeline_or_node=generative_pipeline,
        description="useful for when you need to answer questions about the seven wonders of the world",
        output_variable="answers",
    )
    
    memory = create_memory()

    tools_manager = ToolsManager([search_tool])
    
    conversational_agent = create_agent(agent_prompt, memory, tools_manager)
    conversational_agent.run("What did Rhodes Statue look like?")
    
```

**FAQ Check**
- [x] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: linux
 - GPU/CPU: CPU
 - Haystack version (commit or version number): 1.20.0rc0
 - DocumentStore:
 - Reader:
 - Retriever:



Discussion:

Resolving patch:
diff --git a/haystack/nodes/prompt/invocation_layer/handlers.py b/haystack/nodes/prompt/invocation_layer/handlers.py
--- a/haystack/nodes/prompt/invocation_layer/handlers.py
+++ b/haystack/nodes/prompt/invocation_layer/handlers.py
@@ -47,7 +47,7 @@ def __init__(
         stream_handler: "TokenStreamingHandler",
     ):
         transformers_import.check()
-        super().__init__(tokenizer=tokenizer)  # type: ignore
+        super().__init__(tokenizer=tokenizer, skip_prompt=True)  # type: ignore
         self.token_handler = stream_handler
 
     def on_finalized_text(self, token: str, stream_end: bool = False):


Test patch:
diff --git a/test/prompt/invocation_layer/test_hugging_face.py b/test/prompt/invocation_layer/test_hugging_face.py
--- a/test/prompt/invocation_layer/test_hugging_face.py
+++ b/test/prompt/invocation_layer/test_hugging_face.py
@@ -632,3 +632,17 @@ def test_tokenizer_loading_unsupported_model_with_tokenizer_class_in_config(
         invocation_layer = HFLocalInvocationLayer(model_name_or_path="unsupported_model", trust_remote_code=True)
         assert not mock_tokenizer.called
         assert not caplog.text
+
+
+@pytest.mark.unit
+def test_skip_prompt_is_set_in_hf_text_streamer(mock_pipeline, mock_get_task):
+    """
+    Test that skip_prompt is set in HFTextStreamingHandler. Otherwise, we will output prompt text.
+    """
+    layer = HFLocalInvocationLayer(stream=True)
+
+    layer.invoke(prompt="Tell me hello")
+
+    _, kwargs = layer.pipe.call_args
+    assert "streamer" in kwargs and isinstance(kwargs["streamer"], HFTokenStreamingHandler)
+    assert kwargs["streamer"].skip_prompt