Repository: deepset-ai/haystack

Issue:
Add tool output to final agent result
**Is your feature request related to a problem? Please describe.**
When running an agent on multiple tools, sometimes the agent makes different categories of mistakes:
1- The agent gives the wrong input to the tool
2- The tool returns the wrong output
3- The agent incorrectly synthesizes the final answer
At the moment, with just the agent's final answer as a result of `agent.run()`, it is hard to distinguish between these three cases outside of looking at the log or the transcript.

**Describe the solution you'd like**
What I would like to happen is that I could call `agent.run()` and then have as part of the returned object something like `tools_output`, which would be a `List` of each tool that was used in order, and for each of them, the result of `tool.run()` along with the `tool_name`. This would allow some flexibility in debugging, in returning some of that result to the UI and more.


**Describe alternatives you've considered**
Some people in the Discord have setup a solution where the tool output is gathered in parallel to the agent.run rather than part of the result of agent.run itself. This seems more complex for a less desirable result.

**Additional context**
I'll be able to work on this feature in the coming days.


Discussion:

Resolving patch:
diff --git a/examples/agent_multihop_qa.py b/examples/agent_multihop_qa.py
--- a/examples/agent_multihop_qa.py
+++ b/examples/agent_multihop_qa.py
@@ -2,6 +2,7 @@
 
 from haystack.agents import Agent, Tool
 from haystack.agents.base import ToolsManager
+from haystack.agents.types import AgentToolLogger
 from haystack.nodes import PromptNode, PromptTemplate
 from haystack.nodes.retriever.web import WebRetriever
 from haystack.pipelines import WebQAPipeline
@@ -99,6 +100,7 @@
 agent = Agent(
     prompt_node=prompt_node, prompt_template=few_shot_agent_template, tools_manager=ToolsManager([web_qa_tool])
 )
+atl = AgentToolLogger(agent_events=agent.callback_manager, tool_events=agent.tm.callback_manager)
 
 hotpot_questions = [
     "What year was the father of the Princes in the Tower born?",
@@ -106,7 +108,9 @@
     "Where was the actress who played the niece in the Priest film born?",
     "Which author is English: John Braine or Studs Terkel?",
 ]
-
+verbose = False
 for question in hotpot_questions:
     result = agent.run(query=question)
     print(f"\n{result}")
+    if verbose:
+        print(f"\n{atl.logs}")
diff --git a/haystack/agents/base.py b/haystack/agents/base.py
--- a/haystack/agents/base.py
+++ b/haystack/agents/base.py
@@ -183,6 +183,8 @@ def run_tool(self, llm_response: str, params: Optional[Dict[str, Any]] = None) -
                         observation_prefix="Observation: ",
                         llm_prefix="Thought: ",
                         color=tool.logging_color,
+                        tool_name=tool.name,
+                        tool_input=tool_input,
                     )
                 except Exception as e:
                     self.callback_manager.on_tool_error(e, tool=self.tools[tool_name])
diff --git a/haystack/agents/types.py b/haystack/agents/types.py
--- a/haystack/agents/types.py
+++ b/haystack/agents/types.py
@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Any, Dict, List, Optional
 
 from events import Events
 from haystack.nodes.prompt.invocation_layer.handlers import TokenStreamingHandler
@@ -23,3 +24,21 @@ def __init__(self, events: Events):
     def __call__(self, token_received, **kwargs) -> str:
         self.events.on_new_token(token_received, **kwargs)
         return token_received
+
+
+class AgentToolLogger:
+    def __init__(self, agent_events: Events, tool_events: Events):
+        agent_events.on_agent_start += self.on_agent_start
+        tool_events.on_tool_finish += self.on_tool_finish
+        self.logs: List[Dict[str, Any]] = []
+
+    def on_agent_start(self, **kwargs: Any) -> None:
+        self.logs = []
+
+    def on_tool_finish(
+        self, tool_result: str, tool_name: Optional[str] = None, tool_input: Optional[str] = None, **kwargs: Any
+    ) -> None:
+        self.logs.append({"tool_name": tool_name, "tool_input": tool_input, "tool_output": tool_result})
+
+    def __repr__(self):
+        return f"<DefaultToolLogger with {len(self.logs)} tool log(s)>"


Test patch:
diff --git a/test/agents/test_agent.py b/test/agents/test_agent.py
--- a/test/agents/test_agent.py
+++ b/test/agents/test_agent.py
@@ -4,10 +4,9 @@
 from typing import Tuple
 from unittest.mock import Mock, patch
 
-
 from events import Events
 
-from haystack.agents.types import AgentTokenStreamingHandler
+from haystack.agents.types import AgentTokenStreamingHandler, AgentToolLogger
 from test.conftest import MockRetriever, MockPromptNode
 from unittest import mock
 import pytest
@@ -109,6 +108,33 @@ def test_run_tool():
     assert result == "[]"  # empty list of documents
 
 
+@pytest.mark.unit
+def test_agent_tool_logger():
+    agent = Agent(prompt_node=MockPromptNode())
+    atl = AgentToolLogger(agent_events=agent.callback_manager, tool_events=agent.tm.callback_manager)
+
+    retriever = MockRetriever()
+    agent.add_tool(
+        Tool(
+            name="Retriever",
+            pipeline_or_node=retriever,
+            description="useful for when you need to retrieve documents from your index",
+            output_variable="documents",
+        )
+    )
+    pn_response = "need to find out what city he was born.\nTool: Retriever\nTool Input: Where was Jeremy McKinnon born"
+
+    step = AgentStep(prompt_node_response=pn_response)
+    agent.tm.run_tool(step.prompt_node_response)
+
+    # Check that the AgentToolLogger collected the tool's output
+    assert len(atl.logs) == 1
+    tool_logging_event = atl.logs[0]
+    assert tool_logging_event["tool_name"] == "Retriever"
+    assert tool_logging_event["tool_input"] == "Where was Jeremy McKinnon born"
+    assert tool_logging_event["tool_output"] == "[]"
+
+
 @pytest.mark.unit
 def test_extract_final_answer():
     match_examples = [