Repository: deepset-ai/haystack

Issue:
Refactor serialization/deserialization of `ImageContent`
https://github.com/deepset-ai/haystack/pull/9626#discussion_r2218862047

From @sjrl:
> ... for the breakpoints feature it would be good to have `to_dict` and `from_dict` methods for this dataclass 


Discussion:
Hey @anakin87 , @sjrl ,  happy to help with this issue
Hey @RafaelJohn9, thanks!

For the moment, I'd suggest not working on this because I realized we have to better define the requirements internally.
For example, also `TextContent` is missing `to_dict`/`from_dict` methods and we instead use `_serialize_content_part`/`_deserialize_content_part`.

Hey ,

@anakin87 , thanks for the heads up.



Patch:
diff --git a/haystack/dataclasses/chat_message.py b/haystack/dataclasses/chat_message.py
index 8dec28abf9..5f8e0d89da 100644
--- a/haystack/dataclasses/chat_message.py
+++ b/haystack/dataclasses/chat_message.py
@@ -5,7 +5,7 @@
 import json
 from dataclasses import asdict, dataclass, field
 from enum import Enum
-from typing import Any, Dict, List, Optional, Sequence, Union
+from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
 from haystack import logging
 from haystack.dataclasses.image_content import ImageContent
@@ -131,9 +131,29 @@ class TextContent:
 
     text: str
 
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert TextContent into a dictionary.
+        """
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "TextContent":
+        """
+        Create a TextContent from a dictionary.
+        """
+        return TextContent(**data)
+
 
 ChatMessageContentT = Union[TextContent, ToolCall, ToolCallResult, ImageContent]
 
+_CONTENT_PART_CLASSES_TO_SERIALIZATION_KEYS: Dict[Type, str] = {
+    TextContent: "text",
+    ToolCall: "tool_call",
+    ToolCallResult: "tool_call_result",
+    ImageContent: "image",
+}
+
 
 def _deserialize_content_part(part: Dict[str, Any]) -> ChatMessageContentT:
     """
@@ -146,32 +166,15 @@ def _deserialize_content_part(part: Dict[str, Any]) -> ChatMessageContentT:
     :raises ValueError:
         If the part is not a valid ChatMessageContentT object.
     """
+    # handle flat text format separately
     if "text" in part:
-        return TextContent(text=part["text"])
-    if "tool_call" in part:
-        return ToolCall(**part["tool_call"])
-    if "tool_call_result" in part:
-        result = part["tool_call_result"]["result"]
-        origin = ToolCall(**part["tool_call_result"]["origin"])
-        error = part["tool_call_result"]["error"]
-        tcr = ToolCallResult(result=result, origin=origin, error=error)
-        return tcr
-    if "image" in part:
-        return ImageContent(**part["image"])
-    raise ValueError(f"Unsupported content part in the serialized ChatMessage: `{part}`")
+        return TextContent.from_dict(part)
 
+    for cls, serialization_key in _CONTENT_PART_CLASSES_TO_SERIALIZATION_KEYS.items():
+        if serialization_key in part:
+            return cls.from_dict(part[serialization_key])
 
-def _deserialize_content(serialized_content: List[Dict[str, Any]]) -> List[ChatMessageContentT]:
-    """
-    Deserialize the `content` field of a serialized ChatMessage.
-
-    :param serialized_content:
-        The `content` field of a serialized ChatMessage (a list of dictionaries).
-
-    :returns:
-        Deserialized `content` field as a list of `ChatMessageContentT` objects.
-    """
-    return [_deserialize_content_part(part) for part in serialized_content]
+    raise ValueError(f"Unsupported content part in the serialized ChatMessage: `{part}`")
 
 
 def _serialize_content_part(part: ChatMessageContentT) -> Dict[str, Any]:
@@ -185,15 +188,15 @@ def _serialize_content_part(part: ChatMessageContentT) -> Dict[str, Any]:
     :raises TypeError:
         If the part is not a valid ChatMessageContentT object.
     """
+    serialization_key = _CONTENT_PART_CLASSES_TO_SERIALIZATION_KEYS.get(type(part))
+    if serialization_key is None:
+        raise TypeError(f"Unsupported type in ChatMessage content: `{type(part).__name__}` for `{part}`.")
+
+    # handle flat text format separately
     if isinstance(part, TextContent):
-        return {"text": part.text}
-    elif isinstance(part, ToolCall):
-        return {"tool_call": asdict(part)}
-    elif isinstance(part, ToolCallResult):
-        return {"tool_call_result": asdict(part)}
-    elif isinstance(part, ImageContent):
-        return {"image": asdict(part)}
-    raise TypeError(f"Unsupported type in ChatMessage content: `{type(part).__name__}` for `{part}`.")
+        return part.to_dict()
+
+    return {serialization_key: part.to_dict()}
 
 
 @dataclass
@@ -226,14 +229,6 @@ def __new__(cls, *args, **kwargs):
                 f"{general_msg}"
             )
 
-        allowed_content_types = (TextContent, ToolCall, ToolCallResult)
-        if len(args) > 1 and not isinstance(args[1], allowed_content_types):
-            raise TypeError(
-                "The `_content` parameter of `ChatMessage` must be one of the following types: "
-                f"{', '.join(t.__name__ for t in allowed_content_types)}. "
-                f"{general_msg}"
-            )
-
         return super(ChatMessage, cls).__new__(cls)
 
     def __getattribute__(self, name):
@@ -489,7 +484,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "ChatMessage":
 
             if isinstance(data["content"], list):
                 # current format - the serialized `content` field is a list of dictionaries
-                init_params["_content"] = _deserialize_content(data["content"])
+                init_params["_content"] = [_deserialize_content_part(part) for part in data["content"]]
             elif isinstance(data["content"], str):
                 # pre 2.9.0 format - the `content` field is a string
                 init_params["_content"] = [TextContent(text=data["content"])]
@@ -501,7 +496,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "ChatMessage":
             # format for versions >=2.9.0 and <2.12.0 - the serialized `_content` field is a list of dictionaries
             return cls(
                 _role=ChatRole(data["_role"]),
-                _content=_deserialize_content(data["_content"]),
+                _content=[_deserialize_content_part(part) for part in data["_content"]],
                 _name=data.get("_name"),
                 _meta=data.get("_meta") or {},
             )
diff --git a/haystack/dataclasses/image_content.py b/haystack/dataclasses/image_content.py
index 8513d24b69..068a714eca 100644
--- a/haystack/dataclasses/image_content.py
+++ b/haystack/dataclasses/image_content.py
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import base64
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Dict, Literal, Optional, Tuple, Union
@@ -134,6 +134,19 @@ def show(self) -> None:
         else:
             image.show()
 
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert ImageContent into a dictionary.
+        """
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ImageContent":
+        """
+        Create an ImageContent from a dictionary.
+        """
+        return ImageContent(**data)
+
     @classmethod
     def from_file_path(
         cls,
diff --git a/haystack/utils/jinja2_chat_extension.py b/haystack/utils/jinja2_chat_extension.py
index 1b9ce17bc0..bcd2d8ac1d 100644
--- a/haystack/utils/jinja2_chat_extension.py
+++ b/haystack/utils/jinja2_chat_extension.py
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import json
-from typing import Any, Callable, List, Optional, Union, get_args
+from typing import Any, Callable, List, Optional, Union
 
 from jinja2 import TemplateSyntaxError, nodes
 from jinja2.ext import Extension
@@ -268,11 +268,4 @@ def templatize_part(value: ChatMessageContentT) -> str:
     :return: A JSON string wrapped in special XML content tags
     :raises ValueError: If the value is not an instance of ChatMessageContentT
     """
-    chat_message_content_types = get_args(ChatMessageContentT)
-    if not isinstance(value, chat_message_content_types):
-        chat_message_content_types_str = ", ".join([t.__name__ for t in chat_message_content_types])
-        raise ValueError(
-            f"Value must be an instance of one of the following types: {chat_message_content_types_str}. "
-            f"Got: {type(value).__name__}"
-        )
     return f"{START_TAG}{json.dumps(_serialize_content_part(value))}{END_TAG}"
diff --git a/releasenotes/notes/image-content-serde-f41a29c636e2078b.yaml b/releasenotes/notes/image-content-serde-f41a29c636e2078b.yaml
new file mode 100644
index 0000000000..938370263e
--- /dev/null
+++ b/releasenotes/notes/image-content-serde-f41a29c636e2078b.yaml
@@ -0,0 +1,4 @@
+---
+enhancements:
+  - |
+    Added serialization/deserialization methods for `TextContent` and `ImageContent` parts of `ChatMessage`.


Test patch:
diff --git a/test/dataclasses/test_chat_message.py b/test/dataclasses/test_chat_message.py
index 60f715e512..e978c7c0d3 100644
--- a/test/dataclasses/test_chat_message.py
+++ b/test/dataclasses/test_chat_message.py
@@ -10,6 +10,13 @@
 from haystack.dataclasses.image_content import ImageContent
 
 
+def test_chat_role_from_str():
+    assert ChatRole.from_str("user") == ChatRole.USER
+
+    with pytest.raises(ValueError):
+        ChatRole.from_str("invalid")
+
+
 def test_tool_call_init():
     tc = ToolCall(id="123", tool_name="mytool", arguments={"a": 1})
     assert tc.id == "123"
@@ -17,6 +24,18 @@ def test_tool_call_init():
     assert tc.arguments == {"a": 1}
 
 
+def test_tool_call_to_dict():
+    tc = ToolCall(id="123", tool_name="mytool", arguments={"a": 1})
+    assert tc.to_dict() == {"id": "123", "tool_name": "mytool", "arguments": {"a": 1}}
+
+
+def test_tool_call_from_dict():
+    tc = ToolCall.from_dict({"id": "123", "tool_name": "mytool", "arguments": {"a": 1}})
+    assert tc.id == "123"
+    assert tc.tool_name == "mytool"
+    assert tc.arguments == {"a": 1}
+
+
 def test_tool_call_result_init():
     tcr = ToolCallResult(result="result", origin=ToolCall(id="123", tool_name="mytool", arguments={"a": 1}), error=True)
     assert tcr.result == "result"
@@ -24,11 +43,42 @@ def test_tool_call_result_init():
     assert tcr.error
 
 
+def test_tool_call_result_to_dict():
+    tcr = ToolCallResult(result="result", origin=ToolCall(id="123", tool_name="mytool", arguments={"a": 1}), error=True)
+    assert tcr.to_dict() == {
+        "result": "result",
+        "origin": {"id": "123", "tool_name": "mytool", "arguments": {"a": 1}},
+        "error": True,
+    }
+
+
+def test_tool_call_result_from_dict():
+    tcr = ToolCallResult.from_dict(
+        {"result": "result", "origin": {"id": "123", "tool_name": "mytool", "arguments": {"a": 1}}, "error": True}
+    )
+    assert tcr.result == "result"
+    assert tcr.origin == ToolCall(id="123", tool_name="mytool", arguments={"a": 1})
+    assert tcr.error
+
+    with pytest.raises(ValueError):
+        ToolCallResult.from_dict({"result": "result", "error": False})
+
+
 def test_text_content_init():
     tc = TextContent(text="Hello")
     assert tc.text == "Hello"
 
 
+def test_text_content_to_dict():
+    tc = TextContent(text="Hello")
+    assert tc.to_dict() == {"text": "Hello"}
+
+
+def test_text_content_from_dict():
+    tc = TextContent.from_dict({"text": "Hello"})
+    assert tc.text == "Hello"
+
+
 def test_from_assistant_with_valid_content():
     text = "Hello, how can I assist you?"
     message = ChatMessage.from_assistant(text)
@@ -367,11 +417,6 @@ def test_chat_message_init_parameters_removed():
         ChatMessage(role="irrelevant", content="This is a message")
 
 
-def test_chat_message_init_content_parameter_type():
-    with pytest.raises(TypeError):
-        ChatMessage(ChatRole.USER, "This is a message")
-
-
 def test_to_openai_dict_format():
     message = ChatMessage.from_system("You are good assistant")
     assert message.to_openai_dict_format() == {"role": "system", "content": "You are good assistant"}
@@ -417,13 +462,16 @@ def test_to_openai_dict_format_user_message():
 
 def test_to_openai_dict_format_multimodal_user_message(base64_image_string):
     message = ChatMessage.from_user(
-        content_parts=[TextContent("I have a question"), ImageContent(base64_image=base64_image_string)]
+        content_parts=[TextContent("I have a question"), ImageContent(base64_image=base64_image_string, detail="auto")]
     )
     assert message.to_openai_dict_format() == {
         "role": "user",
         "content": [
             {"type": "text", "text": "I have a question"},
-            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image_string}"}},
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/png;base64,{base64_image_string}", "detail": "auto"},
+            },
         ],
     }
 
diff --git a/test/dataclasses/test_image_content.py b/test/dataclasses/test_image_content.py
index 08bf3ebbc9..ff46157e3a 100644
--- a/test/dataclasses/test_image_content.py
+++ b/test/dataclasses/test_image_content.py
@@ -24,6 +24,36 @@ def test_image_content_init(base64_image_string):
     assert image_content.validation
 
 
+def test_image_content_to_dict(base64_image_string):
+    image_content = ImageContent(
+        base64_image=base64_image_string, mime_type="image/png", detail="auto", meta={"key": "value"}
+    )
+    assert image_content.to_dict() == {
+        "base64_image": base64_image_string,
+        "mime_type": "image/png",
+        "detail": "auto",
+        "meta": {"key": "value"},
+        "validation": True,
+    }
+
+
+def test_image_content_from_dict(base64_image_string):
+    image_content = ImageContent.from_dict(
+        {
+            "base64_image": base64_image_string,
+            "mime_type": "image/png",
+            "detail": "auto",
+            "meta": {"key": "value"},
+            "validation": False,
+        }
+    )
+    assert image_content.base64_image == base64_image_string
+    assert image_content.mime_type == "image/png"
+    assert image_content.detail == "auto"
+    assert image_content.meta == {"key": "value"}
+    assert not image_content.validation
+
+
 def test_image_content_init_with_invalid_base64_string():
     with pytest.raises(ValueError):
         ImageContent(base64_image="invalid_base64_string")
diff --git a/test/utils/test_jinja2_chat_extension.py b/test/utils/test_jinja2_chat_extension.py
index e4018e7b52..e83049002e 100644
--- a/test/utils/test_jinja2_chat_extension.py
+++ b/test/utils/test_jinja2_chat_extension.py
@@ -307,7 +307,7 @@ def test_invalid_role(self, jinja_env):
             jinja_env.from_string(template).render()
 
     def test_templatize_part_filter_with_invalid_type(self):
-        with pytest.raises(ValueError, match="Value must be an instance of one of the following types"):
+        with pytest.raises(TypeError, match="Unsupported type in ChatMessage content"):
             templatize_part(123)
 
     def test_empty_message_content_raises_error(self, jinja_env):
@@ -465,5 +465,5 @@ def test_invalid_tool_message_raises_error(self, jinja_env, base64_image_string)
         {{ image | templatize_part }}
         {% endmessage %}
         """
-        with pytest.raises(ValueError):
+        with pytest.raises(TypeError):
             jinja_env.from_string(template).render(image=image)