Repository: deepset-ai/haystack

Issue:
Match only first half of the mime type in `FileTypeRouter`
In many cases, converter components can handle an entire category of mimetypes, not only a specific one. For example, audio transcribers could handle all `audio` mimetypes, the text converter could handle all `text` mimetypes, and so on.

It would be convenient if FileTypeRouter could match a mime type only by its first half, so not `audio/mpeg`  or `audio/x-wav` but `audio`.


Discussion:
This could be done accepting regexes in the `mime_types: List[str]` parameter - this way it would be backward compatible
Hey @ZanSara , @masci .. 

The following Python script utilizes the mimetypes module to access a dictionary containing all known MIME types and their corresponding file extensions.

```python
import mimetypes
all_mime_types = mimetypes.types_map
s = set()
for mimetype, extension in all_mime_types.items():
    category = extension.split('/')[0]
    s.add((extension, category))
sorted_list = sorted(s, key=lambda x: x[0])
for i in sorted_list:  print(i)


```

Output:

```bash
...
...
...
('audio/3gpp', 'audio')
('audio/3gpp2', 'audio')
('audio/aac', 'audio')
('audio/basic', 'audio')
('audio/mpeg', 'audio')
('audio/opus', 'audio')
('audio/x-aiff', 'audio')
('audio/x-pn-realaudio', 'audio')
('audio/x-wav', 'audio')
('image/gif', 'image')
('image/heic', 'image')
('image/heif', 'image')
('image/ief', 'image')
('image/jpeg', 'image')
('image/png', 'image')
('image/svg+xml', 'image')
...
...
...

```

Resolving patch:
diff --git a/haystack/components/routers/file_type_router.py b/haystack/components/routers/file_type_router.py
--- a/haystack/components/routers/file_type_router.py
+++ b/haystack/components/routers/file_type_router.py
@@ -1,4 +1,5 @@
 import mimetypes
+import re
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Optional, Union
@@ -12,28 +13,39 @@
 @component
 class FileTypeRouter:
     """
-    FileTypeRouter takes a list of data sources (file paths or byte streams) and groups them by their corresponding
-    MIME types.
+    FileTypeRouter groups a list of data sources (file paths or byte streams) by their MIME types, allowing
+    for flexible routing of files to different components based on their content type. It supports both exact MIME type
+    matching and pattern matching using regular expressions.
 
-    For file paths, MIME types are inferred from their extensions, while for byte streams, MIME types
-    are determined from the provided metadata.
+    For file paths, MIME types are inferred from their extensions, while for byte streams, MIME types are determined from
+    the provided metadata. This enables the router to classify a diverse collection of files and data streams for
+    specialized processing.
 
-    The set of MIME types to consider is specified during the initialization of the component.
-
-    This component is useful when you need to classify a large collection of files or data streams according to their
-    MIME types and route them to different components for further processing.
+    The router's flexibility is enhanced by the support for regex patterns in the `mime_types` parameter, allowing users
+    to specify broad categories (e.g., 'audio/*' or 'text/*') or more specific types with regex patterns. This feature
+    is designed to be backward compatible, treating MIME types without regex patterns as exact matches.
 
     Usage example:
     ```python
     from haystack.components.routers import FileTypeRouter
+    from pathlib import Path
+
+    # For exact MIME type matching
+    router = FileTypeRouter(mime_types=["text/plain", "application/pdf"])
 
-    router = FileTypeRouter(mime_types=["text/plain"])
+    # For flexible matching using regex, to handle all audio types
+    router_with_regex = FileTypeRouter(mime_types=[r"audio/.*", r"text/plain"])
 
-    print(router.run(sources=["text_file.txt", "pdf_file.pdf"]))
+    sources = [Path("file.txt"), Path("document.pdf"), Path("song.mp3")]
+    print(router.run(sources=sources))
+    print(router_with_regex.run(sources=sources))
 
-    # defaultdict(<class 'list'>, {'text/plain': [PosixPath('text_file.txt')],
-    #                              'unclassified': [PosixPath('pdf_file.pdf')]})
+    # Expected output:
+    # {'text/plain': [PosixPath('file.txt')], 'application/pdf': [PosixPath('document.pdf')], 'unclassified': [PosixPath('song.mp3')]}
+    # {'audio/.*': [PosixPath('song.mp3')], 'text/plain': [PosixPath('file.txt')], 'unclassified': [PosixPath('document.pdf')]}
     ```
+
+    :param mime_types: A list of MIME types or regex patterns to classify the incoming files or data streams.
     """
 
     def __init__(self, mime_types: List[str]):
@@ -44,11 +56,12 @@ def __init__(self, mime_types: List[str]):
         if not mime_types:
             raise ValueError("The list of mime types cannot be empty.")
 
+        self.mime_type_patterns = []
         for mime_type in mime_types:
             if not self._is_valid_mime_type_format(mime_type):
-                raise ValueError(
-                    f"Unknown mime type: '{mime_type}'. Ensure you passed a list of strings in the 'mime_types' parameter"
-                )
+                raise ValueError(f"Invalid mime type or regex pattern: '{mime_type}'.")
+            pattern = re.compile(mime_type)
+            self.mime_type_patterns.append(pattern)
 
         component.set_output_types(self, unclassified=List[Path], **{mime_type: List[Path] for mime_type in mime_types})
         self.mime_types = mime_types
@@ -66,20 +79,24 @@ def run(self, sources: List[Union[str, Path, ByteStream]]) -> Dict[str, List[Uni
         for source in sources:
             if isinstance(source, str):
                 source = Path(source)
-
             if isinstance(source, Path):
                 mime_type = self._get_mime_type(source)
             elif isinstance(source, ByteStream):
-                mime_type = source.meta.get("content_type")
-            else:
-                raise ValueError(f"Unsupported data source type: {type(source)}")
-
-            if mime_type in self.mime_types:
-                mime_types[mime_type].append(source)
+                mime_type = source.meta.get("content_type", None)
             else:
+                raise ValueError(f"Unsupported data source type: {type(source).__name__}")
+
+            matched = False
+            if mime_type:
+                for pattern in self.mime_type_patterns:
+                    if pattern.fullmatch(mime_type):
+                        mime_types[pattern.pattern].append(source)
+                        matched = True
+                        break
+            if not matched:
                 mime_types["unclassified"].append(source)
 
-        return mime_types
+        return dict(mime_types)
 
     def _get_mime_type(self, path: Path) -> Optional[str]:
         """
@@ -96,13 +113,17 @@ def _get_mime_type(self, path: Path) -> Optional[str]:
 
     def _is_valid_mime_type_format(self, mime_type: str) -> bool:
         """
-        Check if the provided MIME type is in valid format
-
-        :param mime_type: The MIME type to check.
+        Checks if the provided MIME type string is a valid regex pattern.
 
-        :returns: `True` if the provided MIME type is a valid MIME type format, `False` otherwise.
+        :param mime_type: The MIME type or regex pattern to validate.
+        :raises ValueError: If the mime_type is not a valid regex pattern.
+        :returns: Always True because a ValueError is raised for invalid patterns.
         """
-        return mime_type in mimetypes.types_map.values() or mime_type in self._get_custom_mime_mappings().values()
+        try:
+            re.compile(mime_type)
+            return True
+        except re.error:
+            raise ValueError(f"Invalid regex pattern '{mime_type}'.")
 
     @staticmethod
     def _get_custom_mime_mappings() -> Dict[str, str]:


Test patch:
diff --git a/test/components/routers/test_file_router.py b/test/components/routers/test_file_router.py
--- a/test/components/routers/test_file_router.py
+++ b/test/components/routers/test_file_router.py
@@ -1,4 +1,6 @@
+import io
 import sys
+from unittest.mock import mock_open, patch
 
 import pytest
 
@@ -22,13 +24,13 @@ def test_run(self, test_files_path):
             test_files_path / "images" / "apple.jpg",
         ]
 
-        router = FileTypeRouter(mime_types=["text/plain", "audio/x-wav", "image/jpeg"])
+        router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
         output = router.run(sources=file_paths)
         assert output
-        assert len(output["text/plain"]) == 2
-        assert len(output["audio/x-wav"]) == 1
-        assert len(output["image/jpeg"]) == 1
-        assert not output["unclassified"]
+        assert len(output[r"text/plain"]) == 2
+        assert len(output[r"audio/x-wav"]) == 1
+        assert len(output[r"image/jpeg"]) == 1
+        assert not output.get("unclassified")
 
     def test_run_with_bytestreams(self, test_files_path):
         """
@@ -40,14 +42,12 @@ def test_run_with_bytestreams(self, test_files_path):
             test_files_path / "audio" / "the context for this answer is here.wav",
             test_files_path / "images" / "apple.jpg",
         ]
-        mime_types = ["text/plain", "text/plain", "audio/x-wav", "image/jpeg"]
+        mime_types = [r"text/plain", r"text/plain", r"audio/x-wav", r"image/jpeg"]
         # Convert file paths to ByteStream objects and set metadata
         byte_streams = []
         for path, mime_type in zip(file_paths, mime_types):
             stream = ByteStream(path.read_bytes())
-
             stream.meta["content_type"] = mime_type
-
             byte_streams.append(stream)
 
         # add unclassified ByteStream
@@ -55,15 +55,18 @@ def test_run_with_bytestreams(self, test_files_path):
         bs.meta["content_type"] = "unknown_type"
         byte_streams.append(bs)
 
-        router = FileTypeRouter(mime_types=["text/plain", "audio/x-wav", "image/jpeg"])
+        router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
         output = router.run(sources=byte_streams)
         assert output
-        assert len(output["text/plain"]) == 2
-        assert len(output["audio/x-wav"]) == 1
-        assert len(output["image/jpeg"]) == 1
+        assert len(output[r"text/plain"]) == 2
+        assert len(output[r"audio/x-wav"]) == 1
+        assert len(output[r"image/jpeg"]) == 1
         assert len(output.get("unclassified")) == 1
 
     def test_run_with_bytestreams_and_file_paths(self, test_files_path):
+        """
+        Test if the component raises an error for unsupported data source types.
+        """
         file_paths = [
             test_files_path / "txt" / "doc_1.txt",
             test_files_path / "audio" / "the context for this answer is here.wav",
@@ -71,7 +74,7 @@ def test_run_with_bytestreams_and_file_paths(self, test_files_path):
             test_files_path / "images" / "apple.jpg",
             test_files_path / "markdown" / "sample.md",
         ]
-        mime_types = ["text/plain", "audio/x-wav", "text/plain", "image/jpeg", "text/markdown"]
+        mime_types = [r"text/plain", r"audio/x-wav", r"text/plain", r"image/jpeg", r"text/markdown"]
         byte_stream_sources = []
         for path, mime_type in zip(file_paths, mime_types):
             stream = ByteStream(path.read_bytes())
@@ -80,18 +83,18 @@ def test_run_with_bytestreams_and_file_paths(self, test_files_path):
 
         mixed_sources = file_paths[:2] + byte_stream_sources[2:]
 
-        router = FileTypeRouter(mime_types=["text/plain", "audio/x-wav", "image/jpeg", "text/markdown"])
+        router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg", r"text/markdown"])
         output = router.run(sources=mixed_sources)
-        assert len(output["text/plain"]) == 2
-        assert len(output["audio/x-wav"]) == 1
-        assert len(output["image/jpeg"]) == 1
-        assert len(output["text/markdown"]) == 1
+        assert len(output[r"text/plain"]) == 2
+        assert len(output[r"audio/x-wav"]) == 1
+        assert len(output[r"image/jpeg"]) == 1
+        assert len(output[r"text/markdown"]) == 1
 
     def test_no_files(self):
         """
         Test that the component runs correctly when no files are provided.
         """
-        router = FileTypeRouter(mime_types=["text/plain", "audio/x-wav", "image/jpeg"])
+        router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
         output = router.run(sources=[])
         assert not output
 
@@ -104,13 +107,11 @@ def test_unlisted_extensions(self, test_files_path):
             test_files_path / "audio" / "ignored.mp3",
             test_files_path / "audio" / "this is the content of the document.wav",
         ]
-        router = FileTypeRouter(mime_types=["text/plain"])
+        router = FileTypeRouter(mime_types=[r"text/plain"])
         output = router.run(sources=file_paths)
-        assert len(output["text/plain"]) == 1
+        assert len(output[r"text/plain"]) == 1
         assert "mp3" not in output
-        assert len(output["unclassified"]) == 2
-        assert str(output["unclassified"][0]).endswith("ignored.mp3")
-        assert str(output["unclassified"][1]).endswith("this is the content of the document.wav")
+        assert len(output.get("unclassified")) == 2
 
     def test_no_extension(self, test_files_path):
         """
@@ -121,14 +122,61 @@ def test_no_extension(self, test_files_path):
             test_files_path / "txt" / "doc_2",
             test_files_path / "txt" / "doc_2.txt",
         ]
-        router = FileTypeRouter(mime_types=["text/plain"])
+        router = FileTypeRouter(mime_types=[r"text/plain"])
         output = router.run(sources=file_paths)
-        assert len(output["text/plain"]) == 2
-        assert len(output["unclassified"]) == 1
+        assert len(output[r"text/plain"]) == 2
+        assert len(output.get("unclassified")) == 1
+
+    def test_unsupported_source_type(self):
+        """
+        Test if the component raises an error for unsupported data source types.
+        """
+        router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
+        with pytest.raises(ValueError, match="Unsupported data source type:"):
+            router.run(sources=[{"unsupported": "type"}])
+
+    def test_invalid_regex_pattern(self):
+        """
+        Test that the component raises a ValueError for invalid regex patterns.
+        """
+        with pytest.raises(ValueError, match="Invalid regex pattern"):
+            FileTypeRouter(mime_types=["[Invalid-Regex"])
 
-    def test_unknown_mime_type(self):
+    def test_regex_mime_type_matching(self, test_files_path):
         """
-        Test that the component handles files with unknown mime types.
+        Test if the component correctly matches mime types using regex.
         """
-        with pytest.raises(ValueError, match="Unknown mime type:"):
-            FileTypeRouter(mime_types=["type_invalid"])
+        router = FileTypeRouter(mime_types=[r"text\/.*", r"audio\/.*", r"image\/.*"])
+        file_paths = [
+            test_files_path / "txt" / "doc_1.txt",
+            test_files_path / "audio" / "the context for this answer is here.wav",
+            test_files_path / "images" / "apple.jpg",
+        ]
+        output = router.run(sources=file_paths)
+        assert len(output[r"text\/.*"]) == 1, "Failed to match text file with regex"
+        assert len(output[r"audio\/.*"]) == 1, "Failed to match audio file with regex"
+        assert len(output[r"image\/.*"]) == 1, "Failed to match image file with regex"
+
+    @patch("pathlib.Path.open", new_callable=mock_open, read_data=b"Mock file content.")
+    def test_exact_mime_type_matching(self, mock_file):
+        """
+        Test if the component correctly matches mime types exactly, without regex patterns.
+        """
+        txt_stream = ByteStream(io.BytesIO(b"Text file content"), meta={"content_type": "text/plain"})
+        jpg_stream = ByteStream(io.BytesIO(b"JPEG file content"), meta={"content_type": "image/jpeg"})
+        mp3_stream = ByteStream(io.BytesIO(b"MP3 file content"), meta={"content_type": "audio/mpeg"})
+
+        byte_streams = [txt_stream, jpg_stream, mp3_stream]
+
+        router = FileTypeRouter(mime_types=["text/plain", "image/jpeg"])
+
+        output = router.run(sources=byte_streams)
+
+        assert len(output["text/plain"]) == 1, "Failed to match 'text/plain' MIME type exactly"
+        assert txt_stream in output["text/plain"], "'doc_1.txt' ByteStream not correctly classified as 'text/plain'"
+
+        assert len(output["image/jpeg"]) == 1, "Failed to match 'image/jpeg' MIME type exactly"
+        assert jpg_stream in output["image/jpeg"], "'apple.jpg' ByteStream not correctly classified as 'image/jpeg'"
+
+        assert len(output.get("unclassified")) == 1, "Failed to handle unclassified file types"
+        assert mp3_stream in output["unclassified"], "'sound.mp3' ByteStream should be unclassified but is not"