Repository: deepset-ai/haystack

Issue:
Remove frontmatter and add to `meta` in markdown converter
**Is your feature request related to a problem? Please describe.**
We would like to use the markdown converter for the `haystack-tutorials` search

**Describe the solution you'd like**
Frontmatter is often used to store some meta information of markdown files. So it would be a useful feature to be able to auto add these to the `meta` of the created documents
It's also a good idea to remove the fronmatter from the created documents too. 


Discussion:

Resolving patch:
diff --git a/haystack/nodes/file_converter/markdown.py b/haystack/nodes/file_converter/markdown.py
--- a/haystack/nodes/file_converter/markdown.py
+++ b/haystack/nodes/file_converter/markdown.py
@@ -4,6 +4,7 @@
 from typing import Dict, List, Optional, Tuple, Any
 
 try:
+    import frontmatter
     from bs4 import BeautifulSoup, NavigableString
     from markdown import markdown
 except (ImportError, ModuleNotFoundError) as ie:
@@ -27,6 +28,7 @@ def __init__(
         progress_bar: bool = True,
         remove_code_snippets: bool = True,
         extract_headlines: bool = False,
+        add_frontmatter_to_meta: bool = False,
     ):
         """
         :param remove_numeric_tables: Not applicable.
@@ -38,6 +40,7 @@ def __init__(
         :param progress_bar: Show a progress bar for the conversion.
         :param remove_code_snippets: Whether to remove snippets from the markdown file.
         :param extract_headlines: Whether to extract headings from the markdown file.
+        :param add_frontmatter_to_meta: Whether to add the contents of the frontmatter to `meta`.
         """
         super().__init__(
             remove_numeric_tables=remove_numeric_tables,
@@ -48,6 +51,7 @@ def __init__(
 
         self.remove_code_snippets = remove_code_snippets
         self.extract_headlines = extract_headlines
+        self.add_frontmatter_to_meta = add_frontmatter_to_meta
 
     def convert(
         self,
@@ -59,6 +63,7 @@ def convert(
         id_hash_keys: Optional[List[str]] = None,
         remove_code_snippets: Optional[bool] = None,
         extract_headlines: Optional[bool] = None,
+        add_frontmatter_to_meta: Optional[bool] = None,
     ) -> List[Document]:
         """
         Reads text from a markdown file and executes optional preprocessing steps.
@@ -74,14 +79,18 @@ def convert(
             In this case the id will be generated by using the content and the defined metadata.
         :param remove_code_snippets: Whether to remove snippets from the markdown file.
         :param extract_headlines: Whether to extract headings from the markdown file.
+        :param add_frontmatter_to_meta: Whether to add the contents of the frontmatter to `meta`.
         """
 
         id_hash_keys = id_hash_keys if id_hash_keys is not None else self.id_hash_keys
         remove_code_snippets = remove_code_snippets if remove_code_snippets is not None else self.remove_code_snippets
         extract_headlines = extract_headlines if extract_headlines is not None else self.extract_headlines
+        add_frontmatter_to_meta = (
+            add_frontmatter_to_meta if add_frontmatter_to_meta is not None else self.add_frontmatter_to_meta
+        )
 
         with open(file_path, encoding=encoding, errors="ignore") as f:
-            markdown_text = f.read()
+            metadata, markdown_text = frontmatter.parse(f.read())
 
         # md -> html -> text since BeautifulSoup can extract text cleanly
         html = markdown(markdown_text)
@@ -92,6 +101,12 @@ def convert(
             html = re.sub(r"<code>(.*?)</code>", " ", html, flags=re.DOTALL)
         soup = BeautifulSoup(html, "html.parser")
 
+        if add_frontmatter_to_meta:
+            if meta is None:
+                meta = metadata
+            else:
+                meta.update(metadata)
+
         if extract_headlines:
             text, headlines = self._extract_text_and_headlines(soup)
             if meta is None:


Test patch:
diff --git a/test/nodes/test_file_converter.py b/test/nodes/test_file_converter.py
--- a/test/nodes/test_file_converter.py
+++ b/test/nodes/test_file_converter.py
@@ -171,6 +171,13 @@ def test_markdown_converter_headline_extraction():
         assert extracted_headline["headline"] == document.content[start_idx : start_idx + hl_len]
 
 
+def test_markdown_converter_frontmatter_to_meta():
+    converter = MarkdownConverter(add_frontmatter_to_meta=True)
+    document = converter.convert(file_path=SAMPLES_PATH / "markdown" / "sample.md")[0]
+    assert document.meta["type"] == "intro"
+    assert document.meta["date"] == "1.1.2023"
+
+
 def test_azure_converter():
     # Check if Form Recognizer endpoint and credential key in environment variables
     if "AZURE_FORMRECOGNIZER_ENDPOINT" in os.environ and "AZURE_FORMRECOGNIZER_KEY" in os.environ:
diff --git a/test/samples/markdown/sample.md b/test/samples/markdown/sample.md
--- a/test/samples/markdown/sample.md
+++ b/test/samples/markdown/sample.md
@@ -1,3 +1,7 @@
+---
+type: intro
+date: 1.1.2023
+---
 ## What to build with Haystack
 
 - **Ask questions in natural language** and find granular answers in your own documents.