Repository: deepset-ai/haystack

Issue:
Add Crawler support for Indexing Pipelines
In the current implementation, `Crawler.crawl()` writes documents to JSON files and returns their paths.

Adding an option to return documents will enable connecting a Crawler in an Indexing Pipeline where crawled documents can be indexed directly to a document store.
Add Crawler support for Indexing Pipelines
In the current implementation, `Crawler.crawl()` writes documents to JSON files and returns their paths.

Adding an option to return documents will enable connecting a Crawler in an Indexing Pipeline where crawled documents can be indexed directly to a document store.


Discussion:

Resolving patch:
diff --git a/haystack/connector/crawler.py b/haystack/connector/crawler.py
--- a/haystack/connector/crawler.py
+++ b/haystack/connector/crawler.py
@@ -98,22 +98,19 @@ def crawl(self,  output_dir: Union[str, Path, None] = None,
         if not output_dir.exists():
             output_dir.mkdir(parents=True)
 
+        file_paths: list = []
         is_not_empty = len(list(output_dir.rglob("*"))) > 0
         if is_not_empty and not overwrite_existing_files:
             logger.info(
                 f"Found data stored in `{output_dir}`. Delete this first if you really want to fetch new data."
             )
-            return []
         else:
             logger.info(f"Fetching from {urls} to `{output_dir}`")
-
-            filepaths = []
-
             sub_links: Dict[str, List] = {}
 
             # don't go beyond the initial list of urls
             if crawler_depth == 0:
-                filepaths += self._write_to_files(urls, output_dir=output_dir)
+                file_paths += self._write_to_files(urls, output_dir=output_dir)
             # follow one level of sublinks
             elif crawler_depth == 1:
                 for url_ in urls:
@@ -121,9 +118,9 @@ def crawl(self,  output_dir: Union[str, Path, None] = None,
                     sub_links[url_] = list(self._extract_sublinks_from_url(base_url=url_, filter_urls=filter_urls,
                                                                      existed_links=existed_links))
                 for url in sub_links:
-                    filepaths += self._write_to_files(sub_links[url], output_dir=output_dir, base_url=url)
+                    file_paths += self._write_to_files(sub_links[url], output_dir=output_dir, base_url=url)
 
-            return filepaths
+        return file_paths
 
     def _write_to_files(self, urls: List[str], output_dir: Path, base_url: str = None) -> List[Path]:
         paths = []
@@ -148,9 +145,10 @@ def _write_to_files(self, urls: List[str], output_dir: Path, base_url: str = Non
 
         return paths
 
-    def run(self, output_dir: Union[str, Path, None] = None, urls: Optional[List[str]] = None, # type: ignore
-            crawler_depth: Optional[int] = None, filter_urls: Optional[List] = None, # type: ignore
-            overwrite_existing_files: Optional[bool] = None, **kwargs) -> Tuple[Dict, str]: # type: ignore
+    def run(self, output_dir: Union[str, Path, None] = None, urls: Optional[List[str]] = None,  # type: ignore
+            crawler_depth: Optional[int] = None, filter_urls: Optional[List] = None,  # type: ignore
+            overwrite_existing_files: Optional[bool] = None, return_documents: Optional[bool] = False,  # type: ignore
+            **kwargs) -> Tuple[Dict, str]:  # type: ignore
         """
         Method to be executed when the Crawler is used as a Node within a Haystack pipeline.
 
@@ -162,13 +160,18 @@ def run(self, output_dir: Union[str, Path, None] = None, urls: Optional[List[str
         :param filter_urls: Optional list of regular expressions that the crawled URLs must comply with.
                            All URLs not matching at least one of the regular expressions will be dropped.
         :param overwrite_existing_files: Whether to overwrite existing files in output_dir with new content
+        :param return_documents:  Return json files content
 
         :return: Tuple({"paths": List of filepaths, ...}, Name of output edge)
         """
 
-        filepaths = self.crawl(urls=urls, output_dir=output_dir, crawler_depth=crawler_depth, filter_urls=filter_urls,
-                               overwrite_existing_files=overwrite_existing_files)
-        results = {"paths": filepaths}
+        file_paths = self.crawl(urls=urls, output_dir=output_dir, crawler_depth=crawler_depth,
+                                  filter_urls=filter_urls, overwrite_existing_files=overwrite_existing_files)
+        if return_documents:
+            crawled_data = [self._read_json_file(_file) for _file in file_paths]
+            results = {"documents": crawled_data}
+        else:
+            results = {"paths": file_paths}
         results.update(**kwargs)
         return results, "output_1"
 
@@ -207,3 +210,8 @@ def _extract_sublinks_from_url(self, base_url: str,
                         sub_links.add(sub_link)
 
         return sub_links
+
+    def _read_json_file(self, file_path: Path):
+        """Read the json file and return the content"""
+        with open(file_path.absolute(), "r") as read_file:
+            return json.load(read_file)
diff --git a/haystack/pipeline.py b/haystack/pipeline.py
--- a/haystack/pipeline.py
+++ b/haystack/pipeline.py
@@ -34,6 +34,7 @@
 from haystack.translator.base import BaseTranslator
 from haystack.knowledge_graph.base import BaseKnowledgeGraph
 from haystack.graph_retriever.base import BaseGraphRetriever
+from haystack.connector import Crawler
 
 
 logger = logging.getLogger(__name__)


Test patch:
diff --git a/test/test_connector.py b/test/test_connector.py
--- a/test/test_connector.py
+++ b/test/test_connector.py
@@ -92,3 +92,22 @@ def test_crawler_content(tmp_path):
                 for partial_line in partial_content:
                     assert search(partial_line, content['text'])
                     assert partial_line in content['text']
+
+
+def test_crawler_return_document(tmp_path):
+    tmp_dir = tmp_path / "crawled_files"
+    _url = ["https://haystack.deepset.ai/docs/v0.5.0/intromd"]
+
+    crawler = Crawler(output_dir=tmp_dir)
+    docs_path = crawler.crawl(urls=_url, crawler_depth=1)
+    results, _ = crawler.run(urls=_url, crawler_depth=1, return_documents=True)
+    documents = results['documents']
+
+    for json_file, document in zip(docs_path, documents):
+        assert isinstance(json_file, Path)
+        assert isinstance(document, dict)
+
+        with open(json_file.absolute(), "r") as read_file:
+            file_content = json.load(read_file)
+            assert file_content['meta'] == document['meta']
+            assert file_content['text'] == document['text']