Repository: deepset-ai/haystack

Issue:
Bug: Load PreProcessor with `split_by: None` from config fails
### The issue
When trying to load the PreProcessor node with `split_by: None` from config fails.

### Reproduce the issue
To reproduce the issue, execute this script with the latest Haystack version and python 3.8:

```python
from haystack import Pipeline
from haystack.nodes.preprocessor import PreProcessor

processor = PreProcessor(
    split_by=None,
)

pipeline = Pipeline()
pipeline.add_node(processor, name="Preprocessor", inputs=["File"])

config = pipeline.get_config()
pipeline = Pipeline.load_from_config(config, pipeline_name="indexing")

```

### The error

Where it fails:
https://github.com/deepset-ai/haystack/blob/7290196c32abf44e269caa03e1d2178c77992b0e/haystack/pipelines/config.py#L307-L331

The actual error:

```shell
On instance['components'][0]:
    {'name': 'Preprocessor',
     'params': {'split_by': None},
     'type': 'PreProcessor'}
Traceback (most recent call last):
  File "/home/paperspace/repositories/smart-search/env/lib/python3.8/site-packages/haystack/pipelines/config.py", line 338, in validate_schema
    Draft7Validator(schema).validate(instance=pipeline_config)
  File "/home/paperspace/repositories/smart-search/env/lib/python3.8/site-packages/jsonschema/validators.py", line 304, in validate
    raise error
jsonschema.exceptions.ValidationError: {'name': 'Preprocessor', 'type': 'PreProcessor', 'params': {'split_by': None}} is not valid under any of the given schemas

On instance['components'][0]:
    {'name': 'Preprocessor',
     'params': {'split_by': None},
     'type': 'PreProcessor'}

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "haystack_bug_test.py", line 12, in <module>
    pipeline = Pipeline.load_from_config(config, pipeline_name="indexing")
  File "/home/paperspace/repositories/smart-search/env/lib/python3.8/site-packages/haystack/pipelines/base.py", line 1913, in load_from_config
    validate_config(pipeline_config, strict_version_check=strict_version_check)
  File "/home/paperspace/repositories/smart-search/env/lib/python3.8/site-packages/haystack/pipelines/config.py", line 259, in validate_config
    validate_schema(
  File "/home/paperspace/repositories/smart-search/env/lib/python3.8/site-packages/haystack/pipelines/config.py", line 366, in validate_schema
    raise PipelineSchemaError(
haystack.errors.PipelineSchemaError: Node of type PreProcessor found, but it failed validation. Possible causes:
 - The node is missing some mandatory parameter
 - Wrong indentation of some parameter in YAML
See the stacktrace for more information.
```


Discussion:
@ugm2 Thank you also for reporting this bug, Unai! The problem seems to be the type of the `split_by` parameter indicated here: https://github.com/deepset-ai/haystack/blob/main/haystack/nodes/preprocessor/preprocessor.py#L54 and the schema file generated based on that with the following line: https://github.com/deepset-ai/haystack/blob/7290196c32abf44e269caa03e1d2178c77992b0e/haystack/json-schemas/haystack-pipeline-main.schema.json#L4286

Resolving patch:
diff --git a/haystack/nodes/preprocessor/base.py b/haystack/nodes/preprocessor/base.py
--- a/haystack/nodes/preprocessor/base.py
+++ b/haystack/nodes/preprocessor/base.py
@@ -1,5 +1,10 @@
 from typing import List, Optional, Union
 
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
+
 from abc import abstractmethod
 
 from haystack.nodes.base import BaseComponent
@@ -17,7 +22,7 @@ def process(
         clean_header_footer: Optional[bool] = False,
         clean_empty_lines: Optional[bool] = True,
         remove_substrings: List[str] = [],
-        split_by: Optional[str] = "word",
+        split_by: Literal["word", "sentence", "passage", None] = "word",
         split_length: Optional[int] = 1000,
         split_overlap: Optional[int] = None,
         split_respect_sentence_boundary: Optional[bool] = True,
@@ -44,7 +49,7 @@ def clean(
     def split(
         self,
         document: Union[dict, Document],
-        split_by: str,
+        split_by: Literal["word", "sentence", "passage", None],
         split_length: int,
         split_overlap: int,
         split_respect_sentence_boundary: bool,
@@ -57,7 +62,7 @@ def run(  # type: ignore
         clean_whitespace: Optional[bool] = None,
         clean_header_footer: Optional[bool] = None,
         clean_empty_lines: Optional[bool] = None,
-        split_by: Optional[str] = None,
+        split_by: Literal["word", "sentence", "passage", None] = None,
         split_length: Optional[int] = None,
         split_overlap: Optional[int] = None,
         split_respect_sentence_boundary: Optional[bool] = None,
@@ -83,7 +88,7 @@ def run_batch(  # type: ignore
         clean_whitespace: Optional[bool] = None,
         clean_header_footer: Optional[bool] = None,
         clean_empty_lines: Optional[bool] = None,
-        split_by: Optional[str] = None,
+        split_by: Literal["word", "sentence", "passage", None] = None,
         split_length: Optional[int] = None,
         split_overlap: Optional[int] = None,
         split_respect_sentence_boundary: Optional[bool] = None,
diff --git a/haystack/nodes/preprocessor/preprocessor.py b/haystack/nodes/preprocessor/preprocessor.py
--- a/haystack/nodes/preprocessor/preprocessor.py
+++ b/haystack/nodes/preprocessor/preprocessor.py
@@ -4,6 +4,11 @@
 from functools import partial, reduce
 from itertools import chain
 from typing import List, Optional, Generator, Set, Union
+
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
 import warnings
 from pathlib import Path
 from pickle import UnpicklingError
@@ -50,7 +55,7 @@ def __init__(
         clean_header_footer: bool = False,
         clean_empty_lines: bool = True,
         remove_substrings: List[str] = [],
-        split_by: str = "word",
+        split_by: Literal["word", "sentence", "passage", None] = "word",
         split_length: int = 200,
         split_overlap: int = 0,
         split_respect_sentence_boundary: bool = True,
@@ -122,7 +127,7 @@ def process(
         clean_header_footer: Optional[bool] = None,
         clean_empty_lines: Optional[bool] = None,
         remove_substrings: List[str] = [],
-        split_by: Optional[str] = None,
+        split_by: Literal["word", "sentence", "passage", None] = None,
         split_length: Optional[int] = None,
         split_overlap: Optional[int] = None,
         split_respect_sentence_boundary: Optional[bool] = None,
@@ -170,7 +175,7 @@ def _process_single(
         clean_header_footer: Optional[bool] = None,
         clean_empty_lines: Optional[bool] = None,
         remove_substrings: List[str] = [],
-        split_by: Optional[str] = None,
+        split_by: Literal["word", "sentence", "passage", None] = None,
         split_length: Optional[int] = None,
         split_overlap: Optional[int] = None,
         split_respect_sentence_boundary: Optional[bool] = None,
@@ -285,7 +290,7 @@ def clean(
     def split(
         self,
         document: Union[dict, Document],
-        split_by: str,
+        split_by: Literal["word", "sentence", "passage", None],
         split_length: int,
         split_overlap: int,
         split_respect_sentence_boundary: bool,


Test patch: