Repository: deepset-ai/haystack

Issue:
Preprocessor sentence tokenization language.
**Is your feature request related to a problem? Please describe.**
When using the Preprocessor's sentence splitting feature ([here](https://github.com/deepset-ai/haystack/blob/f46b09c756ff6241648cb8ea6138d1f6544b38b1/haystack/preprocessor/preprocessor.py#L177) and [here](https://github.com/deepset-ai/haystack/blob/master/haystack/preprocessor/preprocessor.py#L218)) the language is not passed to the [`sent_tokenize`](https://github.com/nltk/nltk/blob/33ac28cfcad288c611c5901faa9ebc799e7c95dd/nltk/tokenize/__init__.py#L97) call. It could be useful when working with different languages.

**Describe the solution you'd like**
How about passing the language information at Preprocessor's initialization time? It sounds like a fairly easy solution to me.
What do you think?



Discussion:
Yes, this makes total sense. Would you be interested in raising a PR?
hi @tholor, yep, I'll raise a PR soon.

Resolving patch:
diff --git a/haystack/preprocessor/preprocessor.py b/haystack/preprocessor/preprocessor.py
--- a/haystack/preprocessor/preprocessor.py
+++ b/haystack/preprocessor/preprocessor.py
@@ -13,6 +13,28 @@
 logger = logging.getLogger(__name__)
 
 
+iso639_to_nltk = {
+    "ru": "russian",
+    "sl": "slovene",
+    "es": "spanish",
+    "sv": "swedish",
+    "tr": "turkish",
+    "cs": "czech",
+    "da": "danish",
+    "nl": "dutch",
+    "en": "english",
+    "et": "estonian",
+    "fi": "finnish",
+    "fr": "french",
+    "de": "german",
+    "el": "greek",
+    "it": "italian",
+    "no": "norwegian",
+    "pl": "polish",
+    "pt": "portuguese",
+}
+
+
 class PreProcessor(BasePreProcessor):
     def __init__(
         self,
@@ -23,6 +45,7 @@ def __init__(
         split_length: int = 1000,
         split_overlap: int = 0,
         split_respect_sentence_boundary: bool = True,
+        language: str = "en",
     ):
         """
         :param clean_header_footer: Use heuristic to remove footers and headers across different pages by searching
@@ -43,6 +66,7 @@ def __init__(
         :param split_respect_sentence_boundary: Whether to split in partial sentences if split_by -> `word`. If set
                                                 to True, the individual split will always have complete sentences &
                                                 the number of words will be <= split_length.
+        :param language: The language used by "nltk.tokenize.sent_tokenize" in iso639 format. Available options: "en", "es", "de", "fr" & many more.
         """
 
         # save init parameters to enable export of component config as YAML
@@ -56,7 +80,7 @@ def __init__(
             nltk.data.find('tokenizers/punkt')
         except LookupError:
             nltk.download('punkt')
-            
+
         self.clean_whitespace = clean_whitespace
         self.clean_header_footer = clean_header_footer
         self.clean_empty_lines = clean_empty_lines
@@ -64,6 +88,7 @@ def __init__(
         self.split_length = split_length
         self.split_overlap = split_overlap
         self.split_respect_sentence_boundary = split_respect_sentence_boundary
+        self.language = iso639_to_nltk.get(language, language)
 
     def process(
         self,
@@ -166,7 +191,7 @@ def split(
 
         if split_respect_sentence_boundary and split_by == "word":
             # split by words ensuring no sub sentence splits
-            sentences = nltk.tokenize.sent_tokenize(text)
+            sentences = nltk.tokenize.sent_tokenize(text, language=self.language)
             word_count = 0
             list_splits = []
             current_slice: List[str] = []
@@ -207,7 +232,7 @@ def split(
             if split_by == "passage":
                 elements = text.split("\n\n")
             elif split_by == "sentence":
-                elements = nltk.tokenize.sent_tokenize(text)
+                elements = nltk.tokenize.sent_tokenize(text, language=self.language)
             elif split_by == "word":
                 elements = text.split(" ")
             else:


Test patch: