Repository: RasaHQ/rasa
Pull request: Add `split_entities_config` to TEDPolicy
**Proposed changes**:
Closes #7707

**Status (please check what you already did)**:
- [X] added some tests for the functionality
- [x] updated the documentation
- [x] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/master/changelog) for instructions)
- [X] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)


Problem statement:
Add `split_entities_config` to `TEDPolicy` for e2e

**Rasa version**:
2.2.2
**Python version**:
3.7.2
**Operating system** (windows, osx, ...):
OSX

**Issue**:

`TEDPolicy` needs a `split_entities_config` value, to mirror `CRFEntityExtractor` and `DIET`

When we call `convert_predictions_into_entities`, the value of split_entities_config is `None`, which is only a problem if we then call `EntityExtractor._check_is_single_entity`. I think this is somewhat rare, which is why the bug went unnoticed so far.

`TEDPolicy` should have its own value for this, with the same default as `DIET,` and this parameter should be configurable in the same way.

Hints from issue discussion:

Commit message: Fix typo in docs

Patch to review:
diff --git a/changelog/7707.bugfix.md b/changelog/7707.bugfix.md
new file mode 100644
index 000000000000..9d81c3ccd708
--- /dev/null
+++ b/changelog/7707.bugfix.md
@@ -0,0 +1,2 @@
+Add the option to configure whether extracted entities should be split by comma (`","`) or not to TEDPolicy. Fixes
+crash when this parameter is accessed during extraction.
\ No newline at end of file
diff --git a/docs/docs/policies.mdx b/docs/docs/policies.mdx
index 6dcd139907cf..e48a6ee28756 100644
--- a/docs/docs/policies.mdx
+++ b/docs/docs/policies.mdx
@@ -176,6 +176,24 @@ If you want to fine-tune your model, start by modifying the following parameters
   set `weight_sparsity` to 1 as this would result in all kernel weights being 0, i.e. the model is not able
   to learn.
 
+* `split_entities_by_comma`:
+  This parameter defines whether adjacent entities separated by a comma should be treated as one, or split. For example,
+  whether the entities with type ingredients "apple, banana" should be split into "apple" and "banana". Can either be
+  `True`/`False` globally:
+  ```yaml-rasa title="config.yml"
+  policies:
+    - name: TEDPolicy
+      split_entities_by_comma: True
+  ```
+  or set per entity type, such as:
+  ```yaml-rasa title="config.yml"
+  policies:
+    - name: TEDPolicy
+      split_entities_by_comma:
+        address: False
+        ingredients: True
+  ```
+
 The above configuration parameters are the ones you should configure to fit your model to your data.
 However, additional parameters exist that can be adapted.
 
diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py
index f06530b26a54..5991d5fa2c4a 100644
--- a/rasa/core/policies/ted_policy.py
+++ b/rasa/core/policies/ted_policy.py
@@ -32,6 +32,8 @@
     ENTITY_ATTRIBUTE_TYPE,
     ENTITY_TAGS,
     EXTRACTOR,
+    SPLIT_ENTITIES_BY_COMMA,
+    SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
 )
 from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter
 from rasa.core.policies.policy import Policy, PolicyPrediction
@@ -272,6 +274,10 @@ class TEDPolicy(Policy):
         FEATURIZERS: [],
         # If set to true, entities are predicted in user utterances.
         ENTITY_RECOGNITION: True,
+        # Split entities by comma, this makes sense e.g. for a list of
+        # ingredients in a recipe, but it doesn't make sense for the parts of
+        # an address
+        SPLIT_ENTITIES_BY_COMMA: SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
     }
 
     @staticmethod
@@ -280,6 +286,16 @@ def _standard_featurizer(max_history: Optional[int] = None) -> TrackerFeaturizer
             SingleStateFeaturizer(), max_history=max_history
         )
 
+    def init_split_entities(self, split_entities_config):
+        """Initialise the behaviour for splitting entities by comma (or not)."""
+        if isinstance(split_entities_config, bool):
+            split_entities_config = {SPLIT_ENTITIES_BY_COMMA: split_entities_config}
+        else:
+            split_entities_config[SPLIT_ENTITIES_BY_COMMA] = self.defaults[
+                SPLIT_ENTITIES_BY_COMMA
+            ]
+        return split_entities_config
+
     def __init__(
         self,
         featurizer: Optional[TrackerFeaturizer] = None,
@@ -292,6 +308,10 @@ def __init__(
         **kwargs: Any,
     ) -> None:
         """Declare instance variables with default values."""
+        self.split_entities_config = self.init_split_entities(
+            kwargs.get(SPLIT_ENTITIES_BY_COMMA, SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE)
+        )
+
         if not featurizer:
             featurizer = self._standard_featurizer(max_history)
 
@@ -662,7 +682,11 @@ def _create_optional_event_for_entities(
         parsed_message = interpreter.featurize_message(Message(data={TEXT: text}))
         tokens = parsed_message.get(TOKENS_NAMES[TEXT])
         entities = EntityExtractor.convert_predictions_into_entities(
-            text, tokens, predicted_tags, confidences=confidence_values
+            text,
+            tokens,
+            predicted_tags,
+            self.split_entities_config,
+            confidences=confidence_values,
         )
 
         # add the extractor name
diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py
index 997eb5228352..9f6f635e91d8 100644
--- a/tests/core/test_policies.py
+++ b/tests/core/test_policies.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Type, List, Text, Tuple, Optional, Any
+from typing import Type, List, Text, Tuple, Optional, Any, Dict
 from unittest.mock import Mock, patch
 
 import numpy as np
@@ -18,7 +18,12 @@
 from rasa.shared.core.training_data.story_writer.markdown_story_writer import (
     MarkdownStoryWriter,
 )
-from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY
+from rasa.shared.nlu.constants import (
+    ACTION_NAME,
+    INTENT_NAME_KEY,
+    SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
+    SPLIT_ENTITIES_BY_COMMA,
+)
 from rasa.shared.core.constants import (
     USER_INTENT_RESTART,
     USER_INTENT_BACK,
@@ -556,6 +561,33 @@ async def test_gen_batch(self, trained_policy: TEDPolicy, default_domain: Domain
             or batch_slots_sentence_shape[1] == 0
         )
 
+    @pytest.mark.parametrize(
+        "split_entities_config, expected_initialized_config",
+        [
+            (
+                SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
+                {SPLIT_ENTITIES_BY_COMMA: SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE},
+            ),
+            (
+                {"address": False, "ingredients": True},
+                {
+                    "address": False,
+                    "ingredients": True,
+                    SPLIT_ENTITIES_BY_COMMA: SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
+                },
+            ),
+        ],
+    )
+    def test_init_split_entities_config(
+        self,
+        trained_policy: TEDPolicy,
+        split_entities_config: Any,
+        expected_initialized_config: Dict[(str, bool)],
+    ):
+        assert trained_policy.init_split_entities(
+            split_entities_config=split_entities_config
+        )
+
 
 class TestTEDPolicyMargin(TestTEDPolicy):
     def create_policy(


Code review comments:
diff_hunk: @@ -280,6 +286,16 @@ def _standard_featurizer(max_history: Optional[int] = None) -> TrackerFeaturizer
             SingleStateFeaturizer(), max_history=max_history
         )
 
+    def init_split_entities(self, split_entities_config):
path: rasa/core/policies/ted_policy.py
text: @user1:
is the same code used in DIET? could you please move it to `train_utils` to avoid code duplication