Repository: RasaHQ/rasa
Pull request: Remove deprecated tokenizers ConveRTTokenizer and LanguageModelTokenizer.
**Proposed changes**:
Remove deprecated `LanguageModelTokenizer` and `ConveRTTokenizer` classes.
There were two tests in `tests/nlu/featurizers/test_lm_featurizer.py` that tested these tokenizers and I have removed them as well.
Docs entries were removed as well. 

Also removing `HFTransformersNLP` class, tests, and docs entries.

**Status (please check what you already did)**:
- [X] removed tokenizer class definitions.
- [X] updated tests 
- [X] updated the documentation
- [X] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/main/changelog) for instructions)
- [X] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)


Problem statement:
3.0 Deprecation Warning: Remove deprecated tokenizers
**Background**
This warning was found in:
- rasa.nlu.tokenizers.convert_tokenizer:L27
- rasa.nlu.tokenizers.lm_tokenizer.py:L21
```
"f""'{self.__class__.__name__}' is deprecated and ""
f""will be removed in the future. ""
f""It is recommended to use the '{WhitespaceTokenizer.__name__}' or ""
f""another {Tokenizer.__name__} instead."","
```

**Definition of Done**
- [x] resolve warnings in 3.0 version
- [x] Removed the deprecated tokenizers from the codebase.
- [x] Added a changelog entry for 3.0 saying we've removed the previously deprecated tokenizers
- [x] Removed the references in the docs

Hints from issue discussion:
Are we actually removing these tokenisers in 3.0? 🤔

Commit message: Removed unnecessary imports.

Patch to review:
diff --git a/rasa/nlu/registry.py b/rasa/nlu/registry.py
index 4f0891ddda9f..c59af8574e52 100644
--- a/rasa/nlu/registry.py
+++ b/rasa/nlu/registry.py
@@ -33,12 +33,10 @@
 from rasa.nlu.featurizers.sparse_featurizer.regex_featurizer import RegexFeaturizer
 from rasa.nlu.model import Metadata
 from rasa.nlu.selectors.response_selector import ResponseSelector
-from rasa.nlu.tokenizers.convert_tokenizer import ConveRTTokenizer
 from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer
 from rasa.nlu.tokenizers.mitie_tokenizer import MitieTokenizer
 from rasa.nlu.tokenizers.spacy_tokenizer import SpacyTokenizer
 from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
-from rasa.nlu.tokenizers.lm_tokenizer import LanguageModelTokenizer
 from rasa.nlu.utils.mitie_utils import MitieNLP
 from rasa.nlu.utils.spacy_utils import SpacyNLP
 from rasa.nlu.utils.hugging_face.hf_transformers import HFTransformersNLP
@@ -66,9 +64,7 @@
     MitieTokenizer,
     SpacyTokenizer,
     WhitespaceTokenizer,
-    ConveRTTokenizer,
     JiebaTokenizer,
-    LanguageModelTokenizer,
     # extractors
     SpacyEntityExtractor,
     MitieEntityExtractor,
diff --git a/rasa/nlu/tokenizers/convert_tokenizer.py b/rasa/nlu/tokenizers/convert_tokenizer.py
deleted file mode 100644
index 369753791960..000000000000
--- a/rasa/nlu/tokenizers/convert_tokenizer.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from typing import Dict, Text, Any
-
-import rasa.shared.utils.io
-from rasa.nlu.tokenizers.tokenizer import Tokenizer
-from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
-
-
-class ConveRTTokenizer(WhitespaceTokenizer):
-    """This tokenizer is deprecated and will be removed in the future.
-
-    The ConveRTFeaturizer component now sets the sub-token information
-    for dense featurizable attributes of each message object.
-    """
-
-    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
-        """Initializes ConveRTTokenizer with the ConveRT model.
-
-        Args:
-            component_config: Configuration for the component.
-        """
-        super().__init__(component_config)
-        rasa.shared.utils.io.raise_warning(
-            f"'{self.__class__.__name__}' is deprecated and "
-            f"will be removed in the future. "
-            f"It is recommended to use the '{WhitespaceTokenizer.__name__}' or "
-            f"another {Tokenizer.__name__} instead.",
-            category=DeprecationWarning,
-        )
diff --git a/rasa/nlu/tokenizers/lm_tokenizer.py b/rasa/nlu/tokenizers/lm_tokenizer.py
deleted file mode 100644
index fbee73158ef1..000000000000
--- a/rasa/nlu/tokenizers/lm_tokenizer.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from typing import Dict, Text, Any
-
-import rasa.shared.utils.io
-from rasa.nlu.tokenizers.tokenizer import Tokenizer
-from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
-
-
-class LanguageModelTokenizer(WhitespaceTokenizer):
-    """This tokenizer is deprecated and will be removed in the future.
-
-    Use the LanguageModelFeaturizer with any other Tokenizer instead.
-    """
-
-    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
-        """Initializes LanguageModelTokenizer for tokenization.
-
-        Args:
-            component_config: Configuration for the component.
-        """
-        super().__init__(component_config)
-        rasa.shared.utils.io.raise_warning(
-            f"'{self.__class__.__name__}' is deprecated and "
-            f"will be removed in the future. "
-            f"It is recommended to use the '{WhitespaceTokenizer.__name__}' or "
-            f"another {Tokenizer.__name__} instead.",
-            category=DeprecationWarning,
-        )
diff --git a/tests/nlu/featurizers/test_lm_featurizer.py b/tests/nlu/featurizers/test_lm_featurizer.py
index e5d771e81017..452fbca01abd 100644
--- a/tests/nlu/featurizers/test_lm_featurizer.py
+++ b/tests/nlu/featurizers/test_lm_featurizer.py
@@ -10,16 +10,11 @@
 from rasa.nlu.constants import (
     TOKENS_NAMES,
     NUMBER_OF_SUB_TOKENS,
-    SEQUENCE_FEATURES,
-    SENTENCE_FEATURES,
-    LANGUAGE_MODEL_DOCS,
 )
-from rasa.nlu.tokenizers.lm_tokenizer import LanguageModelTokenizer
 from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.nlu.training_data.message import Message
 from rasa.nlu.featurizers.dense_featurizer.lm_featurizer import LanguageModelFeaturizer
-from rasa.nlu.utils.hugging_face.hf_transformers import HFTransformersNLP
 from rasa.shared.nlu.constants import TEXT, INTENT
 
 
@@ -655,52 +650,6 @@ def test_attention_mask(
     assert np.all(mask_zeros == 0)
 
 
-def test_log_deprecation_warning_with_old_config(caplog: LogCaptureFixture):
-    message = Message.build("hi there")
-
-    transformers_nlp = HFTransformersNLP(
-        {"model_name": "bert", "model_weights": "bert-base-uncased"}
-    )
-    transformers_nlp.process(message)
-
-    caplog.set_level(logging.DEBUG)
-    lm_tokenizer = LanguageModelTokenizer()
-    lm_tokenizer.process(message)
-    lm_featurizer = LanguageModelFeaturizer(skip_model_load=True)
-    caplog.clear()
-    with caplog.at_level(logging.DEBUG):
-        lm_featurizer.process(message)
-
-    assert "deprecated component HFTransformersNLP" in caplog.text
-
-
-def test_preserve_sentence_and_sequence_features_old_config():
-    attribute = TEXT
-    message = Message.build("hi there")
-
-    transformers_nlp = HFTransformersNLP(
-        {"model_name": "bert", "model_weights": "bert-base-uncased"}
-    )
-    transformers_nlp.process(message)
-    lm_tokenizer = LanguageModelTokenizer()
-    lm_tokenizer.process(message)
-
-    lm_featurizer = LanguageModelFeaturizer({"model_name": "gpt2"})
-    lm_featurizer.process(message)
-
-    message.set(LANGUAGE_MODEL_DOCS[attribute], None)
-    lm_docs = lm_featurizer._get_docs_for_batch(
-        [message], attribute=attribute, inference_mode=True
-    )[0]
-    hf_docs = transformers_nlp._get_docs_for_batch(
-        [message], attribute=attribute, inference_mode=True
-    )[0]
-    assert not (message.features[0].features == lm_docs[SEQUENCE_FEATURES]).any()
-    assert not (message.features[1].features == lm_docs[SENTENCE_FEATURES]).any()
-    assert (message.features[0].features == hf_docs[SEQUENCE_FEATURES]).all()
-    assert (message.features[1].features == hf_docs[SENTENCE_FEATURES]).all()
-
-
 @pytest.mark.parametrize(
     "text, tokens, expected_feature_tokens",
     [
diff --git a/tests/nlu/test_train.py b/tests/nlu/test_train.py
index 4569798e5260..57bbf007fede 100644
--- a/tests/nlu/test_train.py
+++ b/tests/nlu/test_train.py
@@ -73,8 +73,8 @@ def pipelines_for_tests() -> List[Tuple[Text, List[Dict[Text, Any]]]]:
         (
             "en",
             as_pipeline(
+                "WhitespaceTokenizer",
                 "HFTransformersNLP",
-                "LanguageModelTokenizer",
                 "LanguageModelFeaturizer",
                 "DIETClassifier",
             ),
@@ -129,7 +129,6 @@ def test_all_components_are_in_at_least_one_test_pipeline():
 
     for cls in registry.component_classes:
         if "convert" in cls.name.lower():
-            # TODO
             #   skip ConveRTTokenizer and ConveRTFeaturizer as the ConveRT model is not
             #   publicly available anymore
             #   (see https://github.com/RasaHQ/rasa/issues/6806)


Code review comments:
diff_hunk: @@ -129,7 +129,6 @@ def test_all_components_are_in_at_least_one_test_pipeline():
 
     for cls in registry.component_classes:
         if "convert" in cls.name.lower():
-            # TODO
path: tests/nlu/test_train.py
text: @user1:
this is still a TODO, isn't it?

@author:
thanks for catching that! Accidentally removed it. Its back now.