Repository: RasaHQ/rasa
Pull request: Mark tests that train a model.
**Proposed changes**:
This PR marks all tests that call either `nlu.train`, `core.train`, `Agent.train`, or `Trainer.train`. This can be directly, or indirectly e.g. through a fixture or http request.


Problem statement:
Annotate model training in unit tests
**Description of Problem**:
Currently our unit tests take a very long time to run. Part of the cause of this is superfluous or inefficient model training.

**Overview of the Solution**:
We would also like to annotate all tests that train a model.
Training can occur via many different paths. e.g. importing a certain fixture, or calling a training method. This will help us with future improvements and also allow the possibility for different CI runners or workflows.

**Definition of Done**:
Annotations on all the tests that train a model.

Hints from issue discussion:
Split into annotate, and report tickets.
Should we annotate methods that call a `Policy`/`Component`'s `train` method directly? Or only tests that call `nlu.train` or `core.train`?

- number of tests that call `nlu.train` or `core.train`: 159
- number of tests that call `Policy.train` or `Component.train`: 435


WDYT @wochinge ?
I used a method to track all the times `nlu.train` or `core.train` were called from a unit test: 
```
def write_test_name(filename='train_tests.json'):
    test_name = os.environ.get('PYTEST_CURRENT_TEST').split(' ')[0].split('[')[0]
    try:
        tests_names = rasa.shared.utils.io.read_json_file(filename)
    except:
        tests_names = []
    tests_names = list(set(tests_names).union({test_name}))
    rasa.shared.utils.io.dump_obj_as_json_to_file(filename, tests_names)
```
This method can called from any place in the code base, then if the whole test suite is run we can record which tests call it.

I've attached the file with the list of test names.

[train_tests.txt](https://github.com/RasaHQ/rasa/files/5903179/train_tests.txt)

I also wanted to see which tests caused an invocation of either `Policy.train` or `Component.train`. To avoid having to add `write_test_name` to every subclass I used a metaclass to call this whenever the train method of a subclass is called:
```
class RecordTrainingMetaClass(type):
    method_names = ["train"]

    @staticmethod
    def record_test_name(f):
        @wraps(f)
        def decorated(*args, **kwargs):
            rasa.utils.io.write_test_name(filename='train_tests_meta.json')
            return f(*args, **kwargs)

        return decorated

    def __new__(cls, classname, bases, classdict):
        for attr, item in classdict.items():
            if callable(item) and attr in cls.method_names:
                classdict[attr] = cls.record_test_name(item)  # replace method by wrapper

        return type.__new__(cls, classname, bases, classdict)
```

I've attached this list of test names:

[train_tests_meta.txt](https://github.com/RasaHQ/rasa/files/5903183/train_tests_meta.txt)

Commit message: Merge remote-tracking branch 'origin/main' into 7740-mark_tests_that_train

Patch to review:
diff --git a/tests/core/featurizers/test_single_state_featurizers.py b/tests/core/featurizers/test_single_state_featurizers.py
index 86c518b4a460..0935abee4cd7 100644
--- a/tests/core/featurizers/test_single_state_featurizers.py
+++ b/tests/core/featurizers/test_single_state_featurizers.py
@@ -182,6 +182,7 @@ def test_single_state_featurizer_creates_encoded_all_actions():
 
 
 @pytest.mark.timeout(300)  # these can take a longer time than the default timeout
+@pytest.mark.trains_model
 def test_single_state_featurizer_with_entity_roles_and_groups(
     unpacked_trained_moodbot_path: Text,
 ):
@@ -307,6 +308,7 @@ def test_single_state_featurizer_uses_dtype_float():
 
 
 @pytest.mark.timeout(300)  # these can take a longer time than the default timeout
+@pytest.mark.trains_model
 def test_single_state_featurizer_with_interpreter_state_with_action_listen(
     unpacked_trained_moodbot_path: Text,
 ):
@@ -371,6 +373,7 @@ def test_single_state_featurizer_with_interpreter_state_with_action_listen(
 
 
 @pytest.mark.timeout(300)  # these can take a longer time than the default timeout
+@pytest.mark.trains_model
 def test_single_state_featurizer_with_interpreter_state_not_with_action_listen(
     unpacked_trained_moodbot_path: Text,
 ):
@@ -408,6 +411,7 @@ def test_single_state_featurizer_with_interpreter_state_not_with_action_listen(
 
 
 @pytest.mark.timeout(300)  # these can take a longer time than the default timeout
+@pytest.mark.trains_model
 def test_single_state_featurizer_with_interpreter_state_with_no_action_name(
     unpacked_trained_moodbot_path: Text,
 ):
@@ -471,6 +475,7 @@ def test_to_sparse_sentence_features():
 
 
 @pytest.mark.timeout(300)  # these can take a longer time than the default timeout
+@pytest.mark.trains_model
 def test_single_state_featurizer_uses_regex_interpreter(
     unpacked_trained_moodbot_path: Text,
 ):
diff --git a/tests/core/policies/test_ted_policy.py b/tests/core/policies/test_ted_policy.py
index ea790d422127..1da072ae7366 100644
--- a/tests/core/policies/test_ted_policy.py
+++ b/tests/core/policies/test_ted_policy.py
@@ -69,6 +69,7 @@ def test_diagnostics():
 
 
 class TestTEDPolicy(PolicyTestCollection):
+    @pytest.mark.trains_model
     def test_train_model_checkpointing(self, tmp_path: Path):
         model_name = "core-checkpointed-model"
         best_model_file = tmp_path / (model_name + ".tar.gz")
diff --git a/tests/core/test_agent.py b/tests/core/test_agent.py
index 14ccc455e0e8..94a3e71aa3a0 100644
--- a/tests/core/test_agent.py
+++ b/tests/core/test_agent.py
@@ -74,6 +74,7 @@ async def test_training_data_is_reproducible():
         assert str(x.as_dialogue()) == str(same_training_data[i].as_dialogue())
 
 
+@pytest.mark.trains_model
 async def test_agent_train(trained_moodbot_path: Text):
     moodbot_domain = Domain.load("examples/moodbot/domain.yml")
     loaded = Agent.load(trained_moodbot_path)
@@ -157,6 +158,7 @@ def test_agent_wrong_use_of_load():
         agent.load(training_data_file)
 
 
+@pytest.mark.trains_model
 async def test_agent_with_model_server_in_thread(
     model_server: TestClient, default_domain: Domain, unpacked_trained_rasa_model: Text
 ):
@@ -187,6 +189,7 @@ async def test_agent_with_model_server_in_thread(
     jobs.kill_scheduler()
 
 
+@pytest.mark.trains_model
 async def test_wait_time_between_pulls_without_interval(
     model_server: TestClient, monkeypatch: MonkeyPatch
 ):
@@ -203,6 +206,7 @@ async def test_wait_time_between_pulls_without_interval(
     await rasa.core.agent.load_from_server(agent, model_server=model_endpoint_config)
 
 
+@pytest.mark.trains_model
 async def test_pull_model_with_invalid_domain(
     model_server: TestClient, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture
 ):
@@ -369,6 +373,7 @@ def test_rule_policy_valid(domain: Dict[Text, Any], policy_config: Dict[Text, An
     )
 
 
+@pytest.mark.trains_model
 async def test_agent_update_model_none_domain(trained_rasa_model: Text):
     agent = await load_agent(model_path=trained_rasa_model)
     agent.update_model(
@@ -385,6 +390,7 @@ async def test_agent_update_model_none_domain(trained_rasa_model: Text):
     assert tracker.events[3].intent["name"] == "greet"
 
 
+@pytest.mark.trains_model
 async def test_load_agent_on_not_existing_path():
     agent = await load_agent(model_path="some-random-path")
 
diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py
index 6c24b9bc75f5..5f3ca6a97511 100644
--- a/tests/core/test_evaluation.py
+++ b/tests/core/test_evaluation.py
@@ -107,6 +107,7 @@ async def test_end_to_end_evaluation_script_unknown_entity(default_agent: Agent)
     assert num_stories == 1
 
 
+@pytest.mark.trains_model
 async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
     generator = await _create_data_generator(
         "data/test_evaluations/form-end-to-end-stories.md", form_bot_agent, use_e2e=True
@@ -252,6 +253,7 @@ def test_event_has_proper_implementation(
         ("data/test_yaml_stories/test_base_retrieval_intent_story.yml"),
     ],
 )
+@pytest.mark.trains_model
 async def test_retrieval_intent(response_selector_agent: Agent, test_file: Text):
     generator = await _create_data_generator(
         test_file, response_selector_agent, use_e2e=True,
@@ -272,6 +274,7 @@ async def test_retrieval_intent(response_selector_agent: Agent, test_file: Text)
         ("data/test_yaml_stories/test_base_retrieval_intent_wrong_prediction.yml"),
     ],
 )
+@pytest.mark.trains_model
 async def test_retrieval_intent_wrong_prediction(
     tmpdir: Path, response_selector_agent: Agent, test_file: Text
 ):
diff --git a/tests/core/test_examples.py b/tests/core/test_examples.py
index d1c7c393eb98..e998fceb42ce 100644
--- a/tests/core/test_examples.py
+++ b/tests/core/test_examples.py
@@ -8,6 +8,7 @@
 from rasa.utils.endpoints import ClientResponseError
 
 
+@pytest.mark.trains_model
 async def test_moodbot_example(unpacked_trained_moodbot_path: Text):
     agent = Agent.load(unpacked_trained_moodbot_path)
 
@@ -22,6 +23,7 @@ async def test_moodbot_example(unpacked_trained_moodbot_path: Text):
 
 
 @pytest.mark.timeout(300)
+@pytest.mark.trains_model
 async def test_formbot_example(form_bot_agent: Agent):
     def response_for_slot(slot: Text) -> Dict[Text, Any]:
         if slot:
diff --git a/tests/core/test_nlg.py b/tests/core/test_nlg.py
index ae783142c3e2..373b32b8f9da 100644
--- a/tests/core/test_nlg.py
+++ b/tests/core/test_nlg.py
@@ -43,6 +43,7 @@ def http_nlg(loop, sanic_client):
     return loop.run_until_complete(sanic_client(nlg_app()))
 
 
+@pytest.mark.trains_model
 async def test_nlg(http_nlg, trained_rasa_model):
     sender = str(uuid.uuid1())
 
diff --git a/tests/core/test_restore.py b/tests/core/test_restore.py
index 63f82d3c5d6f..0c52f58f3b19 100644
--- a/tests/core/test_restore.py
+++ b/tests/core/test_restore.py
@@ -1,10 +1,13 @@
 from typing import Text
 
+import pytest
+
 from rasa.core import restore
 from rasa.core.agent import Agent
 from rasa.shared.nlu.constants import ACTION_NAME
 
 
+@pytest.mark.trains_model
 async def test_restoring_tracker(trained_moodbot_path: Text, recwarn):
     tracker_dump = "data/test_trackers/tracker_moodbot.json"
 
diff --git a/tests/core/test_run.py b/tests/core/test_run.py
index 0d9a73a44e30..7568afff6842 100644
--- a/tests/core/test_run.py
+++ b/tests/core/test_run.py
@@ -54,6 +54,7 @@ def test_create_single_input_channels_by_class_wo_credentials():
     assert channels[0].name() == "rest"
 
 
+@pytest.mark.trains_model
 async def test_load_agent_on_start_with_good_model_file(
     trained_rasa_model: Text, rasa_server: Sanic, loop: AbstractEventLoop
 ):
@@ -66,6 +67,7 @@ async def test_load_agent_on_start_with_good_model_file(
     assert isinstance(agent.domain, rasa.shared.core.domain.Domain)
 
 
+@pytest.mark.trains_model
 async def test_load_agent_on_start_with_bad_model_file(
     tmp_path: Path, rasa_server: Sanic, loop: AbstractEventLoop
 ):
diff --git a/tests/core/test_training.py b/tests/core/test_training.py
index ca3b7de32106..6e12da18e467 100644
--- a/tests/core/test_training.py
+++ b/tests/core/test_training.py
@@ -68,6 +68,7 @@ async def test_story_visualization_with_merging(
     assert 20 < len(generated_graph.edges()) < 33
 
 
+@pytest.mark.trains_model
 async def test_training_script(tmp_path: Path):
     await train(
         DEFAULT_DOMAIN_PATH_WITH_SLOTS,
@@ -80,6 +81,7 @@ async def test_training_script(tmp_path: Path):
     assert True
 
 
+@pytest.mark.trains_model
 async def test_training_script_without_max_history_set(tmp_path: Path):
     tmpdir = str(tmp_path)
     await train(
@@ -102,6 +104,7 @@ async def test_training_script_without_max_history_set(tmp_path: Path):
                 assert policy.featurizer.max_history is None
 
 
+@pytest.mark.trains_model
 async def test_training_script_with_max_history_set(tmp_path: Path):
     tmpdir = str(tmp_path)
 
@@ -129,6 +132,7 @@ async def test_training_script_with_max_history_set(tmp_path: Path):
         "data/test_yaml_stories/stories_restart.yml",
     ],
 )
+@pytest.mark.trains_model
 async def test_training_script_with_restart_stories(stories_file: Text, tmp_path: Path):
     await train(
         DEFAULT_DOMAIN_PATH_WITH_SLOTS,
@@ -147,6 +151,7 @@ def configs_for_random_seed_test() -> List[Text]:
 
 
 @pytest.mark.parametrize("config_file", configs_for_random_seed_test())
+@pytest.mark.trains_model
 async def test_random_seed(tmp_path: Path, config_file: Text):
     # set random seed in config file to
     # generate a reproducible training result
@@ -177,6 +182,7 @@ async def test_random_seed(tmp_path: Path, config_file: Text):
     assert probs_1["confidence"] == probs_2["confidence"]
 
 
+@pytest.mark.trains_model
 async def test_trained_interpreter_passed_to_policies(
     tmp_path: Path, monkeypatch: MonkeyPatch
 ):
diff --git a/tests/nlu/classifiers/test_diet_classifier.py b/tests/nlu/classifiers/test_diet_classifier.py
index bb56ee931347..afca14a838c9 100644
--- a/tests/nlu/classifiers/test_diet_classifier.py
+++ b/tests/nlu/classifiers/test_diet_classifier.py
@@ -195,6 +195,7 @@ async def _train_persist_load_with_different_settings(
 
 
 @pytest.mark.skip_on_windows
+@pytest.mark.trains_model
 async def test_train_persist_load_with_different_settings_non_windows(
     component_builder: ComponentBuilder, tmp_path: Path
 ):
@@ -215,6 +216,7 @@ async def test_train_persist_load_with_different_settings_non_windows(
     )
 
 
+@pytest.mark.trains_model
 async def test_train_persist_load_with_different_settings(component_builder, tmpdir):
     pipeline = [
         {"name": "WhitespaceTokenizer"},
@@ -229,6 +231,7 @@ async def test_train_persist_load_with_different_settings(component_builder, tmp
     )
 
 
+@pytest.mark.trains_model
 async def test_train_persist_load_with_only_entity_recognition(
     component_builder, tmpdir
 ):
@@ -250,6 +253,7 @@ async def test_train_persist_load_with_only_entity_recognition(
     )
 
 
+@pytest.mark.trains_model
 async def test_train_persist_load_with_only_intent_classification(
     component_builder, tmpdir
 ):
@@ -271,6 +275,7 @@ async def test_train_persist_load_with_only_intent_classification(
     )
 
 
+@pytest.mark.trains_model
 async def test_raise_error_on_incorrect_pipeline(component_builder, tmp_path: Path):
     _config = RasaNLUModelConfig(
         {
@@ -332,6 +337,7 @@ def as_pipeline(*components):
         ),  # less intents than default ranking_length
     ],
 )
+@pytest.mark.trains_model
 async def test_softmax_normalization(
     component_builder,
     tmp_path,
@@ -371,6 +377,7 @@ async def test_softmax_normalization(
     "classifier_params, output_length",
     [({LOSS_TYPE: "margin", RANDOM_SEED: 42, EPOCHS: 1}, LABEL_RANKING_LENGTH)],
 )
+@pytest.mark.trains_model
 async def test_margin_loss_is_not_normalized(
     monkeypatch, component_builder, tmpdir, classifier_params, output_length
 ):
@@ -405,6 +412,7 @@ async def test_margin_loss_is_not_normalized(
     assert parse_data.get("intent") == intent_ranking[0]
 
 
+@pytest.mark.trains_model
 async def test_set_random_seed(component_builder, tmpdir):
     """test if train result is the same for two runs of tf embedding"""
 
@@ -443,6 +451,7 @@ async def test_set_random_seed(component_builder, tmpdir):
     assert result_a == result_b
 
 
+@pytest.mark.trains_model
 async def test_train_tensorboard_logging(component_builder, tmpdir):
     from pathlib import Path
 
@@ -481,6 +490,7 @@ async def test_train_tensorboard_logging(component_builder, tmpdir):
     assert len(all_files) == 3
 
 
+@pytest.mark.trains_model
 async def test_train_model_checkpointing(
     component_builder: ComponentBuilder, tmpdir: Path
 ):
@@ -533,6 +543,7 @@ async def test_train_model_checkpointing(
         {RANDOM_SEED: 1, EPOCHS: 1, BILOU_FLAG: True},
     ],
 )
+@pytest.mark.trains_model
 async def test_train_persist_load_with_composite_entities(
     classifier_params, component_builder, tmpdir
 ):
diff --git a/tests/nlu/extractors/test_crf_entity_extractor.py b/tests/nlu/extractors/test_crf_entity_extractor.py
index 6aee1f114938..c218c94467f4 100644
--- a/tests/nlu/extractors/test_crf_entity_extractor.py
+++ b/tests/nlu/extractors/test_crf_entity_extractor.py
@@ -20,6 +20,7 @@ def pipeline_from_components(*components: Text) -> List[Dict[Text, Text]]:
     return [{"name": c} for c in components]
 
 
+@pytest.mark.trains_model
 async def test_train_persist_load_with_composite_entities(
     component_builder: ComponentBuilder, tmp_path: Path
 ):
@@ -87,6 +88,7 @@ async def test_train_persist_load_with_composite_entities(
         ),
     ],
 )
+@pytest.mark.trains_model
 async def test_train_persist_with_different_configurations(
     config_params: Dict[Text, Any], component_builder: ComponentBuilder, tmp_path: Path
 ):
diff --git a/tests/nlu/selectors/test_selectors.py b/tests/nlu/selectors/test_selectors.py
index 6d5c4aabea9d..43771ea28864 100644
--- a/tests/nlu/selectors/test_selectors.py
+++ b/tests/nlu/selectors/test_selectors.py
@@ -198,6 +198,7 @@ def test_resolve_intent_response_key_from_label(
     )
 
 
+@pytest.mark.trains_model
 async def test_train_model_checkpointing(
     component_builder: ComponentBuilder, tmpdir: Path
 ):
@@ -274,6 +275,7 @@ async def _train_persist_load_with_different_settings(
 
 
 @pytest.mark.skip_on_windows
+@pytest.mark.trains_model
 async def test_train_persist_load(component_builder: ComponentBuilder, tmpdir: Path):
     pipeline = [
         {"name": "WhitespaceTokenizer"},
diff --git a/tests/nlu/test_components.py b/tests/nlu/test_components.py
index c3098e5df3a3..cde32dd8dc71 100644
--- a/tests/nlu/test_components.py
+++ b/tests/nlu/test_components.py
@@ -93,6 +93,7 @@ def test_builder_load_unknown(component_builder: ComponentBuilder):
     assert "Cannot find class" in str(excinfo.value)
 
 
+@pytest.mark.trains_model
 async def test_example_component(component_builder: ComponentBuilder, tmp_path: Path):
     _config = RasaNLUModelConfig(
         {"pipeline": [{"name": "tests.nlu.example_component.MyComponent"}]}
@@ -190,6 +191,7 @@ def test_can_handle_language_guard_clause(
     assert expected_exec_msg in str(excinfo.value)
 
 
+@pytest.mark.trains_model
 async def test_validate_requirements_raises_exception_on_component_without_name(
     tmp_path: Path,
 ):
@@ -204,6 +206,7 @@ async def test_validate_requirements_raises_exception_on_component_without_name(
         )
 
 
+@pytest.mark.trains_model
 async def test_validate_component_keys_raises_warning_on_invalid_key(tmp_path: Path,):
     _config = RasaNLUModelConfig(
         # config with a component that does not have a `confidence_threshold ` property
diff --git a/tests/nlu/test_evaluation.py b/tests/nlu/test_evaluation.py
index ef7fe33844a4..1d9cf99da1d9 100644
--- a/tests/nlu/test_evaluation.py
+++ b/tests/nlu/test_evaluation.py
@@ -357,6 +357,7 @@ def test_drop_intents_below_freq():
     assert clean_td.intents == {"affirm", "restaurant_search"}
 
 
+@pytest.mark.trains_model
 async def test_run_evaluation(unpacked_trained_moodbot_path: Text):
     result = await run_evaluation(
         DEFAULT_DATA_PATH,
@@ -369,6 +370,7 @@ async def test_run_evaluation(unpacked_trained_moodbot_path: Text):
     assert result.get("intent_evaluation")
 
 
+@pytest.mark.trains_model
 async def test_eval_data(
     component_builder: ComponentBuilder, tmp_path: Path, project: Text
 ):
@@ -919,6 +921,7 @@ def test_label_replacement():
     assert substitute_labels(original_labels, "O", "no_entity") == target_labels
 
 
+@pytest.mark.trains_model
 async def test_nlu_comparison(tmp_path: Path):
     config = {
         "language": "en",
diff --git a/tests/nlu/test_interpreter.py b/tests/nlu/test_interpreter.py
index a9f5da769793..cfff7075a01a 100644
--- a/tests/nlu/test_interpreter.py
+++ b/tests/nlu/test_interpreter.py
@@ -60,6 +60,7 @@ def test_model_is_compatible(metadata):
         {"obj": "not-existing", "endpoint": None, "type": RegexInterpreter},
     ],
 )
+@pytest.mark.trains_model
 def test_create_interpreter(parameters, trained_nlu_model):
     obj = parameters["obj"]
     if obj == "trained_nlu_model":
diff --git a/tests/nlu/test_persistor.py b/tests/nlu/test_persistor.py
index 22677114c05e..56b363c174cb 100644
--- a/tests/nlu/test_persistor.py
+++ b/tests/nlu/test_persistor.py
@@ -12,8 +12,10 @@ class Object:
     pass
 
 
-# noinspection PyPep8Naming
-async def test_list_method_method_in_AWS_persistor(component_builder, tmp_path):
+@pytest.mark.trains_model
+async def test_list_method_method_in_AWS_persistor(  # noinspection PyPep8Naming
+    component_builder, tmp_path
+):
     with mock_s3():
         # artificially create a persisted model
         _config = RasaNLUModelConfig(
diff --git a/tests/nlu/test_train.py b/tests/nlu/test_train.py
index 459a93933950..305ec8154c03 100644
--- a/tests/nlu/test_train.py
+++ b/tests/nlu/test_train.py
@@ -134,6 +134,7 @@ def test_all_components_are_in_at_least_one_test_pipeline():
 
 @pytest.mark.timeout(600)
 @pytest.mark.parametrize("language, pipeline", pipelines_for_tests())
+@pytest.mark.trains_model
 async def test_train_persist_load_parse(language, pipeline, component_builder, tmpdir):
     _config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
 
@@ -155,6 +156,7 @@ async def test_train_persist_load_parse(language, pipeline, component_builder, t
 @pytest.mark.timeout(600)
 @pytest.mark.parametrize("language, pipeline", pipelines_for_non_windows_tests())
 @pytest.mark.skip_on_windows
+@pytest.mark.trains_model
 async def test_train_persist_load_parse_non_windows(
     language, pipeline, component_builder, tmpdir
 ):
@@ -207,6 +209,7 @@ def test_load_and_persist_without_train_non_windows(
     test_load_and_persist_without_train(language, pipeline, component_builder, tmpdir)
 
 
+@pytest.mark.trains_model
 async def test_train_model_empty_pipeline(component_builder):
     _config = RasaNLUModelConfig({"pipeline": None, "language": "en"})
 
@@ -216,6 +219,7 @@ async def test_train_model_empty_pipeline(component_builder):
         )
 
 
+@pytest.mark.trains_model
 async def test_train_named_model(component_builder, tmpdir):
     _config = RasaNLUModelConfig(
         {"pipeline": [{"name": "KeywordIntentClassifier"}], "language": "en"}
@@ -235,6 +239,7 @@ async def test_train_named_model(component_builder, tmpdir):
     assert normalized_path == tmpdir.strpath
 
 
+@pytest.mark.trains_model
 async def test_handles_pipeline_with_non_existing_component(
     component_builder, pretrained_embeddings_spacy_config
 ):
@@ -249,6 +254,7 @@ async def test_handles_pipeline_with_non_existing_component(
     assert "Cannot find class" in str(execinfo.value)
 
 
+@pytest.mark.trains_model
 async def test_train_model_training_data_persisted(component_builder, tmpdir):
     _config = RasaNLUModelConfig(
         {"pipeline": [{"name": "KeywordIntentClassifier"}], "language": "en"}
@@ -270,6 +276,7 @@ async def test_train_model_training_data_persisted(component_builder, tmpdir):
     assert loaded.model_metadata.get("training_data") is not None
 
 
+@pytest.mark.trains_model
 async def test_train_model_no_training_data_persisted(component_builder, tmpdir):
     _config = RasaNLUModelConfig(
         {"pipeline": [{"name": "KeywordIntentClassifier"}], "language": "en"}
diff --git a/tests/shared/importers/test_multi_project.py b/tests/shared/importers/test_multi_project.py
index 0565980c3e8d..eaa85837d80a 100644
--- a/tests/shared/importers/test_multi_project.py
+++ b/tests/shared/importers/test_multi_project.py
@@ -319,6 +319,7 @@ def test_single_additional_file(tmp_path: Path):
     assert selector.is_imported(str(additional_file))
 
 
+@pytest.mark.trains_model
 async def test_multi_project_training(trained_async):
     example_directory = "data/test_multi_domain"
     config_file = os.path.join(example_directory, "config.yml")
diff --git a/tests/test_model.py b/tests/test_model.py
index 52e97902cf95..c585b8e4f2fd 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -55,6 +55,7 @@
 from tests.core.conftest import DEFAULT_DOMAIN_PATH_WITH_MAPPING, DEFAULT_STACK_CONFIG
 
 
+@pytest.mark.trains_model
 def test_get_latest_model(trained_rasa_model: str):
     path_of_latest = os.path.join(os.path.dirname(trained_rasa_model), "latest.tar.gz")
     shutil.copy(trained_rasa_model, path_of_latest)
@@ -64,6 +65,7 @@ def test_get_latest_model(trained_rasa_model: str):
     assert get_latest_model(model_directory) == path_of_latest
 
 
+@pytest.mark.trains_model
 def test_get_model_from_directory(trained_rasa_model: str):
     unpacked = get_model(trained_rasa_model)
 
@@ -71,6 +73,7 @@ def test_get_model_from_directory(trained_rasa_model: str):
     assert os.path.exists(os.path.join(unpacked, "nlu"))
 
 
+@pytest.mark.trains_model
 def test_get_model_context_manager(trained_rasa_model: str):
     with get_model(trained_rasa_model) as unpacked:
         assert os.path.exists(unpacked)
@@ -84,6 +87,7 @@ def test_get_model_exception(model_path: Optional[Text]):
         get_model(model_path)
 
 
+@pytest.mark.trains_model
 def test_get_model_from_directory_with_subdirectories(
     trained_rasa_model: Text, tmp_path: Path
 ):
@@ -97,6 +101,7 @@ def test_get_model_from_directory_with_subdirectories(
         get_model_subdirectories(str(tmp_path))  # temp path should be empty
 
 
+@pytest.mark.trains_model
 def test_get_model_from_directory_nlu_only(trained_rasa_model: Text):
     unpacked = get_model(trained_rasa_model)
     shutil.rmtree(os.path.join(unpacked, DEFAULT_CORE_SUBDIRECTORY_NAME))
@@ -387,6 +392,7 @@ async def test_fingerprinting_additional_action(project: Text):
 
 
 @pytest.mark.parametrize("use_fingerprint", [True, False])
+@pytest.mark.trains_model
 async def test_rasa_packaging(
     trained_rasa_model: Text, project: Text, use_fingerprint: bool, tmp_path: Path
 ):
@@ -467,6 +473,7 @@ async def test_rasa_packaging(
         },
     ],
 )
+@pytest.mark.trains_model
 def test_should_retrain(
     trained_rasa_model: Text, fingerprint: Fingerprint, tmp_path: Path
 ):
@@ -479,6 +486,7 @@ def test_should_retrain(
     assert retrain.should_retrain_nlu() == fingerprint["retrain_nlu"]
 
 
+@pytest.mark.trains_model
 async def test_should_not_retrain_core(default_domain_path: Text, tmp_path: Path):
     # Don't use `default_stories_file` as checkpoints currently break fingerprinting
     story_file = tmp_path / "simple_story.yml"
@@ -560,6 +568,7 @@ def test_fingerprint_comparison_result(
     assert comparison_result.should_retrain_nlu() == retrain_nlu
 
 
+@pytest.mark.trains_model
 async def test_update_with_new_domain(trained_rasa_model: Text, tmpdir: Path):
     _ = model.unpack_model(trained_rasa_model, tmpdir)
 
diff --git a/tests/test_server.py b/tests/test_server.py
index 7a005543ac13..deeea5ae5093 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -125,6 +125,7 @@ async def tear_down_scheduler() -> Generator[None, None, None]:
     rasa.core.jobs.__scheduler = None
 
 
+@pytest.mark.trains_model
 async def test_root(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get("/")
     assert response.status == HTTPStatus.OK
@@ -143,6 +144,7 @@ async def test_root_secured(rasa_secured_app: SanicASGITestClient):
     assert response.text.startswith("Hello from Rasa:")
 
 
+@pytest.mark.trains_model
 async def test_version(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get("/version")
     content = response.json()
@@ -154,6 +156,7 @@ async def test_version(rasa_app: SanicASGITestClient):
     )
 
 
+@pytest.mark.trains_model
 async def test_status(rasa_app: SanicASGITestClient, trained_rasa_model: Text):
     _, response = await rasa_app.get("/status")
     model_file = response.json()["model_file"]
@@ -163,6 +166,7 @@ async def test_status(rasa_app: SanicASGITestClient, trained_rasa_model: Text):
     assert model_file == trained_rasa_model
 
 
+@pytest.mark.trains_model
 async def test_status_nlu_only(
     rasa_app_nlu: SanicASGITestClient, trained_nlu_model: Text
 ):
@@ -179,6 +183,7 @@ async def test_status_secured(rasa_secured_app: SanicASGITestClient):
     assert response.status == HTTPStatus.UNAUTHORIZED
 
 
+@pytest.mark.trains_model
 async def test_status_not_ready_agent(rasa_app: SanicASGITestClient):
     rasa_app.app.agent = None
     _, response = await rasa_app.get("/status")
@@ -359,6 +364,7 @@ def is_server_ready() -> bool:
         ),
     ],
 )
+@pytest.mark.trains_model
 async def test_parse(rasa_app: SanicASGITestClient, response_test: ResponseTest):
     _, response = await rasa_app.post(
         response_test.endpoint, json=response_test.payload
@@ -403,6 +409,7 @@ async def test_parse(rasa_app: SanicASGITestClient, response_test: ResponseTest)
         ),
     ],
 )
+@pytest.mark.trains_model
 async def test_parse_with_different_emulation_mode(
     rasa_app: SanicASGITestClient, response_test: ResponseTest
 ):
@@ -412,6 +419,7 @@ async def test_parse_with_different_emulation_mode(
     assert response.status == HTTPStatus.OK
 
 
+@pytest.mark.trains_model
 async def test_parse_without_nlu_model(rasa_app_core: SanicASGITestClient):
     _, response = await rasa_app_core.post("/model/parse", json={"text": "hello"})
     assert response.status == HTTPStatus.OK
@@ -420,6 +428,7 @@ async def test_parse_without_nlu_model(rasa_app_core: SanicASGITestClient):
     assert all(prop in rjs for prop in ["entities", "intent", "text"])
 
 
+@pytest.mark.trains_model
 async def test_parse_on_invalid_emulation_mode(rasa_app_nlu: SanicASGITestClient):
     _, response = await rasa_app_nlu.post(
         "/model/parse?emulation_mode=ANYTHING", json={"text": "hello"}
@@ -427,6 +436,7 @@ async def test_parse_on_invalid_emulation_mode(rasa_app_nlu: SanicASGITestClient
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_train_stack_success_with_md(
     rasa_app: SanicASGITestClient,
     default_domain_path: Text,
@@ -456,6 +466,7 @@ async def test_train_stack_success_with_md(
     assert os.path.exists(os.path.join(model_path, "fingerprint.json"))
 
 
+@pytest.mark.trains_model
 async def test_train_nlu_success(
     rasa_app: SanicASGITestClient,
     default_stack_config: Text,
@@ -492,6 +503,7 @@ async def test_train_nlu_success(
     assert os.path.exists(os.path.join(model_path, "fingerprint.json"))
 
 
+@pytest.mark.trains_model
 async def test_train_core_success_with(
     rasa_app: SanicASGITestClient,
     default_stack_config: Text,
@@ -522,6 +534,7 @@ async def test_train_core_success_with(
     assert os.path.exists(os.path.join(model_path, "fingerprint.json"))
 
 
+@pytest.mark.trains_model
 async def test_train_with_retrieval_events_success(
     rasa_app: SanicASGITestClient, default_stack_config: Text, tmp_path: Path
 ):
@@ -588,6 +601,7 @@ def test_deprecation_warnings_json_payload(payload: Dict):
         rasa.server._validate_json_training_payload(payload)
 
 
+@pytest.mark.trains_model
 async def test_train_with_yaml(rasa_app: SanicASGITestClient, tmp_path: Path):
     training_data = """
 stories:
@@ -633,6 +647,7 @@ async def test_train_with_yaml(rasa_app: SanicASGITestClient, tmp_path: Path):
     assert_trained_model(response.body, tmp_path)
 
 
+@pytest.mark.trains_model
 async def test_train_with_invalid_yaml(rasa_app: SanicASGITestClient):
     invalid_yaml = """
 rules:
@@ -685,6 +700,7 @@ def test_training_payload_from_yaml_save_to_default_model_directory(
     assert payload.get("output") == expected
 
 
+@pytest.mark.trains_model
 async def test_train_missing_config(rasa_app: SanicASGITestClient):
     payload = dict(domain="domain data", config=None)
 
@@ -692,6 +708,7 @@ async def test_train_missing_config(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_train_missing_training_data(rasa_app: SanicASGITestClient):
     payload = dict(domain="domain data", config="config data")
 
@@ -699,6 +716,7 @@ async def test_train_missing_training_data(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_train_internal_error(rasa_app: SanicASGITestClient):
     payload = dict(domain="domain data", config="config data", nlu="nlu data")
 
@@ -706,6 +724,7 @@ async def test_train_internal_error(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.INTERNAL_SERVER_ERROR
 
 
+@pytest.mark.trains_model
 async def test_evaluate_stories(
     rasa_app: SanicASGITestClient, default_stories_file: Text
 ):
@@ -738,6 +757,7 @@ async def test_evaluate_stories(
     }
 
 
+@pytest.mark.trains_model
 async def test_evaluate_stories_not_ready_agent(
     rasa_app_nlu: SanicASGITestClient, default_stories_file: Text
 ):
@@ -748,6 +768,7 @@ async def test_evaluate_stories_not_ready_agent(
     assert response.status == HTTPStatus.CONFLICT
 
 
+@pytest.mark.trains_model
 async def test_evaluate_stories_end_to_end(
     rasa_app: SanicASGITestClient, end_to_end_test_story_file: Text
 ):
@@ -776,6 +797,7 @@ async def test_evaluate_stories_end_to_end(
     }
 
 
+@pytest.mark.trains_model
 async def test_evaluate_intent(rasa_app: SanicASGITestClient, default_nlu_data: Text):
     nlu_data = rasa.shared.utils.io.read_file(default_nlu_data)
 
@@ -793,6 +815,7 @@ async def test_evaluate_intent(rasa_app: SanicASGITestClient, default_nlu_data:
     }
 
 
+@pytest.mark.trains_model
 async def test_evaluate_intent_on_just_nlu_model(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text
 ):
@@ -812,6 +835,7 @@ async def test_evaluate_intent_on_just_nlu_model(
     }
 
 
+@pytest.mark.trains_model
 async def test_evaluate_intent_with_model_param(
     rasa_app: SanicASGITestClient, trained_nlu_model: Text, default_nlu_data: Text
 ):
@@ -837,6 +861,7 @@ async def test_evaluate_intent_with_model_param(
     assert previous_model_file == response.json()["model_file"]
 
 
+@pytest.mark.trains_model
 async def test_evaluate_intent_with_model_server(
     rasa_app: SanicASGITestClient,
     trained_rasa_model: Text,
@@ -890,6 +915,7 @@ async def test_evaluate_intent_with_model_server(
     assert production_model_server.kwargs.get("wait_time_between_pulls") != 0
 
 
+@pytest.mark.trains_model
 async def test_cross_validation(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text
 ):
@@ -919,6 +945,7 @@ async def test_cross_validation(
         )
 
 
+@pytest.mark.trains_model
 async def test_cross_validation_with_md(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text
 ):
@@ -935,6 +962,7 @@ async def test_cross_validation_with_md(
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_cross_validation_with_callback_success(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text, monkeypatch: MonkeyPatch
 ):
@@ -989,6 +1017,7 @@ async def test_cross_validation_with_callback_success(
             )
 
 
+@pytest.mark.trains_model
 async def test_cross_validation_with_callback_error(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text, monkeypatch: MonkeyPatch
 ):
@@ -1022,6 +1051,7 @@ async def test_cross_validation_with_callback_error(
         assert content["code"] == HTTPStatus.INTERNAL_SERVER_ERROR
 
 
+@pytest.mark.trains_model
 async def test_callback_unexpected_error(
     rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text, monkeypatch: MonkeyPatch
 ):
@@ -1060,6 +1090,7 @@ async def raiseUnexpectedError() -> NoReturn:
         assert content["code"] == HTTPStatus.INTERNAL_SERVER_ERROR
 
 
+@pytest.mark.trains_model
 async def test_predict(rasa_app: SanicASGITestClient):
     data = {
         "Events": {
@@ -1090,6 +1121,7 @@ async def test_predict(rasa_app: SanicASGITestClient):
 
 
 @freeze_time("2018-01-01")
+@pytest.mark.trains_model
 async def test_requesting_non_existent_tracker(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get("/conversations/madeupid/tracker")
     content = response.json()
@@ -1126,6 +1158,7 @@ async def test_requesting_non_existent_tracker(rasa_app: SanicASGITestClient):
 
 
 @pytest.mark.parametrize("event", test_events)
+@pytest.mark.trains_model
 async def test_pushing_event(rasa_app: SanicASGITestClient, event: Event):
     sender_id = str(uuid.uuid1())
     conversation = f"/conversations/{sender_id}"
@@ -1158,6 +1191,7 @@ async def test_pushing_event(rasa_app: SanicASGITestClient, event: Event):
     assert deserialized_events[3].timestamp > time_before_adding_events
 
 
+@pytest.mark.trains_model
 async def test_push_multiple_events(rasa_app: SanicASGITestClient):
     conversation_id = str(uuid.uuid1())
     conversation = f"/conversations/{conversation_id}"
@@ -1186,6 +1220,7 @@ async def test_push_multiple_events(rasa_app: SanicASGITestClient):
 @pytest.mark.parametrize(
     "params", ["?execute_side_effects=true&output_channel=callback", ""]
 )
+@pytest.mark.trains_model
 async def test_pushing_event_while_executing_side_effects(
     rasa_server: Sanic, params: Text
 ):
@@ -1219,6 +1254,7 @@ async def test_pushing_event_while_executing_side_effects(
             assert message_received.get("text") == serialized_event.get("text")
 
 
+@pytest.mark.trains_model
 async def test_post_conversation_id_with_slash(rasa_app: SanicASGITestClient):
     conversation_id = str(uuid.uuid1())
     id_len = len(conversation_id) // 2
@@ -1246,6 +1282,7 @@ async def test_post_conversation_id_with_slash(rasa_app: SanicASGITestClient):
     ] == session_start_sequence + test_events
 
 
+@pytest.mark.trains_model
 async def test_put_tracker(rasa_app: SanicASGITestClient):
     data = [event.as_dict() for event in test_events]
     _, response = await rasa_app.put(
@@ -1265,6 +1302,7 @@ async def test_put_tracker(rasa_app: SanicASGITestClient):
     assert events.deserialise_events(evts) == test_events
 
 
+@pytest.mark.trains_model
 async def test_sorted_predict(rasa_app: SanicASGITestClient):
     await _create_tracker_for_sender(rasa_app, "sortedpredict")
 
@@ -1351,6 +1389,7 @@ def test_list_routes(default_agent: Agent):
     }
 
 
+@pytest.mark.trains_model
 async def test_unload_model_error(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get("/status")
     assert response.status == HTTPStatus.OK
@@ -1360,6 +1399,7 @@ async def test_unload_model_error(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.NO_CONTENT
 
 
+@pytest.mark.trains_model
 async def test_get_domain(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get(
         "/domain", headers={"accept": rasa.server.JSON_CONTENT_TYPE}
@@ -1376,12 +1416,14 @@ async def test_get_domain(rasa_app: SanicASGITestClient):
     assert "actions" in content
 
 
+@pytest.mark.trains_model
 async def test_get_domain_invalid_accept_header(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.get("/domain")
 
     assert response.status == HTTPStatus.NOT_ACCEPTABLE
 
 
+@pytest.mark.trains_model
 async def test_load_model(rasa_app: SanicASGITestClient, trained_core_model: Text):
     _, response = await rasa_app.get("/status")
 
@@ -1403,6 +1445,7 @@ async def test_load_model(rasa_app: SanicASGITestClient, trained_core_model: Tex
     assert old_fingerprint != response.json()["fingerprint"]
 
 
+@pytest.mark.trains_model
 async def test_load_model_from_model_server(
     rasa_app: SanicASGITestClient, trained_core_model: Text, tear_down_scheduler: None
 ):
@@ -1437,12 +1480,14 @@ async def test_load_model_from_model_server(
             assert old_fingerprint != response.json()["fingerprint"]
 
 
+@pytest.mark.trains_model
 async def test_load_model_invalid_request_body(rasa_app: SanicASGITestClient):
     _, response = await rasa_app.put("/model")
 
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_load_model_invalid_configuration(rasa_app: SanicASGITestClient):
     data = {"model_file": "some-random-path"}
     _, response = await rasa_app.put("/model", json=data)
@@ -1450,6 +1495,7 @@ async def test_load_model_invalid_configuration(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_execute(rasa_app: SanicASGITestClient):
     await _create_tracker_for_sender(rasa_app, "test_execute")
 
@@ -1463,6 +1509,7 @@ async def test_execute(rasa_app: SanicASGITestClient):
     assert parsed_content["messages"]
 
 
+@pytest.mark.trains_model
 async def test_execute_with_missing_action_name(rasa_app: SanicASGITestClient):
     test_sender = "test_execute_with_missing_action_name"
     await _create_tracker_for_sender(rasa_app, test_sender)
@@ -1475,6 +1522,7 @@ async def test_execute_with_missing_action_name(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_execute_with_not_existing_action(rasa_app: SanicASGITestClient):
     test_sender = "test_execute_with_not_existing_action"
     await _create_tracker_for_sender(rasa_app, test_sender)
@@ -1487,6 +1535,7 @@ async def test_execute_with_not_existing_action(rasa_app: SanicASGITestClient):
     assert response.status == HTTPStatus.INTERNAL_SERVER_ERROR
 
 
+@pytest.mark.trains_model
 async def test_trigger_intent(rasa_app: SanicASGITestClient):
     data = {INTENT_NAME_KEY: "greet"}
     _, response = await rasa_app.post(
@@ -1500,6 +1549,7 @@ async def test_trigger_intent(rasa_app: SanicASGITestClient):
     assert parsed_content["messages"]
 
 
+@pytest.mark.trains_model
 async def test_trigger_intent_with_missing_intent_name(rasa_app: SanicASGITestClient):
     test_sender = "test_trigger_intent_with_missing_action_name"
 
@@ -1511,6 +1561,7 @@ async def test_trigger_intent_with_missing_intent_name(rasa_app: SanicASGITestCl
     assert response.status == HTTPStatus.BAD_REQUEST
 
 
+@pytest.mark.trains_model
 async def test_trigger_intent_with_not_existing_intent(rasa_app: SanicASGITestClient):
     test_sender = "test_trigger_intent_with_not_existing_intent"
     await _create_tracker_for_sender(rasa_app, test_sender)
@@ -1715,6 +1766,7 @@ class NoInputChannels:
         ([], None, True, 'version: "2.0"'),
     ],
 )
+@pytest.mark.trains_model
 async def test_get_story(
     rasa_app: SanicASGITestClient,
     monkeypatch: MonkeyPatch,
@@ -1748,6 +1800,7 @@ async def test_get_story(
     assert response.content.decode().strip() == expected
 
 
+@pytest.mark.trains_model
 async def test_get_story_does_not_update_conversation_session(
     rasa_app: SanicASGITestClient, monkeypatch: MonkeyPatch
 ):
@@ -1853,6 +1906,7 @@ async def test_get_story_does_not_update_conversation_session(
         ),
     ],
 )
+@pytest.mark.trains_model
 async def test_update_conversation_with_events(
     rasa_app: SanicASGITestClient,
     monkeypatch: MonkeyPatch,
diff --git a/tests/test_test.py b/tests/test_test.py
index aea5aed8b157..e8c0e6769646 100644
--- a/tests/test_test.py
+++ b/tests/test_test.py
@@ -146,6 +146,7 @@ def test_get_label_set(targets, exclude_label, expected):
     assert set(expected) == set(actual)
 
 
+@pytest.mark.trains_model
 async def test_interpreter_passed_to_agent(
     monkeypatch: MonkeyPatch, trained_rasa_model: Text
 ):
@@ -155,6 +156,7 @@ async def test_interpreter_passed_to_agent(
     assert isinstance(agent.interpreter, RasaNLUInterpreter)
 
 
+@pytest.mark.trains_model
 def test_e2e_warning_if_no_nlu_model(
     monkeypatch: MonkeyPatch, trained_core_model: Text, capsys: CaptureFixture
 ):
diff --git a/tests/test_train.py b/tests/test_train.py
index 35e2e167827d..72d370208103 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -38,6 +38,7 @@
         {"model_name": None, "prefix": None},
     ],
 )
+@pytest.mark.trains_model
 def test_package_model(trained_rasa_model: Text, parameters: Dict):
     output_path = tempfile.mkdtemp()
     train_path = rasa.model.unpack_model(trained_rasa_model)
@@ -80,6 +81,7 @@ def count_temp_rasa_files(directory: Text) -> int:
     )
 
 
+@pytest.mark.trains_model
 def test_train_temp_files(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -117,6 +119,7 @@ def test_train_temp_files(
     assert count_temp_rasa_files(tempfile.tempdir) == 0
 
 
+@pytest.mark.trains_model
 def test_train_core_temp_files(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -139,6 +142,7 @@ def test_train_core_temp_files(
     assert count_temp_rasa_files(tempfile.tempdir) == 0
 
 
+@pytest.mark.trains_model
 def test_train_nlu_temp_files(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -173,6 +177,7 @@ def test_train_nlu_wrong_format_error_message(
     assert "Please verify the data format" in captured.out
 
 
+@pytest.mark.trains_model
 def test_train_nlu_with_responses_no_domain_warns(tmp_path: Path):
     data_path = "data/test_nlu_no_responses/nlu_no_responses.yml"
 
@@ -190,6 +195,7 @@ def test_train_nlu_with_responses_no_domain_warns(tmp_path: Path):
     )
 
 
+@pytest.mark.trains_model
 def test_train_nlu_with_responses_and_domain_no_warns(tmp_path: Path):
     data_path = "data/test_nlu_no_responses/nlu_no_responses.yml"
     domain_path = "data/test_nlu_no_responses/domain_with_only_responses.yml"
@@ -226,6 +232,7 @@ def test_train_nlu_no_nlu_file_error_message(
     assert "No NLU data given" in captured.out
 
 
+@pytest.mark.trains_model
 def test_trained_interpreter_passed_to_core_training(
     monkeypatch: MonkeyPatch, tmp_path: Path, unpacked_trained_rasa_model: Text
 ):
@@ -254,6 +261,7 @@ def test_trained_interpreter_passed_to_core_training(
     assert isinstance(kwargs["interpreter"], RasaNLUInterpreter)
 
 
+@pytest.mark.trains_model
 def test_interpreter_of_old_model_passed_to_core_training(
     monkeypatch: MonkeyPatch, tmp_path: Path, trained_rasa_model: Text
 ):
@@ -379,6 +387,7 @@ def new_model_path_in_same_dir(old_model_path: Text) -> Text:
 
 
 class TestE2e:
+    @pytest.mark.trains_model
     def test_e2e_gives_experimental_warning(
         self,
         monkeypatch: MonkeyPatch,
@@ -407,6 +416,7 @@ def test_e2e_gives_experimental_warning(
             ]
         )
 
+    @pytest.mark.trains_model
     def test_models_not_retrained_if_no_new_data(
         self,
         monkeypatch: MonkeyPatch,
@@ -429,6 +439,7 @@ def test_models_not_retrained_if_no_new_data(
         mocked_core_training.assert_not_called()
         mocked_nlu_training.assert_not_called()
 
+    @pytest.mark.trains_model
     def test_retrains_nlu_and_core_if_new_e2e_example(
         self,
         monkeypatch: MonkeyPatch,
@@ -459,6 +470,7 @@ def test_retrains_nlu_and_core_if_new_e2e_example(
         mocked_core_training.assert_called_once()
         mocked_nlu_training.assert_called_once()
 
+    @pytest.mark.trains_model
     def test_retrains_only_core_if_new_e2e_example_seen_before(
         self,
         monkeypatch: MonkeyPatch,
@@ -517,6 +529,7 @@ def make_tmp_model_dir(tmp_path: Path) -> Text:
         output = str(tmp_path / "models")
         return output
 
+    @pytest.mark.trains_model
     def test_new_nlu_data_retrains_core_if_there_are_e2e_stories(
         self,
         monkeypatch: MonkeyPatch,
@@ -547,6 +560,7 @@ def test_new_nlu_data_retrains_core_if_there_are_e2e_stories(
         mocked_core_training.assert_called_once()
         mocked_nlu_training.assert_called_once()
 
+    @pytest.mark.trains_model
     def test_new_nlu_data_does_not_retrain_core_if_there_are_no_e2e_stories(
         self,
         monkeypatch: MonkeyPatch,
@@ -610,6 +624,7 @@ def test_training_core_with_e2e_fails_gracefully(
 
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_latest_model", [True, False])
+@pytest.mark.trains_model
 def test_model_finetuning(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -653,6 +668,7 @@ def test_model_finetuning(
 
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_latest_model", [True, False])
+@pytest.mark.trains_model
 def test_model_finetuning_core(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -707,6 +723,7 @@ def test_model_finetuning_core(
     assert ted.finetune_mode
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_core_with_default_epochs(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_moodbot_path: Text,
 ):
@@ -740,6 +757,7 @@ def test_model_finetuning_core_with_default_epochs(
     assert ted.config[EPOCHS] == TEDPolicy.defaults[EPOCHS] * 2
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_core_new_domain_label(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_moodbot_path: Text,
 ):
@@ -767,6 +785,7 @@ def test_model_finetuning_core_new_domain_label(
     mocked_core_training.assert_not_called()
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_new_domain_label_stops_all_training(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_moodbot_path: Text,
 ):
@@ -801,6 +820,7 @@ def test_model_finetuning_new_domain_label_stops_all_training(
 
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_latest_model", [True, False])
+@pytest.mark.trains_model
 def test_model_finetuning_nlu(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -859,6 +879,7 @@ def test_model_finetuning_nlu(
     assert new_diet_metadata[EPOCHS] == 2
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_nlu_new_label(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_nlu_moodbot_path: Text,
 ):
@@ -885,6 +906,7 @@ def test_model_finetuning_nlu_new_label(
     mocked_nlu_training.assert_not_called()
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_nlu_new_entity(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_nlu_moodbot_path: Text,
 ):
@@ -911,6 +933,7 @@ def test_model_finetuning_nlu_new_entity(
     mocked_nlu_training.assert_not_called()
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_nlu_new_label_already_in_domain(
     tmp_path: Path,
     monkeypatch: MonkeyPatch,
@@ -943,6 +966,7 @@ def test_model_finetuning_nlu_new_label_already_in_domain(
     mocked_nlu_training.assert_not_called()
 
 
+@pytest.mark.trains_model
 def test_model_finetuning_nlu_new_label_to_domain_only(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_nlu_moodbot_path: Text,
 ):
@@ -969,6 +993,7 @@ def test_model_finetuning_nlu_new_label_to_domain_only(
 
 
 @pytest.mark.timeout(200)
+@pytest.mark.trains_model
 def test_model_finetuning_nlu_with_default_epochs(
     tmp_path: Path, monkeypatch: MonkeyPatch, trained_nlu_moodbot_path: Text,
 ):


Code review comments:
diff_hunk: @@ -385,6 +390,7 @@ async def test_agent_update_model_none_domain(trained_rasa_model: Text):
     assert tracker.events[3].intent["name"] == "greet"
 
 
+@pytest.mark.trains_model
path: tests/core/test_agent.py
text: @user1:
why is this one marked?

@author:
oops was meant to be on `test_load_agent`