Repository: RasaHQ/rasa
Pull request: Improve testing output of story evaluation
**Proposed changes**:
closes https://github.com/RasaHQ/rasa/issues/5748

Add option to write down successful story predictions to disk.
Add option to disable writing down incorrect story predictions.
Write down summary report to disk.

**Status (please check what you already did)**:
- [x] added some tests for the functionality
- [ ] updated the documentation
- [x] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/master/changelog) for instructions)
- [x] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)


Problem statement:
Standardize testing output
**Description of Problem**:
For intents we currently have the following testing output: report, error, successes, confusion matrix, histogram. However, for stories, entities, and responses only some of those outputs exists. We should standardize the testing output so that you have the same tool to evaluate intents, entities, responses, and stories.

**Overview of the Solution**:
- [x] For entities we need to add the confusion matrix and the histogram. 
- [x] For responses we need to add everything except the report. 
- [x] For stories we need to actually write the report and add successes.

Hints from issue discussion:

Commit message: resolve doc warnings

Patch to review:
diff --git a/changelog/5748.improvement.rst b/changelog/5748.improvement.rst
index e2eed4ac7724..522a8e9ca0ce 100644
--- a/changelog/5748.improvement.rst
+++ b/changelog/5748.improvement.rst
@@ -1,6 +1,6 @@
-Standardize testing output. The following test output can be produced for intents, responses and entities:
+Standardize testing output. The following test output can be produced for intents, responses, entities and stories:
 - report: a detailed report with testing metrics per label (e.g. precision, recall, accuracy, etc.)
 - errors: a file that contains incorrect predictions
 - successes: a file that contains correct predictions
 - confusion matrix: plot of confusion matrix
-- histogram: plot of confidence distribution
+- histogram: plot of confidence distribution (not available for stories)
diff --git a/docs/conf.py b/docs/conf.py
index 5645fb87d0d5..fe89bf6b4173 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -413,13 +413,13 @@
 
 
 def setup(sphinx):
-    sphinx.add_stylesheet("css/custom.css")
+    sphinx.add_css_file("css/custom.css")
 
     try:
         utils_path = os.path.abspath(os.path.join(__file__, "..", "utils"))
         sys.path.insert(0, utils_path)
         from StoryLexer import StoryLexer
 
-        sphinx.add_lexer("story", StoryLexer())
+        sphinx.add_lexer("story", StoryLexer)
     except ImportError:
         print("No Story Lexer :( Sad times!")
diff --git a/rasa/cli/arguments/test.py b/rasa/cli/arguments/test.py
index 0d5e26545ce0..4a28c848143d 100644
--- a/rasa/cli/arguments/test.py
+++ b/rasa/cli/arguments/test.py
@@ -15,7 +15,6 @@
 
 def set_test_arguments(parser: argparse.ArgumentParser):
     add_model_param(parser, add_positional_arg=False)
-    add_no_plot_param(parser)
 
     core_arguments = parser.add_argument_group("Core Test Arguments")
     add_test_core_argument_group(core_arguments)
@@ -23,6 +22,14 @@ def set_test_arguments(parser: argparse.ArgumentParser):
     nlu_arguments = parser.add_argument_group("NLU Test Arguments")
     add_test_nlu_argument_group(nlu_arguments)
 
+    add_no_plot_param(parser)
+    add_errors_success_params(parser)
+    add_out_param(
+        parser,
+        default=DEFAULT_RESULTS_PATH,
+        help_text="Output path for any files created during the evaluation.",
+    )
+
 
 def set_test_core_arguments(parser: argparse.ArgumentParser):
     add_test_core_model_param(parser)
@@ -83,6 +90,7 @@ def add_test_core_argument_group(
         "and compared against each other.",
     )
     add_no_plot_param(parser)
+    add_errors_success_params(parser)
 
 
 def add_test_nlu_argument_group(
@@ -95,19 +103,6 @@ def add_test_nlu_argument_group(
         default=DEFAULT_RESULTS_PATH,
         help_text="Output path for any files created during the evaluation.",
     )
-
-    parser.add_argument(
-        "--successes",
-        action="store_true",
-        default=False,
-        help="If set successful predictions will be written to a file.",
-    )
-    parser.add_argument(
-        "--no-errors",
-        action="store_true",
-        default=False,
-        help="If set incorrect predictions will NOT be written to a file.",
-    )
     parser.add_argument(
         "-c",
         "--config",
@@ -153,6 +148,7 @@ def add_test_nlu_argument_group(
     )
 
     add_no_plot_param(parser)
+    add_errors_success_params(parser)
 
 
 def add_test_core_model_param(parser: argparse.ArgumentParser):
@@ -180,3 +176,18 @@ def add_no_plot_param(
         help="Don't render evaluation plots.",
         required=required,
     )
+
+
+def add_errors_success_params(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--successes",
+        action="store_true",
+        default=False,
+        help="If set successful predictions will be written to a file.",
+    )
+    parser.add_argument(
+        "--no-errors",
+        action="store_true",
+        default=False,
+        help="If set incorrect predictions will NOT be written to a file.",
+    )
diff --git a/rasa/cli/test.py b/rasa/cli/test.py
index a054c32eb2c9..0666da94adf2 100644
--- a/rasa/cli/test.py
+++ b/rasa/cli/test.py
@@ -69,6 +69,7 @@ def run_core_test(args: argparse.Namespace) -> None:
     stories = cli_utils.get_validated_path(args.stories, "stories", DEFAULT_DATA_PATH)
     stories = data.get_core_directory(stories)
     output = args.out or DEFAULT_RESULTS_PATH
+    args.errors = not args.no_errors
 
     io_utils.create_directory(output)
 
diff --git a/rasa/core/test.py b/rasa/core/test.py
index aec353754d71..59cd65542f0c 100644
--- a/rasa/core/test.py
+++ b/rasa/core/test.py
@@ -3,9 +3,9 @@
 import warnings
 import typing
 from collections import defaultdict, namedtuple
-from typing import Any, Dict, List, Optional, Text, Tuple
+from typing import Any, Dict, List, Optional, Text, Tuple, Union
 
-import rasa.utils.io
+import rasa.utils.io as io_utils
 from rasa.constants import RESULTS_FILE, PERCENTAGE_KEY
 from rasa.core.utils import pad_lists_to_size
 from rasa.core.events import ActionExecuted, UserUttered
@@ -15,30 +15,21 @@
 
 if typing.TYPE_CHECKING:
     from rasa.core.agent import Agent
+    from rasa.core.processor import MessageProcessor
 
-import matplotlib
 
+CONFUSION_MATRIX_STORIES_FILE = "story_confusion_matrix.png"
+REPORT_STORIES_FILE = "story_report.json"
 FAILED_STORIES_FILE = "failed_stories.md"
-
-# At first, matplotlib will be initialized with default OS-specific available backend
-# if that didn't happen, we'll try to set it up manually
-if matplotlib.get_backend() is not None:
-    pass
-else:  # pragma: no cover
-    try:
-        # If the `tkinter` package is available, we can use the `TkAgg` backend
-        import tkinter
-
-        matplotlib.use("TkAgg")
-    except ImportError:
-        matplotlib.use("agg")
+SUCCESSFUL_STORIES_FILE = "successful_stories.md"
 
 
 logger = logging.getLogger(__name__)
 
-StoryEvalution = namedtuple(
+StoryEvaluation = namedtuple(
     "StoryEvaluation",
-    "evaluation_store failed_stories action_list in_training_data_fraction",
+    "evaluation_store failed_stories successful_stories action_list "
+    "in_training_data_fraction",
 )
 
 
@@ -47,10 +38,10 @@ class EvaluationStore:
 
     def __init__(
         self,
-        action_predictions: Optional[List[str]] = None,
-        action_targets: Optional[List[str]] = None,
-        intent_predictions: Optional[List[str]] = None,
-        intent_targets: Optional[List[str]] = None,
+        action_predictions: Optional[List[Text]] = None,
+        action_targets: Optional[List[Text]] = None,
+        intent_predictions: Optional[List[Text]] = None,
+        intent_targets: Optional[List[Text]] = None,
         entity_predictions: Optional[List[Dict[Text, Any]]] = None,
         entity_targets: Optional[List[Dict[Text, Any]]] = None,
     ) -> None:
@@ -63,10 +54,10 @@ def __init__(
 
     def add_to_store(
         self,
-        action_predictions: Optional[List[str]] = None,
-        action_targets: Optional[List[str]] = None,
-        intent_predictions: Optional[List[str]] = None,
-        intent_targets: Optional[List[str]] = None,
+        action_predictions: Optional[Union[Text, List[Text]]] = None,
+        action_targets: Optional[Union[Text, List[Text]]] = None,
+        intent_predictions: Optional[Union[Text, List[Text]]] = None,
+        intent_targets: Optional[Union[Text, List[Text]]] = None,
         entity_predictions: Optional[List[Dict[Text, Any]]] = None,
         entity_targets: Optional[List[Dict[Text, Any]]] = None,
     ) -> None:
@@ -90,7 +81,7 @@ def merge_store(self, other: "EvaluationStore") -> None:
             entity_targets=other.entity_targets,
         )
 
-    def has_prediction_target_mismatch(self):
+    def has_prediction_target_mismatch(self) -> bool:
         return (
             self.intent_predictions != self.intent_targets
             or self.entity_predictions != self.entity_targets
@@ -131,14 +122,12 @@ class WronglyPredictedAction(ActionExecuted):
 
     def __init__(
         self, correct_action, predicted_action, policy, confidence, timestamp=None
-    ):
+    ) -> None:
         self.predicted_action = predicted_action
         super().__init__(correct_action, policy, confidence, timestamp=timestamp)
 
-    def as_story_string(self):
-        return "{}   <!-- predicted: {} -->".format(
-            self.action_name, self.predicted_action
-        )
+    def as_story_string(self) -> Text:
+        return f"{self.action_name}   <!-- predicted: {self.predicted_action} -->"
 
 
 class EndToEndUserUtterance(UserUttered):
@@ -147,7 +136,7 @@ class EndToEndUserUtterance(UserUttered):
     Mostly used to print the full end-to-end user message in the
     `failed_stories.md` output file."""
 
-    def as_story_string(self, e2e=True):
+    def as_story_string(self, e2e: bool = True) -> Text:
         return super().as_story_string(e2e=True)
 
 
@@ -159,7 +148,7 @@ class WronglyClassifiedUserUtterance(UserUttered):
 
     type_name = "wrong_utterance"
 
-    def __init__(self, event: UserUttered, eval_store: EvaluationStore):
+    def __init__(self, event: UserUttered, eval_store: EvaluationStore) -> None:
 
         if not eval_store.intent_predictions:
             self.predicted_intent = None
@@ -178,22 +167,25 @@ def __init__(self, event: UserUttered, eval_store: EvaluationStore):
             event.input_channel,
         )
 
-    def as_story_string(self, e2e=True):
+    def as_story_string(self, e2e: bool = True) -> Text:
         from rasa.core.events import md_format_message
 
         correct_message = md_format_message(self.text, self.intent, self.entities)
         predicted_message = md_format_message(
             self.text, self.predicted_intent, self.predicted_entities
         )
-        return "{}: {}   <!-- predicted: {}: {} -->".format(
-            self.intent.get("name"),
-            correct_message,
-            self.predicted_intent,
-            predicted_message,
+        return (
+            f"{self.intent.get('name')}: {correct_message}   <!-- predicted: "
+            f"{self.predicted_intent}: {predicted_message} -->"
         )
 
 
-async def _generate_trackers(resource_name, agent, max_stories=None, use_e2e=False):
+async def _generate_trackers(
+    resource_name: Text,
+    agent: "Agent",
+    max_stories: Optional[int] = None,
+    use_e2e: bool = False,
+) -> List[Any]:
     from rasa.core.training.generator import TrainingDataGenerator
 
     from rasa.core import training
@@ -271,7 +263,9 @@ def _collect_user_uttered_predictions(
     return user_uttered_eval_store
 
 
-def _emulate_form_rejection(processor, partial_tracker):
+def _emulate_form_rejection(
+    processor: "MessageProcessor", partial_tracker: DialogueStateTracker
+) -> None:
     from rasa.core.policies.form_policy import FormPolicy
     from rasa.core.events import ActionExecutionRejected
 
@@ -290,11 +284,11 @@ def _emulate_form_rejection(processor, partial_tracker):
 
 
 def _collect_action_executed_predictions(
-    processor,
-    partial_tracker,
-    event,
-    fail_on_prediction_errors,
-    circuit_breaker_tripped,
+    processor: "MessageProcessor",
+    partial_tracker: DialogueStateTracker,
+    event: ActionExecuted,
+    fail_on_prediction_errors: bool,
+    circuit_breaker_tripped: bool,
 ):
     from rasa.core.policies.form_policy import FormPolicy
 
@@ -349,9 +343,11 @@ def _collect_action_executed_predictions(
 
 
 def _predict_tracker_actions(
-    tracker, agent: "Agent", fail_on_prediction_errors=False, use_e2e=False
-):
-    from rasa.core.trackers import DialogueStateTracker
+    tracker: DialogueStateTracker,
+    agent: "Agent",
+    fail_on_prediction_errors: bool = False,
+    use_e2e: bool = False,
+) -> Tuple[EvaluationStore, DialogueStateTracker, List[Dict[Text, Any]]]:
 
     processor = agent.create_processor()
     tracker_eval_store = EvaluationStore()
@@ -413,7 +409,7 @@ def _predict_tracker_actions(
     return tracker_eval_store, partial_tracker, tracker_actions
 
 
-def _in_training_data_fraction(action_list):
+def _in_training_data_fraction(action_list: List[Dict[Text, Any]]) -> float:
     """Given a list of action items, returns the fraction of actions
 
     that were predicted using one of the Memoization policies."""
@@ -428,18 +424,19 @@ def _in_training_data_fraction(action_list):
     return len(in_training_data) / len(action_list)
 
 
-def collect_story_predictions(
+def _collect_story_predictions(
     completed_trackers: List["DialogueStateTracker"],
     agent: "Agent",
     fail_on_prediction_errors: bool = False,
     use_e2e: bool = False,
-) -> Tuple[StoryEvalution, int]:
+) -> Tuple[StoryEvaluation, int]:
     """Test the stories from a file, running them through the stored model."""
-    from rasa.nlu.test import get_evaluation_metrics
+    from rasa.test import get_evaluation_metrics
     from tqdm import tqdm
 
     story_eval_store = EvaluationStore()
     failed = []
+    success = []
     correct_dialogues = []
     number_of_stories = len(completed_trackers)
 
@@ -462,6 +459,7 @@ def collect_story_predictions(
             correct_dialogues.append(0)
         else:
             correct_dialogues.append(1)
+            success.append(predicted_tracker)
 
     logger.info("Finished collecting predictions.")
     with warnings.catch_warnings():
@@ -474,7 +472,7 @@ def collect_story_predictions(
 
     in_training_data_fraction = _in_training_data_fraction(action_list)
 
-    log_evaluation_table(
+    _log_evaluation_table(
         [1] * len(completed_trackers),
         "END-TO-END" if use_e2e else "CONVERSATION",
         report,
@@ -486,9 +484,10 @@ def collect_story_predictions(
     )
 
     return (
-        StoryEvalution(
+        StoryEvaluation(
             evaluation_store=story_eval_store,
             failed_stories=failed,
+            successful_stories=success,
             action_list=action_list,
             in_training_data_fraction=in_training_data_fraction,
         ),
@@ -496,19 +495,20 @@ def collect_story_predictions(
     )
 
 
-def log_failed_stories(failed, out_directory):
+def _log_stories(stories: List, filename: Text, out_directory: Text) -> None:
     """Take stories as a list of dicts."""
     if not out_directory:
         return
+
     with open(
-        os.path.join(out_directory, FAILED_STORIES_FILE), "w", encoding=DEFAULT_ENCODING
+        os.path.join(out_directory, filename), "w", encoding=DEFAULT_ENCODING
     ) as f:
-        if len(failed) == 0:
-            f.write("<!-- All stories passed -->")
-        else:
-            for failure in failed:
-                f.write(failure.export_stories(include_source=True))
-                f.write("\n\n")
+        if len(stories) == 0:
+            f.write("<!-- No stories found. -->")
+
+        for story in stories:
+            f.write(story.export_stories(include_source=True))
+            f.write("\n\n")
 
 
 async def test(
@@ -519,13 +519,32 @@ async def test(
     fail_on_prediction_errors: bool = False,
     e2e: bool = False,
     disable_plotting: bool = False,
-):
-    """Run the evaluation of the stories, optionally plot the results."""
-    from rasa.nlu.test import get_evaluation_metrics
+    successes: bool = False,
+    errors: bool = True,
+) -> Dict[Text, Any]:
+    """Run the evaluation of the stories, optionally plot the results.
+
+    Args:
+        stories: the stories to evaulate on
+        agent: the agent
+        max_stories: maximum number of stories to consider
+        out_directory: path to directory to results to
+        fail_on_prediction_errors: boolean indicating whether to fail on prediction
+            errors or not
+        e2e: boolean indicating whether to use end to end evaluation or not
+        disable_plotting: boolean indicating whether to disable plotting or not
+        successes: boolean indicating whether to write down successful predictions or
+            not
+        errors: boolean indicating whether to write down incorrect predictions or not
+
+    Returns:
+        Evaluation summary.
+    """
+    from rasa.test import get_evaluation_metrics
 
     completed_trackers = await _generate_trackers(stories, agent, max_stories, e2e)
 
-    story_evaluation, _ = collect_story_predictions(
+    story_evaluation, _ = _collect_story_predictions(
         completed_trackers, agent, fail_on_prediction_errors, e2e
     )
 
@@ -537,22 +556,46 @@ async def test(
         warnings.simplefilter("ignore", UndefinedMetricWarning)
 
         targets, predictions = evaluation_store.serialise()
-        report, precision, f1, accuracy = get_evaluation_metrics(targets, predictions)
 
-    if out_directory:
-        plot_story_evaluation(
+        if out_directory:
+            report, precision, f1, accuracy = get_evaluation_metrics(
+                targets, predictions, output_dict=True
+            )
+
+            report_filename = os.path.join(out_directory, REPORT_STORIES_FILE)
+            io_utils.dump_obj_as_json_to_file(report_filename, report)
+            logger.info(f"Stories report saved to {report_filename}.")
+        else:
+            report, precision, f1, accuracy = get_evaluation_metrics(
+                targets, predictions, output_dict=True
+            )
+
+    _log_evaluation_table(
+        evaluation_store.action_targets,
+        "ACTION",
+        report,
+        precision,
+        f1,
+        accuracy,
+        story_evaluation.in_training_data_fraction,
+        include_report=False,
+    )
+
+    if not disable_plotting:
+        _plot_story_evaluation(
             evaluation_store.action_targets,
             evaluation_store.action_predictions,
-            report,
-            precision,
-            f1,
-            accuracy,
-            story_evaluation.in_training_data_fraction,
             out_directory,
-            disable_plotting,
         )
 
-    log_failed_stories(story_evaluation.failed_stories, out_directory)
+    if errors:
+        _log_stories(
+            story_evaluation.failed_stories, FAILED_STORIES_FILE, out_directory
+        )
+    if successes:
+        _log_stories(
+            story_evaluation.successful_stories, SUCCESSFUL_STORIES_FILE, out_directory
+        )
 
     return {
         "report": report,
@@ -565,21 +608,19 @@ async def test(
     }
 
 
-def log_evaluation_table(
-    golds,
-    name,
-    report,
-    precision,
-    f1,
-    accuracy,
-    in_training_data_fraction,
-    include_report=True,
+def _log_evaluation_table(
+    golds: List[Any],
+    name: Text,
+    report: Dict[Text, Any],
+    precision: float,
+    f1: float,
+    accuracy: float,
+    in_training_data_fraction: float,
+    include_report: bool = True,
 ):  # pragma: no cover
     """Log the sklearn evaluation metrics."""
     logger.info(f"Evaluation Results on {name} level:")
-    logger.info(
-        "\tCorrect:          {} / {}".format(int(len(golds) * accuracy), len(golds))
-    )
+    logger.info(f"\tCorrect:          {int(len(golds) * accuracy)} / {len(golds)}")
     logger.info(f"\tF1-Score:         {f1:.3f}")
     logger.info(f"\tPrecision:        {precision:.3f}")
     logger.info(f"\tAccuracy:         {accuracy:.3f}")
@@ -589,56 +630,40 @@ def log_evaluation_table(
         logger.info(f"\tClassification report: \n{report}")
 
 
-def plot_story_evaluation(
-    test_y,
-    predictions,
-    report,
-    precision,
-    f1,
-    accuracy,
-    in_training_data_fraction,
-    out_directory,
-    disable_plotting,
-):
-    """Plot the results of story evaluation"""
+def _plot_story_evaluation(
+    targets: List[Text], predictions: List[Text], output_directory: Optional[Text]
+) -> None:
+    """Plot a confusion matrix of story evaluation."""
     from sklearn.metrics import confusion_matrix
     from sklearn.utils.multiclass import unique_labels
-    import matplotlib.pyplot as plt
     from rasa.utils.plotting import plot_confusion_matrix
 
-    log_evaluation_table(
-        test_y,
-        "ACTION",
-        report,
-        precision,
-        f1,
-        accuracy,
-        in_training_data_fraction,
-        include_report=True,
-    )
-
-    if disable_plotting:
-        return
+    confusion_matrix_filename = CONFUSION_MATRIX_STORIES_FILE
+    if output_directory:
+        confusion_matrix_filename = os.path.join(
+            output_directory, confusion_matrix_filename
+        )
 
-    cnf_matrix = confusion_matrix(test_y, predictions)
+    cnf_matrix = confusion_matrix(targets, predictions)
 
     plot_confusion_matrix(
         cnf_matrix,
-        classes=unique_labels(test_y, predictions),
+        classes=unique_labels(targets, predictions),
         title="Action Confusion matrix",
+        output_file=confusion_matrix_filename,
     )
 
-    fig = plt.gcf()
-    fig.set_size_inches(int(20), int(20))
-    fig.savefig(os.path.join(out_directory, "story_confmat.pdf"), bbox_inches="tight")
-
 
 async def compare_models_in_dir(
     model_dir: Text, stories_file: Text, output: Text
 ) -> None:
-    """Evaluates multiple trained models in a directory on a test set."""
-    import rasa.utils.io as io_utils
+    """Evaluate multiple trained models in a directory on a test set.
 
+    Args:
+        model_dir: path to directory that contains the models to evaluate
+        stories_file: path to the story file
+        output: output directory to store results to
+    """
     number_correct = defaultdict(list)
 
     for run in io_utils.list_subdirectories(model_dir):
@@ -657,21 +682,26 @@ async def compare_models_in_dir(
         for k, v in number_correct_in_run.items():
             number_correct[k].append(v)
 
-    rasa.utils.io.dump_obj_as_json_to_file(
+    io_utils.dump_obj_as_json_to_file(
         os.path.join(output, RESULTS_FILE), number_correct
     )
 
 
 async def compare_models(models: List[Text], stories_file: Text, output: Text) -> None:
-    """Evaluates provided trained models on a test set."""
+    """Evaluate provided trained models on a test set.
 
+    Args:
+        models: list of trained model paths
+        stories_file: path to the story file
+        output: output directory to store results to
+    """
     number_correct = defaultdict(list)
 
     for model in models:
         number_of_correct_stories = await _evaluate_core_model(model, stories_file)
         number_correct[os.path.basename(model)].append(number_of_correct_stories)
 
-    rasa.utils.io.dump_obj_as_json_to_file(
+    io_utils.dump_obj_as_json_to_file(
         os.path.join(output, RESULTS_FILE), number_correct
     )
 
@@ -683,89 +713,13 @@ async def _evaluate_core_model(model: Text, stories_file: Text) -> int:
 
     agent = Agent.load(model)
     completed_trackers = await _generate_trackers(stories_file, agent)
-    story_eval_store, number_of_stories = collect_story_predictions(
+    story_eval_store, number_of_stories = _collect_story_predictions(
         completed_trackers, agent
     )
     failed_stories = story_eval_store.failed_stories
     return number_of_stories - len(failed_stories)
 
 
-def plot_nlu_results(output: Text, number_of_examples: List[int]) -> None:
-    """Plot NLU model comparison graph"""
-    graph_path = os.path.join(output, "nlu_model_comparison_graph.pdf")
-
-    _plot_curve(
-        output,
-        number_of_examples,
-        x_label_text="Number of intent examples present during training",
-        y_label_text="Label-weighted average F1 score on test set",
-        graph_path=graph_path,
-    )
-
-
-def plot_core_results(output: Text, number_of_examples: List[int]) -> None:
-    """Plot core model comparison graph"""
-    graph_path = os.path.join(output, "core_model_comparison_graph.pdf")
-
-    _plot_curve(
-        output,
-        number_of_examples,
-        x_label_text="Number of stories present during training",
-        y_label_text="Number of correct test stories",
-        graph_path=graph_path,
-    )
-
-
-def _plot_curve(
-    output: Text,
-    number_of_examples: List[int],
-    x_label_text: Text,
-    y_label_text: Text,
-    graph_path: Text,
-) -> None:
-    """Plot the results from a model comparison.
-
-    Args:
-        output: Output directory to save resulting plots to
-        number_of_examples: Number of examples per run
-        x_label_text: text for the x axis
-        y_label_text: text for the y axis
-        graph_path: output path of the plot
-    """
-    import matplotlib.pyplot as plt
-    import numpy as np
-    import rasa.utils.io
-
-    ax = plt.gca()
-
-    # load results from file
-    data = rasa.utils.io.read_json_file(os.path.join(output, RESULTS_FILE))
-    x = number_of_examples
-
-    # compute mean of all the runs for different configs
-    for label in data.keys():
-        if len(data[label]) == 0:
-            continue
-        mean = np.mean(data[label], axis=0)
-        std = np.std(data[label], axis=0)
-        ax.plot(x, mean, label=label, marker=".")
-        ax.fill_between(
-            x,
-            [m - s for m, s in zip(mean, std)],
-            [m + s for m, s in zip(mean, std)],
-            color="#6b2def",
-            alpha=0.2,
-        )
-    ax.legend(loc=4)
-
-    ax.set_xlabel(x_label_text)
-    ax.set_ylabel(y_label_text)
-
-    plt.savefig(graph_path, format="pdf")
-
-    logger.info(f"Comparison graph saved to '{graph_path}'.")
-
-
 if __name__ == "__main__":
     raise RuntimeError(
         "Calling `rasa.core.test` directly is no longer supported. Please use "
diff --git a/rasa/nlu/test.py b/rasa/nlu/test.py
index 4464d6c7efc5..8b1c93ba8296 100644
--- a/rasa/nlu/test.py
+++ b/rasa/nlu/test.py
@@ -87,62 +87,6 @@ def log_evaluation_table(
     logger.info(f"Classification report: \n{report}")
 
 
-def get_evaluation_metrics(
-    targets: Iterable[Any],
-    predictions: Iterable[Any],
-    output_dict: bool = False,
-    exclude_label: Text = None,
-) -> Tuple[Union[Text, Dict[Text, Dict[Text, float]]], float, float, float]:
-    """Compute the f1, precision, accuracy and summary report from sklearn.
-
-    Args:
-        targets: target labels
-        predictions: predicted labels
-        output_dict: if True sklearn returns a summary report as dict, if False the
-          report is in string format
-        exclude_label: labels to exclude from evaluation
-
-    Returns: a report from sklearn, precision, f1, and accuracy values
-    """
-    from sklearn import metrics
-
-    targets = clean_labels(targets)
-    predictions = clean_labels(predictions)
-
-    labels = get_unique_labels(targets, exclude_label)
-    if not labels:
-        logger.warning("No labels to evaluate. Skip evaluation.")
-        return {}, 0.0, 0.0, 0.0
-
-    report = metrics.classification_report(
-        targets, predictions, labels=labels, output_dict=output_dict
-    )
-    precision = metrics.precision_score(
-        targets, predictions, labels=labels, average="weighted"
-    )
-    f1 = metrics.f1_score(targets, predictions, labels=labels, average="weighted")
-    accuracy = metrics.accuracy_score(targets, predictions)
-
-    return report, precision, f1, accuracy
-
-
-def get_unique_labels(
-    targets: Iterable[Text], exclude_label: Optional[Text]
-) -> List[Text]:
-    """Get unique labels. Exclude 'exclude_label' if specified.
-
-    Args:
-        targets: labels
-        exclude_label: label to exclude
-
-    Returns: unique list of labels
-    """
-    labels = set(targets)
-    if exclude_label and exclude_label in labels:
-        labels.remove(exclude_label)
-    return list(labels)
-
-
 def remove_empty_intent_examples(
     intent_results: List[IntentEvaluationResult],
 ) -> List[IntentEvaluationResult]:
@@ -190,17 +134,6 @@ def remove_empty_response_examples(
     return filtered
 
 
-def clean_labels(labels: Iterable[Text]) -> List[Text]:
-    """Remove `None` labels. sklearn metrics do not support them.
-
-    Args:
-        labels: list of labels
-
-    Returns: cleaned labels
-    """
-    return [label if label is not None else "" for label in labels]
-
-
 def drop_intents_below_freq(
     training_data: TrainingData, cutoff: int = 5
 ) -> TrainingData:
@@ -449,6 +382,7 @@ def evaluate_response_selections(
     """
     import sklearn.metrics
     import sklearn.utils.multiclass
+    from rasa.test import get_evaluation_metrics
 
     # remove empty response targets
     num_examples = len(response_selection_results)
@@ -631,6 +565,7 @@ def evaluate_intents(
     """
     import sklearn.metrics
     import sklearn.utils.multiclass
+    from rasa.test import get_evaluation_metrics
 
     # remove empty intent targets
     num_examples = len(intent_results)
@@ -925,6 +860,7 @@ def evaluate_entities(
     """
     import sklearn.metrics
     import sklearn.utils.multiclass
+    from rasa.test import get_evaluation_metrics
 
     aligned_predictions = align_all_entity_predictions(entity_results, extractors)
     merged_targets = merge_labels(aligned_predictions)
@@ -1971,6 +1907,8 @@ def _compute_metrics(
 
     Returns: metrics
     """
+    from rasa.test import get_evaluation_metrics
+
     # compute fold metrics
     targets, predictions = _targets_predictions_from(
         results, target_key, prediction_key
@@ -1991,6 +1929,8 @@ def _compute_entity_metrics(
 
     Returns: entity metrics
     """
+    from rasa.test import get_evaluation_metrics
+
     entity_metric_results: EntityMetrics = defaultdict(lambda: defaultdict(list))
     extractors = get_entity_extractors(interpreter)
 
diff --git a/rasa/test.py b/rasa/test.py
index 34dfe528b467..5c6b56acb5ec 100644
--- a/rasa/test.py
+++ b/rasa/test.py
@@ -1,7 +1,7 @@
 import asyncio
 import logging
 import os
-from typing import Text, Dict, Optional, List, Any
+from typing import Text, Dict, Optional, List, Any, Iterable, Tuple, Union
 from pathlib import Path
 
 import rasa.utils.io as io_utils
@@ -10,7 +10,7 @@
     RESULTS_FILE,
     NUMBER_OF_TRAINING_STORIES_FILE,
 )
-from rasa.cli.utils import print_error, print_warning
+import rasa.cli.utils as cli_utils
 import rasa.utils.common as utils
 from rasa.exceptions import ModelNotFound
 
@@ -20,7 +20,7 @@
 def test_core_models_in_directory(
     model_directory: Text, stories: Text, output: Text
 ) -> None:
-    from rasa.core.test import compare_models_in_dir, plot_core_results
+    from rasa.core.test import compare_models_in_dir
 
     model_directory = _get_sanitized_model_directory(model_directory)
 
@@ -32,6 +32,21 @@ def test_core_models_in_directory(
     plot_core_results(output, number_of_stories)
 
 
+def plot_core_results(output: Text, number_of_examples: List[int]) -> None:
+    """Plot core model comparison graph"""
+    import rasa.utils.plotting as plotting_utils
+
+    graph_path = os.path.join(output, "core_model_comparison_graph.pdf")
+
+    plotting_utils.plot_curve(
+        output,
+        number_of_examples,
+        x_label_text="Number of stories present during training",
+        y_label_text="Number of correct test stories",
+        graph_path=graph_path,
+    )
+
+
 def _get_sanitized_model_directory(model_directory: Text) -> Text:
     """Adjusts the `--model` argument of `rasa test core` when called with
     `--evaluate-model-directory`.
@@ -52,7 +67,7 @@ def _get_sanitized_model_directory(model_directory: Text) -> Text:
     p = Path(model_directory)
     if p.is_file():
         if model_directory != rasa.model.get_latest_model():
-            print_warning(
+            cli_utils.print_warning(
                 "You passed a file as '--model'. Will use the directory containing "
                 "this file instead."
             )
@@ -107,7 +122,7 @@ def test_core(
     try:
         unpacked_model = rasa.model.get_model(model)
     except ModelNotFound:
-        print_error(
+        cli_utils.print_error(
             "Unable to test: could not find a model. Use 'rasa train' to train a "
             "Rasa model and provide it via the '--model' argument."
         )
@@ -116,7 +131,7 @@ def test_core(
     core_path, nlu_path = rasa.model.get_model_subdirectories(unpacked_model)
 
     if not core_path:
-        print_error(
+        cli_utils.print_error(
             "Unable to test: could not find a Core model. Use 'rasa train' to train a "
             "Rasa model and provide it via the '--model' argument."
         )
@@ -128,7 +143,7 @@ def test_core(
         if nlu_path:
             _interpreter = NaturalLanguageInterpreter.create(_endpoints.nlu or nlu_path)
         else:
-            print_warning(
+            cli_utils.print_warning(
                 "No NLU model found. Using default 'RegexInterpreter' for end-to-end "
                 "evaluation."
             )
@@ -157,7 +172,7 @@ def test_nlu(
     try:
         unpacked_model = get_model(model)
     except ModelNotFound:
-        print_error(
+        cli_utils.print_error(
             "Could not find any model. Use 'rasa train nlu' to train a "
             "Rasa model and provide it via the '--model' argument."
         )
@@ -173,7 +188,7 @@ def test_nlu(
         )
         run_evaluation(nlu_data, nlu_model, output_directory=output_directory, **kwargs)
     else:
-        print_error(
+        cli_utils.print_error(
             "Could not find any model. Use 'rasa train nlu' to train a "
             "Rasa model and provide it via the '--model' argument."
         )
@@ -193,7 +208,6 @@ def compare_nlu_models(
     from rasa.nlu.utils import write_json_to_file
     from rasa.utils.io import create_path
     from rasa.nlu.test import compare_nlu
-    from rasa.core.test import plot_nlu_results
 
     data = load_data(nlu)
     data = drop_intents_below_freq(data, cutoff=5)
@@ -223,6 +237,21 @@ def compare_nlu_models(
     plot_nlu_results(output, training_examples_per_run)
 
 
+def plot_nlu_results(output: Text, number_of_examples: List[int]) -> None:
+    """Plot NLU model comparison graph"""
+    import rasa.utils.plotting as plotting_utils
+
+    graph_path = os.path.join(output, "nlu_model_comparison_graph.pdf")
+
+    plotting_utils.plot_curve(
+        output,
+        number_of_examples,
+        x_label_text="Number of intent examples present during training",
+        y_label_text="Label-weighted average F1 score on test set",
+        graph_path=graph_path,
+    )
+
+
 def perform_nlu_cross_validation(
     config: Text,
     nlu: Text,
@@ -260,3 +289,73 @@ def perform_nlu_cross_validation(
         logger.info("Response Selection evaluation results")
         log_results(response_selection_results.train, "train")
         log_results(response_selection_results.test, "test")
+
+
+def get_evaluation_metrics(
+    targets: Iterable[Any],
+    predictions: Iterable[Any],
+    output_dict: bool = False,
+    exclude_label: Text = None,
+) -> Tuple[Union[Text, Dict[Text, Dict[Text, float]]], float, float, float]:
+    """Compute the f1, precision, accuracy and summary report from sklearn.
+
+    Args:
+        targets: target labels
+        predictions: predicted labels
+        output_dict: if True sklearn returns a summary report as dict, if False the
+          report is in string format
+        exclude_label: labels to exclude from evaluation
+
+    Returns:
+        Report from sklearn, precision, f1, and accuracy values.
+    """
+    from sklearn import metrics
+
+    targets = clean_labels(targets)
+    predictions = clean_labels(predictions)
+
+    labels = get_unique_labels(targets, exclude_label)
+    if not labels:
+        logger.warning("No labels to evaluate. Skip evaluation.")
+        return {}, 0.0, 0.0, 0.0
+
+    report = metrics.classification_report(
+        targets, predictions, labels=labels, output_dict=output_dict
+    )
+    precision = metrics.precision_score(
+        targets, predictions, labels=labels, average="weighted"
+    )
+    f1 = metrics.f1_score(targets, predictions, labels=labels, average="weighted")
+    accuracy = metrics.accuracy_score(targets, predictions)
+
+    return report, precision, f1, accuracy
+
+
+def clean_labels(labels: Iterable[Text]) -> List[Text]:
+    """Remove `None` labels. sklearn metrics do not support them.
+
+    Args:
+        labels: list of labels
+
+    Returns:
+        Cleaned labels.
+    """
+    return [label if label is not None else "" for label in labels]
+
+
+def get_unique_labels(
+    targets: Iterable[Text], exclude_label: Optional[Text]
+) -> List[Text]:
+    """Get unique labels. Exclude 'exclude_label' if specified.
+
+    Args:
+        targets: labels
+        exclude_label: label to exclude
+
+    Returns:
+         Unique labels.
+    """
+    labels = set(targets)
+    if exclude_label and exclude_label in labels:
+        labels.remove(exclude_label)
+    return list(labels)
diff --git a/rasa/utils/plotting.py b/rasa/utils/plotting.py
index 32ed5005c281..6035464e1c55 100644
--- a/rasa/utils/plotting.py
+++ b/rasa/utils/plotting.py
@@ -1,12 +1,31 @@
 import logging
 import itertools
+import os
+
 import numpy as np
 from typing import List, Text, Optional, Union, Any
+import matplotlib
 
+from rasa.constants import RESULTS_FILE
+import rasa.utils.io as io_utils
 
 logger = logging.getLogger(__name__)
 
 
+# At first, matplotlib will be initialized with default OS-specific available backend
+# if that didn't happen, we'll try to set it up manually
+if matplotlib.get_backend() is not None:
+    pass
+else:  # pragma: no cover
+    try:
+        # If the `tkinter` package is available, we can use the `TkAgg` backend
+        import tkinter
+
+        matplotlib.use("TkAgg")
+    except ImportError:
+        matplotlib.use("agg")
+
+
 def plot_confusion_matrix(
     confusion_matrix: np.ndarray,
     classes: Union[np.ndarray, List[Text]],
@@ -112,3 +131,51 @@ def plot_histogram(
         fig = plt.gcf()
         fig.set_size_inches(10, 10)
         fig.savefig(output_file, bbox_inches="tight")
+
+
+def plot_curve(
+    output_directory: Text,
+    number_of_examples: List[int],
+    x_label_text: Text,
+    y_label_text: Text,
+    graph_path: Text,
+) -> None:
+    """Plot the results from a model comparison.
+
+    Args:
+        output_directory: Output directory to save resulting plots to
+        number_of_examples: Number of examples per run
+        x_label_text: text for the x axis
+        y_label_text: text for the y axis
+        graph_path: output path of the plot
+    """
+    import matplotlib.pyplot as plt
+
+    ax = plt.gca()
+
+    # load results from file
+    data = io_utils.read_json_file(os.path.join(output_directory, RESULTS_FILE))
+    x = number_of_examples
+
+    # compute mean of all the runs for different configs
+    for label in data.keys():
+        if len(data[label]) == 0:
+            continue
+        mean = np.mean(data[label], axis=0)
+        std = np.std(data[label], axis=0)
+        ax.plot(x, mean, label=label, marker=".")
+        ax.fill_between(
+            x,
+            [m - s for m, s in zip(mean, std)],
+            [m + s for m, s in zip(mean, std)],
+            color="#6b2def",
+            alpha=0.2,
+        )
+    ax.legend(loc=4)
+
+    ax.set_xlabel(x_label_text)
+    ax.set_ylabel(y_label_text)
+
+    plt.savefig(graph_path, format="pdf")
+
+    logger.info(f"Comparison graph saved to '{graph_path}'.")
diff --git a/tests/cli/test_rasa_test.py b/tests/cli/test_rasa_test.py
index 2e4d7834fef0..823327ef521e 100644
--- a/tests/cli/test_rasa_test.py
+++ b/tests/cli/test_rasa_test.py
@@ -1,5 +1,7 @@
 import os
 from shutil import copyfile
+
+from rasa.core.test import CONFUSION_MATRIX_STORIES_FILE
 from rasa.constants import DEFAULT_RESULTS_PATH, RESULTS_FILE
 from rasa.utils.io import list_files, write_yaml_file
 from typing import Callable
@@ -15,7 +17,7 @@ def test_test_core(run_in_simple_project: Callable[..., RunResult]):
 def test_test_core_no_plot(run_in_simple_project: Callable[..., RunResult]):
     run_in_simple_project("test", "core", "--no-plot")
 
-    assert not os.path.exists("results/story_confmat.pdf")
+    assert not os.path.exists(f"results/{CONFUSION_MATRIX_STORIES_FILE}")
 
 
 def test_test(run_in_simple_project_with_model: Callable[..., RunResult]):
@@ -101,13 +103,11 @@ def test_test_core_comparison_after_train(
     run_in_simple_project: Callable[..., RunResult]
 ):
     write_yaml_file(
-        {"language": "en", "policies": [{"name": "MemoizationPolicy"}],},
-        "config_1.yml",
+        {"language": "en", "policies": [{"name": "MemoizationPolicy"}]}, "config_1.yml"
     )
 
     write_yaml_file(
-        {"language": "en", "policies": [{"name": "MemoizationPolicy"}],},
-        "config_2.yml",
+        {"language": "en", "policies": [{"name": "MemoizationPolicy"}]}, "config_2.yml"
     )
 
     run_in_simple_project(
@@ -155,10 +155,10 @@ def test_test_help(run: Callable[..., RunResult]):
     help_text = """usage: rasa test [-h] [-v] [-vv] [--quiet] [-m MODEL] [-s STORIES]
                  [--max-stories MAX_STORIES] [--endpoints ENDPOINTS]
                  [--fail-on-prediction-errors] [--url URL]
-                 [--evaluate-model-directory] [-u NLU] [--out OUT]
-                 [--successes] [--no-errors] [-c CONFIG [CONFIG ...]]
-                 [--cross-validation] [-f FOLDS] [-r RUNS]
-                 [-p PERCENTAGES [PERCENTAGES ...]] [--no-plot]
+                 [--evaluate-model-directory] [-u NLU]
+                 [-c CONFIG [CONFIG ...]] [--cross-validation] [-f FOLDS]
+                 [-r RUNS] [-p PERCENTAGES [PERCENTAGES ...]] [--no-plot]
+                 [--successes] [--no-errors] [--out OUT]
                  {core,nlu} ..."""
 
     lines = help_text.split("\n")
@@ -171,9 +171,9 @@ def test_test_nlu_help(run: Callable[..., RunResult]):
     output = run("test", "nlu", "--help")
 
     help_text = """usage: rasa test nlu [-h] [-v] [-vv] [--quiet] [-m MODEL] [-u NLU] [--out OUT]
-                     [--successes] [--no-errors] [-c CONFIG [CONFIG ...]]
-                     [--cross-validation] [-f FOLDS] [-r RUNS]
-                     [-p PERCENTAGES [PERCENTAGES ...]] [--no-plot]"""
+                     [-c CONFIG [CONFIG ...]] [--cross-validation] [-f FOLDS]
+                     [-r RUNS] [-p PERCENTAGES [PERCENTAGES ...]] [--no-plot]
+                     [--successes] [--no-errors]"""
 
     lines = help_text.split("\n")
 
@@ -188,7 +188,8 @@ def test_test_core_help(run: Callable[..., RunResult]):
                       [-s STORIES] [--max-stories MAX_STORIES] [--out OUT]
                       [--e2e] [--endpoints ENDPOINTS]
                       [--fail-on-prediction-errors] [--url URL]
-                      [--evaluate-model-directory] [--no-plot]"""
+                      [--evaluate-model-directory] [--no-plot] [--successes]
+                      [--no-errors]"""
 
     lines = help_text.split("\n")
 
diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py
index 899935637b3c..e3260984bd78 100644
--- a/tests/core/test_evaluation.py
+++ b/tests/core/test_evaluation.py
@@ -4,9 +4,12 @@
 import rasa.utils.io
 from rasa.core.test import (
     _generate_trackers,
-    collect_story_predictions,
+    _collect_story_predictions,
     test,
     FAILED_STORIES_FILE,
+    CONFUSION_MATRIX_STORIES_FILE,
+    REPORT_STORIES_FILE,
+    SUCCESSFUL_STORIES_FILE,
 )
 from rasa.core.policies.memoization import MemoizationPolicy
 
@@ -23,9 +26,11 @@
 )
 
 
-async def test_evaluation_image_creation(tmpdir: Path, default_agent: Agent):
-    stories_path = str(tmpdir / FAILED_STORIES_FILE)
-    img_path = str(tmpdir / "story_confmat.pdf")
+async def test_evaluation_file_creation(tmpdir: Path, default_agent: Agent):
+    failed_stories_path = str(tmpdir / FAILED_STORIES_FILE)
+    success_stories_path = str(tmpdir / SUCCESSFUL_STORIES_FILE)
+    report_path = str(tmpdir / REPORT_STORIES_FILE)
+    confusion_matrix_path = str(tmpdir / CONFUSION_MATRIX_STORIES_FILE)
 
     await test(
         stories=DEFAULT_STORIES_FILE,
@@ -33,10 +38,14 @@ async def test_evaluation_image_creation(tmpdir: Path, default_agent: Agent):
         out_directory=str(tmpdir),
         max_stories=None,
         e2e=False,
+        errors=True,
+        successes=True,
     )
 
-    assert os.path.isfile(img_path)
-    assert os.path.isfile(stories_path)
+    assert os.path.isfile(failed_stories_path)
+    assert os.path.isfile(success_stories_path)
+    assert os.path.isfile(report_path)
+    assert os.path.isfile(confusion_matrix_path)
 
 
 async def test_end_to_end_evaluation_script(default_agent: Agent):
@@ -44,7 +53,7 @@ async def test_end_to_end_evaluation_script(default_agent: Agent):
         END_TO_END_STORY_FILE, default_agent, use_e2e=True
     )
 
-    story_evaluation, num_stories = collect_story_predictions(
+    story_evaluation, num_stories = _collect_story_predictions(
         completed_trackers, default_agent, use_e2e=True
     )
 
@@ -81,7 +90,7 @@ async def test_end_to_end_evaluation_script_unknown_entity(default_agent: Agent)
         E2E_STORY_FILE_UNKNOWN_ENTITY, default_agent, use_e2e=True
     )
 
-    story_evaluation, num_stories = collect_story_predictions(
+    story_evaluation, num_stories = _collect_story_predictions(
         completed_trackers, default_agent, use_e2e=True
     )
 
@@ -95,7 +104,7 @@ async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
         "data/test_evaluations/form-end-to-end-stories.md", form_bot_agent, use_e2e=True
     )
 
-    story_evaluation, num_stories = collect_story_predictions(
+    story_evaluation, num_stories = _collect_story_predictions(
         test_stories, form_bot_agent, use_e2e=True
     )
 
@@ -133,7 +142,7 @@ async def test_end_to_evaluation_trips_circuit_breaker():
         E2E_STORY_FILE_TRIPS_CIRCUIT_BREAKER, agent, use_e2e=True
     )
 
-    story_evaluation, num_stories = collect_story_predictions(
+    story_evaluation, num_stories = _collect_story_predictions(
         test_stories, agent, use_e2e=True
     )
 
diff --git a/tests/nlu/test_evaluation.py b/tests/nlu/test_evaluation.py
index 57bceace796c..e6dff4ebfe9f 100644
--- a/tests/nlu/test_evaluation.py
+++ b/tests/nlu/test_evaluation.py
@@ -33,8 +33,6 @@
     evaluate_intents,
     evaluate_entities,
     evaluate_response_selections,
-    get_unique_labels,
-    get_evaluation_metrics,
     NO_ENTITY,
     collect_successful_entity_predictions,
     collect_incorrect_entity_predictions,
@@ -50,7 +48,6 @@
 import json
 import os
 from rasa.nlu import training_data
-from tests.nlu import utilities
 from tests.nlu.conftest import DEFAULT_DATA_PATH
 from rasa.nlu.selectors.response_selector import ResponseSelector
 from rasa.nlu.test import is_response_selector_present
@@ -834,61 +831,6 @@ def test_label_replacement():
     assert substitute_labels(original_labels, "O", "no_entity") == target_labels
 
 
-@pytest.mark.parametrize(
-    "targets,exclude_label,expected",
-    [
-        (
-            ["no_entity", "location", "location", "location", "person"],
-            NO_ENTITY,
-            ["location", "person"],
-        ),
-        (
-            ["no_entity", "location", "location", "location", "person"],
-            None,
-            ["no_entity", "location", "person"],
-        ),
-        (["no_entity"], NO_ENTITY, []),
-        (["location", "location", "location"], NO_ENTITY, ["location"]),
-        ([], None, []),
-    ],
-)
-def test_get_label_set(targets, exclude_label, expected):
-    actual = get_unique_labels(targets, exclude_label)
-    assert set(expected) == set(actual)
-
-
-@pytest.mark.parametrize(
-    "targets,predictions,expected_precision,expected_fscore,expected_accuracy",
-    [
-        (
-            ["no_entity", "location", "no_entity", "location", "no_entity"],
-            ["no_entity", "location", "no_entity", "no_entity", "person"],
-            1.0,
-            0.6666666666666666,
-            3 / 5,
-        ),
-        (
-            ["no_entity", "no_entity", "no_entity", "no_entity", "person"],
-            ["no_entity", "no_entity", "no_entity", "no_entity", "no_entity"],
-            0.0,
-            0.0,
-            4 / 5,
-        ),
-    ],
-)
-def test_get_evaluation_metrics(
-    targets, predictions, expected_precision, expected_fscore, expected_accuracy
-):
-    report, precision, f1, accuracy = get_evaluation_metrics(
-        targets, predictions, True, exclude_label=NO_ENTITY
-    )
-
-    assert f1 == expected_fscore
-    assert precision == expected_precision
-    assert accuracy == expected_accuracy
-    assert NO_ENTITY not in report
-
-
 def test_nlu_comparison(tmpdir, config_path, config_path_duplicate):
     # the configs need to be at a different path, otherwise the results are
     # combined on the same dictionary key and cannot be plotted properly
diff --git a/tests/test_test.py b/tests/test_test.py
index 1bc338770096..b07b801e4d05 100644
--- a/tests/test_test.py
+++ b/tests/test_test.py
@@ -1,9 +1,12 @@
 from pathlib import Path
+
+import pytest
 from _pytest.capture import CaptureFixture
 from _pytest.monkeypatch import MonkeyPatch
 
 import rasa.model
 import rasa.cli.utils
+from rasa.nlu.test import NO_ENTITY
 
 
 def monkeypatch_get_latest_model(tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
@@ -64,3 +67,62 @@ def test_get_sanitized_model_directory_when_passing_other_input(
     captured = capsys.readouterr()
     assert not captured.out
     assert new_modeldir == modeldir
+
+
+@pytest.mark.parametrize(
+    "targets,predictions,expected_precision,expected_fscore,expected_accuracy",
+    [
+        (
+            ["no_entity", "location", "no_entity", "location", "no_entity"],
+            ["no_entity", "location", "no_entity", "no_entity", "person"],
+            1.0,
+            0.6666666666666666,
+            3 / 5,
+        ),
+        (
+            ["no_entity", "no_entity", "no_entity", "no_entity", "person"],
+            ["no_entity", "no_entity", "no_entity", "no_entity", "no_entity"],
+            0.0,
+            0.0,
+            4 / 5,
+        ),
+    ],
+)
+def test_get_evaluation_metrics(
+    targets, predictions, expected_precision, expected_fscore, expected_accuracy
+):
+    from rasa.test import get_evaluation_metrics
+
+    report, precision, f1, accuracy = get_evaluation_metrics(
+        targets, predictions, True, exclude_label=NO_ENTITY
+    )
+
+    assert f1 == expected_fscore
+    assert precision == expected_precision
+    assert accuracy == expected_accuracy
+    assert NO_ENTITY not in report
+
+
+@pytest.mark.parametrize(
+    "targets,exclude_label,expected",
+    [
+        (
+            ["no_entity", "location", "location", "location", "person"],
+            NO_ENTITY,
+            ["location", "person"],
+        ),
+        (
+            ["no_entity", "location", "location", "location", "person"],
+            None,
+            ["no_entity", "location", "person"],
+        ),
+        (["no_entity"], NO_ENTITY, []),
+        (["location", "location", "location"], NO_ENTITY, ["location"]),
+        ([], None, []),
+    ],
+)
+def test_get_label_set(targets, exclude_label, expected):
+    from rasa.test import get_unique_labels
+
+    actual = get_unique_labels(targets, exclude_label)
+    assert set(expected) == set(actual)


Code review comments:
diff_hunk: @@ -290,11 +284,11 @@ def _emulate_form_rejection(processor, partial_tracker):
 
 
 def _collect_action_executed_predictions(
-    processor,
-    partial_tracker,
-    event,
-    fail_on_prediction_errors,
-    circuit_breaker_tripped,
+    processor: "MessageProcessor",
+    partial_tracker: DialogueStateTracker,
+    event: ActionExecuted,
+    fail_on_prediction_errors: bool,
+    circuit_breaker_tripped: bool,
 ):
path: rasa/core/test.py
text: @user1:
```suggestion
) -> Tuple[EvaluationStore, "Policy", float]
```

diff_hunk: @@ -486,29 +484,31 @@ def collect_story_predictions(
     )
 
     return (
-        StoryEvalution(
+        StoryEvaluation(
             evaluation_store=story_eval_store,
             failed_stories=failed,
+            successful_stories=success,
             action_list=action_list,
             in_training_data_fraction=in_training_data_fraction,
         ),
         number_of_stories,
     )
 
 
-def log_failed_stories(failed, out_directory):
+def _log_stories(stories: List, filename: Text, out_directory: Text) -> None:
     """Take stories as a list of dicts."""
path: rasa/core/test.py
text: @user1:
Given the changes to this function, is this docstring still accurate?

diff_hunk: @@ -565,21 +608,19 @@ async def test(
     }
 
 
-def log_evaluation_table(
-    golds,
-    name,
-    report,
-    precision,
-    f1,
-    accuracy,
-    in_training_data_fraction,
-    include_report=True,
+def _log_evaluation_table(
+    golds: List[Any],
+    name: Text,
+    report: Dict[Text, Any],
+    precision: float,
+    f1: float,
+    accuracy: float,
+    in_training_data_fraction: float,
+    include_report: bool = True,
 ):  # pragma: no cover
path: rasa/core/test.py
text: @user1:
```suggestion
) -> None:  # pragma: no cover
```

diff_hunk: @@ -260,3 +289,73 @@ def perform_nlu_cross_validation(
         logger.info("Response Selection evaluation results")
         log_results(response_selection_results.train, "train")
         log_results(response_selection_results.test, "test")
+
+
+def get_evaluation_metrics(
+    targets: Iterable[Any],
+    predictions: Iterable[Any],
+    output_dict: bool = False,
+    exclude_label: Text = None,
path: rasa/test.py
text: @user1:
```suggestion
    exclude_label: Optional[Text] = None,
```

diff_hunk: @@ -413,13 +413,13 @@
 
 
 def setup(sphinx):
-    sphinx.add_stylesheet("css/custom.css")
+    sphinx.add_css_file("css/custom.css")
 
     try:
         utils_path = os.path.abspath(os.path.join(__file__, "..", "utils"))
         sys.path.insert(0, utils_path)
         from StoryLexer import StoryLexer
 
-        sphinx.add_lexer("story", StoryLexer())
+        sphinx.add_lexer("story", StoryLexer)
path: docs/conf.py
text: @user1:
Are these changes linked to the others? From what I understand they are documentation-related

@author:
Yes, can also remove them. The documentation built was failing on GH actions and it reported that those function are deprecated and that we should switch to xyz. That's why I updated them. But I guess it was just a random failure of GH actions and I should be able to undo those changes. Might be better to update the deprecated functions in a separate PR.

diff_hunk: @@ -15,30 +15,21 @@
 
 if typing.TYPE_CHECKING:
     from rasa.core.agent import Agent
+    from rasa.core.processor import MessageProcessor
 
-import matplotlib
 
+CONFUSION_MATRIX_STORIES_FILE = "story_confusion_matrix.png"
+REPORT_STORIES_FILE = "story_report.json"
 FAILED_STORIES_FILE = "failed_stories.md"
-
-# At first, matplotlib will be initialized with default OS-specific available backend
-# if that didn't happen, we'll try to set it up manually
-if matplotlib.get_backend() is not None:
-    pass
-else:  # pragma: no cover
-    try:
-        # If the `tkinter` package is available, we can use the `TkAgg` backend
-        import tkinter
-
-        matplotlib.use("TkAgg")
-    except ImportError:
-        matplotlib.use("agg")
+SUCCESSFUL_STORIES_FILE = "successful_stories.md"
 
 
 logger = logging.getLogger(__name__)
 
-StoryEvalution = namedtuple(
+StoryEvaluation = namedtuple(
     "StoryEvaluation",
-    "evaluation_store failed_stories action_list in_training_data_fraction",
+    "evaluation_store failed_stories successful_stories action_list "
+    "in_training_data_fraction",
path: rasa/core/test.py
text: @user1:
Let's change this to a list of strings, so that we can avoid having to manually split the string across lines leaving a space at the end. e.g `["evaluation_store", "failed_stories", ...]`

diff_hunk: @@ -486,29 +484,31 @@ def collect_story_predictions(
     )
 
     return (
-        StoryEvalution(
+        StoryEvaluation(
             evaluation_store=story_eval_store,
             failed_stories=failed,
+            successful_stories=success,
             action_list=action_list,
             in_training_data_fraction=in_training_data_fraction,
         ),
         number_of_stories,
     )
 
 
-def log_failed_stories(failed, out_directory):
+def _log_stories(stories: List, filename: Text, out_directory: Text) -> None:
     """Take stories as a list of dicts."""
     if not out_directory:
         return
+
     with open(
-        os.path.join(out_directory, FAILED_STORIES_FILE), "w", encoding=DEFAULT_ENCODING
+        os.path.join(out_directory, filename), "w", encoding=DEFAULT_ENCODING
     ) as f:
-        if len(failed) == 0:
-            f.write("<!-- All stories passed -->")
-        else:
-            for failure in failed:
-                f.write(failure.export_stories(include_source=True))
-                f.write("\n\n")
+        if len(stories) == 0:
path: rasa/core/test.py
text: @user1:
Optional:
```suggestion
        if not stories:
```

diff_hunk: @@ -486,29 +484,31 @@ def collect_story_predictions(
     )
 
     return (
-        StoryEvalution(
+        StoryEvaluation(
             evaluation_store=story_eval_store,
             failed_stories=failed,
+            successful_stories=success,
             action_list=action_list,
             in_training_data_fraction=in_training_data_fraction,
         ),
         number_of_stories,
     )
 
 
-def log_failed_stories(failed, out_directory):
+def _log_stories(stories: List, filename: Text, out_directory: Text) -> None:
path: rasa/core/test.py
text: @user1:
```suggestion
def _log_stories(stories: List[DialogueStateTracker], filename: Text, out_directory: Text) -> None:
```

diff_hunk: @@ -180,3 +176,18 @@ def add_no_plot_param(
         help="Don't render evaluation plots.",
         required=required,
     )
+
+
+def add_errors_success_params(parser: argparse.ArgumentParser):
path: rasa/cli/arguments/test.py
text: @user1:
```suggestion
def add_errors_success_params(parser: argparse.ArgumentParser) -> None:
```