Repository: rasahq/rasa
Issue: RulePolicy core fallback config not loaded from model

<!-- THIS INFORMATION IS MANDATORY - YOUR ISSUE WILL BE CLOSED IF IT IS MISSING. If you don't know your Rasa version, use `rasa --version`. 
Please format any code or console output with three ticks ``` above and below.
If you are asking a usage question (e.g. "How do I do xyz") please post your question on https://forum.rasa.com instead -->

**Rasa version**: 2.0.0a2

**Rasa SDK version** (if used & relevant):

**Rasa X version** (if used & relevant):

**Python version**: 3.6.8

**Operating system** (windows, osx, ...): Ubuntu 18.04

**Issue**:
RulePolicy fallback configuration does not seem to be loaded from model. It predicts `action_default_fallback` even though I configured another action.
I added logging to check and when running `rasa train`, the RulePolicy is correctly instantiated with the parameters in the config file, but when running `rasa shell`, it is instantiated with the default parameters (at least the action name, I didn't check the other parameters). My custom policy is instantiated correctly with the same action and it works. I think I could submit a PR for this, the problem seems to be the inherited `load` method from the MemoizationPolicy in the RulePolicy.

**Content of configuration file (config.yml)** (if relevant):
```yml
# Configuration for Rasa NLU.
# https://rasa.com/docs/rasa/nlu/components/
language: en

pipeline:
  - name: ConveRTTokenizer
  - name: ConveRTFeaturizer
    case_sensitive: false
  - name: RegexFeaturizer
  - name: LexicalSyntacticFeaturizer
    case_sensitive: false
  - name: CountVectorsFeaturizer
    case_sensitive: false
  - name: CountVectorsFeaturizer
    case_sensitive: false
    analyzer: "char_wb"
    min_ngram: 1
    max_ngram: 4
  - name: DIETClassifier
    epochs: 100
  - name: EntitySynonymMapper
  - name: ResponseSelector
    epochs: 100
# Configuration for Rasa Core.
# https://rasa.com/docs/rasa/core/policies/
policies:
  - name: RulePolicy
    core_fallback_threshold: 0.01
    core_fallback_action_name: "action_my_fallback"
    enable_fallback_prediction: true
  - name: core.policies.unsupported_intent_policy.UnsupportedIntentPolicy
    fallback_action_name: "action_my_fallback"
```

**Content of domain file (domain.yml)** (if relevant):
```yml
...
actions:
  ...
  - action_my_fallback
...
```


Changed files:
rasa/core/policies/two_stage_fallback.py, rasa/core/policies/form_policy.py, rasa/core/policies/sklearn_policy.py, rasa/core/featurizers/tracker_featurizers.py, rasa/core/policies/mapping_policy.py, rasa/core/policies/rule_policy.py, rasa/core/policies/ensemble.py, rasa/core/policies/ted_policy.py, rasa/core/actions/action.py, rasa/core/policies/fallback.py, tests/core/policies/test_rule_policy.py, tests/core/test_policies.py, rasa/core/policies/policy.py, rasa/core/policies/memoization.py

diff --git a/rasa/core/actions/action.py b/rasa/core/actions/action.py
index 0c5d14af72b..65b901631fe 100644
--- a/rasa/core/actions/action.py
+++ b/rasa/core/actions/action.py
@@ -1,8 +1,7 @@
 import copy
 import json
 import logging
-import typing
-from typing import List, Text, Optional, Dict, Any, Set
+from typing import List, Text, Optional, Dict, Any, Set, TYPE_CHECKING
 
 import aiohttp
 
@@ -52,7 +51,8 @@ from rasa.shared.core.events import (
 from rasa.utils.endpoints import EndpointConfig, ClientResponseError
 from rasa.shared.core.domain import Domain
 
-if typing.TYPE_CHECKING:
+
+if TYPE_CHECKING:
     from rasa.shared.core.trackers import DialogueStateTracker
     from rasa.core.nlg import NaturalLanguageGenerator
     from rasa.core.channels.channel import OutputChannel
@@ -665,7 +665,7 @@ class RemoteAction(Action):
 
 class ActionExecutionRejection(Exception):
     """Raising this exception will allow other policies
-        to predict a different action"""
+    to predict a different action"""
 
     def __init__(self, action_name: Text, message: Optional[Text] = None) -> None:
         self.action_name = action_name
@@ -680,10 +680,10 @@ class ActionExecutionRejection(Exception):
 class ActionRevertFallbackEvents(Action):
     """Reverts events which were done during the `TwoStageFallbackPolicy`.
 
-       This reverts user messages and bot utterances done during a fallback
-       of the `TwoStageFallbackPolicy`. By doing so it is not necessary to
-       write custom stories for the different paths, but only of the happy
-       path.
+    This reverts user messages and bot utterances done during a fallback
+    of the `TwoStageFallbackPolicy`. By doing so it is not necessary to
+    write custom stories for the different paths, but only of the happy
+    path.
     """
 
     def name(self) -> Text:
@@ -760,9 +760,9 @@ def _revert_rephrasing_events() -> List[Event]:
 class ActionDefaultAskAffirmation(Action):
     """Default implementation which asks the user to affirm his intent.
 
-       It is suggested to overwrite this default action with a custom action
-       to have more meaningful prompts for the affirmations. E.g. have a
-       description of the intent instead of its identifier name.
+    It is suggested to overwrite this default action with a custom action
+    to have more meaningful prompts for the affirmations. E.g. have a
+    description of the intent instead of its identifier name.
     """
 
     def name(self) -> Text:
diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py
index 603d4eaf458..dc6d763f165 100644
--- a/rasa/core/featurizers/tracker_featurizers.py
+++ b/rasa/core/featurizers/tracker_featurizers.py
@@ -1,10 +1,11 @@
+from pathlib import Path
+
 import jsonpickle
 import logging
-import os
 
 from rasa.shared.nlu.constants import TEXT
 from tqdm import tqdm
-from typing import Tuple, List, Optional, Dict, Text
+from typing import Tuple, List, Optional, Dict, Text, Union
 import numpy as np
 
 from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
@@ -16,6 +17,8 @@ from rasa.shared.core.constants import USER
 import rasa.shared.utils.io
 from rasa.shared.nlu.training_data.features import Features
 
+FEATURIZER_FILE = "featurizer.json"
+
 logger = logging.getLogger(__name__)
 
 
@@ -177,13 +180,13 @@ class TrackerFeaturizer:
         trackers_as_states = self.prediction_states(trackers, domain)
         return self._featurize_states(trackers_as_states, interpreter)
 
-    def persist(self, path: Text) -> None:
+    def persist(self, path: Union[Text, Path]) -> None:
         """Persist the tracker featurizer to the given path.
 
         Args:
             path: The path to persist the tracker featurizer to.
         """
-        featurizer_file = os.path.join(path, "featurizer.json")
+        featurizer_file = Path(path) / FEATURIZER_FILE
         rasa.shared.utils.io.create_directory_for_file(featurizer_file)
 
         # noinspection PyTypeChecker
@@ -201,8 +204,8 @@ class TrackerFeaturizer:
         Returns:
             The loaded tracker featurizer.
         """
-        featurizer_file = os.path.join(path, "featurizer.json")
-        if os.path.isfile(featurizer_file):
+        featurizer_file = Path(path) / FEATURIZER_FILE
+        if featurizer_file.is_file():
             return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file))
 
         logger.error(
diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py
index 290b853bf4c..4d4aa8cc31c 100644
--- a/rasa/core/policies/ensemble.py
+++ b/rasa/core/policies/ensemble.py
@@ -5,7 +5,8 @@ import os
 import sys
 from collections import defaultdict
 from datetime import datetime
-from typing import Text, Optional, Any, List, Dict, Tuple, Set, NamedTuple
+from pathlib import Path
+from typing import Text, Optional, Any, List, Dict, Tuple, Set, NamedTuple, Union
 
 import rasa.core
 import rasa.shared.utils.common
@@ -286,14 +287,14 @@ class PolicyEnsemble:
 
         rasa.shared.utils.io.dump_obj_as_json_to_file(domain_spec_path, metadata)
 
-    def persist(self, path: Text) -> None:
+    def persist(self, path: Union[Text, Path]) -> None:
         """Persists the policy to storage."""
 
         self._persist_metadata(path)
 
         for i, policy in enumerate(self.policies):
             dir_name = "policy_{}_{}".format(i, type(policy).__name__)
-            policy_path = os.path.join(path, dir_name)
+            policy_path = Path(path) / dir_name
             policy.persist(policy_path)
 
     @classmethod
@@ -336,7 +337,7 @@ class PolicyEnsemble:
             )
 
     @classmethod
-    def load(cls, path: Text) -> "PolicyEnsemble":
+    def load(cls, path: Union[Text, Path]) -> "PolicyEnsemble":
         """Loads policy and domain specification from storage"""
 
         metadata = cls.load_metadata(path)
diff --git a/rasa/core/policies/fallback.py b/rasa/core/policies/fallback.py
index de9432426eb..431bf24cd04 100644
--- a/rasa/core/policies/fallback.py
+++ b/rasa/core/policies/fallback.py
@@ -1,6 +1,4 @@
-import json
 import logging
-import os
 from typing import Any, List, Text, Optional, Dict, Tuple
 
 import rasa.shared.utils.common
@@ -8,9 +6,6 @@ import rasa.shared.utils.io
 from rasa.constants import DOCS_URL_MIGRATION_GUIDE
 from rasa.shared.core.constants import ACTION_LISTEN_NAME, ACTION_DEFAULT_FALLBACK_NAME
 
-import rasa.utils.io
-from rasa.utils import common as common_utils
-
 from rasa.shared.core.domain import Domain
 from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter
 from rasa.core.policies.policy import Policy
@@ -26,7 +21,7 @@ class FallbackPolicy(Policy):
 
     A fallback can be triggered by a low confidence score on a
     NLU prediction or by a low confidence score on an action
-    prediction. """
+    prediction."""
 
     @staticmethod
     def _standard_featurizer() -> None:
@@ -191,26 +186,15 @@ class FallbackPolicy(Policy):
 
         return result
 
-    def persist(self, path: Text) -> None:
-        """Persists the policy to storage."""
-
-        config_file = os.path.join(path, "fallback_policy.json")
-        meta = {
+    def _metadata(self) -> Dict[Text, Any]:
+        return {
             "priority": self.priority,
             "nlu_threshold": self.nlu_threshold,
             "ambiguity_threshold": self.ambiguity_threshold,
             "core_threshold": self.core_threshold,
             "fallback_action_name": self.fallback_action_name,
         }
-        rasa.shared.utils.io.create_directory_for_file(config_file)
-        rasa.shared.utils.io.dump_obj_as_json_to_file(config_file, meta)
 
     @classmethod
-    def load(cls, path: Text) -> "FallbackPolicy":
-        meta = {}
-        if os.path.exists(path):
-            meta_path = os.path.join(path, "fallback_policy.json")
-            if os.path.isfile(meta_path):
-                meta = json.loads(rasa.shared.utils.io.read_file(meta_path))
-
-        return cls(**meta)
+    def _metadata_filename(cls) -> Text:
+        return "fallback_policy.json"
diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py
index cf8a4388a06..b0cc04a539b 100644
--- a/rasa/core/policies/form_policy.py
+++ b/rasa/core/policies/form_policy.py
@@ -163,3 +163,9 @@ class FormPolicy(MemoizationPolicy):
             logger.debug("There is no active form")
 
         return result
+
+    def _metadata(self) -> Dict[Text, Any]:
+        return {
+            "priority": self.priority,
+            "lookup": self.lookup,
+        }
diff --git a/rasa/core/policies/mapping_policy.py b/rasa/core/policies/mapping_policy.py
index e75702fcf06..066287e780b 100644
--- a/rasa/core/policies/mapping_policy.py
+++ b/rasa/core/policies/mapping_policy.py
@@ -1,8 +1,5 @@
 import logging
-import json
-import os
-import typing
-from typing import Any, List, Text, Optional
+from typing import Any, List, Text, Optional, Dict, TYPE_CHECKING
 
 import rasa.shared.utils.common
 import rasa.utils.io
@@ -28,7 +25,7 @@ from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.core.generator import TrackerWithCachedStates
 from rasa.core.constants import MAPPING_POLICY_PRIORITY
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from rasa.core.policies.ensemble import PolicyEnsemble
 
 
@@ -174,22 +171,9 @@ class MappingPolicy(Policy):
             )
         return result
 
-    def persist(self, path: Text) -> None:
-        """Only persists the priority."""
-
-        config_file = os.path.join(path, "mapping_policy.json")
-        meta = {"priority": self.priority}
-        rasa.shared.utils.io.create_directory_for_file(config_file)
-        rasa.shared.utils.io.dump_obj_as_json_to_file(config_file, meta)
+    def _metadata(self) -> Dict[Text, Any]:
+        return {"priority": self.priority}
 
     @classmethod
-    def load(cls, path: Text) -> "MappingPolicy":
-        """Returns the class with the configured priority."""
-
-        meta = {}
-        if os.path.exists(path):
-            meta_path = os.path.join(path, "mapping_policy.json")
-            if os.path.isfile(meta_path):
-                meta = json.loads(rasa.shared.utils.io.read_file(meta_path))
-
-        return cls(**meta)
+    def _metadata_filename(cls) -> Text:
+        return "mapping_policy.json"
diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py
index 6652d75dce8..9748ca05c42 100644
--- a/rasa/core/policies/memoization.py
+++ b/rasa/core/policies/memoization.py
@@ -3,11 +3,10 @@ import zlib
 import base64
 import json
 import logging
-import os
+
 from tqdm import tqdm
 from typing import Optional, Any, Dict, List, Text
 
-import rasa.shared.utils.io
 import rasa.utils.io
 import rasa.shared.utils.io
 from rasa.shared.constants import DOCS_URL_POLICIES
@@ -33,23 +32,23 @@ OLD_DEFAULT_MAX_HISTORY = 5
 
 class MemoizationPolicy(Policy):
     """The policy that remembers exact examples of
-        `max_history` turns from training stories.
-
-        Since `slots` that are set some time in the past are
-        preserved in all future feature vectors until they are set
-        to None, this policy implicitly remembers and most importantly
-        recalls examples in the context of the current dialogue
-        longer than `max_history`.
-
-        This policy is not supposed to be the only policy in an ensemble,
-        it is optimized for precision and not recall.
-        It should get a 100% precision because it emits probabilities of 1.1
-        along it's predictions, which makes every mistake fatal as
-        no other policy can overrule it.
-
-        If it is needed to recall turns from training dialogues where
-        some slots might not be set during prediction time, and there are
-        training stories for this, use AugmentedMemoizationPolicy.
+    `max_history` turns from training stories.
+
+    Since `slots` that are set some time in the past are
+    preserved in all future feature vectors until they are set
+    to None, this policy implicitly remembers and most importantly
+    recalls examples in the context of the current dialogue
+    longer than `max_history`.
+
+    This policy is not supposed to be the only policy in an ensemble,
+    it is optimized for precision and not recall.
+    It should get a 100% precision because it emits probabilities of 1.1
+    along it's predictions, which makes every mistake fatal as
+    no other policy can overrule it.
+
+    If it is needed to recall turns from training dialogues where
+    some slots might not be set during prediction time, and there are
+    training stories for this, use AugmentedMemoizationPolicy.
     """
 
     ENABLE_FEATURE_STRING_COMPRESSION = True
@@ -158,7 +157,6 @@ class MemoizationPolicy(Policy):
         return lookup
 
     def _create_feature_key(self, states: List[State]) -> Text:
-
         # we sort keys to make sure that the same states
         # represented as dictionaries have the same json strings
         # quotes are removed for aesthetic reasons
@@ -240,59 +238,39 @@ class MemoizationPolicy(Policy):
 
         return result
 
-    def persist(self, path: Text) -> None:
-
-        self.featurizer.persist(path)
-
-        memorized_file = os.path.join(path, "memorized_turns.json")
-        data = {
+    def _metadata(self) -> Dict[Text, Any]:
+        return {
             "priority": self.priority,
             "max_history": self.max_history,
             "lookup": self.lookup,
         }
-        rasa.shared.utils.io.create_directory_for_file(memorized_file)
-        rasa.shared.utils.io.dump_obj_as_json_to_file(memorized_file, data)
 
     @classmethod
-    def load(cls, path: Text) -> "MemoizationPolicy":
-
-        featurizer = TrackerFeaturizer.load(path)
-        memorized_file = os.path.join(path, "memorized_turns.json")
-        if os.path.isfile(memorized_file):
-            data = json.loads(rasa.shared.utils.io.read_file(memorized_file))
-            return cls(
-                featurizer=featurizer, priority=data["priority"], lookup=data["lookup"]
-            )
-        else:
-            logger.info(
-                "Couldn't load memoization for policy. "
-                "File '{}' doesn't exist. Falling back to empty "
-                "turn memory.".format(memorized_file)
-            )
-            return cls()
+    def _metadata_filename(cls) -> Text:
+        return "memorized_turns.json"
 
 
 class AugmentedMemoizationPolicy(MemoizationPolicy):
     """The policy that remembers examples from training stories
-        for `max_history` turns.
-
-        If it is needed to recall turns from training dialogues
-        where some slots might not be set during prediction time,
-        add relevant stories without such slots to training data.
-        E.g. reminder stories.
-
-        Since `slots` that are set some time in the past are
-        preserved in all future feature vectors until they are set
-        to None, this policy has a capability to recall the turns
-        up to `max_history` from training stories during prediction
-        even if additional slots were filled in the past
-        for current dialogue.
+    for `max_history` turns.
+
+    If it is needed to recall turns from training dialogues
+    where some slots might not be set during prediction time,
+    add relevant stories without such slots to training data.
+    E.g. reminder stories.
+
+    Since `slots` that are set some time in the past are
+    preserved in all future feature vectors until they are set
+    to None, this policy has a capability to recall the turns
+    up to `max_history` from training stories during prediction
+    even if additional slots were filled in the past
+    for current dialogue.
     """
 
     @staticmethod
     def _back_to_the_future_again(tracker) -> Optional[DialogueStateTracker]:
         """Send Marty to the past to get
-            the new featurization for the future"""
+        the new featurization for the future"""
 
         idx_of_first_action = None
         idx_of_second_action = None
@@ -322,7 +300,7 @@ class AugmentedMemoizationPolicy(MemoizationPolicy):
 
     def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]:
         """Recursively go to the past to correctly forget slots,
-            and then back to the future to recall."""
+        and then back to the future to recall."""
 
         logger.debug("Launch DeLorean...")
         mcfly_tracker = self._back_to_the_future_again(tracker)
diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py
index e1f21eec4eb..bf91b1f86cb 100644
--- a/rasa/core/policies/policy.py
+++ b/rasa/core/policies/policy.py
@@ -1,24 +1,38 @@
 import copy
+import json
 import logging
 from enum import Enum
-import typing
-from typing import Any, List, Optional, Text, Dict, Callable, Type, Union, Tuple
+from pathlib import Path
+from typing import (
+    Any,
+    List,
+    Optional,
+    Text,
+    Dict,
+    Callable,
+    Type,
+    Union,
+    Tuple,
+    TYPE_CHECKING,
+)
 import numpy as np
 
 import rasa.shared.utils.common
 import rasa.utils.common
+import rasa.shared.utils.io
 from rasa.shared.core.domain import Domain
 from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
 from rasa.core.featurizers.tracker_featurizers import (
     TrackerFeaturizer,
     MaxHistoryTrackerFeaturizer,
+    FEATURIZER_FILE,
 )
 from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.core.generator import TrackerWithCachedStates
 from rasa.core.constants import DEFAULT_POLICY_PRIORITY
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from rasa.shared.nlu.training_data.features import Features
 
 
@@ -144,7 +158,7 @@ class Policy:
             - a dictionary of attribute (INTENT, TEXT, ACTION_NAME, ACTION_TEXT,
               ENTITIES, SLOTS, FORM) to a list of features for all dialogue turns in
               all training trackers
-            - the label ids (e.g. action ids) for every dialuge turn in all training
+            - the label ids (e.g. action ids) for every dialogue turn in all training
               trackers
         """
 
@@ -202,26 +216,70 @@ class Policy:
 
         raise NotImplementedError("Policy must have the capacity to predict.")
 
-    def persist(self, path: Text) -> None:
-        """Persists the policy to a storage.
+    def _metadata(self) -> Optional[Dict[Text, Any]]:
+        """Returns this policy's attributes that should be persisted.
+
+        Policies following the default `persist()` and `load()` templates must
+        implement the `_metadata()` method."
+
+        Returns:
+            The policy metadata.
+        """
+        pass
+
+    @classmethod
+    def _metadata_filename(cls) -> Optional[Text]:
+        """Returns the filename of the persisted policy metadata.
+
+        Policies following the default `persist()` and `load()` templates must
+        implement the `_metadata_filename()` method.
+
+        Returns:
+            The filename of the persisted policy metadata.
+        """
+        pass
+
+    def persist(self, path: Union[Text, Path]) -> None:
+        """Persists the policy to storage.
 
         Args:
-            path: the path where to save the policy to
+            path: Path to persist policy to.
         """
+        # not all policies have a featurizer
+        if self.featurizer is not None:
+            self.featurizer.persist(path)
 
-        raise NotImplementedError("Policy must have the capacity to persist itself.")
+        file = Path(path) / self._metadata_filename()
 
-    @classmethod
-    def load(cls, path: Text) -> "Policy":
-        """Loads a policy from the storage.
+        rasa.shared.utils.io.create_directory_for_file(file)
+        rasa.shared.utils.io.dump_obj_as_json_to_file(file, self._metadata())
 
-        Needs to load its featurizer.
+    @classmethod
+    def load(cls, path: Union[Text, Path]) -> "Policy":
+        """Loads a policy from path.
 
         Args:
-            path: the path from where to load the policy
+            path: Path to load policy from.
+
+        Returns:
+            An instance of `Policy`.
         """
+        metadata_file = Path(path) / cls._metadata_filename()
+
+        if metadata_file.is_file():
+            data = json.loads(rasa.shared.utils.io.read_file(metadata_file))
+
+            if (Path(path) / FEATURIZER_FILE).is_file():
+                featurizer = TrackerFeaturizer.load(path)
+                data["featurizer"] = featurizer
 
-        raise NotImplementedError("Policy must have the capacity to load itself.")
+            return cls(**data)
+
+        logger.info(
+            f"Couldn't load metadata for policy '{cls.__name__}'. "
+            f"File '{metadata_file}' doesn't exist."
+        )
+        return cls()
 
     @staticmethod
     def _default_predictions(domain: Domain) -> List[float]:
diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py
index 68d08925d4d..35fe1fa9bd1 100644
--- a/rasa/core/policies/rule_policy.py
+++ b/rasa/core/policies/rule_policy.py
@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import List, Dict, Text, Optional, Any, Set, TYPE_CHECKING
+from typing import List, Dict, Text, Optional, Any, Set, TYPE_CHECKING, Union
 
 from rasa.shared.core.events import FormValidation
 from rasa.core.featurizers.tracker_featurizers import TrackerFeaturizer
@@ -31,6 +31,7 @@ from rasa.shared.core.constants import (
 from rasa.shared.core.domain import InvalidDomain, State, Domain
 from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY
 
+
 if TYPE_CHECKING:
     from rasa.core.policies.ensemble import PolicyEnsemble  # pytype: disable=pyi-error
 
@@ -59,7 +60,7 @@ class RulePolicy(MemoizationPolicy):
         """The type of data supported by this policy.
 
         Returns:
-            The data type supported by this policy (rule data).
+            The data type supported by this policy (ML and rule data).
         """
         return SupportedData.ML_AND_RULE_DATA
 
@@ -485,3 +486,16 @@ class RulePolicy(MemoizationPolicy):
                 domain.index_for_action(self._fallback_action_name)
             ] = self._core_fallback_threshold
         return result
+
+    def _metadata(self) -> Dict[Text, Any]:
+        return {
+            "priority": self.priority,
+            "lookup": self.lookup,
+            "core_fallback_threshold": self._core_fallback_threshold,
+            "core_fallback_action_name": self._fallback_action_name,
+            "enable_fallback_prediction": self._enable_fallback_prediction,
+        }
+
+    @classmethod
+    def _metadata_filename(cls) -> Text:
+        return "rule_policy.json"
diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py
index 5c0d2cd783b..29d2fc66b53 100644
--- a/rasa/core/policies/sklearn_policy.py
+++ b/rasa/core/policies/sklearn_policy.py
@@ -1,8 +1,8 @@
 import json
 import logging
-import os
 import typing
-from typing import Any, Callable, Dict, List, Optional, Text, Tuple
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Union
 from collections import defaultdict, OrderedDict
 import scipy.sparse
 
@@ -273,20 +273,23 @@ class SklearnPolicy(Policy):
         y_proba = self.model.predict_proba(Xt)
         return self._postprocess_prediction(y_proba, domain)
 
-    def persist(self, path: Text) -> None:
+    def persist(self, path: Union[Text, Path]) -> None:
 
         if self.model:
             self.featurizer.persist(path)
 
             meta = {"priority": self.priority}
+            path = Path(path)
 
-            meta_file = os.path.join(path, "sklearn_policy.json")
+            meta_file = path / "sklearn_policy.json"
             rasa.shared.utils.io.dump_obj_as_json_to_file(meta_file, meta)
 
-            filename = os.path.join(path, "sklearn_model.pkl")
-            io_utils.pickle_dump(filename, self._state)
-            zero_features_filename = os.path.join(path, "zero_state_features.pkl")
+            filename = path / "sklearn_model.pkl"
+            rasa.utils.io.pickle_dump(filename, self._state)
+
+            zero_features_filename = path / "zero_state_features.pkl"
             io_utils.pickle_dump(zero_features_filename, self.zero_state_features)
+
         else:
             rasa.shared.utils.io.raise_warning(
                 "Persist called without a trained model present. "
@@ -294,22 +297,22 @@ class SklearnPolicy(Policy):
             )
 
     @classmethod
-    def load(cls, path: Text) -> Policy:
-        filename = os.path.join(path, "sklearn_model.pkl")
-        zero_features_filename = os.path.join(path, "zero_state_features.pkl")
-        if not os.path.exists(path):
+    def load(cls, path: Union[Text, Path]) -> Policy:
+        filename = Path(path) / "sklearn_model.pkl"
+        zero_features_filename = Path(path) / "zero_state_features.pkl"
+        if not Path(path).exists():
             raise OSError(
-                "Failed to load dialogue model. Path {} "
-                "doesn't exist".format(os.path.abspath(filename))
+                f"Failed to load dialogue model. Path {filename.absolute()} "
+                f"doesn't exist."
             )
 
         featurizer = TrackerFeaturizer.load(path)
         assert isinstance(featurizer, MaxHistoryTrackerFeaturizer), (
-            "Loaded featurizer of type {}, should be "
-            "MaxHistoryTrackerFeaturizer.".format(type(featurizer).__name__)
+            f"Loaded featurizer of type {type(featurizer).__name__}, should be "
+            f"MaxHistoryTrackerFeaturizer."
         )
 
-        meta_file = os.path.join(path, "sklearn_policy.json")
+        meta_file = Path(path) / "sklearn_policy.json"
         meta = json.loads(rasa.shared.utils.io.read_file(meta_file))
         zero_state_features = io_utils.pickle_load(zero_features_filename)
 
diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py
index 45aa0861a60..7ecb0c202c7 100644
--- a/rasa/core/policies/ted_policy.py
+++ b/rasa/core/policies/ted_policy.py
@@ -1,6 +1,5 @@
 import copy
 import logging
-import os
 from pathlib import Path
 from collections import defaultdict
 
@@ -390,7 +389,7 @@ class TEDPolicy(Policy):
 
         return confidence.tolist()
 
-    def persist(self, path: Text) -> None:
+    def persist(self, path: Union[Text, Path]) -> None:
         """Persists the policy to a storage."""
 
         if self.model is None:
@@ -429,18 +428,18 @@ class TEDPolicy(Policy):
         )
 
     @classmethod
-    def load(cls, path: Text) -> "TEDPolicy":
+    def load(cls, path: Union[Text, Path]) -> "TEDPolicy":
         """Loads a policy from the storage.
         **Needs to load its featurizer**
         """
+        model_path = Path(path)
 
-        if not os.path.exists(path):
+        if not model_path.exists():
             raise Exception(
                 f"Failed to load TED policy model. Path "
-                f"'{os.path.abspath(path)}' doesn't exist."
+                f"'{model_path.absolute()}' doesn't exist."
             )
 
-        model_path = Path(path)
         tf_model_file = model_path / f"{SAVE_MODEL_FILE_NAME}.tf_model"
 
         featurizer = TrackerFeaturizer.load(path)
diff --git a/rasa/core/policies/two_stage_fallback.py b/rasa/core/policies/two_stage_fallback.py
index b7b8c10979f..bde2bf86d17 100644
--- a/rasa/core/policies/two_stage_fallback.py
+++ b/rasa/core/policies/two_stage_fallback.py
@@ -1,11 +1,5 @@
-import json
 import logging
-import os
-import typing
-from typing import List, Text, Optional, Any
-
-import rasa.shared.utils.io
-import rasa.utils.io
+from typing import List, Text, Optional, Any, TYPE_CHECKING, Dict
 
 from rasa.shared.core.events import UserUttered, ActionExecuted
 
@@ -25,7 +19,7 @@ from rasa.shared.core.constants import (
 from rasa.shared.core.domain import InvalidDomain, Domain
 from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from rasa.core.policies.ensemble import PolicyEnsemble
 
 
@@ -37,21 +31,21 @@ def has_user_rephrased(tracker: DialogueStateTracker) -> bool:
 
 
 class TwoStageFallbackPolicy(FallbackPolicy):
-    """ This policy handles low NLU confidence in multiple stages.
-
-        If a NLU prediction has a low confidence score,
-        the user is asked to affirm whether they really had this intent.
-        If they affirm, the story continues as if the intent was classified
-        with high confidence from the beginning.
-        If they deny, the user is asked to rephrase his intent.
-        If the classification for the rephrased intent was confident, the story
-        continues as if the user had this intent from the beginning.
-        If the rephrased intent was not classified with high confidence,
-        the user is asked to affirm the classified intent.
-        If the user affirm the intent, the story continues as if the user had
-        this intent from the beginning.
-        If the user denies, an ultimate fallback action is triggered
-        (e.g. a hand-off to a human).
+    """This policy handles low NLU confidence in multiple stages.
+
+    If a NLU prediction has a low confidence score,
+    the user is asked to affirm whether they really had this intent.
+    If they affirm, the story continues as if the intent was classified
+    with high confidence from the beginning.
+    If they deny, the user is asked to rephrase his intent.
+    If the classification for the rephrased intent was confident, the story
+    continues as if the user had this intent from the beginning.
+    If the rephrased intent was not classified with high confidence,
+    the user is asked to affirm the classified intent.
+    If the user affirm the intent, the story continues as if the user had
+    this intent from the beginning.
+    If the user denies, an ultimate fallback action is triggered
+    (e.g. a hand-off to a human).
     """
 
     def __init__(
@@ -215,10 +209,8 @@ class TwoStageFallbackPolicy(FallbackPolicy):
         else:
             return confidence_scores_for(ACTION_DEFAULT_ASK_REPHRASE_NAME, 1.0, domain)
 
-    def persist(self, path: Text) -> None:
-        """Persists the policy to storage."""
-        config_file = os.path.join(path, "two_stage_fallback_policy.json")
-        meta = {
+    def _metadata(self) -> Dict[Text, Any]:
+        return {
             "priority": self.priority,
             "nlu_threshold": self.nlu_threshold,
             "ambiguity_threshold": self.ambiguity_threshold,
@@ -227,15 +219,7 @@ class TwoStageFallbackPolicy(FallbackPolicy):
             "fallback_nlu_action_name": self.fallback_nlu_action_name,
             "deny_suggestion_intent_name": self.deny_suggestion_intent_name,
         }
-        rasa.shared.utils.io.create_directory_for_file(config_file)
-        rasa.shared.utils.io.dump_obj_as_json_to_file(config_file, meta)
 
     @classmethod
-    def load(cls, path: Text) -> "FallbackPolicy":
-        meta = {}
-        if os.path.exists(path):
-            meta_path = os.path.join(path, "two_stage_fallback_policy.json")
-            if os.path.isfile(meta_path):
-                meta = json.loads(rasa.shared.utils.io.read_file(meta_path))
-
-        return cls(**meta)
+    def _metadata_filename(cls) -> Text:
+        return "two_stage_fallback_policy.json"
diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py
index 3f37e607d91..c5b28ec0710 100644
--- a/tests/core/policies/test_rule_policy.py
+++ b/tests/core/policies/test_rule_policy.py
@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import List, Text
 
 import pytest
@@ -96,6 +97,30 @@ def test_rule_policy_has_max_history_none():
     assert policy.featurizer.max_history is None
 
 
+def test_all_policy_attributes_are_persisted(tmpdir: Path):
+    priority = 5
+    lookup = {"a": "b"}
+    core_fallback_threshold = 0.451231
+    core_fallback_action_name = "action_some_fallback"
+    enable_fallback_prediction = False
+
+    policy = RulePolicy(
+        priority=priority,
+        lookup=lookup,
+        core_fallback_threshold=core_fallback_threshold,
+        core_fallback_action_name=core_fallback_action_name,
+        enable_fallback_prediction=enable_fallback_prediction,
+    )
+    policy.persist(tmpdir)
+
+    persisted_policy = RulePolicy.load(tmpdir)
+    assert persisted_policy.priority == priority
+    assert persisted_policy.lookup == lookup
+    assert persisted_policy._core_fallback_threshold == core_fallback_threshold
+    assert persisted_policy._fallback_action_name == core_fallback_action_name
+    assert persisted_policy._enable_fallback_prediction == enable_fallback_prediction
+
+
 def test_faq_rule():
     domain = Domain.from_yaml(
         f"""
diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py
index 9353e7d9071..4e9186b6c95 100644
--- a/tests/core/test_policies.py
+++ b/tests/core/test_policies.py
@@ -122,6 +122,7 @@ class PolicyTestCollection:
         self, trained_policy: Policy, default_domain: Domain, tmp_path: Path
     ):
         trained_policy.persist(str(tmp_path))
+        print("have trained policy", type(trained_policy))
         loaded = trained_policy.__class__.load(str(tmp_path))
         trackers = await train_trackers(default_domain, augmentation_factor=20)
 
@@ -1035,7 +1036,7 @@ def test_supported_data(policy: Type[Policy], supported_data: SupportedData):
 
 
 class OnlyRulePolicy(Policy):
-    """Test policy that supports both rule-based and ML-based training data."""
+    """Test policy that supports only rule-based training data."""
 
     @staticmethod
     def supported_data() -> SupportedData: