Repository: sktime/sktime

Issue:
[ENH] list tag values in `all_estimators` return table
`all_estimators` currently returns tuples/columns (classname, class), but it would be useful to users, e.g., for listing, or for display in the sktime docs, if additional columns return values of certain tags.

I would propose to add an additional `return_tags` argument which can be a str or list ot str of tag names (see `all_tags` for valid tags). Default None which retrieves no tags in addition.

The utility would then return the value of those tags in a tuple (classname, class, tagvalue1, tagvalue2, etc), with tagvalue being empty (None) value if the estimator does not have the tag.


Discussion:
Hey!  I agree this would be a useful change!  I am planning to take a crack at implementing this over the next few days! :)  

Resolving patch:
diff --git a/sktime/registry/_lookup.py b/sktime/registry/_lookup.py
--- a/sktime/registry/_lookup.py
+++ b/sktime/registry/_lookup.py
@@ -45,6 +45,7 @@ def all_estimators(
     exclude_estimators=None,
     return_names=True,
     as_dataframe=False,
+    return_tags=None,
     suppress_import_stdout=True,
 ):
     """Get a list of all estimators from sktime.
@@ -57,7 +58,7 @@ def all_estimators(
 
     Parameters
     ----------
-    estimator_types: string, list of string, optional (default=None)
+    estimator_types: str, list of str, optional (default=None)
         Which kind of estimators should be returned.
         if None, no filter is applied and all estimators are returned.
         if str or list of str, strings define scitypes specified in search
@@ -65,9 +66,13 @@ def all_estimators(
             possible str values are entries of registry.BASE_CLASS_REGISTER (first col)
                 for instance 'classifier', 'regressor', 'transformer', 'forecaster'
     return_names: bool, optional (default=True)
-        If True, return estimators as list of (name, estimator class) tuples.
-        If False, return list of estimators classes.
+        if True, estimator class name is included in the all_estimators()
+            return in the order: name, estimator class, optional tags, either as
+            a tuple or as pandas.DataFrame columns
+        if False, estimator class name is removed from the all_estimators()
+            return.
     filter_tags: dict of (str or list of str), optional (default=None)
+        For a list of valid tag strings, use the registry.all_tags utility.
         subsets the returned estimators as follows:
             each key/value pair is statement in "and"/conjunction
                 key is tag name to sub-set on
@@ -76,23 +81,45 @@ def all_estimators(
     exclude_estimators: str, list of str, optional (default=None)
         Names of estimators to exclude.
     as_dataframe: bool, optional (default=False)
-                if False, return is as described below;
-                if True, return is converted into a DataFrame for pretty display
+        if True, all_estimators will return a pandas.DataFrame with named
+            columns for all of the attributes being returned.
+        if False, all_estimators will return a list (either a list of
+            estimators or a list of tuples, see Returns)
+    return_tags: str or list of str, optional (default=None)
+        Names of tags to fetch and return each estimator's value of.
+        For a list of valid tag strings, use the registry.all_tags utility.
+        if str or list of str,
+            the tag values named in return_tags will be fetched for each
+            estimator and will be appended as either columns or tuple entries.
     suppress_import_stdout : bool, optional. Default=True
         whether to suppress stdout printout upon import.
 
     Returns
     -------
-    estimators: list of class, if return_names=False,
-            or list of tuples (str, class), if return_names=True
+    all_estimators will return one of the following:
+        1. list of estimators, if return_names=False, and return_tags is None
+        2. list of tuples (optional estimator name, class, ~optional estimator
+                tags), if return_names=True or return_tags is not None.
+        3. pandas.DataFrame if as_dataframe = True
         if list of estimators:
-            entries are estimator classes matching the query,
-            in alphabetical order of class name
+            entries are estimators matching the query,
+            in alphabetical order of estimator name
         if list of tuples:
-            list of (name, class) matching the query,
-            in alphabetical order of class name, where
-            ``name`` is the estimator class name as string
-            ``class`` is the actual class
+            list of (optional estimator name, estimator, optional estimator
+            tags) matching the query, in alphabetical order of estimator name,
+            where
+            ``name`` is the estimator name as string, and is an
+                optional return
+            ``estimator`` is the actual estimator
+            ``tags`` are the estimator's values for each tag in return_tags
+                and is an optional return.
+        if dataframe:
+            all_estimators will return a pandas.DataFrame.
+            column names represent the attributes contained in each column.
+            "estimators" will be the name of the column of estimators, "names"
+            will be the name of the column of estimator class names and the string(s)
+            passed in return_tags will serve as column names for all columns of
+            tags that were optionally requested.
 
     References
     ----------
@@ -183,7 +210,7 @@ def _is_in_estimator_types(estimator, estimator_types):
             ]
         )
 
-    if estimator_types is not None:
+    if estimator_types:
         estimator_types = _check_estimator_types(estimator_types)
         all_estimators = [
             (name, estimator)
@@ -192,13 +219,10 @@ def _is_in_estimator_types(estimator, estimator_types):
         ]
 
     # Filter based on given exclude list
-    if exclude_estimators is not None:
-        if not isinstance(exclude_estimators, list):
-            exclude_estimators = [exclude_estimators]  # make iterable
-        if not all([isinstance(estimator, str) for estimator in exclude_estimators]):
-            raise ValueError(
-                "Please specify `exclude_estimators` as a list of strings."
-            )
+    if exclude_estimators:
+        exclude_estimators = _check_list_of_str_or_error(
+            exclude_estimators, "exclude_estimators"
+        )
         all_estimators = [
             (name, estimator)
             for name, estimator in all_estimators
@@ -210,7 +234,7 @@ def _is_in_estimator_types(estimator, estimator_types):
     # the tuple
     all_estimators = sorted(all_estimators, key=itemgetter(0))
 
-    if filter_tags is not None:
+    if filter_tags:
         all_estimators = [
             (n, est) for (n, est) in all_estimators if _check_tag_cond(est, filter_tags)
         ]
@@ -222,13 +246,81 @@ def _is_in_estimator_types(estimator, estimator_types):
     else:
         columns = ["name", "estimator"]
 
-    # convert to pd.DataFrame if as_dataframe=True
+    # add new tuple entries to all_estimators for each tag in return_tags:
+    if return_tags:
+        return_tags = _check_list_of_str_or_error(return_tags, "return_tags")
+        # enrich all_estimators by adding the values for all return_tags tags:
+        if all_estimators:
+            if isinstance(all_estimators[0], tuple):
+                all_estimators = [
+                    (name, est) + _get_return_tags(est, return_tags)
+                    for (name, est) in all_estimators
+                ]
+            else:
+                all_estimators = [
+                    tuple([est]) + _get_return_tags(est, return_tags)
+                    for est in all_estimators
+                ]
+        columns = columns + return_tags
+
+    # convert to pandas.DataFrame if as_dataframe=True
     if as_dataframe:
         all_estimators = pd.DataFrame(all_estimators, columns=columns)
 
     return all_estimators
 
 
+def _check_list_of_str_or_error(arg_to_check, arg_name):
+    """Check that certain arguments are str or list of str.
+
+    Parameters
+    ----------
+    arg_to_check: argument we are testing the type of
+    arg_name: str,
+        name of the argument we are testing, will be added to the error if
+        ``arg_to_check`` is not a str or a list of str
+
+    Returns
+    -------
+    arg_to_check: list of str,
+        if arg_to_check was originally a str it converts it into a list of str
+        so that it can be iterated over.
+
+    Raises
+    ------
+    TypeError if arg_to_check is not a str or list of str
+    """
+    # check that return_tags has the right type:
+    if isinstance(arg_to_check, str):
+        arg_to_check = [arg_to_check]
+    if not isinstance(arg_to_check, list) or not all(
+        isinstance(value, str) for value in arg_to_check
+    ):
+        raise TypeError(
+            f"Error in all_estimators!  Argument {arg_name} must be either\
+             a str or list of str"
+        )
+    return arg_to_check
+
+
+def _get_return_tags(estimator, return_tags):
+    """Fetch a list of all tags for every_entry of all_estimators.
+
+    Parameters
+    ----------
+    estimator:  BaseEstimator, an sktime estimator
+    return_tags: list of str,
+        names of tags to get values for the estimator
+
+    Returns
+    -------
+    tags: a tuple with all the estimators values for all tags in return tags.
+        a value is None if it is not a valid tag for the estimator provided.
+    """
+    tags = tuple(estimator.get_class_tag(tag) for tag in return_tags)
+    return tags
+
+
 def _check_tag_cond(estimator, filter_tags=None, as_dataframe=True):
     """Check whether estimator satisfies filter_tags condition.
 
@@ -243,7 +335,8 @@ def _check_tag_cond(estimator, filter_tags=None, as_dataframe=True):
                 condition is "key must be equal to value, or in set(value)"
     as_dataframe: bool, default=False
                 if False, return is as described below;
-                if True, return is converted into a DataFrame for pretty display
+                if True, return is converted into a pandas.DataFrame for pretty
+                display
 
     Returns
     -------
@@ -280,7 +373,8 @@ def all_tags(
         these to get the estimators that fit at least one of the types.
     as_dataframe: bool, optional (default=False)
                 if False, return is as described below;
-                if True, return is converted into a DataFrame for pretty display
+                if True, return is converted into a pandas.DataFrame for pretty
+                display
 
     Returns
     -------
@@ -301,13 +395,8 @@ def all_tags(
 
     def is_tag_for_type(tag, estimator_types):
         tag_types = tag[1]
-        if isinstance(tag_types, str):
-            tag_types = [tag_types]
-        elif not isinstance(tag_types, list):
-            raise ValueError(
-                "Error in ESTIMATOR_TAG_REGISTER, "
-                "2nd entries of register tuples must be list or list of str"
-            )
+        tag_types = _check_list_of_str_or_error(tag_types, "tag_types")
+
         if isinstance(estimator_types, str):
             estimator_types = [estimator_types]
 
@@ -319,7 +408,7 @@ def is_tag_for_type(tag, estimator_types):
 
     all_tags = ESTIMATOR_TAG_REGISTER
 
-    if estimator_types is not None:
+    if estimator_types:
         # checking, but not using the return since that is classes, not strings
         _check_estimator_types(estimator_types)
         all_tags = [tag for tag in all_tags if is_tag_for_type(tag, estimator_types)]


Test patch:
diff --git a/sktime/registry/tests/test_lookup.py b/sktime/registry/tests/test_lookup.py
--- a/sktime/registry/tests/test_lookup.py
+++ b/sktime/registry/tests/test_lookup.py
@@ -157,6 +157,69 @@ def test_all_estimators_exclude_estimators(exclude_estimators):
         assert estimator not in names
 
 
+def _get_tag_fixture():
+    """Generate a simple list of test cases for optional return_tags."""
+    # just picked a few valid tags to try out as valid str return_tags args:
+    test_str_as_arg = [
+        "X-y-must-have-same-index",
+        "capability:pred_var",
+        "skip-inverse-transform",
+    ]
+
+    # we can also make them into a list to test list of str as a valid arg:
+    test_list_as_arg = [test_str_as_arg]
+    # Note - I don't include None explicitly as a test case - tested elsewhere
+    return test_str_as_arg + test_list_as_arg
+
+
+# test that all_estimators returns as expected if given correct return_tags:
+@pytest.mark.parametrize("return_tags", _get_tag_fixture())
+@pytest.mark.parametrize("return_names", [True, False])
+def test_all_estimators_return_tags(return_tags, return_names):
+    """Test ability to return estimator value of passed tags."""
+    estimators = all_estimators(
+        return_tags=return_tags,
+        return_names=return_names,
+    )
+    # Helps us keep track of estimator index within the tuple:
+    ESTIMATOR_INDEX = 1 if return_names else 0
+    TAG_START_INDEX = ESTIMATOR_INDEX + 1
+
+    assert isinstance(estimators[0], tuple)
+    # check length of tuple is what we expect:
+    if isinstance(return_tags, str):
+        assert len(estimators[0]) == TAG_START_INDEX + 1
+    else:
+        assert len(estimators[0]) == len(return_tags) + TAG_START_INDEX
+
+    # check that for each estimator the value for that tag is correct:
+    for est_tuple in estimators:
+        est = est_tuple[ESTIMATOR_INDEX]
+        if isinstance(return_tags, str):
+            assert est.get_class_tag(return_tags) == est_tuple[TAG_START_INDEX]
+        else:
+            for tag_index, tag in enumerate(return_tags):
+                assert est.get_class_tag(tag) == est_tuple[TAG_START_INDEX + tag_index]
+
+
+def _get_bad_return_tags():
+    """Get return_tags arguments that should throw an exception."""
+    # case not a str or a list:
+    is_int = [12]
+    # case is a list, but not all elements are str:
+    is_not_all_str = [["this", "is", "a", "test", 12, "!"]]
+
+    return is_int + is_not_all_str
+
+
+# test that all_estimators breaks as expected if given bad return_tags:
+@pytest.mark.parametrize("return_tags", _get_bad_return_tags())
+def test_all_estimators_return_tags_bad_arg(return_tags):
+    """Test ability to catch bad arguments of return_tags."""
+    with pytest.raises(TypeError):
+        _ = all_estimators(return_tags=return_tags)
+
+
 @pytest.mark.parametrize("estimator_scitype", BASE_CLASS_SCITYPE_LIST)
 def test_scitype_inference(estimator_scitype):
     """Check that scitype inverts _check_estimator_types."""