Repository: ray-project/ray

Issue:
[tune] Configure parameters in output table
We want to be able to configure the hyperparameters printed during the experiment execution.

cc @krfricke 


Discussion:

Resolving patch:
diff --git a/python/ray/tune/progress_reporter.py b/python/ray/tune/progress_reporter.py
--- a/python/ray/tune/progress_reporter.py
+++ b/python/ray/tune/progress_reporter.py
@@ -56,6 +56,11 @@ class TuneReporterBase(ProgressReporter):
             include in progress table. If this is a dict, the keys should
             be metric names and the values should be the displayed names.
             If this is a list, the metric name is used directly.
+        parameter_columns (dict[str, str]|list[str]): Names of parameters to
+            include in progress table. If this is a dict, the keys should
+            be parameter names and the values should be the displayed names.
+            If this is a list, the parameter name is used directly. If empty,
+            defaults to all available parameters.
         max_progress_rows (int): Maximum number of rows to print
             in the progress table. The progress table describes the
             progress of each trial. Defaults to 20.
@@ -78,10 +83,12 @@ class TuneReporterBase(ProgressReporter):
 
     def __init__(self,
                  metric_columns=None,
+                 parameter_columns=None,
                  max_progress_rows=20,
                  max_error_rows=20,
                  max_report_frequency=5):
         self._metric_columns = metric_columns or self.DEFAULT_COLUMNS
+        self._parameter_columns = parameter_columns or []
         self._max_progress_rows = max_progress_rows
         self._max_error_rows = max_error_rows
 
@@ -117,6 +124,29 @@ def add_metric_column(self, metric, representation=None):
                     "of metric columns.")
             self._metric_columns.append(metric)
 
+    def add_parameter_column(self, parameter, representation=None):
+        """Adds a parameter to the existing columns.
+
+        Args:
+            parameter (str): Parameter to add. This must be a parameter
+                specified in the configuration.
+            representation (str): Representation to use in table. Defaults to
+                `parameter`.
+        """
+        if parameter in self._parameter_columns:
+            raise ValueError("Column {} already exists.".format(parameter))
+
+        if isinstance(self._parameter_columns, Mapping):
+            representation = representation or parameter
+            self._parameter_columns[parameter] = representation
+        else:
+            if representation is not None and representation != parameter:
+                raise ValueError(
+                    "`representation` cannot differ from `parameter` "
+                    "if this reporter was initialized with a list "
+                    "of metric columns.")
+            self._parameter_columns.append(parameter)
+
     def _progress_str(self, trials, done, *sys_info, fmt="psql", delim="\n"):
         """Returns full progress string.
 
@@ -142,6 +172,7 @@ def _progress_str(self, trials, done, *sys_info, fmt="psql", delim="\n"):
             trial_progress_str(
                 trials,
                 metric_columns=self._metric_columns,
+                parameter_columns=self._parameter_columns,
                 fmt=fmt,
                 max_rows=max_progress))
         messages.append(trial_errors_str(trials, fmt=fmt, max_rows=max_error))
@@ -157,6 +188,11 @@ class JupyterNotebookReporter(TuneReporterBase):
             include in progress table. If this is a dict, the keys should
             be metric names and the values should be the displayed names.
             If this is a list, the metric name is used directly.
+        parameter_columns (dict[str, str]|list[str]): Names of parameters to
+            include in progress table. If this is a dict, the keys should
+            be parameter names and the values should be the displayed names.
+            If this is a list, the parameter name is used directly. If empty,
+            defaults to all available parameters.
         max_progress_rows (int): Maximum number of rows to print
             in the progress table. The progress table describes the
             progress of each trial. Defaults to 20.
@@ -170,12 +206,13 @@ class JupyterNotebookReporter(TuneReporterBase):
     def __init__(self,
                  overwrite,
                  metric_columns=None,
+                 parameter_columns=None,
                  max_progress_rows=20,
                  max_error_rows=20,
                  max_report_frequency=5):
-        super(JupyterNotebookReporter,
-              self).__init__(metric_columns, max_progress_rows, max_error_rows,
-                             max_report_frequency)
+        super(JupyterNotebookReporter, self).__init__(
+            metric_columns, parameter_columns, max_progress_rows,
+            max_error_rows, max_report_frequency)
         self._overwrite = overwrite
 
     def report(self, trials, done, *sys_info):
@@ -196,6 +233,11 @@ class CLIReporter(TuneReporterBase):
             include in progress table. If this is a dict, the keys should
             be metric names and the values should be the displayed names.
             If this is a list, the metric name is used directly.
+        parameter_columns (dict[str, str]|list[str]): Names of parameters to
+            include in progress table. If this is a dict, the keys should
+            be parameter names and the values should be the displayed names.
+            If this is a list, the parameter name is used directly. If empty,
+            defaults to all available parameters.
         max_progress_rows (int): Maximum number of rows to print
             in the progress table. The progress table describes the
             progress of each trial. Defaults to 20.
@@ -208,12 +250,14 @@ class CLIReporter(TuneReporterBase):
 
     def __init__(self,
                  metric_columns=None,
+                 parameter_columns=None,
                  max_progress_rows=20,
                  max_error_rows=20,
                  max_report_frequency=5):
 
-        super(CLIReporter, self).__init__(metric_columns, max_progress_rows,
-                                          max_error_rows, max_report_frequency)
+        super(CLIReporter, self).__init__(metric_columns, parameter_columns,
+                                          max_progress_rows, max_error_rows,
+                                          max_report_frequency)
 
     def report(self, trials, done, *sys_info):
         print(self._progress_str(trials, done, *sys_info))
@@ -241,7 +285,11 @@ def memory_debug_str():
                 "(or ray[debug]) to resolve)")
 
 
-def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
+def trial_progress_str(trials,
+                       metric_columns,
+                       parameter_columns=None,
+                       fmt="psql",
+                       max_rows=None):
     """Returns a human readable message for printing to the console.
 
     This contains a table where each row represents a trial, its parameters
@@ -253,6 +301,11 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
             If this is a dict, the keys are metric names and the values are
             the names to use in the message. If this is a list, the metric
             name is used in the message directly.
+        parameter_columns (dict[str, str]|list[str]): Names of parameters to
+            include. If this is a dict, the keys are parameter names and the
+            values are the names to use in the message. If this is a list,
+            the parameter name is used in the message directly. If this is
+            empty, all parameters are used in the message.
         fmt (str): Output format (see tablefmt in tabulate API).
         max_rows (int): Maximum number of rows in the trial table. Defaults to
             unlimited.
@@ -297,22 +350,39 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
 
     # Pre-process trials to figure out what columns to show.
     if isinstance(metric_columns, Mapping):
-        keys = list(metric_columns.keys())
+        metric_keys = list(metric_columns.keys())
     else:
-        keys = metric_columns
-    keys = [
-        k for k in keys if any(
+        metric_keys = metric_columns
+    metric_keys = [
+        k for k in metric_keys if any(
             t.last_result.get(k) is not None for t in trials)
     ]
+
+    if not parameter_columns:
+        parameter_keys = sorted(
+            set().union(*[t.evaluated_params for t in trials]))
+    elif isinstance(parameter_columns, Mapping):
+        parameter_keys = list(parameter_columns.keys())
+    else:
+        parameter_keys = parameter_columns
+
     # Build trial rows.
-    params = sorted(set().union(*[t.evaluated_params for t in trials]))
-    trial_table = [_get_trial_info(trial, params, keys) for trial in trials]
+    trial_table = [
+        _get_trial_info(trial, parameter_keys, metric_keys) for trial in trials
+    ]
     # Format column headings
     if isinstance(metric_columns, Mapping):
-        formatted_columns = [metric_columns[k] for k in keys]
+        formatted_metric_columns = [metric_columns[k] for k in metric_keys]
+    else:
+        formatted_metric_columns = metric_keys
+    if isinstance(parameter_columns, Mapping):
+        formatted_parameter_columns = [
+            parameter_columns[k] for k in parameter_keys
+        ]
     else:
-        formatted_columns = keys
-    columns = (["Trial name", "status", "loc"] + params + formatted_columns)
+        formatted_parameter_columns = parameter_keys
+    columns = (["Trial name", "status", "loc"] + formatted_parameter_columns +
+               formatted_metric_columns)
     # Tabulate.
     messages.append(
         tabulate(trial_table, headers=columns, tablefmt=fmt, showindex=False))


Test patch:
diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py
--- a/python/ray/tune/tests/test_progress_reporter.py
+++ b/python/ray/tune/tests/test_progress_reporter.py
@@ -12,13 +12,13 @@
 
 EXPECTED_RESULT_1 = """Result logdir: /foo
 Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
-+--------------+------------+-------+-----+-----+
-|   Trial name | status     | loc   |   a |   b |
-|--------------+------------+-------+-----+-----|
-|        00001 | PENDING    | here  |   1 |   2 |
-|        00002 | RUNNING    | here  |   2 |   4 |
-|        00000 | TERMINATED | here  |   0 |   0 |
-+--------------+------------+-------+-----+-----+
++--------------+------------+-------+-----+-----+------------+
+|   Trial name | status     | loc   |   a |   b |   metric_1 |
+|--------------+------------+-------+-----+-----+------------|
+|        00001 | PENDING    | here  |   1 |   2 |        0.5 |
+|        00002 | RUNNING    | here  |   2 |   4 |        1   |
+|        00000 | TERMINATED | here  |   0 |   0 |        0   |
++--------------+------------+-------+-----+-----+------------+
 ... 2 more trials not shown (2 RUNNING)"""
 
 EXPECTED_RESULT_2 = """Result logdir: /foo
@@ -33,6 +33,17 @@
 |        00004 | RUNNING    | here  |   4 |   8 |
 +--------------+------------+-------+-----+-----+"""
 
+EXPECTED_RESULT_3 = """Result logdir: /foo
+Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
++--------------+------------+-------+-----+------------+------------+
+|   Trial name | status     | loc   |   A |   Metric 1 |   Metric 2 |
+|--------------+------------+-------+-----+------------+------------|
+|        00001 | PENDING    | here  |   1 |        0.5 |       0.25 |
+|        00002 | RUNNING    | here  |   2 |        1   |       0.5  |
+|        00000 | TERMINATED | here  |   0 |        0   |       0    |
++--------------+------------+-------+-----+------------+------------+
+... 2 more trials not shown (2 RUNNING)"""
+
 END_TO_END_COMMAND = """
 import ray
 from ray import tune
@@ -125,6 +136,42 @@ def f(config):
 | f_xxxxx_00029 | TERMINATED |       |     |     |   9 |
 +---------------+------------+-------+-----+-----+-----+"""
 
+EXPECTED_END_TO_END_AC = """Number of trials: 30 (30 TERMINATED)
++---------------+------------+-------+-----+-----+-----+
+| Trial name    | status     | loc   |   a |   b |   c |
+|---------------+------------+-------+-----+-----+-----|
+| f_xxxxx_00000 | TERMINATED |       |   0 |     |     |
+| f_xxxxx_00001 | TERMINATED |       |   1 |     |     |
+| f_xxxxx_00002 | TERMINATED |       |   2 |     |     |
+| f_xxxxx_00003 | TERMINATED |       |   3 |     |     |
+| f_xxxxx_00004 | TERMINATED |       |   4 |     |     |
+| f_xxxxx_00005 | TERMINATED |       |   5 |     |     |
+| f_xxxxx_00006 | TERMINATED |       |   6 |     |     |
+| f_xxxxx_00007 | TERMINATED |       |   7 |     |     |
+| f_xxxxx_00008 | TERMINATED |       |   8 |     |     |
+| f_xxxxx_00009 | TERMINATED |       |   9 |     |     |
+| f_xxxxx_00010 | TERMINATED |       |     |   0 |     |
+| f_xxxxx_00011 | TERMINATED |       |     |   1 |     |
+| f_xxxxx_00012 | TERMINATED |       |     |   2 |     |
+| f_xxxxx_00013 | TERMINATED |       |     |   3 |     |
+| f_xxxxx_00014 | TERMINATED |       |     |   4 |     |
+| f_xxxxx_00015 | TERMINATED |       |     |   5 |     |
+| f_xxxxx_00016 | TERMINATED |       |     |   6 |     |
+| f_xxxxx_00017 | TERMINATED |       |     |   7 |     |
+| f_xxxxx_00018 | TERMINATED |       |     |   8 |     |
+| f_xxxxx_00019 | TERMINATED |       |     |   9 |     |
+| f_xxxxx_00020 | TERMINATED |       |     |     |   0 |
+| f_xxxxx_00021 | TERMINATED |       |     |     |   1 |
+| f_xxxxx_00022 | TERMINATED |       |     |     |   2 |
+| f_xxxxx_00023 | TERMINATED |       |     |     |   3 |
+| f_xxxxx_00024 | TERMINATED |       |     |     |   4 |
+| f_xxxxx_00025 | TERMINATED |       |     |     |   5 |
+| f_xxxxx_00026 | TERMINATED |       |     |     |   6 |
+| f_xxxxx_00027 | TERMINATED |       |     |     |   7 |
+| f_xxxxx_00028 | TERMINATED |       |     |     |   8 |
+| f_xxxxx_00029 | TERMINATED |       |     |     |   9 |
++---------------+------------+-------+-----+-----+-----+"""
+
 
 class ProgressReporterTest(unittest.TestCase):
     def mock_trial(self, status, i):
@@ -202,17 +249,38 @@ def testProgressStr(self):
             t.location = "here"
             t.config = {"a": i, "b": i * 2}
             t.evaluated_params = t.config
-            t.last_result = {"config": {"a": i, "b": i * 2}}
+            t.last_result = {
+                "config": {
+                    "a": i,
+                    "b": i * 2
+                },
+                "metric_1": i / 2,
+                "metric_2": i / 4
+            }
             t.__str__ = lambda self: self.trial_id
             trials.append(t)
-        prog1 = trial_progress_str(trials, ["a", "b"], fmt="psql", max_rows=3)
+        # One metric, all parameters
+        prog1 = trial_progress_str(
+            trials, ["metric_1"], None, fmt="psql", max_rows=3)
         print(prog1)
         assert prog1 == EXPECTED_RESULT_1
-        prog2 = trial_progress_str(
-            trials, ["a", "b"], fmt="psql", max_rows=None)
+
+        # No metric, all parameters
+        prog2 = trial_progress_str(trials, [], None, fmt="psql", max_rows=None)
         print(prog2)
         assert prog2 == EXPECTED_RESULT_2
 
+        # Both metrics, one parameter, all with custom representation
+        prog3 = trial_progress_str(
+            trials, {
+                "metric_1": "Metric 1",
+                "metric_2": "Metric 2"
+            }, {"a": "A"},
+            fmt="psql",
+            max_rows=3)
+        print(prog3)
+        assert prog3 == EXPECTED_RESULT_3
+
     def testEndToEndReporting(self):
         try:
             os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx"