Repository: ray-project/ray

Issue:
[tune] Ea scope avg n step
<!-- Thank you for your contribution! Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->

Continuing #8147, this adds `last-n-avg`, specifically `last-5-avg` and `last-10-avg` scopes to the collected experiment metrics. These _n_ are currently hardcoded since a variable number of steps would need an extension of the API for trials and the experiment runner.

<!-- Please give a short summary of the change and the problem this solves. -->

Closes #8147.

<!-- For example: "Closes #1234" -->

## Checks

- [X] I've run `scripts/format.sh` to lint the changes in this PR.
- [X] I've included any doc changes needed for https://docs.ray.io/en/latest/.
- [X] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failure rates at https://ray-travis-tracker.herokuapp.com/.
- Testing Strategy
   - [X] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested (please justify below)



Discussion:
Can one of the admins verify this patch?
Test FAILed.
Refer to this link for build results (access rights to CI server needed): 
https://amplab.cs.berkeley.edu/jenkins//job/Ray-PRB/25977/
Test FAILed.
Looks like tune tests are failing?
There seems to be an error when restoring a run where the `self.metric_n_steps` don't get initialized: https://github.com/ray-project/ray/runs/681419121#step:3:2390 I'll fix this tomorrow.
The error was in restoring the n-step metrics. The dict keys were restored as strings, while they were originally created as integers. We should talk about whether this is expected behavior. For the meantime, we're now converting the integers to strings for the dict keys.
Test PASSed.
Refer to this link for build results (access rights to CI server needed): 
https://amplab.cs.berkeley.edu/jenkins//job/Ray-PRB/26084/
Test PASSed.
ah can we fix up the PR? looks like the merge history is broken
I rebased the PR against the current master and resolved any merge conflicts. I hope it works now. Alternatively, I could re-create the changes on a fresh branch to avoid the messy commit history.
Test PASSed.
Refer to this link for build results (access rights to CI server needed): 
https://amplab.cs.berkeley.edu/jenkins//job/Ray-PRB/26574/
Test PASSed.

Resolving patch:
diff --git a/python/ray/tune/analysis/experiment_analysis.py b/python/ray/tune/analysis/experiment_analysis.py
--- a/python/ray/tune/analysis/experiment_analysis.py
+++ b/python/ray/tune/analysis/experiment_analysis.py
@@ -220,34 +220,35 @@ def get_best_trial(self, metric, mode="max", scope="all"):
         Args:
             metric (str): Key for trial info to order on.
             mode (str): One of [min, max].
-            scope (str): One of [all, last, avg]. If `scope=last`, only look at
-                each trial's final step for `metric`, and compare across
-                trials based on `mode=[min,max]`. If `scope=avg`, consider the
-                simple average over all steps for `metric` and compare across
-                trials based on `mode=[min,max]`. If `scope=all`, find each
-                trial's min/max score for `metric` based on `mode`, and
-                compare trials based on `mode=[min,max]`.
+            scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
+                If `scope=last`, only look at each trial's final step for
+                `metric`, and compare across trials based on `mode=[min,max]`.
+                If `scope=avg`, consider the simple average over all steps
+                for `metric` and compare across trials based on
+                `mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
+                consider the simple average over the last 5 or 10 steps for
+                `metric` and compare across trials based on `mode=[min,max]`.
+                If `scope=all`, find each trial's min/max score for `metric`
+                based on `mode`, and compare trials based on `mode=[min,max]`.
         """
         if mode not in ["max", "min"]:
             raise ValueError(
                 "ExperimentAnalysis: attempting to get best trial for "
                 "metric {} for mode {} not in [\"max\", \"min\"]".format(
                     metric, mode))
-        if scope not in ["all", "last", "avg"]:
+        if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:
             raise ValueError(
                 "ExperimentAnalysis: attempting to get best trial for "
-                "metric {} for scope {} not in [\"all\", \"last\", \"avg\"]".
-                format(metric, scope))
+                "metric {} for scope {} not in [\"all\", \"last\", \"avg\", "
+                "\"last-5-avg\", \"last-10-avg\"]".format(metric, scope))
         best_trial = None
         best_metric_score = None
         for trial in self.trials:
             if metric not in trial.metric_analysis:
                 continue
 
-            if scope == "last":
-                metric_score = trial.metric_analysis[metric]["last"]
-            elif scope == "avg":
-                metric_score = trial.metric_analysis[metric]["avg"]
+            if scope in ["last", "avg", "last-5-avg", "last-10-avg"]:
+                metric_score = trial.metric_analysis[metric][scope]
             else:
                 metric_score = trial.metric_analysis[metric][mode]
 
@@ -273,13 +274,16 @@ def get_best_config(self, metric, mode="max", scope="all"):
         Args:
             metric (str): Key for trial info to order on.
             mode (str): One of [min, max].
-            scope (str): One of [all, last, avg]. If `scope=last`, only look at
-                each trial's final step for `metric`, and compare across
-                trials based on `mode=[min,max]`. If `scope=avg`, consider the
-                simple average over all steps for `metric` and compare across
-                trials based on `mode=[min,max]`. If `scope=all`, find each
-                trial's min/max score for `metric` based on `mode`, and
-                compare trials based on `mode=[min,max]`.
+            scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
+                If `scope=last`, only look at each trial's final step for
+                `metric`, and compare across trials based on `mode=[min,max]`.
+                If `scope=avg`, consider the simple average over all steps
+                for `metric` and compare across trials based on
+                `mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
+                consider the simple average over the last 5 or 10 steps for
+                `metric` and compare across trials based on `mode=[min,max]`.
+                If `scope=all`, find each trial's min/max score for `metric`
+                based on `mode`, and compare trials based on `mode=[min,max]`.
         """
         best_trial = self.get_best_trial(metric, mode, scope)
         return best_trial.config if best_trial else None
@@ -292,13 +296,16 @@ def get_best_logdir(self, metric, mode="max", scope="all"):
         Args:
             metric (str): Key for trial info to order on.
             mode (str): One of [min, max].
-            scope (str): One of [all, last, avg]. If `scope=last`, only look at
-                each trial's final step for `metric`, and compare across
-                trials based on `mode=[min,max]`. If `scope=avg`, consider the
-                simple average over all steps for `metric` and compare across
-                trials based on `mode=[min,max]`. If `scope=all`, find each
-                trial's min/max score for `metric` based on `mode`, and
-                compare trials based on `mode=[min,max]`.
+            scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
+                If `scope=last`, only look at each trial's final step for
+                `metric`, and compare across trials based on `mode=[min,max]`.
+                If `scope=avg`, consider the simple average over all steps
+                for `metric` and compare across trials based on
+                `mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
+                consider the simple average over the last 5 or 10 steps for
+                `metric` and compare across trials based on `mode=[min,max]`.
+                If `scope=all`, find each trial's min/max score for `metric`
+                based on `mode`, and compare trials based on `mode=[min,max]`.
         """
         best_trial = self.get_best_trial(metric, mode, scope)
         return best_trial.logdir if best_trial else None
diff --git a/python/ray/tune/trial.py b/python/ray/tune/trial.py
--- a/python/ray/tune/trial.py
+++ b/python/ray/tune/trial.py
@@ -1,4 +1,5 @@
 import ray.cloudpickle as cloudpickle
+from collections import deque
 import copy
 from datetime import datetime
 import logging
@@ -214,9 +215,14 @@ def __init__(self,
         self.last_result = {}
         self.last_update_time = -float("inf")
 
-        # stores in memory max/min/avg/last result for each metric by trial
+        # stores in memory max/min/avg/last-n-avg/last result for each
+        # metric by trial
         self.metric_analysis = {}
 
+        # keep a moving average over these last n steps
+        self.n_steps = [5, 10]
+        self.metric_n_steps = {}
+
         self.export_formats = export_formats
         self.status = Trial.PENDING
         self.start_time = None
@@ -470,6 +476,7 @@ def update_last_result(self, result, terminate=False):
         self.last_result = result
         self.last_update_time = time.time()
         self.result_logger.on_result(self.last_result)
+
         for metric, value in flatten_dict(result).items():
             if isinstance(value, Number):
                 if metric not in self.metric_analysis:
@@ -479,6 +486,13 @@ def update_last_result(self, result, terminate=False):
                         "avg": value,
                         "last": value
                     }
+                    self.metric_n_steps[metric] = {}
+                    for n in self.n_steps:
+                        key = "last-{:d}-avg".format(n)
+                        self.metric_analysis[metric][key] = value
+                        # Store n as string for correct restore.
+                        self.metric_n_steps[metric][str(n)] = deque(
+                            [value], maxlen=n)
                 else:
                     step = result["training_iteration"] or 1
                     self.metric_analysis[metric]["max"] = max(
@@ -490,6 +504,13 @@ def update_last_result(self, result, terminate=False):
                         (step - 1) * self.metric_analysis[metric]["avg"])
                     self.metric_analysis[metric]["last"] = value
 
+                    for n in self.n_steps:
+                        key = "last-{:d}-avg".format(n)
+                        self.metric_n_steps[metric][str(n)].append(value)
+                        self.metric_analysis[metric][key] = sum(
+                            self.metric_n_steps[metric][str(n)]) / len(
+                                self.metric_n_steps[metric][str(n)])
+
     def get_trainable_cls(self):
         return get_trainable_cls(self.trainable_name)
 


Test patch:
diff --git a/python/ray/tune/tests/test_experiment_analysis_mem.py b/python/ray/tune/tests/test_experiment_analysis_mem.py
--- a/python/ray/tune/tests/test_experiment_analysis_mem.py
+++ b/python/ray/tune/tests/test_experiment_analysis_mem.py
@@ -14,11 +14,11 @@ class ExperimentAnalysisInMemorySuite(unittest.TestCase):
     def setUp(self):
         class MockTrainable(Trainable):
             scores_dict = {
-                0: [5, 4, 0],
-                1: [4, 3, 1],
-                2: [2, 1, 8],
-                3: [9, 7, 6],
-                4: [7, 5, 3]
+                0: [5, 4, 4, 4, 4, 4, 4, 4, 0],
+                1: [4, 3, 3, 3, 3, 3, 3, 3, 1],
+                2: [2, 1, 1, 1, 1, 1, 1, 1, 8],
+                3: [9, 7, 7, 7, 7, 7, 7, 7, 6],
+                4: [7, 5, 5, 5, 5, 5, 5, 5, 3]
             }
 
             def _setup(self, config):
@@ -53,7 +53,7 @@ def testCompareTrials(self):
             self.MockTrainable,
             name="analysis_exp",
             local_dir=self.test_dir,
-            stop={"training_iteration": 3},
+            stop={"training_iteration": len(scores[0])},
             num_samples=1,
             config={"id": grid_search(list(range(5)))})
 
@@ -67,12 +67,33 @@ def testCompareTrials(self):
                                     "avg").metric_analysis["score"]["avg"]
         min_avg = ea.get_best_trial("score", "min",
                                     "avg").metric_analysis["score"]["avg"]
+        max_avg_5 = ea.get_best_trial(
+            "score", "max",
+            "last-5-avg").metric_analysis["score"]["last-5-avg"]
+        min_avg_5 = ea.get_best_trial(
+            "score", "min",
+            "last-5-avg").metric_analysis["score"]["last-5-avg"]
+        max_avg_10 = ea.get_best_trial(
+            "score", "max",
+            "last-10-avg").metric_analysis["score"]["last-10-avg"]
+        min_avg_10 = ea.get_best_trial(
+            "score", "min",
+            "last-10-avg").metric_analysis["score"]["last-10-avg"]
         self.assertEqual(max_all, max(scores_all))
         self.assertEqual(min_all, min(scores_all))
         self.assertEqual(max_last, max(scores_last))
+        self.assertNotEqual(max_last, max(scores_all))
+
         self.assertAlmostEqual(max_avg, max(np.mean(scores, axis=1)))
         self.assertAlmostEqual(min_avg, min(np.mean(scores, axis=1)))
-        self.assertNotEqual(max_last, max(scores_all))
+
+        self.assertAlmostEqual(max_avg_5, max(np.mean(scores[:, -5:], axis=1)))
+        self.assertAlmostEqual(min_avg_5, min(np.mean(scores[:, -5:], axis=1)))
+
+        self.assertAlmostEqual(max_avg_10, max(
+            np.mean(scores[:, -10:], axis=1)))
+        self.assertAlmostEqual(min_avg_10, min(
+            np.mean(scores[:, -10:], axis=1)))
 
 
 class AnalysisSuite(unittest.TestCase):