Repository: elastic/rally

Issue:
Add a unique race id
Currently we use a combination of track, challenge, car and race timestamp to select documents in reporting. We can simplify the code (and also make it more robust) by generating a unique id per race and using that instead.

Discussion:

Patch:
diff --git a/docs/metrics.rst b/docs/metrics.rst
index 99f804b3..9f952e04 100644
--- a/docs/metrics.rst
+++ b/docs/metrics.rst
@@ -11,6 +11,10 @@ Here is a typical metrics record::
 
     {
           "environment": "nightly",
+          "trial-timestamp": "20160421T042749Z",
+          "trial-id": "6ebc6e53-ee20-4b0c-99b4-09697987e9f4",
+          "@timestamp": 1461213093093,
+          "relative-time": 10507328,
           "track": "geonames",
           "track-params": {
             "shard-count": 3
@@ -18,9 +22,6 @@ Here is a typical metrics record::
           "challenge": "append-no-conflicts",
           "car": "defaults",
           "sample-type": "normal",
-          "trial-timestamp": "20160421T042749Z",
-          "@timestamp": 1461213093093,
-          "relative-time": 10507328,
           "name": "throughput",
           "value": 27385,
           "unit": "docs/s",
@@ -38,7 +39,7 @@ Here is a typical metrics record::
             "node_name": "rally-node0",
             "source_revision": "a6c0a81",
             "distribution_version": "5.0.0-SNAPSHOT",
-            "tag_reference": "Github ticket 1234",
+            "tag_reference": "Github ticket 1234"
           }
         }
 
@@ -66,7 +67,12 @@ Rally runs warmup trials but records all samples. Normally, we are just interest
 trial-timestamp
 ~~~~~~~~~~~~~~~
 
-A constant timestamp (always in UTC) that is determined when Rally is invoked. It is intended to group all samples of a benchmark trial.
+A constant timestamp (always in UTC) that is determined when Rally is invoked.
+
+trial-id
+~~~~~~~~
+
+A UUID that changes on every invocation of Rally. It is intended to group all samples of a benchmark trial.
 
 @timestamp
 ~~~~~~~~~~
diff --git a/esrally/driver/driver.py b/esrally/driver/driver.py
index 9b9defbb..cfb50622 100644
--- a/esrally/driver/driver.py
+++ b/esrally/driver/driver.py
@@ -326,9 +326,12 @@ class Driver:
         self.challenge = select_challenge(self.config, self.track)
         self.quiet = self.config.opts("system", "quiet.mode", mandatory=False, default_value=False)
         self.throughput_calculator = ThroughputCalculator()
-        # create - but do not yet open - the metrics store as an internal timer starts when we open it.
-        cls = metrics.metrics_store_class(self.config)
-        self.metrics_store = cls(cfg=self.config, meta_info=metrics_meta_info, lap=lap)
+        self.metrics_store = metrics.metrics_store(cfg=self.config,
+                                                   track=self.track.name,
+                                                   challenge=self.challenge.name,
+                                                   meta_info=metrics_meta_info,
+                                                   lap=lap,
+                                                   read_only=False)
         for host in self.config.opts("driver", "load_driver_hosts"):
             if host != "localhost":
                 self.load_driver_hosts.append(net.resolve(host))
@@ -339,13 +342,9 @@ class Driver:
         self.target.on_prepare_track(preps, self.config, self.track)
 
     def after_track_prepared(self):
-        track_name = self.track.name
-        challenge_name = self.challenge.name
-        car_name = self.config.opts("mechanic", "car.names")
-
-        logger.info("Benchmark for track [%s], challenge [%s] and car %s is about to start." % (track_name, challenge_name, car_name))
-        invocation = self.config.opts("system", "time.start")
-        self.metrics_store.open(invocation, track_name, challenge_name, car_name)
+        logger.info("Benchmark is about to start.")
+        # ensure relative time starts when the benchmark starts.
+        self.reset_relative_time()
 
         allocator = Allocator(self.challenge.schedule)
         self.allocations = allocator.allocations
diff --git a/esrally/metrics.py b/esrally/metrics.py
index 4c95cf66..c3d23e5a 100644
--- a/esrally/metrics.py
+++ b/esrally/metrics.py
@@ -190,7 +190,7 @@ class MetaInfoScope(Enum):
     """
 
 
-def metrics_store(cfg, read_only=True, invocation=None, track=None, challenge=None, car=None):
+def metrics_store(cfg, read_only=True, track=None, challenge=None, car=None, meta_info=None, lap=None):
     """
     Creates a proper metrics store based on the current configuration.
 
@@ -199,13 +199,14 @@ def metrics_store(cfg, read_only=True, invocation=None, track=None, challenge=No
     :return: A metrics store implementation.
     """
     cls = metrics_store_class(cfg)
-    store = cls(cfg)
+    store = cls(cfg=cfg, meta_info=meta_info, lap=lap)
     logger.info("Creating %s" % str(store))
 
-    selected_invocation = cfg.opts("system", "time.start") if invocation is None else invocation
+    trial_id = cfg.opts("system", "trial.id")
+    trial_timestamp = cfg.opts("system", "time.start")
     selected_car = cfg.opts("mechanic", "car.names") if car is None else car
 
-    store.open(selected_invocation, track, challenge, selected_car, create=not read_only)
+    store.open(trial_id, trial_timestamp, track, challenge, selected_car, create=not read_only)
     return store
 
 
@@ -267,7 +268,8 @@ class MetricsStore:
         :param lap: This parameter is optional and intended for creating a metrics store with a previously serialized lap.
         """
         self._config = cfg
-        self._invocation = None
+        self._trial_id = None
+        self._trial_timestamp = None
         self._track = None
         self._track_params = cfg.opts("track", "params")
         self._challenge = None
@@ -286,11 +288,12 @@ class MetricsStore:
         self._clock = clock
         self._stop_watch = self._clock.stop_watch()
 
-    def open(self, invocation=None, track_name=None, challenge_name=None, car_name=None, ctx=None, create=False):
+    def open(self, trial_id=None, trial_timestamp=None, track_name=None, challenge_name=None, car_name=None, ctx=None, create=False):
         """
-        Opens a metrics store for a specific invocation, track, challenge and car.
+        Opens a metrics store for a specific trial, track, challenge and car.
 
-        :param invocation: The invocation (timestamp).
+        :param trial_id: The trial id. This attribute is sufficient to uniquely identify a challenge.
+        :param trial_timestamp: The trial timestamp as a datetime.
         :param track_name: Track name.
         :param challenge_name: Challenge name.
         :param car_name: Car name.
@@ -299,24 +302,27 @@ class MetricsStore:
         False when it is just opened for reading (as we can assume all necessary indices exist at this point).
         """
         if ctx:
-            self._invocation = ctx["invocation"]
+            self._trial_id = ctx["trial-id"]
+            self._trial_timestamp = ctx["trial-timestamp"]
             self._track = ctx["track"]
             self._challenge = ctx["challenge"]
             self._car = ctx["car"]
         else:
-            self._invocation = time.to_iso8601(invocation)
+            self._trial_id = trial_id
+            self._trial_timestamp = time.to_iso8601(trial_timestamp)
             self._track = track_name
             self._challenge = challenge_name
             self._car = car_name
-        assert self._invocation is not None, "Attempting to open metrics store without an invocation"
+        assert self._trial_id is not None, "Attempting to open metrics store without a trial id"
+        assert self._trial_timestamp is not None, "Attempting to open metrics store without a trial timestamp"
         assert self._track is not None, "Attempting to open metrics store without a track"
         assert self._challenge is not None, "Attempting to open metrics store without a challenge"
         assert self._car is not None, "Attempting to open metrics store without a car"
 
         self._car_name = "+".join(self._car) if isinstance(self._car, list) else self._car
 
-        logger.info("Opening metrics store for invocation=[%s], track=[%s], challenge=[%s], car=[%s]" %
-                    (self._invocation, self._track, self._challenge, self._car))
+        logger.info("Opening metrics store for trial timestamp=[%s], track=[%s], challenge=[%s], car=[%s]" %
+                    (self._trial_timestamp, self._track, self._challenge, self._car))
 
         user_tags = extract_user_tags_from_config(self._config)
         for k, v in user_tags.items():
@@ -412,7 +418,8 @@ class MetricsStore:
     @property
     def open_context(self):
         return {
-            "invocation": self._invocation,
+            "trial-id": self._trial_id,
+            "trial-timestamp": self._trial_timestamp,
             "track": self._track,
             "challenge": self._challenge,
             "car": self._car
@@ -526,7 +533,8 @@ class MetricsStore:
         doc = {
             "@timestamp": time.to_epoch_millis(absolute_time),
             "relative-time": int(relative_time * 1000 * 1000),
-            "trial-timestamp": self._invocation,
+            "trial-id": self._trial_id,
+            "trial-timestamp": self._trial_timestamp,
             "environment": self._environment_name,
             "track": self._track,
             "lap": self._lap,
@@ -728,9 +736,9 @@ class EsMetricsStore(MetricsStore):
         self._index_template_provider = index_template_provider_class(cfg)
         self._docs = None
 
-    def open(self, invocation=None, track_name=None, challenge_name=None, car_name=None, ctx=None, create=False):
+    def open(self, trial_id=None, trial_timestamp=None, track_name=None, challenge_name=None, car_name=None, ctx=None, create=False):
         self._docs = []
-        MetricsStore.open(self, invocation, track_name, challenge_name, car_name, ctx, create)
+        MetricsStore.open(self, trial_id, trial_timestamp, track_name, challenge_name, car_name, ctx, create)
         self._index = self.index_name()
         # reduce a bit of noise in the metrics cluster log
         if create:
@@ -742,7 +750,7 @@ class EsMetricsStore(MetricsStore):
         self._client.refresh(index=self._index)
 
     def index_name(self):
-        ts = time.from_is8601(self._invocation)
+        ts = time.from_is8601(self._trial_timestamp)
         return "rally-metrics-%04d-%02d" % (ts.year, ts.month)
 
     def _get_template(self):
@@ -751,8 +759,8 @@ class EsMetricsStore(MetricsStore):
     def flush(self, refresh=True):
         if self._docs:
             self._client.bulk_index(index=self._index, doc_type=EsMetricsStore.METRICS_DOC_TYPE, items=self._docs)
-            logger.info("Successfully added %d metrics documents for invocation=[%s], track=[%s], challenge=[%s], car=[%s]." %
-                        (len(self._docs), self._invocation, self._track, self._challenge, self._car))
+            logger.info("Successfully added %d metrics documents for trial timestamp=[%s], track=[%s], challenge=[%s], car=[%s]." %
+                        (len(self._docs), self._trial_timestamp, self._track, self._challenge, self._car))
         self._docs = []
         # ensure we can search immediately after flushing
         if refresh:
@@ -862,27 +870,7 @@ class EsMetricsStore(MetricsStore):
                 "filter": [
                     {
                         "term": {
-                            "trial-timestamp": self._invocation
-                        }
-                    },
-                    {
-                        "term": {
-                            "environment": self._environment_name
-                        }
-                    },
-                    {
-                        "term": {
-                            "track": self._track
-                        }
-                    },
-                    {
-                        "term": {
-                            "challenge": self._challenge
-                        }
-                    },
-                    {
-                        "term": {
-                            "car": self._car_name
+                            "trial-id": self._trial_id
                         }
                     },
                     {
@@ -1076,7 +1064,8 @@ def list_races(cfg):
 
 def create_race(cfg, track, challenge):
     car = cfg.opts("mechanic", "car.names")
-    environment_name = cfg.opts("system", "env.name")
+    environment = cfg.opts("system", "env.name")
+    trial_id = cfg.opts("system", "trial.id")
     trial_timestamp = cfg.opts("system", "time.start")
     total_laps = cfg.opts("race", "laps")
     user_tags = extract_user_tags_from_config(cfg)
@@ -1084,11 +1073,11 @@ def create_race(cfg, track, challenge):
     track_params = cfg.opts("track", "params")
     rally_version = version.version()
 
-    return Race(rally_version, environment_name, trial_timestamp, pipeline, user_tags, track, track_params, challenge, car, total_laps)
+    return Race(rally_version, environment, trial_id, trial_timestamp, pipeline, user_tags, track, track_params, challenge, car, total_laps)
 
 
 class Race:
-    def __init__(self, rally_version, environment_name, trial_timestamp, pipeline, user_tags, track, track_params, challenge, car,
+    def __init__(self, rally_version, environment_name, trial_id, trial_timestamp, pipeline, user_tags, track, track_params, challenge, car,
                  total_laps, cluster=None, lap_results=None, results=None):
         if results is None:
             results = {}
@@ -1096,6 +1085,7 @@ class Race:
             lap_results = []
         self.rally_version = rally_version
         self.environment_name = environment_name
+        self.trial_id = trial_id
         self.trial_timestamp = trial_timestamp
         self.pipeline = pipeline
         self.user_tags = user_tags
@@ -1104,7 +1094,7 @@ class Race:
         self.challenge = challenge
         self.car = car
         self.total_laps = total_laps
-        # will be set later - contains hosts, revision, distribution_version, ...s
+        # will be set later - contains hosts, revision, distribution_version, ...
         self.cluster = cluster
         self.lap_results = lap_results
         self.results = results
@@ -1143,6 +1133,7 @@ class Race:
         d = {
             "rally-version": self.rally_version,
             "environment": self.environment_name,
+            "trial-id": self.trial_id,
             "trial-timestamp": time.to_iso8601(self.trial_timestamp),
             "pipeline": self.pipeline,
             "user-tags": self.user_tags,
@@ -1164,6 +1155,7 @@ class Race:
         result_template = {
             "rally-version": self.rally_version,
             "environment": self.environment_name,
+            "trial-id": self.trial_id,
             "trial-timestamp": time.to_iso8601(self.trial_timestamp),
             "distribution-version": self.cluster.distribution_version,
             "distribution-major-version": versions.major_version(self.cluster.distribution_version),
@@ -1206,7 +1198,7 @@ class Race:
 
         # Don't restore a few properties like cluster because they (a) cannot be reconstructed easily without knowledge of other modules
         # and (b) it is not necessary for this use case.
-        return Race(d["rally-version"], d["environment"], time.from_is8601(d["trial-timestamp"]), d["pipeline"], user_tags,
+        return Race(d["rally-version"], d["environment"], d["trial-id"], time.from_is8601(d["trial-timestamp"]), d["pipeline"], user_tags,
                     d["track"], d.get("track-params"), d["challenge"], d["car"], d["total-laps"], results=d["results"])
 
 
@@ -1359,10 +1351,10 @@ class EsRaceStore(RaceStore):
 
     def find_by_timestamp(self, timestamp):
         filters = [{
-            "term": {
-                "environment": self.environment_name
-            }
-        },
+                "term": {
+                    "environment": self.environment_name
+                }
+            },
             {
                 "term": {
                     "trial-timestamp": timestamp
diff --git a/esrally/rally.py b/esrally/rally.py
index 2fe5c91f..483a86ec 100644
--- a/esrally/rally.py
+++ b/esrally/rally.py
@@ -5,6 +5,7 @@ import logging.handlers
 import os
 import sys
 import time
+import uuid
 
 from esrally import version, actor, config, paths, racecontrol, reporter, metrics, track, chart_generator, exceptions, time as rtime
 from esrally import PROGRAM_NAME, DOC_LINK, BANNER, SKULL, check_python_version
@@ -662,6 +663,7 @@ def main():
         cfg.add(config.Scope.application, "system", "time.start.user_provided", False)
 
     cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet)
+    cfg.add(config.Scope.applicationOverride, "system", "trial.id", str(uuid.uuid4()))
 
     # per node?
     cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline)
diff --git a/esrally/reporter.py b/esrally/reporter.py
index 9e089a5a..ee0b997a 100644
--- a/esrally/reporter.py
+++ b/esrally/reporter.py
@@ -543,8 +543,8 @@ class ComparisonReporter:
         self.plain = False
 
     def report(self, r1, r2):
-        logger.info("Generating comparison report for baseline (invocation=[%s], track=[%s], challenge=[%s], car=[%s]) and "
-                    "contender (invocation=[%s], track=[%s], challenge=[%s], car=[%s])" %
+        logger.info("Generating comparison report for baseline (trial timestamp=[%s], track=[%s], challenge=[%s], car=[%s]) and "
+                    "contender (trial timestamp=[%s], track=[%s], challenge=[%s], car=[%s])" %
                     (r1.trial_timestamp, r1.track, r1.challenge, r1.car,
                      r2.trial_timestamp, r2.track, r2.challenge, r2.car))
         # we don't verify anything about the races as it is possible that the user benchmarks two different tracks intentionally
diff --git a/esrally/resources/metrics-template.json b/esrally/resources/metrics-template.json
index 20155504..ea7f62d6 100644
--- a/esrally/resources/metrics-template.json
+++ b/esrally/resources/metrics-template.json
@@ -32,6 +32,9 @@
         "relative-time": {
           "type": "long"
         },
+        "trial-id": {
+          "type": "keyword"
+        },
         "trial-timestamp": {
           "type": "date",
           "format": "basic_date_time_no_millis",
diff --git a/esrally/resources/races-template.json b/esrally/resources/races-template.json
index cc158693..67eea774 100644
--- a/esrally/resources/races-template.json
+++ b/esrally/resources/races-template.json
@@ -25,6 +25,9 @@
         "enabled": true
       },
       "properties": {
+        "trial-id": {
+          "type": "keyword"
+        },
         "trial-timestamp": {
           "type": "date",
           "format": "basic_date_time_no_millis",
diff --git a/esrally/resources/results-template.json b/esrally/resources/results-template.json
index 9239c817..ac5896b0 100644
--- a/esrally/resources/results-template.json
+++ b/esrally/resources/results-template.json
@@ -25,6 +25,9 @@
         "enabled": true
       },
       "properties": {
+        "trial-id": {
+          "type": "keyword"
+        },
         "trial-timestamp": {
           "type": "date",
           "format": "basic_date_time_no_millis",


Test patch:
diff --git a/tests/driver/driver_test.py b/tests/driver/driver_test.py
index c2ccfdc2..fbfcc08a 100644
--- a/tests/driver/driver_test.py
+++ b/tests/driver/driver_test.py
@@ -57,6 +57,7 @@ class DriverTests(TestCase):
         self.cfg = config.Config()
         self.cfg.add(config.Scope.application, "system", "env.name", "unittest")
         self.cfg.add(config.Scope.application, "system", "time.start", datetime(year=2017, month=8, day=20, hour=1, minute=0, second=0))
+        self.cfg.add(config.Scope.application, "system", "trial.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
         self.cfg.add(config.Scope.application, "track", "challenge.name", "default")
         self.cfg.add(config.Scope.application, "track", "params", {})
         self.cfg.add(config.Scope.application, "track", "test.mode.enabled", True)
diff --git a/tests/metrics_test.py b/tests/metrics_test.py
index fab49911..5c754bfe 100644
--- a/tests/metrics_test.py
+++ b/tests/metrics_test.py
@@ -167,6 +167,7 @@ class EsClientTests(TestCase):
 
 class EsMetricsTests(TestCase):
     TRIAL_TIMESTAMP = datetime.datetime(2016, 1, 31)
+    TRIAL_ID = "6ebc6e53-ee20-4b0c-99b4-09697987e9f4"
 
     def setUp(self):
         self.cfg = config.Config()
@@ -182,12 +183,13 @@ class EsMetricsTests(TestCase):
 
     def test_put_value_without_meta_info(self):
         throughput = 5000
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append", "defaults", create=True)
         self.metrics_store.lap = 1
 
         self.metrics_store.put_count_cluster_level("indexing_throughput", throughput, "docs/s")
         expected_doc = {
             "@timestamp": StaticClock.NOW * 1000,
+            "trial-id": EsMetricsTests.TRIAL_ID,
             "trial-timestamp": "20160131T000000Z",
             "relative-time": 0,
             "environment": "unittest",
@@ -197,7 +199,7 @@ class EsMetricsTests(TestCase):
                 "shard-count": 3
             },
             "lap": 1,
-            "challenge": "append-no-conflicts",
+            "challenge": "append",
             "car": "defaults",
             "name": "indexing_throughput",
             "value": throughput,
@@ -211,13 +213,14 @@ class EsMetricsTests(TestCase):
 
     def test_put_value_with_explicit_timestamps(self):
         throughput = 5000
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append", "defaults", create=True)
         self.metrics_store.lap = 1
 
         self.metrics_store.put_count_cluster_level(name="indexing_throughput", count=throughput, unit="docs/s",
                                                    absolute_time=0, relative_time=10)
         expected_doc = {
             "@timestamp": 0,
+            "trial-id": EsMetricsTests.TRIAL_ID,
             "trial-timestamp": "20160131T000000Z",
             "relative-time": 10000000,
             "environment": "unittest",
@@ -227,7 +230,7 @@ class EsMetricsTests(TestCase):
                 "shard-count": 3
             },
             "lap": 1,
-            "challenge": "append-no-conflicts",
+            "challenge": "append",
             "car": "defaults",
             "name": "indexing_throughput",
             "value": throughput,
@@ -243,7 +246,7 @@ class EsMetricsTests(TestCase):
         throughput = 5000
         # add a user-defined tag
         self.cfg.add(config.Scope.application, "race", "user.tag", "intention:testing,disk_type:hdd")
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append", "defaults", create=True)
         self.metrics_store.lap = 1
 
         # Ensure we also merge in cluster level meta info
@@ -257,6 +260,7 @@ class EsMetricsTests(TestCase):
         self.metrics_store.put_value_node_level("node0", "indexing_throughput", throughput, "docs/s")
         expected_doc = {
             "@timestamp": StaticClock.NOW * 1000,
+            "trial-id": EsMetricsTests.TRIAL_ID,
             "trial-timestamp": "20160131T000000Z",
             "relative-time": 0,
             "environment": "unittest",
@@ -266,7 +270,7 @@ class EsMetricsTests(TestCase):
                 "shard-count": 3
             },
             "lap": 1,
-            "challenge": "append-no-conflicts",
+            "challenge": "append",
             "car": "defaults",
             "name": "indexing_throughput",
             "value": throughput,
@@ -301,7 +305,7 @@ class EsMetricsTests(TestCase):
         }
         self.es_mock.search = mock.MagicMock(return_value=search_result)
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
 
         expected_query = {
             "query": {
@@ -309,27 +313,7 @@ class EsMetricsTests(TestCase):
                     "filter": [
                         {
                             "term": {
-                                "trial-timestamp": "20160131T000000Z"
-                            }
-                        },
-                        {
-                            "term": {
-                                "environment": "unittest"
-                            }
-                        },
-                        {
-                            "term": {
-                                "track": "test"
-                            }
-                        },
-                        {
-                            "term": {
-                                "challenge": "append-no-conflicts"
-                            }
-                        },
-                        {
-                            "term": {
-                                "car": "defaults"
+                                "trial-id": EsMetricsTests.TRIAL_ID
                             }
                         },
                         {
@@ -369,7 +353,7 @@ class EsMetricsTests(TestCase):
         }
         self.es_mock.search = mock.MagicMock(return_value=search_result)
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
 
         expected_query = {
             "query": {
@@ -377,27 +361,7 @@ class EsMetricsTests(TestCase):
                     "filter": [
                         {
                             "term": {
-                                "trial-timestamp": "20160131T000000Z"
-                            }
-                        },
-                        {
-                            "term": {
-                                "environment": "unittest"
-                            }
-                        },
-                        {
-                            "term": {
-                                "track": "test"
-                            }
-                        },
-                        {
-                            "term": {
-                                "challenge": "append-no-conflicts"
-                            }
-                        },
-                        {
-                            "term": {
-                                "car": "defaults"
+                                "trial-id": EsMetricsTests.TRIAL_ID
                             }
                         },
                         {
@@ -523,7 +487,7 @@ class EsMetricsTests(TestCase):
         }
         self.es_mock.search = mock.MagicMock(return_value=search_result)
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(EsMetricsTests.TRIAL_ID, EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
 
         expected_query = {
             "query": {
@@ -531,27 +495,7 @@ class EsMetricsTests(TestCase):
                     "filter": [
                         {
                             "term": {
-                                "trial-timestamp": "20160131T000000Z"
-                            }
-                        },
-                        {
-                            "term": {
-                                "environment": "unittest"
-                            }
-                        },
-                        {
-                            "term": {
-                                "track": "test"
-                            }
-                        },
-                        {
-                            "term": {
-                                "challenge": "append-no-conflicts"
-                            }
-                        },
-                        {
-                            "term": {
-                                "car": "defaults"
+                                "trial-id": EsMetricsTests.TRIAL_ID
                             }
                         },
                         {
@@ -589,6 +533,7 @@ class EsMetricsTests(TestCase):
 
 class EsRaceStoreTests(TestCase):
     TRIAL_TIMESTAMP = datetime.datetime(2016, 1, 31)
+    TRIAL_ID = "6ebc6e53-ee20-4b0c-99b4-09697987e9f4"
 
     class DictHolder:
         def __init__(self, d):
@@ -617,7 +562,8 @@ class EsRaceStoreTests(TestCase):
                         indices=[track.Index(name="tests", types=["test-type"])],
                         challenges=[track.Challenge(name="index", default=True, schedule=schedule)])
 
-        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_timestamp=EsRaceStoreTests.TRIAL_TIMESTAMP,
+        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_id=EsRaceStoreTests.TRIAL_ID,
+                            trial_timestamp=EsRaceStoreTests.TRIAL_TIMESTAMP,
                             pipeline="from-sources", user_tags={"os": "Linux"}, track=t, track_params={"shard-count": 3},
                             challenge=t.default_challenge, car="4gheap",
                             total_laps=12,
@@ -653,6 +599,7 @@ class EsRaceStoreTests(TestCase):
         expected_doc = {
             "rally-version": "0.4.4",
             "environment": "unittest",
+            "trial-id": EsRaceStoreTests.TRIAL_ID,
             "trial-timestamp": "20160131T000000Z",
             "pipeline": "from-sources",
             "user-tags": {
@@ -697,6 +644,7 @@ class EsRaceStoreTests(TestCase):
 
 class EsResultsStoreTests(TestCase):
     TRIAL_TIMESTAMP = datetime.datetime(2016, 1, 31)
+    TRIAL_ID = "6ebc6e53-ee20-4b0c-99b4-09697987e9f4"
 
     def setUp(self):
         self.cfg = config.Config()
@@ -727,7 +675,8 @@ class EsResultsStoreTests(TestCase):
         node = c.add_node("localhost", "rally-node-0")
         node.plugins.append("x-pack")
 
-        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_timestamp=EsResultsStoreTests.TRIAL_TIMESTAMP,
+        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_id=EsResultsStoreTests.TRIAL_ID,
+                            trial_timestamp=EsResultsStoreTests.TRIAL_TIMESTAMP,
                             pipeline="from-sources", user_tags={"os": "Linux"}, track=t, track_params=None,
                             challenge=t.default_challenge, car="4gheap",
                             total_laps=12,
@@ -764,6 +713,7 @@ class EsResultsStoreTests(TestCase):
             {
                 "rally-version": "0.4.4",
                 "environment": "unittest",
+                "trial-id": EsResultsStoreTests.TRIAL_ID,
                 "trial-timestamp": "20160131T000000Z",
                 "distribution-version": "5.0.0",
                 "distribution-major-version": 5,
@@ -784,6 +734,7 @@ class EsResultsStoreTests(TestCase):
             {
                 "rally-version": "0.4.4",
                 "environment": "unittest",
+                "trial-id": EsResultsStoreTests.TRIAL_ID,
                 "trial-timestamp": "20160131T000000Z",
                 "distribution-version": "5.0.0",
                 "distribution-major-version": 5,
@@ -805,6 +756,7 @@ class EsResultsStoreTests(TestCase):
             {
                 "rally-version": "0.4.4",
                 "environment": "unittest",
+                "trial-id": EsResultsStoreTests.TRIAL_ID,
                 "trial-timestamp": "20160131T000000Z",
                 "distribution-version": "5.0.0",
                 "distribution-major-version": 5,
@@ -830,6 +782,7 @@ class EsResultsStoreTests(TestCase):
             {
                 "rally-version": "0.4.4",
                 "environment": "unittest",
+                "trial-id": EsResultsStoreTests.TRIAL_ID,
                 "trial-timestamp": "20160131T000000Z",
                 "distribution-version": "5.0.0",
                 "distribution-major-version": 5,
@@ -852,6 +805,9 @@ class EsResultsStoreTests(TestCase):
 
 
 class InMemoryMetricsStoreTests(TestCase):
+    TRIAL_TIMESTAMP = datetime.datetime(2016, 1, 31)
+    TRIAL_ID = "6ebc6e53-ee20-4b0c-99b4-09697987e9f4"
+
     def setUp(self):
         self.cfg = config.Config()
         self.cfg.add(config.Scope.application, "system", "env.name", "unittest")
@@ -864,7 +820,8 @@ class InMemoryMetricsStoreTests(TestCase):
 
     def test_get_value(self):
         throughput = 5000
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.put_count_cluster_level("indexing_throughput", 1, "docs/s", sample_type=metrics.SampleType.Warmup)
         self.metrics_store.put_count_cluster_level("indexing_throughput", throughput, "docs/s")
@@ -872,20 +829,23 @@ class InMemoryMetricsStoreTests(TestCase):
 
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assertEqual(1, self.metrics_store.get_one("indexing_throughput", sample_type=metrics.SampleType.Warmup))
         self.assertEqual(throughput, self.metrics_store.get_one("indexing_throughput", sample_type=metrics.SampleType.Normal))
 
     def test_get_percentile(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         for i in range(1, 1001):
             self.metrics_store.put_value_cluster_level("query_latency", float(i), "ms")
 
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assert_equal_percentiles("query_latency", [100.0], {100.0: 1000.0})
         self.assert_equal_percentiles("query_latency", [99.0], {99.0: 990.0})
@@ -895,14 +855,16 @@ class InMemoryMetricsStoreTests(TestCase):
         self.assert_equal_percentiles("query_latency", [99, 99.9, 100], {99: 990.0, 99.9: 999.0, 100: 1000.0})
 
     def test_get_median(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         for i in range(1, 1001):
             self.metrics_store.put_value_cluster_level("query_latency", float(i), "ms")
 
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assertAlmostEqual(500.5, self.metrics_store.get_median("query_latency", lap=1))
 
@@ -914,7 +876,8 @@ class InMemoryMetricsStoreTests(TestCase):
                                    msg=str(percentile) + "th percentile differs")
 
     def test_externalize_and_bulk_add(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.put_count_cluster_level("final_index_size", 1000, "GB")
 
@@ -932,7 +895,8 @@ class InMemoryMetricsStoreTests(TestCase):
         self.assertEqual(1000, self.metrics_store.get_one("final_index_size"))
 
     def test_meta_data_per_document(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.add_meta_info(metrics.MetaInfoScope.cluster, None, "cluster-name", "test")
 
@@ -955,16 +919,19 @@ class InMemoryMetricsStoreTests(TestCase):
         }, self.metrics_store.docs[1]["meta"])
 
     def test_get_error_rate_zero_without_samples(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assertEqual(0.0, self.metrics_store.get_error_rate("term-query", sample_type=metrics.SampleType.Normal))
 
     def test_get_error_rate_by_sample_type(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.put_value_cluster_level("service_time", 3.0, "ms", task="term-query", sample_type=metrics.SampleType.Warmup,
                                                    meta_data={"success": False})
@@ -973,13 +940,15 @@ class InMemoryMetricsStoreTests(TestCase):
 
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assertEqual(1.0, self.metrics_store.get_error_rate("term-query", sample_type=metrics.SampleType.Warmup))
         self.assertEqual(0.0, self.metrics_store.get_error_rate("term-query", sample_type=metrics.SampleType.Normal))
 
     def test_get_error_rate_mixed(self):
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults", create=True)
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults", create=True)
         self.metrics_store.lap = 1
         self.metrics_store.put_value_cluster_level("service_time", 3.0, "ms", task="term-query", sample_type=metrics.SampleType.Normal,
                                                    meta_data={"success": True})
@@ -994,7 +963,8 @@ class InMemoryMetricsStoreTests(TestCase):
 
         self.metrics_store.close()
 
-        self.metrics_store.open(EsMetricsTests.TRIAL_TIMESTAMP, "test", "append-no-conflicts", "defaults")
+        self.metrics_store.open(InMemoryMetricsStoreTests.TRIAL_ID, InMemoryMetricsStoreTests.TRIAL_TIMESTAMP,
+                                "test", "append-no-conflicts", "defaults")
 
         self.assertEqual(0.0, self.metrics_store.get_error_rate("term-query", sample_type=metrics.SampleType.Warmup))
         self.assertEqual(0.2, self.metrics_store.get_error_rate("term-query", sample_type=metrics.SampleType.Normal))
@@ -1002,6 +972,7 @@ class InMemoryMetricsStoreTests(TestCase):
 
 class FileRaceStoreTests(TestCase):
     TRIAL_TIMESTAMP = datetime.datetime(2016, 1, 31)
+    TRIAL_ID = "6ebc6e53-ee20-4b0c-99b4-09697987e9f4"
 
     class DictHolder:
         def __init__(self, d):
@@ -1030,7 +1001,8 @@ class FileRaceStoreTests(TestCase):
                         indices=[track.Index(name="tests", types=["test-type"])],
                         challenges=[track.Challenge(name="index", default=True, schedule=schedule)])
 
-        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
+        race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_id=FileRaceStoreTests.TRIAL_ID,
+                            trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
                             pipeline="from-sources", user_tags={"os": "Linux"}, track=t, track_params={"clients": 12},
                             challenge=t.default_challenge, car="4gheap",
                             total_laps=12,
diff --git a/tests/reporter_test.py b/tests/reporter_test.py
index 7750f7c6..7dc623dd 100644
--- a/tests/reporter_test.py
+++ b/tests/reporter_test.py
@@ -10,6 +10,7 @@ class StatsCalculatorTests(TestCase):
         cfg = config.Config()
         cfg.add(config.Scope.application, "system", "env.name", "unittest")
         cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now())
+        cfg.add(config.Scope.application, "system", "trial.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
         cfg.add(config.Scope.application, "reporting", "datastore.type", "in-memory")
         cfg.add(config.Scope.application, "mechanic", "car.names", ["unittest_car"])
         cfg.add(config.Scope.application, "race", "laps", 1)