Repository: elastic/rally

Issue:
Allow to use track parameters also in index / template definitions
Currently index definitions are read as is. However, it can be useful to allow track parameters also for index definition files. For consistency, we should allow the same for index templates.

Technically, this means that we will treat index definition files also as Jinja templates.

Discussion:

Patch:
diff --git a/esrally/config.py b/esrally/config.py
index a78ebf77..b8e0d79d 100644
--- a/esrally/config.py
+++ b/esrally/config.py
@@ -447,6 +447,7 @@ class ConfigFactory:
             # the Elasticsearch directory is just the last path component (relative to the source root directory)
             config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir)
 
+        if gradle_bin:
             config["build"] = {}
             config["build"]["gradle.bin"] = gradle_bin
 
diff --git a/esrally/track/loader.py b/esrally/track/loader.py
index 513b1e4f..d04de55e 100644
--- a/esrally/track/loader.py
+++ b/esrally/track/loader.py
@@ -757,9 +757,10 @@ class TrackSpecificationReader:
     Creates a track instances based on its parsed JSON description.
     """
 
-    def __init__(self, override_auto_manage_indices=None, source=io.FileSource):
+    def __init__(self, override_auto_manage_indices=None, track_params=None, source=io.FileSource):
         self.name = None
         self.override_auto_manage_indices = override_auto_manage_indices
+        self.track_params = track_params if track_params else {}
         self.source = source
         self.index_op_type_warning_issued = False
 
@@ -770,7 +771,7 @@ class TrackSpecificationReader:
         meta_data = self._r(track_specification, "meta", mandatory=False)
         indices = [self._create_index(idx, mapping_dir)
                    for idx in self._r(track_specification, "indices", mandatory=False, default_value=[])]
-        templates = [self._create_template(tpl, mapping_dir)
+        templates = [self._create_index_template(tpl, mapping_dir)
                      for tpl in self._r(track_specification, "templates", mandatory=False, default_value=[])]
         corpora = self._create_corpora(self._r(track_specification, "corpora", mandatory=False, default_value=[]), indices)
         # TODO: Remove this in Rally 0.10.0
@@ -816,7 +817,7 @@ class TrackSpecificationReader:
         body_file = self._r(index_spec, "body", mandatory=False)
         if body_file:
             with self.source(os.path.join(mapping_dir, body_file), "rt") as f:
-                body = json.load(f)
+                body = self._load_template(f.read(), index_name)
         else:
             body = None
 
@@ -832,15 +833,25 @@ class TrackSpecificationReader:
 
         return track.Index(name=index_name, body=body, auto_managed=auto_managed, types=types)
 
-    def _create_template(self, tpl_spec, mapping_dir):
+    def _create_index_template(self, tpl_spec, mapping_dir):
         name = self._r(tpl_spec, "name")
         index_pattern = self._r(tpl_spec, "index-pattern")
         delete_matching_indices = self._r(tpl_spec, "delete-matching-indices", mandatory=False, default_value=True)
         template_file = os.path.join(mapping_dir, self._r(tpl_spec, "template"))
         with self.source(template_file, "rt") as f:
-            template_content = json.load(f)
+            template_content = self._load_template(f.read(), name)
         return track.IndexTemplate(name, index_pattern, template_content, delete_matching_indices)
 
+    def _load_template(self, contents, description):
+        try:
+            rendered = render_template(loader=jinja2.DictLoader({"default": contents}),
+                                       template_name="default",
+                                       template_vars=self.track_params)
+            return json.loads(rendered)
+        except (json.JSONDecodeError, jinja2.exceptions.TemplateError) as e:
+            logger.exception("Could not load file template for %s." % description)
+            raise TrackSyntaxError("Could not load file template for '%s'" % description, str(e))
+
     def _create_corpora(self, corpora_specs, indices):
         document_corpora = []
         known_corpora_names = set()


Test patch:
diff --git a/tests/track/loader_test.py b/tests/track/loader_test.py
index 4f9e524e..ea6b4996 100644
--- a/tests/track/loader_test.py
+++ b/tests/track/loader_test.py
@@ -1426,8 +1426,20 @@ class TrackSpecificationReaderTests(TestCase):
                 }
             ]
         }
-        reader = loader.TrackSpecificationReader(source=io.DictStringFileSourceFactory({
-            "/mappings/body.json": ['{"mappings": {"main": "empty-for-test", "secondary": "empty-for-test"}}']
+        reader = loader.TrackSpecificationReader(
+            track_params={"number_of_shards": 3},
+            source=io.DictStringFileSourceFactory({
+            "/mappings/body.json": ["""
+            {
+                "settings": {
+                    "number_of_shards": {{ number_of_shards }}
+                },            
+                "mappings": {
+                    "main": "empty-for-test", 
+                    "secondary": "empty-for-test"
+                }
+            }
+            """]
         }))
         resulting_track = reader("unittest", track_specification, "/mappings")
         self.assertEqual("unittest", resulting_track.name)
@@ -1435,7 +1447,16 @@ class TrackSpecificationReaderTests(TestCase):
         # indices
         self.assertEqual(1, len(resulting_track.indices))
         self.assertEqual("index-historical", resulting_track.indices[0].name)
-        self.assertDictEqual({"mappings": {"main": "empty-for-test", "secondary": "empty-for-test"}}, resulting_track.indices[0].body)
+        self.assertDictEqual({
+            "settings": {
+                "number_of_shards": 3
+            },
+            "mappings":
+                {
+                    "main": "empty-for-test",
+                    "secondary": "empty-for-test"
+                }
+        }, resulting_track.indices[0].body)
         self.assertEqual(2, len(resulting_track.indices[0].types))
         self.assertEqual("main", resulting_track.indices[0].types[0].name)
         self.assertEqual("secondary", resulting_track.indices[0].types[1].name)
@@ -1492,8 +1513,17 @@ class TrackSpecificationReaderTests(TestCase):
             "operations": [],
             "challenges": []
         }
-        reader = loader.TrackSpecificationReader(source=io.DictStringFileSourceFactory({
-            "/mappings/default-template.json": ['{"some-index-template": "empty-for-test"}'],
+        reader = loader.TrackSpecificationReader(
+            track_params={"index_pattern": "*"},
+            source=io.DictStringFileSourceFactory({
+                "/mappings/default-template.json": ["""
+                {
+                    "index_patterns": [ "{{index_pattern}}"],
+                    "settings": {
+                        "number_of_shards": {{ number_of_shards | default(1) }} 
+                    }                    
+                }
+                """],
         }))
         resulting_track = reader("unittest", track_specification, "/mappings")
         self.assertEqual("unittest", resulting_track.name)
@@ -1502,7 +1532,13 @@ class TrackSpecificationReaderTests(TestCase):
         self.assertEqual(1, len(resulting_track.templates))
         self.assertEqual("my-index-template", resulting_track.templates[0].name)
         self.assertEqual("*", resulting_track.templates[0].pattern)
-        self.assertEqual({"some-index-template": "empty-for-test"}, resulting_track.templates[0].content)
+        self.assertDictEqual(
+            {
+                "index_patterns": ["*"],
+                "settings": {
+                    "number_of_shards": 1
+                }
+            }, resulting_track.templates[0].content)
         self.assertEqual(0, len(resulting_track.challenges))
 
     def test_types_are_optional_for_user_managed_indices(self):