Repository: elastic/rally

Issue:
Allow simpler configuration of cluster configuration
If we want to customize the benchmarked Elasticsearch cluster, we currently need to create a custom team repository which requires git. For simple use cases, we should allow users to just provide a path to a configuration directory (similarly to `--track-path` for tracks). We will therefore add a new command line parameter `--team-path` for that purpose.

Discussion:

Patch:
diff --git a/docs/car.rst b/docs/car.rst
index 5d7f21c2..855d38e8 100644
--- a/docs/car.rst
+++ b/docs/car.rst
@@ -94,6 +94,11 @@ These values are derived by Rally internally based on command line flags and you
 
 If you specify multiple configurations, e.g. ``--car="4gheap,ea"``, Rally will apply them in order. It will first read all variables in ``4gheap.ini``, then in ``ea.ini``. Afterwards, it will copy all configuration files from the corresponding config base of ``4gheap`` and *append* all configuration files from ``ea``. This also shows when to define a separate "car" and when to define a "mixin": If you need to amend configuration files, use a mixin, if you need to have a specific configuration, define a car.
 
+Simple customizations
+^^^^^^^^^^^^^^^^^^^^^
+
+For simple customizations you can create the directory hierarchy as outlined above and use the ``--team-path`` command line parameter to refer to this configuration. For more complex use cases and distributed multi-node benchmarks, we recommend to use custom team repositories.
+
 Custom Team Repositories
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/docs/command_line_reference.rst b/docs/command_line_reference.rst
index 7b33c618..16aa80e7 100644
--- a/docs/command_line_reference.rst
+++ b/docs/command_line_reference.rst
@@ -141,6 +141,14 @@ You can use ``--include-tasks`` to specify a comma-separated list of tasks that
 
 Selects the team repository that Rally should use to resolve cars. By default the ``default`` team repository is used, which is available in the Github project `rally-teams <https://github.com/elastic/rally-teams>`__. See the documentation about :doc:`cars </car>` on how to add your own team repositories.
 
+``team-path``
+~~~~~~~~~~~~~
+
+A directory that contains a team configuration. ``--team-path`` and ``--team-repository`` are mutually exclusive. See the :doc:`car reference </car>` for the required directory structure.
+
+Example::
+
+   esrally --team-path=~/Projects/es-teams
 
 ``car``
 ~~~~~~~
diff --git a/esrally/mechanic/mechanic.py b/esrally/mechanic/mechanic.py
index 1e40957d..b14ec1ca 100644
--- a/esrally/mechanic/mechanic.py
+++ b/esrally/mechanic/mechanic.py
@@ -589,9 +589,9 @@ def create(cfg, metrics_store, all_node_ips, cluster_settings=None, sources=Fals
         car = None
         plugins = []
     else:
-        repo = team.team_repo(cfg)
-        car = team.load_car(repo, cfg.opts("mechanic", "car.names"), cfg.opts("mechanic", "car.params"))
-        plugins = team.load_plugins(repo, cfg.opts("mechanic", "car.plugins"), cfg.opts("mechanic", "plugin.params"))
+        team_path = team.team_path(cfg)
+        car = team.load_car(team_path, cfg.opts("mechanic", "car.names"), cfg.opts("mechanic", "car.params"))
+        plugins = team.load_plugins(team_path, cfg.opts("mechanic", "car.plugins"), cfg.opts("mechanic", "plugin.params"))
 
     if sources or distribution:
         s = supplier.create(cfg, sources, distribution, build, challenge_root_path, plugins)
diff --git a/esrally/mechanic/team.py b/esrally/mechanic/team.py
index e097194f..b56160f2 100644
--- a/esrally/mechanic/team.py
+++ b/esrally/mechanic/team.py
@@ -11,7 +11,7 @@ logger = logging.getLogger("rally.team")
 
 
 def list_cars(cfg):
-    loader = CarLoader(team_repo(cfg))
+    loader = CarLoader(team_path(cfg))
     cars = []
     for name in loader.car_names():
         cars.append(loader.load_car(name))
@@ -50,7 +50,7 @@ def load_car(repo, name, car_params={}):
 
 
 def list_plugins(cfg):
-    plugins = PluginLoader(team_repo(cfg)).plugins()
+    plugins = PluginLoader(team_path(cfg)).plugins()
     if plugins:
         console.println("Available Elasticsearch plugins:\n")
         console.println(tabulate.tabulate([[p.name, p.config] for p in plugins], headers=["Name", "Configuration"]))
@@ -83,25 +83,27 @@ def load_plugins(repo, plugin_names, plugin_params={}):
     return plugins
 
 
-def team_repo(cfg, update=True):
-    distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
-    repo_name = cfg.opts("mechanic", "repository.name")
-    offline = cfg.opts("system", "offline.mode")
-    remote_url = cfg.opts("teams", "%s.url" % repo_name, mandatory=False)
-    root = cfg.opts("node", "root.dir")
-    team_repositories = cfg.opts("mechanic", "team.repository.dir")
-    teams_dir = os.path.join(root, team_repositories)
-
-    current_team_repo = repo.RallyRepository(remote_url, teams_dir, repo_name, "teams", offline)
-    if update:
+def team_path(cfg):
+    root_path = cfg.opts("mechanic", "team.path", mandatory=False)
+    if root_path:
+        return root_path
+    else:
+        distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
+        repo_name = cfg.opts("mechanic", "repository.name")
+        offline = cfg.opts("system", "offline.mode")
+        remote_url = cfg.opts("teams", "%s.url" % repo_name, mandatory=False)
+        root = cfg.opts("node", "root.dir")
+        team_repositories = cfg.opts("mechanic", "team.repository.dir")
+        teams_dir = os.path.join(root, team_repositories)
+
+        current_team_repo = repo.RallyRepository(remote_url, teams_dir, repo_name, "teams", offline)
         current_team_repo.update(distribution_version)
-    return current_team_repo
+        return current_team_repo.repo_dir
 
 
 class CarLoader:
-    def __init__(self, repo):
-        self.repo = repo
-        self.cars_dir = os.path.join(self.repo.repo_dir, "cars")
+    def __init__(self, team_root_path):
+        self.cars_dir = os.path.join(team_root_path, "cars")
 
     def car_names(self):
         def __car_name(path):
@@ -199,9 +201,8 @@ class Car:
 
 
 class PluginLoader:
-    def __init__(self, repo):
-        self.repo = repo
-        self.plugins_root_path = os.path.join(self.repo.repo_dir, "plugins")
+    def __init__(self, team_root_path):
+        self.plugins_root_path = os.path.join(team_root_path, "plugins")
 
     def plugins(self):
         known_plugins = self._core_plugins() + self._configured_plugins()
diff --git a/esrally/rally.py b/esrally/rally.py
index 15b04b27..131e89a2 100644
--- a/esrally/rally.py
+++ b/esrally/rally.py
@@ -278,6 +278,9 @@ def create_arg_parser():
         p.add_argument(
             "--challenge",
             help="define the challenge to use. List possible challenges for tracks with `%s list tracks`" % PROGRAM_NAME)
+        p.add_argument(
+            "--team-path",
+            help="define the path to the car and plugin configurations to use.")
         p.add_argument(
             "--car",
             help="define the car to use. List possible cars with `%s list cars` (default: defaults)." % PROGRAM_NAME,
@@ -676,8 +679,12 @@ def main():
     if args.distribution_version:
         cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.version", args.distribution_version)
     cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.repository", args.distribution_repository)
-    cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository)
     cfg.add(config.Scope.applicationOverride, "mechanic", "car.names", csv_to_list(args.car))
+    if args.team_path:
+        cfg.add(config.Scope.applicationOverride, "mechanic", "team.path", os.path.abspath(io.normalize_path(args.team_path)))
+        cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", None)
+    else:
+        cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository)
     cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", csv_to_list(args.elasticsearch_plugins))
     cfg.add(config.Scope.applicationOverride, "mechanic", "car.params", kv_to_map(csv_to_list(args.car_params)))
     cfg.add(config.Scope.applicationOverride, "mechanic", "plugin.params", kv_to_map(csv_to_list(args.plugin_params)))


Test patch:
diff --git a/tests/mechanic/team_test.py b/tests/mechanic/team_test.py
index ea717b0e..37f80635 100644
--- a/tests/mechanic/team_test.py
+++ b/tests/mechanic/team_test.py
@@ -7,41 +7,36 @@ from esrally.mechanic import team
 current_dir = os.path.dirname(os.path.abspath(__file__))
 
 
-class UnitTestRepo:
-    def __init__(self, repo_dir):
-        self.repo_dir = repo_dir
-
-
 class CarLoaderTests(TestCase):
     def __init__(self, args):
         super().__init__(args)
-        self.repo = None
+        self.team_dir = None
         self.loader = None
 
     def setUp(self):
-        self.repo = UnitTestRepo(os.path.join(current_dir, "data"))
-        self.loader = team.CarLoader(self.repo)
+        self.team_dir = os.path.join(current_dir, "data")
+        self.loader = team.CarLoader(self.team_dir)
 
     def test_lists_car_names(self):
         # contrary to the name this assertion compares contents but does not care about order.
         self.assertCountEqual(["default", "32gheap", "missing_config_base", "empty_config_base", "ea", "verbose"], self.loader.car_names())
 
     def test_load_known_car(self):
-        car = team.load_car(self.repo, ["default"], car_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]})
+        car = team.load_car(self.team_dir, ["default"], car_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]})
         self.assertEqual("default", car.name)
         self.assertEqual([os.path.join(current_dir, "data", "cars", "vanilla")], car.config_paths)
         self.assertDictEqual({"heap_size": "1g", "data_paths": ["/mnt/disk0", "/mnt/disk1"]}, car.variables)
         self.assertEqual({}, car.env)
 
     def test_load_car_with_mixin_single_config_base(self):
-        car = team.load_car(self.repo, ["32gheap", "ea"])
+        car = team.load_car(self.team_dir, ["32gheap", "ea"])
         self.assertEqual("32gheap+ea", car.name)
         self.assertEqual([os.path.join(current_dir, "data", "cars", "vanilla")], car.config_paths)
         self.assertEqual({"heap_size": "32g", "assertions": "true"}, car.variables)
         self.assertEqual({"JAVA_TOOL_OPTS": "A B C D E F"}, car.env)
 
     def test_load_car_with_mixin_multiple_config_bases(self):
-        car = team.load_car(self.repo, ["32gheap", "ea", "verbose"])
+        car = team.load_car(self.team_dir, ["32gheap", "ea", "verbose"])
         self.assertEqual("32gheap+ea+verbose", car.name)
         self.assertEqual([
             os.path.join(current_dir, "data", "cars", "vanilla"),
@@ -52,17 +47,17 @@ class CarLoaderTests(TestCase):
 
     def test_raises_error_on_unknown_car(self):
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
-            team.load_car(self.repo, ["don_t-know-you"])
+            team.load_car(self.team_dir, ["don_t-know-you"])
         self.assertRegex(ctx.exception.args[0], r"Unknown car \[don_t-know-you\]. List the available cars with [^\s]+ list cars.")
 
     def test_raises_error_on_empty_config_base(self):
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
-            team.load_car(self.repo, ["empty_config_base"])
+            team.load_car(self.team_dir, ["empty_config_base"])
         self.assertEqual("At least one config base is required for car ['empty_config_base']", ctx.exception.args[0])
 
     def test_raises_error_on_missing_config_base(self):
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
-            team.load_car(self.repo, ["missing_config_base"])
+            team.load_car(self.team_dir, ["missing_config_base"])
         self.assertEqual("At least one config base is required for car ['missing_config_base']", ctx.exception.args[0])
 
 
@@ -72,8 +67,7 @@ class PluginLoaderTests(TestCase):
         self.loader = None
 
     def setUp(self):
-        repo = UnitTestRepo(os.path.join(current_dir, "data"))
-        self.loader = team.PluginLoader(repo)
+        self.loader = team.PluginLoader(os.path.join(current_dir, "data"))
 
     def test_lists_plugins(self):
         self.assertCountEqual(