Repository: elastic/rally

Issue:
Allow first Rally run from non master branch
Current Rally won't run from a source checkout with [disabled automatic updates](http://esrally.readthedocs.io/en/stable/developing.html?highlight=developing#automatic-updates) if the user is running it for the first time.

This is because the execution of [setup.py](https://github.com/elastic/rally/blob/master/run.sh#L65-L68) only happens when updates are enabled.


Discussion:

Patch:
diff --git a/docs/configuration.rst b/docs/configuration.rst
index b4f524ed..08430670 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -66,7 +66,7 @@ As you can see above, Rally autodetects if git, Gradle and a JDK are installed.
 
 As you can see, Rally tells you that you cannot build Elasticsearch from sources but you can still benchmark official binary distributions.
 
-It's also possible that Rally cannot automatically find your JDK 8 or JDK 9 home directory. In that case, it will ask you later in the configuration process. If you do not provide a JDK home directory, Rally cannot start Elasticsearch on this machine but you can still use it as a load generator to :doc:`benchmark remote clusters </recipes>`.
+It's also possible that Rally cannot automatically find your JDK 8 or JDK 10 home directory. In that case, it will ask you later in the configuration process. If you do not provide a JDK home directory, Rally cannot start Elasticsearch on this machine but you can still use it as a load generator to :doc:`benchmark remote clusters </recipes>`.
 
 After running the initial detection, Rally will try to autodetect your Elasticsearch project directory (either in the current directory or in ``../elasticsearch``) or will choose a default directory::
 
@@ -122,7 +122,7 @@ Rally will ask you a few more things in the advanced setup:
 
 * **Benchmark data directory**: Rally stores all benchmark related data in this directory which can take up to several tens of GB. If you want to use a dedicated partition, you can specify a different data directory here.
 * **Elasticsearch project directory**: This is the directory where the Elasticsearch sources are located. If you don't actively develop on Elasticsearch you can just leave the default but if you want to benchmark local changes you should point Rally to your project directory. Note that Rally will run builds with Gradle in this directory (it runs ``gradle clean`` and ``gradle :distribution:tar:assemble``).
-* **JDK root directory**: Rally will only ask this if it could not autodetect the JDK home by itself. Just enter the root directory of the JDK you want to use. By default, Rally will choose Java 8 if available and fallback to Java 9.
+* **JDK root directory**: Rally will only ask this if it could not autodetect the JDK home by itself. Just enter the root directory of the JDK you want to use. By default, Rally will choose Java 8 if available and fallback to Java 10.
 * **Metrics store type**: You can choose between ``in-memory`` which requires no additional setup or ``elasticsearch`` which requires that you start a dedicated Elasticsearch instance to store metrics but gives you much more flexibility to analyse results.
 * **Metrics store settings** (only for metrics store type ``elasticsearch``): Provide the connection details to the Elasticsearch metrics store. This should be an instance that you use just for Rally but it can be a rather small one. A single node cluster with default setting should do it. When using self-signed certificates on the Elasticsearch metrics store, certificate verification can be turned off by setting the ``datastore.ssl.verification_mode`` setting to ``none``. Alternatively you can enter the path to the certificate authority's signing certificate in ``datastore.ssl.certificate_authorities``. Both settings are optional.
 * **Name for this benchmark environment** (only for metrics store type ``elasticsearch``): You can use the same metrics store for multiple environments (e.g. local, continuous integration etc.) so you can separate metrics from different environments by choosing a different name.
diff --git a/esrally/config.py b/esrally/config.py
index b8e0d79d..32348a67 100644
--- a/esrally/config.py
+++ b/esrally/config.py
@@ -106,7 +106,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl
 
 
 class Config:
-    CURRENT_CONFIG_VERSION = 13
+    CURRENT_CONFIG_VERSION = 14
 
     """
     Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on
@@ -305,13 +305,13 @@ class ConfigFactory:
         gradle_bin = "./gradlew" if use_gradle_wrapper else io.guess_install_location("gradle")
 
         java_8_home = runtime_java_home if runtime_java_home else io.guess_java_home(major_version=8)
-        java_9_home = java_home if java_home else io.guess_java_home(major_version=9)
+        java_10_home = java_home if java_home else io.guess_java_home(major_version=10)
         from esrally.utils import jvm
         if java_8_home:
             auto_detected_java_home = java_8_home
         # Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to.
-        elif java_9_home and not jvm.is_early_access_release(java_9_home):
-            auto_detected_java_home = java_9_home
+        elif java_10_home and not jvm.is_early_access_release(java_10_home):
+            auto_detected_java_home = java_10_home
         else:
             auto_detected_java_home = None
 
@@ -342,14 +342,14 @@ class ConfigFactory:
             self.o("* Setting up benchmark data directory in %s" % root_dir)
 
         if benchmark_from_sources:
-            if not java_9_home or jvm.is_early_access_release(java_9_home):
-                raw_java_9_home = self._ask_property("Enter the JDK 9 root directory", check_path_exists=True, mandatory=False)
-                if raw_java_9_home and jvm.major_version(raw_java_9_home) == 9 and not jvm.is_early_access_release(raw_java_9_home):
-                    java_9_home = io.normalize_path(raw_java_9_home) if raw_java_9_home else None
+            if not java_10_home or jvm.is_early_access_release(java_10_home):
+                raw_java_10_home = self._ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False)
+                if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home):
+                    java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None
                 else:
                     benchmark_from_sources = False
                     self.o("********************************************************************************")
-                    self.o("You don't have a valid JDK 9 installation and cannot benchmark source builds.")
+                    self.o("You don't have a valid JDK 10 installation and cannot benchmark source builds.")
                     self.o("")
                     self.o("You can still benchmark binary distributions with e.g.:")
                     self.o("")
@@ -454,8 +454,8 @@ class ConfigFactory:
         config["runtime"] = {}
         if java_home:
             config["runtime"]["java.home"] = java_home
-        if java_9_home:
-            config["runtime"]["java9.home"] = java_9_home
+        if java_10_home:
+            config["runtime"]["java10.home"] = java_10_home
 
         config["benchmarks"] = {}
         config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data"
@@ -829,6 +829,44 @@ def migrate(config_file, current_version, target_version, out=print, i=input):
         current_version = 13
         config["meta"]["config.version"] = str(current_version)
 
+    if current_version == 13 and target_version > current_version:
+        # This version replaced java9.home with java10.home
+        if "build" in config and "gradle.bin" in config["build"]:
+            java_10_home = io.guess_java_home(major_version=10)
+            from esrally.utils import jvm
+            if java_10_home and not jvm.is_early_access_release(java_10_home):
+                logger.debug("Autodetected a JDK 10 installation at [%s]" % java_10_home)
+                if "runtime" not in config:
+                    config["runtime"] = {}
+                config["runtime"]["java10.home"] = java_10_home
+            else:
+                logger.debug("Could not autodetect a JDK 10 installation. Checking [java.home] already points to a JDK 10.")
+                detected = False
+                if "runtime" in config:
+                    java_home = config["runtime"]["java.home"]
+                    if jvm.major_version(java_home) == 10 and not jvm.is_early_access_release(java_home):
+                        config["runtime"]["java10.home"] = java_home
+                        detected = True
+
+                if not detected:
+                    logger.debug("Could not autodetect a JDK 10 installation. Asking user.")
+                    raw_java_10_home = prompter.ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False)
+                    if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home):
+                        java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None
+                        config["runtime"]["java10.home"] = java_10_home
+                    else:
+                        out("********************************************************************************")
+                        out("You don't have a valid JDK 10 installation and cannot benchmark source builds.")
+                        out("")
+                        out("You can still benchmark binary distributions with e.g.:")
+                        out("")
+                        out("  %s --distribution-version=6.0.0" % PROGRAM_NAME)
+                        out("********************************************************************************")
+                        out("")
+
+        current_version = 14
+        config["meta"]["config.version"] = str(current_version)
+
     # all migrations done
     config_file.store(config)
     logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)
diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py
index 5646a0b2..4a1c3b38 100644
--- a/esrally/mechanic/provisioner.py
+++ b/esrally/mechanic/provisioner.py
@@ -7,12 +7,13 @@ from enum import Enum
 import jinja2
 
 from esrally import exceptions
-from esrally.utils import io, console, process, modules
+from esrally.utils import io, console, process, modules, versions
 
 logger = logging.getLogger("rally.provisioner")
 
 
 def local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, target_root, node_id):
+    distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
     ip = cfg.opts("provisioning", "node.ip")
     http_port = cfg.opts("provisioning", "node.http.port")
     node_name_prefix = cfg.opts("provisioning", "node.name.prefix")
@@ -24,7 +25,7 @@ def local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, target_
     es_installer = ElasticsearchInstaller(car, node_name, node_root_dir, all_node_ips, ip, http_port)
     plugin_installers = [PluginInstaller(plugin) for plugin in plugins]
 
-    return BareProvisioner(cluster_settings, es_installer, plugin_installers, preserve)
+    return BareProvisioner(cluster_settings, es_installer, plugin_installers, preserve, distribution_version=distribution_version)
 
 
 def no_op_provisioner():
@@ -143,11 +144,12 @@ class BareProvisioner:
     of the benchmark candidate to the appropriate place.
     """
 
-    def __init__(self, cluster_settings, es_installer, plugin_installers, preserve, apply_config=_apply_config):
+    def __init__(self, cluster_settings, es_installer, plugin_installers, preserve, distribution_version=None, apply_config=_apply_config):
         self.preserve = preserve
         self._cluster_settings = cluster_settings
         self.es_installer = es_installer
         self.plugin_installers = plugin_installers
+        self.distribution_version = distribution_version
         self.apply_config = apply_config
 
     def prepare(self, binary):
@@ -184,7 +186,17 @@ class BareProvisioner:
         plugin_variables = {}
         mandatory_plugins = []
         for installer in self.plugin_installers:
-            mandatory_plugins.append(installer.plugin_name)
+            # For Elasticsearch < 6.3 more specific plugin names are required for mandatory plugin check
+            # Details in: https://github.com/elastic/elasticsearch/pull/28710
+            # TODO: Remove this section with Elasticsearch <6.3 becomes EOL.
+            try:
+                major, minor, _, _ = versions.components(self.distribution_version)
+                if (major == 6 and minor < 3) or major < 6:
+                    mandatory_plugins.append(installer.sub_plugin_name)
+                else:
+                    mandatory_plugins.append(installer.plugin_name)
+            except (TypeError, exceptions.InvalidSyntax):
+                mandatory_plugins.append(installer.plugin_name)
             plugin_variables.update(installer.variables)
 
         cluster_settings = {}
@@ -373,6 +385,11 @@ class PluginInstaller:
     def plugin_name(self):
         return self.plugin.name
 
+    @property
+    def sub_plugin_name(self):
+        # if a plugin consists of multiple plugins (e.g. x-pack) we're interested in that name
+        return self.variables.get("plugin_name", self.plugin_name)
+
 
 class NoOpProvisioner:
     def __init__(self, *args):
diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py
index 7cbd4b70..b32106c4 100644
--- a/esrally/mechanic/supplier.py
+++ b/esrally/mechanic/supplier.py
@@ -27,9 +27,9 @@ def create(cfg, sources, distribution, build, challenge_root_path, plugins):
 
     if build_needed:
         gradle = cfg.opts("build", "gradle.bin")
-        java9_home = _java9_home(cfg)
+        java10_home = _java10_home(cfg)
         es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir"))
-        builder = Builder(es_src_dir, gradle, java9_home, challenge_root_path)
+        builder = Builder(es_src_dir, gradle, java10_home, challenge_root_path)
     else:
         builder = None
 
@@ -68,14 +68,14 @@ def create(cfg, sources, distribution, build, challenge_root_path, plugins):
     return CompositeSupplier(suppliers)
 
 
-def _java9_home(cfg):
+def _java10_home(cfg):
     from esrally import config
     try:
-        return cfg.opts("runtime", "java9.home")
+        return cfg.opts("runtime", "java10.home")
     except config.ConfigError:
-        logger.exception("Cannot determine Java 9 home.")
-        raise exceptions.SystemSetupError("No JDK 9 is configured. You cannot benchmark source builds of Elasticsearch on this machine. "
-                                          "Please install a JDK 9 and reconfigure Rally with %s configure" % PROGRAM_NAME)
+        logger.exception("Cannot determine Java 10 home.")
+        raise exceptions.SystemSetupError("No JDK 10 is configured. You cannot benchmark source builds of Elasticsearch on this machine. "
+                                          "Please install a JDK 10 and reconfigure Rally with %s configure" % PROGRAM_NAME)
 
 
 def _required_version(version):
@@ -427,13 +427,6 @@ class SourceRepository:
 
 
 class Builder:
-    # Tested with Gradle 4.1 on Java 9-ea+161
-    JAVA_9_GRADLE_OPTS = "--add-opens=java.base/java.io=ALL-UNNAMED " \
-                         "--add-opens=java.base/java.lang=ALL-UNNAMED " \
-                         "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED " \
-                         "--add-opens=java.base/java.util=ALL-UNNAMED " \
-                         "--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED " \
-                         "--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED"
     """
     A builder is responsible for creating an installable binary from the source files.
 
@@ -460,14 +453,8 @@ class Builder:
         log_file = "%s/build.log" % self.log_dir
 
         # we capture all output to a dedicated build log file
-        jvm_major_version = jvm.major_version(self.java_home)
-        if jvm_major_version > 8:
-            logger.info("Detected JVM with major version [%d]. Adjusting JDK module access options for the build." % jvm_major_version)
-            gradle_opts = "export GRADLE_OPTS=\"%s\"; " % Builder.JAVA_9_GRADLE_OPTS
-        else:
-            gradle_opts = ""
 
-        build_cmd = "%sexport JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (gradle_opts, self.java_home, src_dir, self.gradle, task, log_file)
+        build_cmd = "export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (self.java_home, src_dir, self.gradle, task, log_file)
         logger.info("Running build command [%s]" % build_cmd)
 
         if process.run_subprocess(build_cmd):
diff --git a/esrally/mechanic/team.py b/esrally/mechanic/team.py
index b56160f2..e37e5c3a 100644
--- a/esrally/mechanic/team.py
+++ b/esrally/mechanic/team.py
@@ -341,5 +341,3 @@ class PluginDescriptor:
 
     def __eq__(self, other):
         return isinstance(other, type(self)) and (self.name, self.config, self.core_plugin) == (other.name, other.config, other.core_plugin)
-
-
diff --git a/rally b/rally
index 2ae34e28..ba83469d 100755
--- a/rally
+++ b/rally
@@ -26,4 +26,5 @@ cd "${RALLY_SRC_HOME}" >/dev/null 2>&1
 __RALLY_INTERNAL_BINARY_NAME="esrally"
 __RALLY_INTERNAL_HUMAN_NAME="Rally"
 
-source run.sh
\ No newline at end of file
+source run.sh
+
diff --git a/run.sh b/run.sh
old mode 100644
new mode 100755
index ad5c9cb6..7893e2d8
--- a/run.sh
+++ b/run.sh
@@ -11,13 +11,33 @@
 readonly BINARY_NAME="${__RALLY_INTERNAL_BINARY_NAME}"
 readonly HUMAN_NAME="${__RALLY_INTERNAL_HUMAN_NAME}"
 
+function install_esrally_with_setuptools() {
+    # Check if optional parameter with Rally binary path, points to an existing executable file.
+    if [[ -n $1 ]]; then
+     if [[ -f $1 && -x $1 ]]; then return; fi
+    fi
+
+    if [[ ${IN_VIRTUALENV} == 0 ]]; then
+        python3 setup.py -q develop --user
+    else
+        python3 setup.py -q develop
+    fi
+}
+
 # Attempt to update Rally itself by default but allow user to skip it.
 SELF_UPDATE=YES
 # Assume that the "main remote" is called "origin"
 REMOTE="origin"
 
 # While we could also check via the presence of `VIRTUAL_ENV` this is a bit more reliable.
-python3 -c 'import sys; print(sys.real_prefix)' >/dev/null 2>&1 && IN_VIRTUALENV=1 || IN_VIRTUALENV=0
+# Check for both pyvenv and normal venv environments
+# https://www.python.org/dev/peps/pep-0405/
+if python3 -c 'import os, sys; sys.exit(0) if "VIRTUAL_ENV" in os.environ else sys.exit(1)' >/dev/null 2>&1
+then
+    IN_VIRTUALENV=1
+else
+    IN_VIRTUALENV=0
+fi
 
 # Check for parameters that are intended for this script. Note that they only work if they're specified at the beginning (due to how
 # the shell builtin `shift` works. We could make it work for arbitrary positions but that's not worth the complexity for such an
@@ -45,7 +65,7 @@ case ${i} in
 esac
 done
 
-if [ ${SELF_UPDATE} == YES ]
+if [[ $SELF_UPDATE == YES ]]
 then
     # see http://unix.stackexchange.com/a/155077
     if output=$(git status --porcelain) && [ -z "$output" ] && on_master=$(git rev-parse --abbrev-ref HEAD) && [ "$on_master" == "master" ]
@@ -56,17 +76,11 @@ then
       git fetch ${REMOTE} --quiet >/dev/null 2>&1
       exit_code=$?
       set -e
-      if [ ${exit_code} == 0 ]
+      if [[ $exit_code == 0 ]]
       then
         echo "Auto-updating Rally from ${REMOTE}"
         git rebase ${REMOTE}/master --quiet
-        # if we're running in a virtualenv then the --user prefix is not defined (and makes no sense)
-        if [ ${IN_VIRTUALENV} == 0 ]
-        then
-            python3 setup.py -q develop --user
-        else
-            python3 setup.py -q develop
-        fi
+        install_esrally_with_setuptools
       #else
       # offline - skipping update
       fi
@@ -85,16 +99,18 @@ export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800}
 
 # Provide a consistent binary name to the user and hide the fact that we call another binary under the hood.
 export RALLY_ALTERNATIVE_BINARY_NAME=$(basename "$0")
-if [ ${IN_VIRTUALENV} == 0 ]
+if [[ $IN_VIRTUALENV == 0 ]]
 then
     RALLY_ROOT=$(python3 -c "import site; print(site.USER_BASE)")
     RALLY_BIN=${RALLY_ROOT}/bin/${BINARY_NAME}
-    if [ -x "$RALLY_BIN" ]
-    then
+    install_esrally_with_setuptools "${RALLY_BIN}"
+    if [[ -x $RALLY_BIN ]]; then
         ${RALLY_BIN} "$@"
     else
         echo "Cannot execute ${HUMAN_NAME} in ${RALLY_BIN}."
     fi
 else
+    install_esrally_with_setuptools "${BINARY_NAME}"
+
     ${BINARY_NAME} "$@"
 fi


Test patch:
diff --git a/tests/config_test.py b/tests/config_test.py
index 57bafef2..91fc9b3e 100644
--- a/tests/config_test.py
+++ b/tests/config_test.py
@@ -238,7 +238,7 @@ class ConfigFactoryTests(TestCase):
     @mock.patch("esrally.utils.io.guess_install_location")
     def test_create_simple_config(self, guess_install_location, guess_java_home, is_ea_release, working_copy):
         guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"]
-        guess_java_home.return_value = "/tests/java9/home"
+        guess_java_home.return_value = "/tests/java10/home"
         is_ea_release.return_value = False
         # Rally checks in the parent and sibling directories whether there is an ES working copy. We don't want this detection logic
         # to succeed spuriously (e.g. on developer machines).
@@ -256,7 +256,7 @@ class ConfigFactoryTests(TestCase):
                 print("%s::%s: %s" % (section, k, v))
 
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("13", config_store.config["meta"]["config.version"])
+        self.assertEqual("14", config_store.config["meta"]["config.version"])
 
         self.assertTrue("system" in config_store.config)
         self.assertEqual("local", config_store.config["system"]["env.name"])
@@ -273,8 +273,8 @@ class ConfigFactoryTests(TestCase):
         self.assertEqual("/tests/usr/bin/gradle", config_store.config["build"]["gradle.bin"])
 
         self.assertTrue("runtime" in config_store.config)
-        self.assertEqual("/tests/java9/home", config_store.config["runtime"]["java.home"])
-        self.assertEqual("/tests/java9/home", config_store.config["runtime"]["java9.home"])
+        self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java.home"])
+        self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"])
 
         self.assertTrue("benchmarks" in config_store.config)
         self.assertEqual("${node:root.dir}/data", config_store.config["benchmarks"]["local.dataset.cache"])
@@ -316,13 +316,13 @@ class ConfigFactoryTests(TestCase):
                                                    major_jvm_version, jvm_is_early_access_release):
         guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"]
         guess_java_home.return_value = None
-        normalize_path.side_effect = ["/home/user/.rally/benchmarks", "/tests/java9/home", "/tests/java8/home",
+        normalize_path.side_effect = ["/home/user/.rally/benchmarks", "/tests/java10/home", "/tests/java8/home",
                                       "/home/user/.rally/benchmarks/src"]
-        major_jvm_version.return_value = 9
+        major_jvm_version.return_value = 10
         jvm_is_early_access_release.return_value = False
         path_exists.return_value = True
 
-        f = config.ConfigFactory(i=MockInput(["/tests/java9/home", "/Projects/elasticsearch/src", "/tests/java8/home"]), o=null_output)
+        f = config.ConfigFactory(i=MockInput(["/tests/java10/home", "/Projects/elasticsearch/src", "/tests/java8/home"]), o=null_output)
 
         config_store = InMemoryConfigStore("test")
         f.create_config(config_store)
@@ -337,7 +337,7 @@ class ConfigFactoryTests(TestCase):
         guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"]
         guess_java_home.return_value = None
 
-        # the input is the question for the JDK home and the JDK 9 home directory - the user does not define one
+        # the input is the question for the JDK home and the JDK 10 home directory - the user does not define one
         f = config.ConfigFactory(i=MockInput(["", ""]), o=null_output)
 
         config_store = InMemoryConfigStore("test")
@@ -345,14 +345,14 @@ class ConfigFactoryTests(TestCase):
 
         self.assertIsNotNone(config_store.config)
         self.assertFalse("java.home" in config_store.config["runtime"])
-        self.assertFalse("java9.home" in config_store.config["runtime"])
+        self.assertFalse("java10.home" in config_store.config["runtime"])
 
     @mock.patch("esrally.utils.jvm.is_early_access_release")
     @mock.patch("esrally.utils.io.guess_java_home")
     @mock.patch("esrally.utils.io.guess_install_location")
     def test_create_advanced_config(self, guess_install_location, guess_java_home, is_ea_release):
         guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"]
-        guess_java_home.side_effect = ["/tests/java8/home", "/tests/java9/home"]
+        guess_java_home.side_effect = ["/tests/java8/home", "/tests/java10/home"]
         is_ea_release.return_value = False
 
         f = config.ConfigFactory(i=MockInput([
@@ -381,7 +381,7 @@ class ConfigFactoryTests(TestCase):
 
         self.assertIsNotNone(config_store.config)
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("13", config_store.config["meta"]["config.version"])
+        self.assertEqual("14", config_store.config["meta"]["config.version"])
         self.assertTrue("system" in config_store.config)
         self.assertEqual("unittest-env", config_store.config["system"]["env.name"])
         self.assertTrue("node" in config_store.config)
@@ -391,7 +391,7 @@ class ConfigFactoryTests(TestCase):
         self.assertEqual("/tests/usr/bin/gradle", config_store.config["build"]["gradle.bin"])
         self.assertTrue("runtime" in config_store.config)
         self.assertEqual("/tests/java8/home", config_store.config["runtime"]["java.home"])
-        self.assertEqual("/tests/java9/home", config_store.config["runtime"]["java9.home"])
+        self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"])
         self.assertTrue("benchmarks" in config_store.config)
 
         self.assertTrue("reporting" in config_store.config)
@@ -919,3 +919,129 @@ class ConfigMigrationTests(TestCase):
         self.assertEqual("13", config_file.config["meta"]["config.version"])
         self.assertEqual("/usr/lib/java8", config_file.config["runtime"]["java.home"])
         self.assertEqual("/usr/lib/java9", config_file.config["runtime"]["java9.home"])
+
+    def test_migrate_from_13_to_14_without_gradle(self):
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 13
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 13, 14, out=null_output)
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("14", config_file.config["meta"]["config.version"])
+
+    @mock.patch("esrally.utils.io.guess_java_home")
+    @mock.patch("esrally.utils.jvm.is_early_access_release")
+    def test_migrate_from_13_to_14_with_gradle_and_jdk8_autodetect_jdk10(self, is_early_access_release, guess_java_home):
+        guess_java_home.return_value = "/usr/lib/java10"
+        is_early_access_release.return_value = False
+
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 13
+            },
+            "build": {
+                "gradle.bin": "/usr/local/bin/gradle"
+            },
+            "runtime": {
+                "java.home": "/usr/lib/java8"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 13, 14, out=null_output)
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("14", config_file.config["meta"]["config.version"])
+        self.assertEqual("/usr/lib/java8", config_file.config["runtime"]["java.home"])
+        self.assertEqual("/usr/lib/java10", config_file.config["runtime"]["java10.home"])
+
+    @mock.patch("esrally.utils.io.guess_java_home")
+    @mock.patch("esrally.utils.jvm.is_early_access_release")
+    @mock.patch("esrally.utils.jvm.major_version")
+    def test_migrate_from_13_to_14_with_gradle_and_jdk10(self, major_version, is_early_access_release, guess_java_home):
+        guess_java_home.return_value = None
+        is_early_access_release.return_value = False
+        major_version.return_value = 10
+
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 13
+            },
+            "build": {
+                "gradle.bin": "/usr/local/bin/gradle"
+            },
+            "runtime": {
+                "java.home": "/usr/lib/java10"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 13, 14, out=null_output)
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("14", config_file.config["meta"]["config.version"])
+        self.assertEqual("/usr/lib/java10", config_file.config["runtime"]["java.home"])
+        self.assertEqual("/usr/lib/java10", config_file.config["runtime"]["java10.home"])
+
+    @mock.patch("esrally.utils.io.guess_java_home")
+    @mock.patch("esrally.utils.jvm.is_early_access_release")
+    @mock.patch("esrally.utils.jvm.major_version")
+    def test_migrate_from_13_to_14_with_gradle_and_jdk8_ask_user_and_skip(self, major_version, is_early_access_release, guess_java_home):
+        guess_java_home.return_value = None
+        is_early_access_release.return_value = False
+        major_version.return_value = 8
+
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 13
+            },
+            "build": {
+                "gradle.bin": "/usr/local/bin/gradle"
+            },
+            "runtime": {
+                "java.home": "/usr/lib/java8"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 13, 14, out=null_output, i=MockInput(inputs=[""]))
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("14", config_file.config["meta"]["config.version"])
+        self.assertEqual("/usr/lib/java8", config_file.config["runtime"]["java.home"])
+        self.assertTrue("java10.home" not in config_file.config["runtime"])
+
+    @mock.patch("esrally.utils.io.exists")
+    @mock.patch("esrally.utils.io.guess_java_home")
+    @mock.patch("esrally.utils.jvm.is_early_access_release")
+    @mock.patch("esrally.utils.jvm.major_version")
+    def test_migrate_from_13_to_14_with_gradle_and_jdk8_ask_user_enter_valid(self, major_version, is_early_access_release, guess_java_home,
+                                                                             path_exists):
+        guess_java_home.return_value = None
+        is_early_access_release.return_value = False
+        major_version.side_effect = [8, 10]
+        path_exists.return_value = True
+
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 13
+            },
+            "build": {
+                "gradle.bin": "/usr/local/bin/gradle"
+            },
+            "runtime": {
+                "java.home": "/usr/lib/java8"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 13, 14, out=null_output, i=MockInput(inputs=["/usr/lib/java10"]))
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("14", config_file.config["meta"]["config.version"])
+        self.assertEqual("/usr/lib/java8", config_file.config["runtime"]["java.home"])
+        self.assertEqual("/usr/lib/java10", config_file.config["runtime"]["java10.home"])
diff --git a/tests/mechanic/provisioner_test.py b/tests/mechanic/provisioner_test.py
index 426a329c..7c29095a 100644
--- a/tests/mechanic/provisioner_test.py
+++ b/tests/mechanic/provisioner_test.py
@@ -67,6 +67,194 @@ class BareProvisionerTests(TestCase):
         }, config_vars)
 
 
+    class NoopHookHandler:
+        def __init__(self, plugin):
+            self.hook_calls = {}
+
+        def can_load(self):
+            return False
+
+        def invoke(self, phase, variables):
+            self.hook_calls[phase] = variables
+
+    class MockRallyTeamXPackPlugin:
+        """
+        Mock XPackPlugin settings as found in rally-team repo:
+        https://github.com/elastic/rally-teams/blob/6/plugins/x_pack/security.ini
+        """
+        def __init__(self):
+            self.name = "x-pack"
+            self.core_plugin = False
+            self.config = {
+                'base': 'internal_base,security'
+            }
+            self.root_path = None
+            self.config_paths = []
+            self.variables = {
+                'xpack_security_enabled': True,
+                'plugin_name': 'x-pack-security'
+            }
+
+        def __str__(self):
+            return "Plugin descriptor for [%s]" % self.name
+
+        def __repr__(self):
+            r = []
+            for prop, value in vars(self).items():
+                r.append("%s = [%s]" % (prop, repr(value)))
+            return ", ".join(r)
+
+    @mock.patch("glob.glob", lambda p: ["/opt/elasticsearch-5.0.0"])
+    @mock.patch("esrally.utils.io.decompress")
+    @mock.patch("esrally.utils.io.ensure_dir")
+    @mock.patch("esrally.mechanic.provisioner.PluginInstaller.install")
+    @mock.patch("shutil.rmtree")
+    def test_prepare_distribution_lt_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress):
+        """
+        Test that plugin.mandatory is set to the specific plugin name (e.g. `x-pack-security`) and not
+        the meta plugin name (e.g. `x-pack`) for Elasticsearch <6.3
+
+        See: https://github.com/elastic/elasticsearch/pull/28710
+        """
+        apply_config_calls = []
+
+        def null_apply_config(source_root_path, target_root_path, config_vars):
+            apply_config_calls.append((source_root_path, target_root_path, config_vars))
+
+        installer = provisioner.ElasticsearchInstaller(car=
+        team.Car(
+            name="unit-test-car",
+            config_paths=["~/.rally/benchmarks/teams/default/my-car"],
+            variables={"heap": "4g"}),
+            node_name="rally-node-0",
+            node_root_dir="~/.rally/benchmarks/races/unittest",
+            all_node_ips=["10.17.22.22", "10.17.22.23"],
+            ip="10.17.22.23",
+            http_port=9200)
+
+        p = provisioner.BareProvisioner(cluster_settings={"indices.query.bool.max_clause_count": 50000},
+                                        es_installer=installer,
+                                        plugin_installers=[
+                                            provisioner.PluginInstaller(BareProvisionerTests.MockRallyTeamXPackPlugin(),
+                                                                        hook_handler_class=BareProvisionerTests.NoopHookHandler)
+                                        ],
+                                        preserve=True,
+                                        distribution_version="6.2.3",
+                                        apply_config=null_apply_config)
+
+        node_config = p.prepare({"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"})
+        self.assertEqual(installer.car, node_config.car)
+        self.assertEqual("/opt/elasticsearch-5.0.0", node_config.binary_path)
+        self.assertEqual(["/opt/elasticsearch-5.0.0/data"], node_config.data_paths)
+
+        self.assertEqual(1, len(apply_config_calls))
+        source_root_path, target_root_path, config_vars = apply_config_calls[0]
+
+        self.assertEqual("~/.rally/benchmarks/teams/default/my-car", source_root_path)
+        self.assertEqual("/opt/elasticsearch-5.0.0", target_root_path)
+
+        self.maxDiff = None
+
+        self.assertEqual({
+            "cluster_settings": {
+                "indices.query.bool.max_clause_count": 50000,
+                "plugin.mandatory": ["x-pack-security"]
+            },
+            "heap": "4g",
+            "cluster_name": "rally-benchmark",
+            "node_name": "rally-node-0",
+            "data_paths": ["/opt/elasticsearch-5.0.0/data"],
+            "log_path": "~/.rally/benchmarks/races/unittest/logs/server",
+            "heap_dump_path": "~/.rally/benchmarks/races/unittest/heapdump",
+            "node_ip": "10.17.22.23",
+            "network_host": "10.17.22.23",
+            "http_port": "9200-9300",
+            "transport_port": "9300-9400",
+            "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
+            "minimum_master_nodes": 2,
+            "node_count_per_host": 1,
+            "install_root_path": "/opt/elasticsearch-5.0.0",
+            "plugin_name": "x-pack-security",
+            "xpack_security_enabled": True
+
+        }, config_vars)
+
+    @mock.patch("glob.glob", lambda p: ["/opt/elasticsearch-6.3.0"])
+    @mock.patch("esrally.utils.io.decompress")
+    @mock.patch("esrally.utils.io.ensure_dir")
+    @mock.patch("esrally.mechanic.provisioner.PluginInstaller.install")
+    @mock.patch("shutil.rmtree")
+    def test_prepare_distribution_ge_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress):
+        """
+        Test that plugin.mandatory is set to the meta plugin name (e.g. `x-pack`) and not
+        the specific plugin name (e.g. `x-pack-security`) for Elasticsearch >=6.3.0
+
+        See: https://github.com/elastic/elasticsearch/pull/28710
+        """
+        apply_config_calls = []
+
+        def null_apply_config(source_root_path, target_root_path, config_vars):
+            apply_config_calls.append((source_root_path, target_root_path, config_vars))
+
+        installer = provisioner.ElasticsearchInstaller(car=
+        team.Car(
+            name="unit-test-car",
+            config_paths=["~/.rally/benchmarks/teams/default/my-car"],
+            variables={"heap": "4g"}),
+            node_name="rally-node-0",
+            node_root_dir="~/.rally/benchmarks/races/unittest",
+            all_node_ips=["10.17.22.22", "10.17.22.23"],
+            ip="10.17.22.23",
+            http_port=9200)
+
+        p = provisioner.BareProvisioner(cluster_settings={"indices.query.bool.max_clause_count": 50000},
+                                        es_installer=installer,
+                                        plugin_installers=[
+                                            provisioner.PluginInstaller(BareProvisionerTests.MockRallyTeamXPackPlugin(),
+                                                                        hook_handler_class=BareProvisionerTests.NoopHookHandler)
+                                        ],
+                                        preserve=True,
+                                        distribution_version="6.3.0",
+                                        apply_config=null_apply_config)
+
+        node_config = p.prepare({"elasticsearch": "/opt/elasticsearch-6.3.0.tar.gz"})
+        self.assertEqual(installer.car, node_config.car)
+        self.assertEqual("/opt/elasticsearch-6.3.0", node_config.binary_path)
+        self.assertEqual(["/opt/elasticsearch-6.3.0/data"], node_config.data_paths)
+
+        self.assertEqual(1, len(apply_config_calls))
+        source_root_path, target_root_path, config_vars = apply_config_calls[0]
+
+        self.assertEqual("~/.rally/benchmarks/teams/default/my-car", source_root_path)
+        self.assertEqual("/opt/elasticsearch-6.3.0", target_root_path)
+
+        self.maxDiff = None
+
+        self.assertEqual({
+            "cluster_settings": {
+                "indices.query.bool.max_clause_count": 50000,
+                "plugin.mandatory": ["x-pack"]
+            },
+            "heap": "4g",
+            "cluster_name": "rally-benchmark",
+            "node_name": "rally-node-0",
+            "data_paths": ["/opt/elasticsearch-6.3.0/data"],
+            "log_path": "~/.rally/benchmarks/races/unittest/logs/server",
+            "heap_dump_path": "~/.rally/benchmarks/races/unittest/heapdump",
+            "node_ip": "10.17.22.23",
+            "network_host": "10.17.22.23",
+            "http_port": "9200-9300",
+            "transport_port": "9300-9400",
+            "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
+            "minimum_master_nodes": 2,
+            "node_count_per_host": 1,
+            "install_root_path": "/opt/elasticsearch-6.3.0",
+            "plugin_name": "x-pack-security",
+            "xpack_security_enabled": True
+
+        }, config_vars)
+
+
 class ElasticsearchInstallerTests(TestCase):
     @mock.patch("shutil.rmtree")
     @mock.patch("os.path.exists")
diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py
index be8ae8e2..cb184505 100644
--- a/tests/mechanic/supplier_test.py
+++ b/tests/mechanic/supplier_test.py
@@ -134,20 +134,19 @@ class BuilderTests(TestCase):
 
     @mock.patch("esrally.utils.process.run_subprocess")
     @mock.patch("esrally.utils.jvm.major_version")
-    def test_build_on_jdk_9(self, jvm_major_version, mock_run_subprocess):
-        jvm_major_version.return_value = 9
+    def test_build_on_jdk_10(self, jvm_major_version, mock_run_subprocess):
+        jvm_major_version.return_value = 10
         mock_run_subprocess.return_value = False
 
-        b = supplier.Builder(src_dir="/src", gradle="/usr/local/gradle", java_home="/opt/jdk9", log_dir="logs")
+        b = supplier.Builder(src_dir="/src", gradle="/usr/local/gradle", java_home="/opt/jdk10", log_dir="logs")
         b.build([supplier.CLEAN_TASK, supplier.ASSEMBLE_TASK])
 
         calls = [
             # Actual call
-            mock.call("export GRADLE_OPTS=\"%s\"; export JAVA_HOME=/opt/jdk9; cd /src; /usr/local/gradle clean >> logs/build.log 2>&1" %
-                      supplier.Builder.JAVA_9_GRADLE_OPTS),
+            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; /usr/local/gradle clean >> logs/build.log 2>&1"),
             # Return value check
-            mock.call("export GRADLE_OPTS=\"%s\"; export JAVA_HOME=/opt/jdk9; cd /src; /usr/local/gradle "
-                      ":distribution:archives:tar:assemble >> logs/build.log 2>&1" % supplier.Builder.JAVA_9_GRADLE_OPTS),
+            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; /usr/local/gradle "
+                      ":distribution:archives:tar:assemble >> logs/build.log 2>&1"),
         ]
 
         mock_run_subprocess.assert_has_calls(calls)
@@ -335,7 +334,7 @@ class CreateSupplierTests(TestCase):
         cfg.add(config.Scope.application, "distributions", "release.url",
                 "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
         cfg.add(config.Scope.application, "distributions", "release.cache", True)
-        cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
+        cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/")
         cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
 
         composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", plugins=[])
@@ -352,7 +351,7 @@ class CreateSupplierTests(TestCase):
         cfg.add(config.Scope.application, "distributions", "release.url",
                 "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
         cfg.add(config.Scope.application, "distributions", "release.cache", True)
-        cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
+        cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/")
         cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
         cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin")
 
@@ -383,7 +382,7 @@ class CreateSupplierTests(TestCase):
         cfg.add(config.Scope.application, "distributions", "release.url",
                 "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
         cfg.add(config.Scope.application, "distributions", "release.cache", True)
-        cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
+        cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/")
         cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
 
         core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
@@ -406,7 +405,7 @@ class CreateSupplierTests(TestCase):
         cfg.add(config.Scope.application, "distributions", "release.url",
                 "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
         cfg.add(config.Scope.application, "distributions", "release.cache", True)
-        cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
+        cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/")
         cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
         cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
         cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle")
@@ -437,7 +436,7 @@ class CreateSupplierTests(TestCase):
         cfg.add(config.Scope.application, "distributions", "release.url",
                 "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
         cfg.add(config.Scope.application, "distributions", "release.cache", True)
-        cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
+        cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/")
         cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
         cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
         cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle")