Repository: elastic/rally

Issue:
Allow .ini definable in environment variable or pass as command line runtime value instead of persisting it in .ini file
<!--
If you are filing a feature request, please remove the above bug
report block and provide responses for all of the below items.
-->

**Describe the feature**:

Currently, metrics store password has to be defined in the `.ini` file. This is fine, but some users would prefer not to persist the password in the .ini file, so either allowing the password to be configured as an environment variable or allowing it to be passed as a runtime flag would be helpful in this use case scenario. 

The workaround for now is to move the .ini file in the `/dev/shm` directory and symlink it to avoid persisting the value in the disk.  

Thank you for the consideration! 

Discussion:

Patch:
diff --git a/docs/configuration.rst b/docs/configuration.rst
index a3d55992..03703e0f 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -72,7 +72,7 @@ The following settings are applicable only if ``datastore.type`` is set to "elas
 * ``datastore.ssl.verification_mode`` (default: "full"): By default the metric store's SSL certificate is checked ("full"). To disable certificate verification set this value to "none".
 * ``datastore.ssl.certificate_authorities`` (default: empty): Determines the path on the local file system to the certificate authority's signing certificate.
 * ``datastore.user``: Sets the name of the Elasticsearch user for the metrics store.
-* ``datastore.password``: Sets the password of the Elasticsearch user for the metrics store.
+* ``datastore.password``: Sets the password of the Elasticsearch user for the metrics store. Alternatively, this password can be configured using the ``RALLY_REPORTING_DATASTORE_PASSWORD`` environment variable, which avoids storing credentials in a plain text file. The environment variable will take precedence over the config file if both define a password.
 * ``datastore.probe.cluster_version`` (default: true): Enables automatic detection of the metric store's version.
 * ``datastore.number_of_shards`` (default: `Elasticsearch default value <https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#_static_index_settings>`_): The number of primary shards that the ``rally-*`` indices should have. Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices.
 * ``datastore.number_of_replicas`` (default: `Elasticsearch default value <https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#_static_index_settings>`_): The number of replicas each primary shard has. Defaults to . Any updates to this setting after initial index creation will only be applied to new ``rally-*`` indices.
diff --git a/esrally/metrics.py b/esrally/metrics.py
index f542070f..b6482de5 100644
--- a/esrally/metrics.py
+++ b/esrally/metrics.py
@@ -204,7 +204,16 @@ class EsClientFactory:
         port = self._config.opts("reporting", "datastore.port")
         secure = convert.to_bool(self._config.opts("reporting", "datastore.secure"))
         user = self._config.opts("reporting", "datastore.user")
-        password = self._config.opts("reporting", "datastore.password")
+        try:
+            password = os.environ["RALLY_REPORTING_DATASTORE_PASSWORD"]
+        except KeyError:
+            try:
+                password = self._config.opts("reporting", "datastore.password")
+            except exceptions.ConfigError:
+                raise exceptions.ConfigError(
+                    "No password configured through [reporting] configuration or RALLY_REPORTING_DATASTORE_PASSWORD environment variable."
+                ) from None
+
         verify = self._config.opts("reporting", "datastore.ssl.verification_mode", default_value="full", mandatory=False) != "none"
         ca_path = self._config.opts("reporting", "datastore.ssl.certificate_authorities", default_value=None, mandatory=False)
         self.probe_version = self._config.opts("reporting", "datastore.probe.cluster_version", default_value=True, mandatory=False)


Test patch:
diff --git a/tests/metrics_test.py b/tests/metrics_test.py
index 33d3d2f9..b15ccff1 100644
--- a/tests/metrics_test.py
+++ b/tests/metrics_test.py
@@ -119,8 +119,9 @@ class TestEsClient:
         def __init__(self, hosts):
             self.transport = TestEsClient.TransportMock(hosts)
 
+    @pytest.mark.parametrize("password_configuration", [None, "config", "environment"])
     @mock.patch("esrally.client.EsClientFactory")
-    def test_config_opts_parsing(self, client_esclientfactory):
+    def test_config_opts_parsing(self, client_esclientfactory, password_configuration, monkeypatch):
         cfg = config.Config()
 
         _datastore_host = ".".join([str(random.randint(1, 254)) for _ in range(4)])
@@ -134,11 +135,27 @@ class TestEsClient:
         cfg.add(config.Scope.applicationOverride, "reporting", "datastore.port", _datastore_port)
         cfg.add(config.Scope.applicationOverride, "reporting", "datastore.secure", _datastore_secure)
         cfg.add(config.Scope.applicationOverride, "reporting", "datastore.user", _datastore_user)
-        cfg.add(config.Scope.applicationOverride, "reporting", "datastore.password", _datastore_password)
+
+        if password_configuration == "config":
+            cfg.add(config.Scope.applicationOverride, "reporting", "datastore.password", _datastore_password)
+        elif password_configuration == "environment":
+            monkeypatch.setenv("RALLY_REPORTING_DATASTORE_PASSWORD", _datastore_password)
+
         if not _datastore_verify_certs:
             cfg.add(config.Scope.applicationOverride, "reporting", "datastore.ssl.verification_mode", "none")
 
-        metrics.EsClientFactory(cfg)
+        try:
+            metrics.EsClientFactory(cfg)
+        except exceptions.ConfigError as e:
+            if password_configuration is not None:
+                raise
+
+            assert (
+                e.message
+                == "No password configured through [reporting] configuration or RALLY_REPORTING_DATASTORE_PASSWORD environment variable."
+            )
+            return
+
         expected_client_options = {
             "use_ssl": True,
             "timeout": 120,