Repository: elastic/rally

Issue:
Warn when enabling node-stats
When `docs` / `completion` is requested via node stats (which is done by the node stats telemetry device) it will trigger additional refreshes in Elasticsearch which will skew results and highly likely make benchmark results invalid. We should document that and maybe even emit a warning when the node stats telemetry device is turned on.

Discussion:

Patch:
diff --git a/docs/Makefile b/docs/Makefile
index 7333b7a1..dd64bf1e 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -6,6 +6,7 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
+CUR_RELDIR    := $(notdir $(shell pwd))
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
@@ -56,7 +57,7 @@ clean:
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+	@echo "Build finished. The HTML pages are in $(CUR_RELDIR)/$(BUILDDIR)/html."
 
 .PHONY: serve
 serve: html
@@ -68,13 +69,13 @@ serve: html
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+	@echo "Build finished. The HTML pages are in $(CUR_RELDIR)/$(BUILDDIR)/dirhtml."
 
 .PHONY: singlehtml
 singlehtml:
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+	@echo "Build finished. The HTML page is in $(CUR_RELDIR)/$(BUILDDIR)/singlehtml."
 
 .PHONY: pickle
 pickle:
@@ -93,23 +94,23 @@ htmlhelp:
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+	      ".hhp project file in $(CUR_RELDIR)/$(BUILDDIR)/htmlhelp."
 
 .PHONY: qthelp
 qthelp:
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	      ".qhcp project file in $(CUR_RELDIR)/$(BUILDDIR)/qthelp, like this:"
 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Rally.qhcp"
 	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Rally.qhc"
+	@echo "# assistant -collectionFile $(CUR_RELDIR)/$(BUILDDIR)/qthelp/Rally.qhc"
 
 .PHONY: applehelp
 applehelp:
 	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
 	@echo
-	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "Build finished. The help book is in $(CUR_RELDIR)/$(BUILDDIR)/applehelp."
 	@echo "N.B. You won't be able to view it unless you put it in" \
 	      "~/Library/Documentation/Help or install it in your application" \
 	      "bundle."
@@ -121,20 +122,20 @@ devhelp:
 	@echo "Build finished."
 	@echo "To view the help file:"
 	@echo "# mkdir -p $$HOME/.local/share/devhelp/Rally"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Rally"
+	@echo "# ln -s $(CUR_RELDIR)/$(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Rally"
 	@echo "# devhelp"
 
 .PHONY: epub
 epub:
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+	@echo "Build finished. The epub file is in $(CUR_RELDIR)/$(BUILDDIR)/epub."
 
 .PHONY: latex
 latex:
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Build finished; the LaTeX files are in $(CUR_RELDIR)/$(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
@@ -143,32 +144,32 @@ latexpdf:
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+	@echo "pdflatex finished; the PDF files are in $(CUR_RELDIR)/$(BUILDDIR)/latex."
 
 .PHONY: latexpdfja
 latexpdfja:
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through platex and dvipdfmx..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+	@echo "pdflatex finished; the PDF files are in $(CUR_RELDIR)/$(BUILDDIR)/latex."
 
 .PHONY: text
 text:
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+	@echo "Build finished. The text files are in $(CUR_RELDIR)/$(BUILDDIR)/text."
 
 .PHONY: man
 man:
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+	@echo "Build finished. The manual pages are in $(CUR_RELDIR)/$(BUILDDIR)/man."
 
 .PHONY: texinfo
 texinfo:
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo
-	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Build finished. The Texinfo files are in $(CUR_RELDIR)/$(BUILDDIR)/texinfo."
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."
 
@@ -177,47 +178,47 @@ info:
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
-	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+	@echo "makeinfo finished; the Info files are in $(CUR_RELDIR)/$(BUILDDIR)/texinfo."
 
 .PHONY: gettext
 gettext:
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
-	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+	@echo "Build finished. The message catalogs are in $(CUR_RELDIR)/$(BUILDDIR)/locale."
 
 .PHONY: changes
 changes:
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
+	@echo "The overview file is in $(CUR_RELDIR)/$(BUILDDIR)/changes."
 
 .PHONY: linkcheck
 linkcheck:
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
+	      "or in $(CUR_RELDIR)/$(BUILDDIR)/linkcheck/output.txt."
 
 .PHONY: doctest
 doctest:
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
+	      "results in $(CUR_RELDIR)/$(BUILDDIR)/doctest/output.txt."
 
 .PHONY: coverage
 coverage:
 	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
 	@echo "Testing of coverage in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/coverage/python.txt."
+	      "results in $(CUR_RELDIR)/$(BUILDDIR)/coverage/python.txt."
 
 .PHONY: xml
 xml:
 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
 	@echo
-	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+	@echo "Build finished. The XML files are in $(CUR_RELDIR)/$(BUILDDIR)/xml."
 
 .PHONY: pseudoxml
 pseudoxml:
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
-	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+	@echo "Build finished. The pseudo-XML files are in $(CUR_RELDIR)/$(BUILDDIR)/pseudoxml."
diff --git a/docs/developing.rst b/docs/developing.rst
index 3097f0a3..14945ad3 100644
--- a/docs/developing.rst
+++ b/docs/developing.rst
@@ -9,7 +9,8 @@ Install the following software packages:
 * Python 3.4 or better available as ``python3`` on the path (verify with: ``python3 --version``)
 * ``pip3`` available on the path (verify with ``pip3 --version``)
 * ``virtualenv`` available on the path (verify with ``virtualenv --version``)
-* JDK 8 or 9
+* Pyenv installed and ``eval "$(pyenv init -)"`` is added to the shell configuration file. For more details please refer to the PyEnv `installation instructions <https://github.com/pyenv/pyenv#installation>`_.
+* JDK version required to build Elasticsearch. Please refer to the `build setup requirements <https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-to-the-elasticsearch-codebase>`_.
 * git 1.9 or better
 
 Check the :doc:`installation guide </install>` for detailed installation instructions for these packages.
@@ -23,7 +24,8 @@ Installation Instructions for Development
 
     git clone https://github.com/elastic/rally.git
     cd rally
-    virtualenv -p python3 .venv
+    make prereq
+    make install
     source .venv/bin/activate
     ./rally
 
diff --git a/docs/install.rst b/docs/install.rst
index 0389cb11..4da488c2 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -142,6 +142,13 @@ You can also use Virtualenv to install Rally into an isolated Python environment
 
 Whenever you want to use Rally, run the activation script (step 2 above) first.  When you are done, simply execute ``deactivate`` in the shell to exit the virtual environment.
 
+PyEnv Install
+------------------
+
+Rally can be tested with different Python versions and it uses pyenv to manage them. 
+
+Please refer to PyEnv `installation instructions <https://github.com/pyenv/pyenv#installation>`_.
+
 .. _install_offline-install:
 
 Offline Install
diff --git a/docs/telemetry.rst b/docs/telemetry.rst
index 0989b627..2711f6d7 100644
--- a/docs/telemetry.rst
+++ b/docs/telemetry.rst
@@ -75,7 +75,8 @@ node-stats
 
 .. warning::
 
-    This telemetry device will record a lot of metrics and likely skew your measurement results.
+    Using this telemetry device will skew your results because the node-stats API triggers additional refreshes.
+    Additionally a lot of metrics get recorded impacting the measurement results even further.
 
 The node-stats telemetry devices regularly calls the `cluster node-stats API <https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html>`_ and records metrics from the following sections:
 
diff --git a/esrally/mechanic/telemetry.py b/esrally/mechanic/telemetry.py
index 14796043..35ad1d96 100644
--- a/esrally/mechanic/telemetry.py
+++ b/esrally/mechanic/telemetry.py
@@ -443,14 +443,18 @@ class CcrStatsRecorder:
 
 
 class NodeStats(TelemetryDevice):
+    """
+    Gathers different node stats.
+    """
+
     internal = False
     command = "node-stats"
     human_name = "Node Stats"
     help = "Regularly samples node stats"
-
-    """
-    Gathers different node stats.
+    warning = """You have enabled the node-stats telemetry device, but requests to the _nodes/stats Elasticsearch endpoint
+          trigger additional refreshes and WILL SKEW results.
     """
+
     def __init__(self, telemetry_params, clients, metrics_store):
         super().__init__()
         self.telemetry_params = telemetry_params
@@ -464,6 +468,8 @@ class NodeStats(TelemetryDevice):
         super().attach_to_cluster(cluster)
 
     def on_benchmark_start(self):
+        console.warn(NodeStats.warning, logger=self.logger)
+
         recorder = []
         for cluster_name in self.specified_cluster_names:
             recorder = NodeStatsRecorder(self.telemetry_params, cluster_name, self.clients[cluster_name], self.metrics_store)


Test patch:
diff --git a/tests/mechanic/telemetry_test.py b/tests/mechanic/telemetry_test.py
index 31ed92d5..7134cca8 100644
--- a/tests/mechanic/telemetry_test.py
+++ b/tests/mechanic/telemetry_test.py
@@ -24,6 +24,7 @@ from unittest import TestCase
 from esrally import config, metrics, exceptions
 from esrally.mechanic import telemetry, team, cluster
 from esrally.metrics import MetaInfoScope
+from esrally.utils import console
 
 
 def create_config():
@@ -624,6 +625,30 @@ class CcrStatsRecorderTests(TestCase):
         )
 
 
+class NodeStatsTests(TestCase):
+    warning = """You have enabled the node-stats telemetry device, but requests to the _nodes/stats Elasticsearch endpoint
+          trigger additional refreshes and WILL SKEW results.
+    """
+
+    @mock.patch("esrally.mechanic.telemetry.NodeStatsRecorder", mock.Mock())
+    @mock.patch("esrally.mechanic.telemetry.SamplerThread", mock.Mock())
+    def test_prints_warning_using_node_stats(self):
+        clients = {"default": Client()}
+        cfg = create_config()
+        metrics_store = metrics.EsMetricsStore(cfg)
+        telemetry_params = {
+            "node-stats-sample-interval": random.randint(1, 100)
+        }
+        t = telemetry.NodeStats(telemetry_params, clients, metrics_store)
+
+        with mock.patch.object(console, "warn") as mocked_console_warn:
+            t.on_benchmark_start()
+        mocked_console_warn.assert_called_once_with(
+            NodeStatsTests.warning,
+            logger=t.logger
+        )
+
+
 class NodeStatsRecorderTests(TestCase):
     node_stats_response = {
         "cluster_name": "elasticsearch",