Repository: HypothesisWorks/hypothesis

Issue:
Move Travis jobs checking documentation and requirements to whole repo tests
This will mean we only have five jobs in the precheck stage, and can run them all in parallel.  I'd consider also moving the lint and check-format jobs, as worker startup time dominates the runtime for most of our prechecks.


Discussion:

Resolving patch:
diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py
--- a/tooling/src/hypothesistooling/__main__.py
+++ b/tooling/src/hypothesistooling/__main__.py
@@ -244,8 +244,7 @@ def check_format():
                 print(
                     '%s has incorrect start %r' % (f, start), file=sys.stderr)
                 bad = True
-    if bad:
-        sys.exit(1)
+    assert not bad
     check_not_changed()
 
 
@@ -265,11 +264,10 @@ def compile_requirements(upgrade=False):
     else:
         extra = []
 
-    os.chdir(tools.ROOT)
-
     for f in glob(os.path.join('requirements', '*.in')):
         base, _ = os.path.splitext(f)
-        pip_tool('pip-compile', *extra, f, '--output-file', base + '.txt')
+        pip_tool('pip-compile', *extra, f, '--output-file', base + '.txt',
+                 cwd=tools.ROOT)
 
 
 @task()
@@ -329,26 +327,24 @@ def check_requirements():
 
     if tools.has_uncommitted_changes('requirements'):
         push_pyup_requirements_commit()
-        sys.exit(1)
-    else:
-        sys.exit(0)
+        raise RuntimeError('Pushed new requirements; check next build.')
 
 
 @task(if_changed=hp.HYPOTHESIS_PYTHON)
 def documentation():
-    os.chdir(hp.HYPOTHESIS_PYTHON)
     try:
         if hp.has_release():
             hp.update_changelog_and_version()
         pip_tool(
             # See http://www.sphinx-doc.org/en/stable/man/sphinx-build.html
             'sphinx-build', '-n', '-W', '--keep-going', '-T', '-E',
-            '-b', 'html', 'docs', 'docs/_build/html'
+            '-b', 'html', 'docs', 'docs/_build/html',
+            cwd=hp.HYPOTHESIS_PYTHON
         )
     finally:
         subprocess.check_call([
             'git', 'checkout', 'docs/changes.rst', 'src/hypothesis/version.py'
-        ])
+        ], cwd=hp.HYPOTHESIS_PYTHON)
 
 
 def run_tox(task, version):
@@ -362,14 +358,13 @@ def run_tox(task, version):
     except FileExistsError:
         pass
 
-    os.chdir(hp.HYPOTHESIS_PYTHON)
     env = dict(os.environ)
     python = install.python_executable(version)
 
     env['PATH'] = os.path.dirname(python) + ':' + env['PATH']
     print(env['PATH'])
 
-    pip_tool('tox', '-e', task, env=env)
+    pip_tool('tox', '-e', task, env=env, cwd=hp.HYPOTHESIS_PYTHON)
 
 
 PY27 = '2.7.14'


Test patch:
diff --git a/whole-repo-tests/test_documentation.py b/whole-repo-tests/test_documentation.py
new file mode 100644
--- /dev/null
+++ b/whole-repo-tests/test_documentation.py
@@ -0,0 +1,24 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis-python
+#
+# Most of this work is copyright (C) 2013-2018 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import division, print_function, absolute_import
+
+from hypothesistooling.__main__ import documentation
+
+
+def test_documentation():
+    documentation()
diff --git a/whole-repo-tests/test_requirements.py b/whole-repo-tests/test_requirements.py
new file mode 100644
--- /dev/null
+++ b/whole-repo-tests/test_requirements.py
@@ -0,0 +1,24 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis-python
+#
+# Most of this work is copyright (C) 2013-2018 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import division, print_function, absolute_import
+
+from hypothesistooling.__main__ import check_requirements
+
+
+def test_requirements():
+    check_requirements()