Repository: HypothesisWorks/hypothesis

Issue:
Hypothesis intermittently fails when running tests in parallel
When running tests under py.test using pytest-xdist and Hypothesis, the test run occasionally fails. The error output can be seen below.

This occurs because Hypothesis writes out source code it generates. When running tests in parallel this can encounter conflicts. That's not really ideal: we should fix it!

Error:

```
cory@corymbp:hyper-h2/ % tox -e py27
GLOB sdist-make: /Users/cory/Documents/Python/hyper-h2/setup.py
py27 inst-nodeps: /Users/cory/Documents/Python/hyper-h2/.tox/dist/h2-0.1.0.zip
py27 runtests: PYTHONHASHSEED='3864724170'
py27 runtests: commands[0] | py.test -n 4 --cov h2 /Users/cory/Documents/Python/hyper-h2/test/
============================= test session starts ==============================
platform darwin -- Python 2.7.10 -- py-1.4.30 -- pytest-2.7.2
rootdir: /Users/cory/Documents/Python/hyper-h2, inifile: 
plugins: hypothesis-pytest, cov, xdist, xdist, xdist
gw0 [33] / gw1 [31] / gw2 [31] / gw3 [33]
scheduling tests via LoadScheduling
collecting 0 items / 4 errors
==================================== ERRORS ====================================
_________________ ERROR collecting test/test_state_machines.py _________________
test/test_state_machines.py:18: in <module>
    class TestConnectionStateMachine(object):
test/test_state_machines.py:23: in TestConnectionStateMachine
    input_=sampled_from(h2.connection.ConnectionInputs))
.tox/py27/lib/python2.7/site-packages/hypothesis/core.py:456: in run_test_with_generator
    test.__name__, argspec
.tox/py27/lib/python2.7/site-packages/hypothesis/internal/reflection.py:431: in accept
    u'invocation': u', '.join(invocation_parts)
.tox/py27/lib/python2.7/site-packages/hypothesis/internal/reflection.py:359: in source_exec_as_module
    assert r.read() == source
E   assert '' == 'from hypothesis.utils.conventions import not_set\n\ndef accept(f):\n    def test_state_transitions(self, state=not_set, input_=not_set):\n        return f(self, state, input_)\n    return test_state_transitions\n'
E    +  where '' = <built-in method read of file object at 0x106b51030>()
E    +    where <built-in method read of file object at 0x106b51030> = <open file u'/Users/cory/Documents/Python/hyper-h2/.hypothesis/eval_source/hyp...y_module_29641d823094e91fb6f7a85e6fe7aca197c90b0e.py', mode 'r' at 0x106b51030>.read
------------------------------- Captured stderr --------------------------------
/Users/cory/Documents/Python/hyper-h2/.tox/py27/lib/python2.7/site-packages/hypothesis/core.py:662: HypothesisDeprecationWarning: You have an obsolete version of the hypothesis-pytest plugin installed. Please update to a more recent version.
  load_entry_points()
_________________ ERROR collecting test/test_state_machines.py _________________
test/test_state_machines.py:14: in <module>
    from hypothesis import given
.tox/py27/lib/python2.7/site-packages/hypothesis/__init__.py:29: in <module>
    from hypothesis.core import given, assume, find, example
.tox/py27/lib/python2.7/site-packages/hypothesis/core.py:32: in <module>
    import hypothesis.strategies as sd
.tox/py27/lib/python2.7/site-packages/hypothesis/strategies.py:179: in <module>
    def floats(min_value=None, max_value=None):
.tox/py27/lib/python2.7/site-packages/hypothesis/strategies.py:58: in defines_strategy
    @proxies(strategy_definition)
.tox/py27/lib/python2.7/site-packages/hypothesis/internal/reflection.py:443: in accept
    copy_argspec(target.__name__, inspect.getargspec(target))(proxy))
.tox/py27/lib/python2.7/site-packages/hypothesis/internal/reflection.py:431: in accept
    u'invocation': u', '.join(invocation_parts)
.tox/py27/lib/python2.7/site-packages/hypothesis/internal/reflection.py:359: in source_exec_as_module
    assert r.read() == source
E   assert '' == 'from hypothesis.utils.conventions import not_set\n\ndef accept(f):\n    def floats(min_value=not_set, max_value=not_set):\n        return f(min_value, max_value)\n    return floats\n'
E    +  where '' = <built-in method read of file object at 0x106e10930>()
E    +    where <built-in method read of file object at 0x106e10930> = <open file u'/Users/cory/Documents/Python/hyper-h2/.hypothesis/eval_source/hyp...y_module_1eba03841185292c6018ae372c9f399bb5b99054.py', mode 'r' at 0x106e10930>.read
_____________________________ ERROR collecting gw1 _____________________________
Different tests were collected between gw3 and gw1. The difference is:
--- gw3

+++ gw1

@@ -29,5 +29,3 @@

 test/test_basic_logic.py::TestBasicServer::()::test_no_preamble_errors
 test/test_basic_logic.py::TestBasicServer::()::test_headers_event
 test/test_basic_logic.py::TestBasicServer::()::test_data_event
-test/test_state_machines.py::TestConnectionStateMachine::()::test_state_transitions
-test/test_state_machines.py::TestStreamStateMachine::()::test_state_transitions
_____________________________ ERROR collecting gw2 _____________________________
Different tests were collected between gw3 and gw2. The difference is:
--- gw3

+++ gw2

@@ -29,5 +29,3 @@

 test/test_basic_logic.py::TestBasicServer::()::test_no_preamble_errors
 test/test_basic_logic.py::TestBasicServer::()::test_headers_event
 test/test_basic_logic.py::TestBasicServer::()::test_data_event
-test/test_state_machines.py::TestConnectionStateMachine::()::test_state_transitions
-test/test_state_machines.py::TestStreamStateMachine::()::test_state_transitions
--------------- coverage: platform darwin, python 2.7.10-final-0 ---------------
Name              Stmts   Miss  Cover
-------------------------------------
h2/__init__           2      0   100%
h2/connection       162    109    33%
h2/events            17      8    53%
h2/exceptions         3      0   100%
h2/frame_buffer      20     13    35%
h2/stream           142     96    32%
-------------------------------------
TOTAL               346    226    35%
=========================== 4 error in 3.09 seconds ============================
ERROR: InvocationError: '/Users/cory/Documents/Python/hyper-h2/.tox/py27/bin/py.test -n 4 --cov h2 /Users/cory/Documents/Python/hyper-h2/test/'
___________________________________ summary ____________________________________
ERROR:   py27: commands failed
```



Discussion:
Thanks for reporting this.

To clarify: There's not strictly speaking a conflict, because they will both be writing out the same thing. It's essentially content addressable storage - the file name is a hash of the source that's going to be written.

The problem is that Hypothesis can see that the file already exists and try to read it while the other process is still writing to it and instead reads an empty file. This is obviously silly and shouldn't happen I agree.

There are a number of ways this could be dealt with. I think my preferred option is to write to a temp file and then move it to the right location once it's written (note that this needs to be atomic, so we probably don't want to use /tmp/ because it's often going to be mounted on a different file system)

Hmm. pytest-xdist uses separate processes, so we could use the PID as a distinguishing feature for the temp file name? Or alternatively use it as a salt and then hash with some stuff?

I'd just use a long random string. Using the pid seems needlessly complicated and would break if you were e.g. using threads instead

Fair enough, that works too.


Resolving patch:
diff --git a/src/hypothesis/internal/reflection.py b/src/hypothesis/internal/reflection.py
--- a/src/hypothesis/internal/reflection.py
+++ b/src/hypothesis/internal/reflection.py
@@ -24,6 +24,7 @@
 import re
 import ast
 import sys
+import uuid
 import types
 import hashlib
 import inspect
@@ -347,18 +348,37 @@ def source_exec_as_module(source):
 
     d = eval_directory()
     add_directory_to_path(d)
-    name = u'hypothesis_temporary_module_%s' % (
+    final_name = u'hypothesis_temporary_module_%s' % (
         hashlib.sha1(source.encode(u'utf-8')).hexdigest(),
     )
-    filepath = os.path.join(d, name + u'.py')
-    f = open(filepath, u'w')
+    temporary_name = u'hypothesis_temporary_module_%s_%s' % (
+        hashlib.sha1(source.encode(u'utf-8')).hexdigest(),
+        uuid.uuid4(),
+    )
+    temporary_filepath = os.path.join(d, temporary_name + u'.py')
+    final_filepath = os.path.join(d, final_name + u'.py')
+    f = open(temporary_filepath, u'w')
     f.write(source)
     f.close()
-    assert os.path.exists(filepath)
-    with open(filepath) as r:
+    assert os.path.exists(temporary_filepath)
+
+    try:
+        os.rename(temporary_filepath, final_filepath)
+    except OSError:  # pragma: no cover
+        # The odds of final_filepath being a directory are basically zero, and
+        # it's basically impossible for them to be on different filesystems, so
+        # if this is raised it's because the destination already exists on
+        # Windows. That's fine, it won't be different, so just keep going,
+        # deleting our tempfile.
+        assert not os.path.isdir(final_filepath)
+        os.remove(temporary_filepath)
+
+    assert os.path.exists(final_filepath)
+    assert not os.path.exists(temporary_filepath)
+    with open(final_filepath) as r:
         assert r.read() == source
     importlib_invalidate_caches()
-    result = __import__(name)
+    result = __import__(final_name)
     eval_cache[source] = result
     return result
 


Test patch: