Repository: HypothesisWorks/hypothesis

Issue:
Use our own global Random instance instead of random
As per discussion in #2129, currently if users seed the global PRNG before each test they will get bad results because tests will be weirdly correlated with eachother. We have plenty of mechanisms for controlling determinism if they want to use them, they shouldn't be using this one.

It is still important that we respect `random.seed` at a *global* level - i.e. that calling it once makes the whole test suite deterministic - but we don't want to respect it at a *per test* level.

My recommended solution is as follows:

1. We store a single `Random` instance in `hypothesis.core` which we use as our source of random seeds instead of the global random number generator.
2. Initially it is set to `None`. When it is first accessed it will be initialized with a seed from `random`. This ensures that we respect any seeding that occurs before the first hypothesis test is run.
3. We fix all of the places in our test suite where we call `random.seed`.

The third will be the fiddly bit. The first two should be relatively straightforward.

We may also optionally want to think about exposing this as a property that people can set. For example, if we exposed a `hypothesis.global_random` module attribute that people could set and we used that for everything, this would allow more fine grained control. However I think we don't need to do that to start with - we can just do the first bits and see if anyone complains.


Discussion:
Is this related to #2871 ? If so are you aware of a workaround?
I'm pretty sure it's unrelated, but discussion on that issue.

There's no workaround for this one beyond "don't reset the global random seed before every test".

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -20,7 +20,6 @@
 import datetime
 import inspect
 import io
-import random as rnd_module
 import sys
 import time
 import traceback
@@ -120,6 +119,7 @@
 
 running_under_pytest = False
 global_force_seed = None
+_hypothesis_global_random = None
 
 
 @attr.s()
@@ -419,7 +419,10 @@ def get_random_for_wrapped_test(test, wrapped_test):
     elif global_force_seed is not None:
         return Random(global_force_seed)
     else:
-        seed = rnd_module.getrandbits(128)
+        global _hypothesis_global_random
+        if _hypothesis_global_random is None:
+            _hypothesis_global_random = Random()
+        seed = _hypothesis_global_random.getrandbits(128)
         wrapped_test._hypothesis_internal_use_generated_seed = seed
         return Random(seed)
 
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py
@@ -238,6 +238,7 @@ def normalize(
     required_successes=100,
     allowed_to_update=False,
     max_dfas=10,
+    random=None,
 ):
     """Attempt to ensure that this test function successfully normalizes - i.e.
     whenever it declares a test case to be interesting, we are able
@@ -267,6 +268,7 @@ def normalize(
         test_function,
         settings=settings(database=None, suppress_health_check=HealthCheck.all()),
         ignore_limits=True,
+        random=random,
     )
 
     seen = set()


Test patch:
diff --git a/hypothesis-python/tests/cover/test_fuzz_one_input.py b/hypothesis-python/tests/cover/test_fuzz_one_input.py
--- a/hypothesis-python/tests/cover/test_fuzz_one_input.py
+++ b/hypothesis-python/tests/cover/test_fuzz_one_input.py
@@ -14,7 +14,6 @@
 # END HEADER
 
 import io
-import random
 from operator import attrgetter
 
 import pytest
@@ -24,6 +23,11 @@
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal.conjecture.shrinker import sort_key
 
+try:
+    from random import randbytes
+except ImportError:  # New in Python 3.9
+    from secrets import token_bytes as randbytes
+
 
 @pytest.mark.parametrize(
     "buffer_type",
@@ -53,7 +57,7 @@ def test(s):
     # find a failing example.
     with pytest.raises(AssertionError):
         for _ in range(1000):
-            buf = bytes(random.getrandbits(8) for _ in range(1000))
+            buf = randbytes(1000)
             seeds.append(buf)
             test.hypothesis.fuzz_one_input(buffer_type(buf))
 
@@ -95,7 +99,7 @@ def test(s):
         raise AssertionError("Unreachable because there are no valid examples")
 
     for _ in range(100):
-        buf = bytes(random.getrandbits(8) for _ in range(3))
+        buf = randbytes(3)
         ret = test.hypothesis.fuzz_one_input(buf)
         assert ret is None
 
diff --git a/hypothesis-python/tests/cover/test_random_module.py b/hypothesis-python/tests/cover/test_random_module.py
--- a/hypothesis-python/tests/cover/test_random_module.py
+++ b/hypothesis-python/tests/cover/test_random_module.py
@@ -17,7 +17,7 @@
 
 import pytest
 
-from hypothesis import find, given, register_random, reporting, strategies as st
+from hypothesis import core, find, given, register_random, reporting, strategies as st
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal import entropy
 from hypothesis.internal.entropy import deterministic_PRNG
@@ -120,11 +120,14 @@ def test(r):
 
         test()
         state_a = random.getstate()
+        state_a2 = core._hypothesis_global_random.getstate()
 
         test()
         state_b = random.getstate()
+        state_b2 = core._hypothesis_global_random.getstate()
 
-        assert state_a != state_b
+        assert state_a == state_b
+        assert state_a2 != state_b2
 
 
 def test_find_does_not_pollute_state():
@@ -132,8 +135,11 @@ def test_find_does_not_pollute_state():
 
         find(st.random_module(), lambda r: True)
         state_a = random.getstate()
+        state_a2 = core._hypothesis_global_random.getstate()
 
         find(st.random_module(), lambda r: True)
         state_b = random.getstate()
+        state_b2 = core._hypothesis_global_random.getstate()
 
-        assert state_a != state_b
+        assert state_a == state_b
+        assert state_a2 != state_b2
diff --git a/hypothesis-python/tests/nocover/test_randomization.py b/hypothesis-python/tests/nocover/test_randomization.py
--- a/hypothesis-python/tests/nocover/test_randomization.py
+++ b/hypothesis-python/tests/nocover/test_randomization.py
@@ -13,20 +13,18 @@
 #
 # END HEADER
 
-import random
-
 from pytest import raises
 
-from hypothesis import Verbosity, find, given, settings, strategies as st
+from hypothesis import Verbosity, core, find, given, settings, strategies as st
 
 from tests.common.utils import no_shrink
 
 
-def test_seeds_off_random():
+def test_seeds_off_internal_random():
     s = settings(phases=no_shrink, database=None)
-    r = random.getstate()
+    r = core._hypothesis_global_random.getstate()
     x = find(st.integers(), lambda x: True, settings=s)
-    random.setstate(r)
+    core._hypothesis_global_random.setstate(r)
     y = find(st.integers(), lambda x: True, settings=s)
     assert x == y
 
diff --git a/hypothesis-python/tests/quality/test_normalization.py b/hypothesis-python/tests/quality/test_normalization.py
--- a/hypothesis-python/tests/quality/test_normalization.py
+++ b/hypothesis-python/tests/quality/test_normalization.py
@@ -14,6 +14,7 @@
 # END HEADER
 
 from itertools import islice
+from random import Random
 
 import pytest
 
@@ -56,10 +57,6 @@ def test_function(data):
 
 @pytest.mark.parametrize("strategy", [st.emails(), st.complex_numbers()], ids=repr)
 def test_harder_strategies_normalize_to_minimal(strategy, normalize_kwargs):
-    import random
-
-    random.seed(0)
-
     def test_function(data):
         try:
             v = data.draw(strategy)
@@ -68,4 +65,4 @@ def test_function(data):
         data.output = repr(v)
         data.mark_interesting()
 
-    dfas.normalize(repr(strategy), test_function, **normalize_kwargs)
+    dfas.normalize(repr(strategy), test_function, random=Random(0), **normalize_kwargs)
diff --git a/hypothesis-python/tests/quality/test_zig_zagging.py b/hypothesis-python/tests/quality/test_zig_zagging.py
--- a/hypothesis-python/tests/quality/test_zig_zagging.py
+++ b/hypothesis-python/tests/quality/test_zig_zagging.py
@@ -13,8 +13,8 @@
 #
 # END HEADER
 
-import random
 from math import log
+from random import Random
 
 from hypothesis import (
     HealthCheck,
@@ -77,8 +77,6 @@ def problem(draw):
 def test_avoids_zig_zag_trap(p):
     b, marker, lower_bound = p
 
-    random.seed(0)
-
     n_bits = 8 * (len(b) + 1)
 
     def test_function(data):
@@ -95,6 +93,7 @@ def test_function(data):
         test_function,
         database_key=None,
         settings=settings(base_settings, phases=(Phase.generate, Phase.shrink)),
+        random=Random(0),
     )
 
     runner.cached_test_function(b + bytes([0]) + b + bytes([1]) + marker)