Repository: optuna/optuna

Issue:
Non-determinism in TPE sampler when `group=True`
TPE sampler is non-deterministic in different runs (with the same seed, no parallelization) when group decomposed search space is enabled.

## Expected behavior

Consider the script below:
```python

import optuna

sampler = optuna.samplers.TPESampler(n_startup_trials=1, seed=2, multivariate=True, group=True)
study = optuna.create_study(sampler=sampler)

for _ in range(10):
    trial = study.ask()
    picked = [i for i in range(10) if trial.suggest_int(str(i), 0, 1) == 1]
    print(picked)
    study.tell(trial, len(picked))

```

Different runs of this script are expected to yield the same output (because the seed is fixed, the objective function is deterministic, and we are running in sequential mode), but they do not. For example, two consecutive runs of this script yielded the following outputs (your output might differ) :

First run:
```
[2, 7]
[2, 7]
[2, 7]
[5, 6, 7, 8, 9]
[1, 5]
[0, 2, 3, 4, 7]
[2, 3, 8]
[1, 2, 3, 6, 7]
[0, 1, 2, 4]
[3, 5, 7]
```

Second run:
```
[2, 7]
[2, 7, 9]
[2, 7]
[1, 6]
[2, 3, 6, 7, 8]
[0, 2, 3, 5, 7]
[1, 7, 8]
[0, 1, 2, 6, 7]
[3]
[3, 5]
```

Note that if we set `group=False` when initializing the TPE sampler, we get deterministic outputs.

## Environment

- Optuna version: 2.91
- Python version: 3.7.4
- OS: Linux

## Error messages, stack traces, or logs

Not applicable.

## Steps to reproduce

Run the script above multiple times. Observe that the output changes everytime.




Discussion:
Thank you for pointing it out! I found this bug caused by the randomness of Python's hash seed. This line is not deterministic. https://github.com/optuna/optuna/blob/2a9f8639c1703f455fdbad211e7fa8ab53628a21/optuna/samplers/_search_space/group_decomposed.py#L21

Resolving patch:
diff --git a/optuna/samplers/_tpe/sampler.py b/optuna/samplers/_tpe/sampler.py
--- a/optuna/samplers/_tpe/sampler.py
+++ b/optuna/samplers/_tpe/sampler.py
@@ -303,7 +303,8 @@ def infer_relative_search_space(
             assert self._group_decomposed_search_space is not None
             self._search_space_group = self._group_decomposed_search_space.calculate(study)
             for sub_space in self._search_space_group.search_spaces:
-                for name, distribution in sub_space.items():
+                # Sort keys because Python's string hashing is nondeterministic.
+                for name, distribution in sorted(sub_space.items()):
                     if distribution.single():
                         continue
                     search_space[name] = distribution
@@ -340,7 +341,8 @@ def sample_relative(
             params = {}
             for sub_space in self._search_space_group.search_spaces:
                 search_space = {}
-                for name, distribution in sub_space.items():
+                # Sort keys because Python's string hashing is nondeterministic.
+                for name, distribution in sorted(sub_space.items()):
                     if not distribution.single():
                         search_space[name] = distribution
                 params.update(self._sample_relative(study, trial, search_space))


Test patch:
diff --git a/tests/samplers_tests/tpe_tests/test_sampler.py b/tests/samplers_tests/tpe_tests/test_sampler.py
--- a/tests/samplers_tests/tpe_tests/test_sampler.py
+++ b/tests/samplers_tests/tpe_tests/test_sampler.py
@@ -1,6 +1,8 @@
+import multiprocessing
 import random
 from typing import Callable
 from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Union
 from unittest.mock import Mock
@@ -16,6 +18,7 @@
 from optuna import TrialPruned
 from optuna.samplers import _tpe
 from optuna.samplers import TPESampler
+from optuna.study.study import create_study
 from optuna.trial import Trial
 
 
@@ -1054,6 +1057,36 @@ def test_group_experimental_warning() -> None:
         _ = TPESampler(multivariate=True, group=True)
 
 
+# This function is used only in test_group_deterministic_iteration, but declared at top-level
+# because local function cannot be pickled, which occurs within multiprocessing.
+def run_tpe(k: int, sequence_dict: Dict[int, List[int]], hash_dict: Dict[int, int]) -> None:
+    hash_dict[k] = hash("nondeterministic hash")
+    sampler = TPESampler(n_startup_trials=1, seed=2, multivariate=True, group=True)
+    study = create_study(sampler=sampler)
+    study.optimize(
+        lambda t: np.sum([t.suggest_int(f"x{i}", 0, 10) for i in range(10)]), n_trials=2
+    )
+    sequence_dict[k] = list(study.trials[-1].params.values())
+
+
+def test_group_deterministic_iteration() -> None:
+    # Multiprocessing supports three way to start a process.
+    # We use `spawn` option to create a child process as a fresh python process.
+    # For more detail, see https://github.com/optuna/optuna/pull/3187#issuecomment-997673037.
+    multiprocessing.set_start_method("spawn", force=True)
+    manager = multiprocessing.Manager()
+    sequence_dict: Dict[int, List[int]] = manager.dict()
+    hash_dict: Dict[int, int] = manager.dict()
+    for i in range(3):
+        p = multiprocessing.Process(target=run_tpe, args=(i, sequence_dict, hash_dict))
+        p.start()
+        p.join()
+    # Hashes are expected to be different because string hashing is nondeterministic per process.
+    assert not (hash_dict[0] == hash_dict[1] == hash_dict[2])
+    # But the sequences are expected to be the same.
+    assert sequence_dict[0] == sequence_dict[1] == sequence_dict[2]
+
+
 @pytest.mark.parametrize("direction", ["minimize", "maximize"])
 @pytest.mark.parametrize("multivariate", [True, False])
 def test_constant_liar_observation_pairs(direction: str, multivariate: bool) -> None: