Repository: optuna/optuna

Issue:
Spurious test failures related to timestamp ordering on s390x
### Expected behavior

Tests should pass on s390x just as they do on other architectures. On other architectures they pass fine, on s390x they *sometimes* (around 20% of the time by my observation) fail with an error relating to the ordering of time points.

### Environment

- Optuna version: 3.1.0
- Python version: 3.11.2
- OS: Linux-5.10.0-21-s390x-s390x-with-glibc2.36


### Error messages, stack traces, or logs

```
============================================================== test session starts ===============================================================
platform linux -- Python 3.11.2, pytest-7.2.1, pluggy-1.0.0+repack -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /home/gspr/tmp/optuna-3.1.0, configfile: pyproject.toml
collected 212 items                                                                                                                              

test_storages.py::test_get_storage PASSED                                                                                                  [  0%]
test_storages.py::test_create_new_study[inmemory] PASSED                                                                                   [  0%]
test_storages.py::test_create_new_study[sqlite] PASSED                                                                                     [  1%]
test_storages.py::test_create_new_study[cached_sqlite] PASSED                                                                              [  1%]
test_storages.py::test_create_new_study[journal] PASSED                                                                                    [  2%]
test_storages.py::test_create_new_study[journal_redis] PASSED                                                                              [  2%]
test_storages.py::test_create_new_study[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                        [  3%]
test_storages.py::test_create_new_study_unique_id[inmemory] PASSED                                                                         [  3%]
test_storages.py::test_create_new_study_unique_id[sqlite] PASSED                                                                           [  4%]
test_storages.py::test_create_new_study_unique_id[cached_sqlite] PASSED                                                                    [  4%]
test_storages.py::test_create_new_study_unique_id[journal] PASSED                                                                          [  5%]
test_storages.py::test_create_new_study_unique_id[journal_redis] PASSED                                                                    [  5%]
test_storages.py::test_create_new_study_unique_id[dask] SKIPPED (distributed doesn't yet support Python 3.11)                              [  6%]
test_storages.py::test_create_new_study_with_name[inmemory] PASSED                                                                         [  6%]
test_storages.py::test_create_new_study_with_name[sqlite] PASSED                                                                           [  7%]
test_storages.py::test_create_new_study_with_name[cached_sqlite] PASSED                                                                    [  7%]
test_storages.py::test_create_new_study_with_name[journal] PASSED                                                                          [  8%]
test_storages.py::test_create_new_study_with_name[journal_redis] PASSED                                                                    [  8%]
test_storages.py::test_create_new_study_with_name[dask] SKIPPED (distributed doesn't yet support Python 3.11)                              [  8%]
test_storages.py::test_delete_study[inmemory] PASSED                                                                                       [  9%]
test_storages.py::test_delete_study[sqlite] PASSED                                                                                         [  9%]
test_storages.py::test_delete_study[cached_sqlite] PASSED                                                                                  [ 10%]
test_storages.py::test_delete_study[journal] PASSED                                                                                        [ 10%]
test_storages.py::test_delete_study[journal_redis] PASSED                                                                                  [ 11%]
test_storages.py::test_delete_study[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                            [ 11%]
test_storages.py::test_delete_study_after_create_multiple_studies[inmemory] PASSED                                                         [ 12%]
test_storages.py::test_delete_study_after_create_multiple_studies[sqlite] PASSED                                                           [ 12%]
test_storages.py::test_delete_study_after_create_multiple_studies[cached_sqlite] PASSED                                                    [ 13%]
test_storages.py::test_delete_study_after_create_multiple_studies[journal] PASSED                                                          [ 13%]
test_storages.py::test_delete_study_after_create_multiple_studies[journal_redis] PASSED                                                    [ 14%]
test_storages.py::test_delete_study_after_create_multiple_studies[dask] SKIPPED (distributed doesn't yet support Python 3.11)              [ 14%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[inmemory] PASSED                                                  [ 15%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[sqlite] PASSED                                                    [ 15%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[cached_sqlite] PASSED                                             [ 16%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[journal] PASSED                                                   [ 16%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[journal_redis] PASSED                                             [ 16%]
test_storages.py::test_get_study_id_from_name_and_get_study_name_from_id[dask] SKIPPED (distributed doesn't yet support Python 3.11)       [ 17%]
test_storages.py::test_set_and_get_study_directions[inmemory] PASSED                                                                       [ 17%]
test_storages.py::test_set_and_get_study_directions[sqlite] PASSED                                                                         [ 18%]
test_storages.py::test_set_and_get_study_directions[cached_sqlite] PASSED                                                                  [ 18%]
test_storages.py::test_set_and_get_study_directions[journal] PASSED                                                                        [ 19%]
test_storages.py::test_set_and_get_study_directions[journal_redis] PASSED                                                                  [ 19%]
test_storages.py::test_set_and_get_study_directions[dask] SKIPPED (distributed doesn't yet support Python 3.11)                            [ 20%]
test_storages.py::test_set_and_get_study_user_attrs[inmemory] PASSED                                                                       [ 20%]
test_storages.py::test_set_and_get_study_user_attrs[sqlite] PASSED                                                                         [ 21%]
test_storages.py::test_set_and_get_study_user_attrs[cached_sqlite] PASSED                                                                  [ 21%]
test_storages.py::test_set_and_get_study_user_attrs[journal] PASSED                                                                        [ 22%]
test_storages.py::test_set_and_get_study_user_attrs[journal_redis] PASSED                                                                  [ 22%]
test_storages.py::test_set_and_get_study_user_attrs[dask] SKIPPED (distributed doesn't yet support Python 3.11)                            [ 23%]
test_storages.py::test_set_and_get_study_system_attrs[inmemory] PASSED                                                                     [ 23%]
test_storages.py::test_set_and_get_study_system_attrs[sqlite] PASSED                                                                       [ 24%]
test_storages.py::test_set_and_get_study_system_attrs[cached_sqlite] PASSED                                                                [ 24%]
test_storages.py::test_set_and_get_study_system_attrs[journal] PASSED                                                                      [ 25%]
test_storages.py::test_set_and_get_study_system_attrs[journal_redis] PASSED                                                                [ 25%]
test_storages.py::test_set_and_get_study_system_attrs[dask] SKIPPED (distributed doesn't yet support Python 3.11)                          [ 25%]
test_storages.py::test_study_user_and_system_attrs_confusion[inmemory] PASSED                                                              [ 26%]
test_storages.py::test_study_user_and_system_attrs_confusion[sqlite] PASSED                                                                [ 26%]
test_storages.py::test_study_user_and_system_attrs_confusion[cached_sqlite] PASSED                                                         [ 27%]
test_storages.py::test_study_user_and_system_attrs_confusion[journal] PASSED                                                               [ 27%]
test_storages.py::test_study_user_and_system_attrs_confusion[journal_redis] PASSED                                                         [ 28%]
test_storages.py::test_study_user_and_system_attrs_confusion[dask] SKIPPED (distributed doesn't yet support Python 3.11)                   [ 28%]
test_storages.py::test_create_new_trial[inmemory] PASSED                                                                                   [ 29%]
test_storages.py::test_create_new_trial[sqlite] PASSED                                                                                     [ 29%]
test_storages.py::test_create_new_trial[cached_sqlite] PASSED                                                                              [ 30%]
test_storages.py::test_create_new_trial[journal] FAILED                                                                                    [ 30%]
test_storages.py::test_create_new_trial[journal_redis] PASSED                                                                              [ 31%]
test_storages.py::test_create_new_trial[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                        [ 31%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-inmemory] PASSED                                    [ 32%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-sqlite] PASSED                                      [ 32%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-cached_sqlite] PASSED                               [ 33%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-journal] PASSED                                     [ 33%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-journal_redis] PASSED                               [ 33%]
test_storages.py::test_create_new_trial_with_template_trial[start_time0-complete_time0-dask] SKIPPED (distributed doesn't yet support ...) [ 34%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-inmemory] PASSED                                    [ 34%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-sqlite] PASSED                                      [ 35%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-cached_sqlite] PASSED                               [ 35%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-journal] PASSED                                     [ 36%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-journal_redis] PASSED                               [ 36%]
test_storages.py::test_create_new_trial_with_template_trial[start_time1-complete_time1-dask] SKIPPED (distributed doesn't yet support ...) [ 37%]
test_storages.py::test_get_trial_number_from_id[inmemory] PASSED                                                                           [ 37%]
test_storages.py::test_get_trial_number_from_id[sqlite] PASSED                                                                             [ 38%]
test_storages.py::test_get_trial_number_from_id[cached_sqlite] PASSED                                                                      [ 38%]
test_storages.py::test_get_trial_number_from_id[journal] PASSED                                                                            [ 39%]
test_storages.py::test_get_trial_number_from_id[journal_redis] PASSED                                                                      [ 39%]
test_storages.py::test_get_trial_number_from_id[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                [ 40%]
test_storages.py::test_set_trial_state_values_for_state[inmemory] PASSED                                                                   [ 40%]
test_storages.py::test_set_trial_state_values_for_state[sqlite] PASSED                                                                     [ 41%]
test_storages.py::test_set_trial_state_values_for_state[cached_sqlite] PASSED                                                              [ 41%]
test_storages.py::test_set_trial_state_values_for_state[journal] PASSED                                                                    [ 41%]
test_storages.py::test_set_trial_state_values_for_state[journal_redis] PASSED                                                              [ 42%]
test_storages.py::test_set_trial_state_values_for_state[dask] SKIPPED (distributed doesn't yet support Python 3.11)                        [ 42%]
test_storages.py::test_get_trial_param_and_get_trial_params[inmemory] PASSED                                                               [ 43%]
test_storages.py::test_get_trial_param_and_get_trial_params[sqlite] PASSED                                                                 [ 43%]
test_storages.py::test_get_trial_param_and_get_trial_params[cached_sqlite] PASSED                                                          [ 44%]
test_storages.py::test_get_trial_param_and_get_trial_params[journal] PASSED                                                                [ 44%]
test_storages.py::test_get_trial_param_and_get_trial_params[journal_redis] PASSED                                                          [ 45%]
test_storages.py::test_get_trial_param_and_get_trial_params[dask] SKIPPED (distributed doesn't yet support Python 3.11)                    [ 45%]
test_storages.py::test_set_trial_param[inmemory] PASSED                                                                                    [ 46%]
test_storages.py::test_set_trial_param[sqlite] PASSED                                                                                      [ 46%]
test_storages.py::test_set_trial_param[cached_sqlite] PASSED                                                                               [ 47%]
test_storages.py::test_set_trial_param[journal] PASSED                                                                                     [ 47%]
test_storages.py::test_set_trial_param[journal_redis] PASSED                                                                               [ 48%]
test_storages.py::test_set_trial_param[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                         [ 48%]
test_storages.py::test_set_trial_state_values_for_values[inmemory] PASSED                                                                  [ 49%]
test_storages.py::test_set_trial_state_values_for_values[sqlite] PASSED                                                                    [ 49%]
test_storages.py::test_set_trial_state_values_for_values[cached_sqlite] PASSED                                                             [ 50%]
test_storages.py::test_set_trial_state_values_for_values[journal] PASSED                                                                   [ 50%]
test_storages.py::test_set_trial_state_values_for_values[journal_redis] PASSED                                                             [ 50%]
test_storages.py::test_set_trial_state_values_for_values[dask] SKIPPED (distributed doesn't yet support Python 3.11)                       [ 51%]
test_storages.py::test_set_trial_intermediate_value[inmemory] PASSED                                                                       [ 51%]
test_storages.py::test_set_trial_intermediate_value[sqlite] PASSED                                                                         [ 52%]
test_storages.py::test_set_trial_intermediate_value[cached_sqlite] PASSED                                                                  [ 52%]
test_storages.py::test_set_trial_intermediate_value[journal] PASSED                                                                        [ 53%]
test_storages.py::test_set_trial_intermediate_value[journal_redis] PASSED                                                                  [ 53%]
test_storages.py::test_set_trial_intermediate_value[dask] SKIPPED (distributed doesn't yet support Python 3.11)                            [ 54%]
test_storages.py::test_get_trial_user_attrs[inmemory] PASSED                                                                               [ 54%]
test_storages.py::test_get_trial_user_attrs[sqlite] PASSED                                                                                 [ 55%]
test_storages.py::test_get_trial_user_attrs[cached_sqlite] PASSED                                                                          [ 55%]
test_storages.py::test_get_trial_user_attrs[journal] PASSED                                                                                [ 56%]
test_storages.py::test_get_trial_user_attrs[journal_redis] PASSED                                                                          [ 56%]
test_storages.py::test_get_trial_user_attrs[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                    [ 57%]
test_storages.py::test_set_trial_user_attr[inmemory] PASSED                                                                                [ 57%]
test_storages.py::test_set_trial_user_attr[sqlite] PASSED                                                                                  [ 58%]
test_storages.py::test_set_trial_user_attr[cached_sqlite] PASSED                                                                           [ 58%]
test_storages.py::test_set_trial_user_attr[journal] PASSED                                                                                 [ 58%]
test_storages.py::test_set_trial_user_attr[journal_redis] PASSED                                                                           [ 59%]
test_storages.py::test_set_trial_user_attr[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                     [ 59%]
test_storages.py::test_get_trial_system_attrs[inmemory] PASSED                                                                             [ 60%]
test_storages.py::test_get_trial_system_attrs[sqlite] PASSED                                                                               [ 60%]
test_storages.py::test_get_trial_system_attrs[cached_sqlite] PASSED                                                                        [ 61%]
test_storages.py::test_get_trial_system_attrs[journal] PASSED                                                                              [ 61%]
test_storages.py::test_get_trial_system_attrs[journal_redis] PASSED                                                                        [ 62%]
test_storages.py::test_get_trial_system_attrs[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                  [ 62%]
test_storages.py::test_set_trial_system_attr[inmemory] PASSED                                                                              [ 63%]
test_storages.py::test_set_trial_system_attr[sqlite] PASSED                                                                                [ 63%]
test_storages.py::test_set_trial_system_attr[cached_sqlite] PASSED                                                                         [ 64%]
test_storages.py::test_set_trial_system_attr[journal] PASSED                                                                               [ 64%]
test_storages.py::test_set_trial_system_attr[journal_redis] PASSED                                                                         [ 65%]
test_storages.py::test_set_trial_system_attr[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                   [ 65%]
test_storages.py::test_get_all_studies[inmemory] PASSED                                                                                    [ 66%]
test_storages.py::test_get_all_studies[sqlite] PASSED                                                                                      [ 66%]
test_storages.py::test_get_all_studies[cached_sqlite] PASSED                                                                               [ 66%]
test_storages.py::test_get_all_studies[journal] PASSED                                                                                     [ 67%]
test_storages.py::test_get_all_studies[journal_redis] PASSED                                                                               [ 67%]
test_storages.py::test_get_all_studies[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                         [ 68%]
test_storages.py::test_get_trial[inmemory] PASSED                                                                                          [ 68%]
test_storages.py::test_get_trial[sqlite] PASSED                                                                                            [ 69%]
test_storages.py::test_get_trial[cached_sqlite] PASSED                                                                                     [ 69%]
test_storages.py::test_get_trial[journal] PASSED                                                                                           [ 70%]
test_storages.py::test_get_trial[journal_redis] PASSED                                                                                     [ 70%]
test_storages.py::test_get_trial[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                               [ 71%]
test_storages.py::test_get_all_trials[inmemory] PASSED                                                                                     [ 71%]
test_storages.py::test_get_all_trials[sqlite] PASSED                                                                                       [ 72%]
test_storages.py::test_get_all_trials[cached_sqlite] PASSED                                                                                [ 72%]
test_storages.py::test_get_all_trials[journal] PASSED                                                                                      [ 73%]
test_storages.py::test_get_all_trials[journal_redis] PASSED                                                                                [ 73%]
test_storages.py::test_get_all_trials[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                          [ 74%]
test_storages.py::test_get_all_trials_deepcopy_option[inmemory] PASSED                                                                     [ 74%]
test_storages.py::test_get_all_trials_deepcopy_option[sqlite] PASSED                                                                       [ 75%]
test_storages.py::test_get_all_trials_deepcopy_option[cached_sqlite] PASSED                                                                [ 75%]
test_storages.py::test_get_all_trials_deepcopy_option[journal] PASSED                                                                      [ 75%]
test_storages.py::test_get_all_trials_deepcopy_option[journal_redis] PASSED                                                                [ 76%]
test_storages.py::test_get_all_trials_deepcopy_option[dask] SKIPPED (distributed doesn't yet support Python 3.11)                          [ 76%]
test_storages.py::test_get_all_trials_state_option[inmemory] PASSED                                                                        [ 77%]
test_storages.py::test_get_all_trials_state_option[sqlite] PASSED                                                                          [ 77%]
test_storages.py::test_get_all_trials_state_option[cached_sqlite] PASSED                                                                   [ 78%]
test_storages.py::test_get_all_trials_state_option[journal] PASSED                                                                         [ 78%]
test_storages.py::test_get_all_trials_state_option[journal_redis] PASSED                                                                   [ 79%]
test_storages.py::test_get_all_trials_state_option[dask] SKIPPED (distributed doesn't yet support Python 3.11)                             [ 79%]
test_storages.py::test_get_n_trials[inmemory] PASSED                                                                                       [ 80%]
test_storages.py::test_get_n_trials[sqlite] PASSED                                                                                         [ 80%]
test_storages.py::test_get_n_trials[cached_sqlite] PASSED                                                                                  [ 81%]
test_storages.py::test_get_n_trials[journal] PASSED                                                                                        [ 81%]
test_storages.py::test_get_n_trials[journal_redis] PASSED                                                                                  [ 82%]
test_storages.py::test_get_n_trials[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                            [ 82%]
test_storages.py::test_get_n_trials_state_option[inmemory] PASSED                                                                          [ 83%]
test_storages.py::test_get_n_trials_state_option[sqlite] PASSED                                                                            [ 83%]
test_storages.py::test_get_n_trials_state_option[cached_sqlite] PASSED                                                                     [ 83%]
test_storages.py::test_get_n_trials_state_option[journal] PASSED                                                                           [ 84%]
test_storages.py::test_get_n_trials_state_option[journal_redis] PASSED                                                                     [ 84%]
test_storages.py::test_get_n_trials_state_option[dask] SKIPPED (distributed doesn't yet support Python 3.11)                               [ 85%]
test_storages.py::test_get_best_trial[inmemory] PASSED                                                                                     [ 85%]
test_storages.py::test_get_best_trial[sqlite] PASSED                                                                                       [ 86%]
test_storages.py::test_get_best_trial[cached_sqlite] PASSED                                                                                [ 86%]
test_storages.py::test_get_best_trial[journal] PASSED                                                                                      [ 87%]
test_storages.py::test_get_best_trial[journal_redis] PASSED                                                                                [ 87%]
test_storages.py::test_get_best_trial[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                          [ 88%]
test_storages.py::test_get_trials_excluded_trial_ids PASSED                                                                                [ 88%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[inmemory] PASSED                                                    [ 89%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[sqlite] PASSED                                                      [ 89%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[cached_sqlite] PASSED                                               [ 90%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[journal] PASSED                                                     [ 90%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[journal_redis] PASSED                                               [ 91%]
test_storages.py::test_get_best_trial_for_multi_objective_optimization[dask] SKIPPED (distributed doesn't yet support Python 3.11)         [ 91%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[inmemory] PASSED                                                            [ 91%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[sqlite] PASSED                                                              [ 92%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[cached_sqlite] PASSED                                                       [ 92%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[journal] PASSED                                                             [ 93%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[journal_redis] PASSED                                                       [ 93%]
test_storages.py::test_get_trial_id_from_study_id_trial_number[dask] SKIPPED (distributed doesn't yet support Python 3.11)                 [ 94%]
test_storages.py::test_pickle_storage[inmemory] PASSED                                                                                     [ 94%]
test_storages.py::test_pickle_storage[sqlite] PASSED                                                                                       [ 95%]
test_storages.py::test_pickle_storage[cached_sqlite] PASSED                                                                                [ 95%]
test_storages.py::test_pickle_storage[journal] PASSED                                                                                      [ 96%]
test_storages.py::test_pickle_storage[journal_redis] SKIPPED (The `fakeredis` does not support multi instances.)                           [ 96%]
test_storages.py::test_pickle_storage[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                          [ 97%]
test_storages.py::test_check_trial_is_updatable[inmemory] PASSED                                                                           [ 97%]
test_storages.py::test_check_trial_is_updatable[sqlite] PASSED                                                                             [ 98%]
test_storages.py::test_check_trial_is_updatable[cached_sqlite] PASSED                                                                      [ 98%]
test_storages.py::test_check_trial_is_updatable[journal] PASSED                                                                            [ 99%]
test_storages.py::test_check_trial_is_updatable[journal_redis] PASSED                                                                      [ 99%]
test_storages.py::test_check_trial_is_updatable[dask] SKIPPED (distributed doesn't yet support Python 3.11)                                [100%]

==================================================================== FAILURES ====================================================================
_________________________________________________________ test_create_new_trial[journal] _________________________________________________________

storage_mode = 'journal'

    @pytest.mark.parametrize("storage_mode", STORAGE_MODES)
    def test_create_new_trial(storage_mode: str) -> None:
        def _check_trials(
            trials: List[FrozenTrial],
            idx: int,
            trial_id: int,
            time_before_creation: datetime,
            time_after_creation: datetime,
        ) -> None:
            assert len(trials) == idx + 1
            assert len({t._trial_id for t in trials}) == idx + 1
            assert trial_id in {t._trial_id for t in trials}
            assert {t.number for t in trials} == set(range(idx + 1))
            assert all(t.state == TrialState.RUNNING for t in trials)
            assert all(t.params == {} for t in trials)
            assert all(t.intermediate_values == {} for t in trials)
            assert all(t.user_attrs == {} for t in trials)
            assert all(t.system_attrs == {} for t in trials)
            assert all(
                t.datetime_start < time_before_creation
                for t in trials
                if t._trial_id != trial_id and t.datetime_start is not None
            )
            assert all(
                time_before_creation < t.datetime_start < time_after_creation
                for t in trials
                if t._trial_id == trial_id and t.datetime_start is not None
            )
            assert all(t.datetime_complete is None for t in trials)
            assert all(t.value is None for t in trials)
    
        with StorageSupplier(storage_mode) as storage:
    
            study_id = storage.create_new_study(directions=[StudyDirection.MINIMIZE])
            n_trial_in_study = 3
            for i in range(n_trial_in_study):
                time_before_creation = datetime.now()
                trial_id = storage.create_new_trial(study_id)
                time_after_creation = datetime.now()
    
                trials = storage.get_all_trials(study_id)
>               _check_trials(trials, i, trial_id, time_before_creation, time_after_creation)

test_storages.py:315: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

trials = [FrozenTrial(number=0, state=0, values=None, datetime_start=datetime.datetime(2023, 2, 21, 15, 41, 3, 765620), datetim...ete=None, params={}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={}, trial_id=1, value=None)]
idx = 1, trial_id = 1, time_before_creation = datetime.datetime(2023, 2, 21, 15, 41, 3, 765722)
time_after_creation = datetime.datetime(2023, 2, 21, 15, 41, 3, 765767)

    def _check_trials(
        trials: List[FrozenTrial],
        idx: int,
        trial_id: int,
        time_before_creation: datetime,
        time_after_creation: datetime,
    ) -> None:
        assert len(trials) == idx + 1
        assert len({t._trial_id for t in trials}) == idx + 1
        assert trial_id in {t._trial_id for t in trials}
        assert {t.number for t in trials} == set(range(idx + 1))
        assert all(t.state == TrialState.RUNNING for t in trials)
        assert all(t.params == {} for t in trials)
        assert all(t.intermediate_values == {} for t in trials)
        assert all(t.user_attrs == {} for t in trials)
        assert all(t.system_attrs == {} for t in trials)
        assert all(
            t.datetime_start < time_before_creation
            for t in trials
            if t._trial_id != trial_id and t.datetime_start is not None
        )
>       assert all(
            time_before_creation < t.datetime_start < time_after_creation
            for t in trials
            if t._trial_id == trial_id and t.datetime_start is not None
        )
E       assert False
E        +  where False = all(<generator object test_create_new_trial.<locals>._check_trials.<locals>.<genexpr> at 0x3ffb6dff880>)

test_storages.py:297: AssertionError
-------------------------------------------------------------- Captured stderr call --------------------------------------------------------------
[I 2023-02-21 15:41:03,765] A new study created in Journal with name: no-name-709f1cda-a544-45f7-a7e3-a6b170171323
============================================================ short test summary info =============================================================
FAILED test_storages.py::test_create_new_trial[journal] - assert False
=================================================== 1 failed, 175 passed, 36 skipped in 9.61s ====================================================
```


### Steps to reproduce

On an s390x machine:
1. Install Optuna
2. Run `python3 -B -m pytest -v tests/storages_tests/test_storage.py` a few times.
3. Observe that it usually passes, but sometimes fails as in the quoted error message, indicating an incorrect ordering of time points.


### Additional context (optional)

I might be missing something regarding the design of these tests, but they seem to rely on `datetime.now()` being monotonically increasing. If I'm not misremembering, there is no guarantee that the return value of that function is less than the value of an earlier call. This would make the tests fail for no useful reason.

This comment is not specific to the tests failing only on s390x, but seems appropriate to make (see also issue #4455). Please let me know if I've misunderstood something about the assumptions in the tests though!


Discussion:

Resolving patch:
diff --git a/optuna/integration/botorch.py b/optuna/integration/botorch.py
--- a/optuna/integration/botorch.py
+++ b/optuna/integration/botorch.py
@@ -652,7 +652,7 @@ def sample_independent(
     def reseed_rng(self) -> None:
         self._independent_sampler.reseed_rng()
         if self._seed is not None:
-            self._seed = numpy.random.RandomState().randint(2**60)
+            self._seed = numpy.random.RandomState().randint(numpy.iinfo(numpy.int32).max)
 
     def after_trial(
         self,
diff --git a/optuna/integration/skopt.py b/optuna/integration/skopt.py
--- a/optuna/integration/skopt.py
+++ b/optuna/integration/skopt.py
@@ -139,7 +139,7 @@ def __init__(
         self._rng: Optional[np.random.RandomState] = None
 
     def reseed_rng(self) -> None:
-        self._skopt_kwargs["random_state"] = random.randint(1, 2**32)
+        self._skopt_kwargs["random_state"] = random.randint(1, np.iinfo(np.int32).max)
         self._independent_sampler.reseed_rng()
 
     def infer_relative_search_space(


Test patch:
diff --git a/optuna/testing/storages.py b/optuna/testing/storages.py
--- a/optuna/testing/storages.py
+++ b/optuna/testing/storages.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-import tempfile
 from types import TracebackType
 from typing import Any
 from typing import IO
@@ -14,14 +13,13 @@
 
 import optuna
 from optuna.storages import JournalFileStorage
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 
 
 try:
     import distributed
-
-    _has_distributed = True
 except ImportError:
-    _has_distributed = False
+    pass
 
 STORAGE_MODES: list[Any] = [
     "inmemory",
@@ -53,9 +51,9 @@
 class StorageSupplier:
     def __init__(self, storage_specifier: str, **kwargs: Any) -> None:
         self.storage_specifier = storage_specifier
-        self.tempfile: Optional[IO[Any]] = None
         self.extra_args = kwargs
         self.dask_client: Optional["distributed.Client"] = None
+        self.tempfile: IO[Any] | None = None
 
     def __enter__(
         self,
@@ -71,7 +69,7 @@ def __enter__(
                 raise ValueError("InMemoryStorage does not accept any arguments!")
             return optuna.storages.InMemoryStorage()
         elif "sqlite" in self.storage_specifier:
-            self.tempfile = tempfile.NamedTemporaryFile()
+            self.tempfile = NamedTemporaryFilePool().tempfile()
             url = "sqlite:///{}".format(self.tempfile.name)
             rdb_storage = optuna.storages.RDBStorage(
                 url,
@@ -90,7 +88,8 @@ def __enter__(
             )
             return optuna.storages.JournalStorage(journal_redis_storage)
         elif "journal" in self.storage_specifier:
-            file_storage = JournalFileStorage(tempfile.NamedTemporaryFile().name)
+            self.tempfile = NamedTemporaryFilePool().tempfile()
+            file_storage = JournalFileStorage(self.tempfile.name)
             return optuna.storages.JournalStorage(file_storage)
         elif self.storage_specifier == "dask":
             self.dask_client = distributed.Client()  # type: ignore[no-untyped-call]
diff --git a/optuna/testing/tempfile_pool.py b/optuna/testing/tempfile_pool.py
new file mode 100644
--- /dev/null
+++ b/optuna/testing/tempfile_pool.py
@@ -0,0 +1,48 @@
+# On Windows, temporary file shold delete "after" storage was deleted
+# NamedTemporaryFilePool ensures tempfile delete after tests.
+
+from __future__ import annotations
+
+import atexit
+import gc
+import os
+import tempfile
+from types import TracebackType
+from typing import Any
+from typing import IO
+from typing import List
+from typing import Type
+
+
+class NamedTemporaryFilePool:
+    tempfile_pool: List[IO[Any]] = []
+
+    def __new__(cls, **kwargs: Any) -> "NamedTemporaryFilePool":
+        if not hasattr(cls, "_instance"):
+            cls._instance = super(NamedTemporaryFilePool, cls).__new__(cls)
+            atexit.register(cls._instance.cleanup)
+        return cls._instance
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.kwargs = kwargs
+
+    def tempfile(self) -> IO[Any]:
+        self._tempfile = tempfile.NamedTemporaryFile(delete=False, **self.kwargs)
+        self.tempfile_pool.append(self._tempfile)
+        return self._tempfile
+
+    def cleanup(self) -> None:
+        gc.collect()
+        for i in self.tempfile_pool:
+            os.unlink(i.name)
+
+    def __enter__(self) -> IO[Any]:
+        return self.tempfile()
+
+    def __exit__(
+        self,
+        exc_type: Type[BaseException],
+        exc_val: BaseException,
+        exc_tb: TracebackType,
+    ) -> None:
+        self._tempfile.close()
diff --git a/tests/integration_tests/test_dask.py b/tests/integration_tests/test_dask.py
--- a/tests/integration_tests/test_dask.py
+++ b/tests/integration_tests/test_dask.py
@@ -1,5 +1,4 @@
 from contextlib import contextmanager
-import tempfile
 import time
 from typing import Iterator
 
@@ -10,6 +9,7 @@
 from optuna._imports import try_import
 from optuna.integration.dask import _OptunaSchedulerExtension
 from optuna.integration.dask import DaskStorage
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 from optuna.trial import Trial
 
 
@@ -34,7 +34,7 @@ def get_storage_url(specifier: str) -> Iterator:
         if specifier == "inmemory":
             url = None
         elif specifier == "sqlite":
-            tmpfile = tempfile.NamedTemporaryFile()
+            tmpfile = NamedTemporaryFilePool().tempfile()
             url = "sqlite:///{}".format(tmpfile.name)
         else:
             raise ValueError(
diff --git a/tests/samplers_tests/test_cmaes.py b/tests/samplers_tests/test_cmaes.py
--- a/tests/samplers_tests/test_cmaes.py
+++ b/tests/samplers_tests/test_cmaes.py
@@ -66,7 +66,7 @@ def test_init_cmaes_opts(
         assert np.array_equal(actual_kwargs["mean"], np.array([0.5, 0.5]))
         assert actual_kwargs["sigma"] == 0.1
         assert np.allclose(actual_kwargs["bounds"], np.array([(0, 1), (0, 1)]))
-        assert actual_kwargs["seed"] == np.random.RandomState(1).randint(1, 2**32)
+        assert actual_kwargs["seed"] == np.random.RandomState(1).randint(1, np.iinfo(np.int32).max)
         assert actual_kwargs["n_max_resampling"] == 10 * 2
         assert actual_kwargs["population_size"] == popsize
 
@@ -100,7 +100,7 @@ def test_init_cmaes_opts_with_margin(popsize: Optional[int]) -> None:
         assert actual_kwargs["sigma"] == 0.1
         assert np.allclose(actual_kwargs["bounds"], np.array([(0, 1), (0, 1)]))
         assert np.allclose(actual_kwargs["steps"], np.array([0.0, 1.0]))
-        assert actual_kwargs["seed"] == np.random.RandomState(1).randint(1, 2**32)
+        assert actual_kwargs["seed"] == np.random.RandomState(1).randint(1, np.iinfo(np.int32).max)
         assert actual_kwargs["n_max_resampling"] == 10 * 2
         assert actual_kwargs["population_size"] == popsize
 
diff --git a/tests/storages_tests/rdb_tests/test_storage.py b/tests/storages_tests/rdb_tests/test_storage.py
--- a/tests/storages_tests/rdb_tests/test_storage.py
+++ b/tests/storages_tests/rdb_tests/test_storage.py
@@ -1,4 +1,5 @@
 import os
+import platform
 import shutil
 import sys
 import tempfile
@@ -21,6 +22,7 @@
 from optuna.storages._rdb.models import SCHEMA_VERSION
 from optuna.storages._rdb.models import VersionInfoModel
 from optuna.storages._rdb.storage import _create_scoped_session
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 
 from .create_db import mo_objective_test_upgrade
 from .create_db import objective_test_upgrade
@@ -51,7 +53,7 @@ def test_init() -> None:
 
 
 def test_init_url_template() -> None:
-    with tempfile.NamedTemporaryFile(suffix="{SCHEMA_VERSION}") as tf:
+    with NamedTemporaryFilePool(suffix="{SCHEMA_VERSION}") as tf:
         storage = RDBStorage("sqlite:///" + tf.name)
         assert storage.engine.url.database.endswith(str(SCHEMA_VERSION))
 
@@ -59,13 +61,13 @@ def test_init_url_template() -> None:
 def test_init_url_that_contains_percent_character() -> None:
     # Alembic's ini file regards '%' as the special character for variable expansion.
     # We checks `RDBStorage` does not raise an error even if a storage url contains the character.
-    with tempfile.NamedTemporaryFile(suffix="%") as tf:
+    with NamedTemporaryFilePool(suffix="%") as tf:
         RDBStorage("sqlite:///" + tf.name)
 
-    with tempfile.NamedTemporaryFile(suffix="%foo") as tf:
+    with NamedTemporaryFilePool(suffix="%foo") as tf:
         RDBStorage("sqlite:///" + tf.name)
 
-    with tempfile.NamedTemporaryFile(suffix="%foo%%bar") as tf:
+    with NamedTemporaryFilePool(suffix="%foo%%bar") as tf:
         RDBStorage("sqlite:///" + tf.name)
 
 
@@ -157,6 +159,7 @@ def test_upgrade_identity() -> None:
     assert old_version == new_version
 
 
+@pytest.mark.skipif(platform.system() == "Windows", reason="Skip on Windows")
 @pytest.mark.parametrize(
     "optuna_version",
     [
@@ -180,7 +183,11 @@ def test_upgrade_single_objective_optimization(optuna_version: str) -> None:
         shutil.copyfile(src_db_file, f"{workdir}/sqlite.db")
         storage_url = f"sqlite:///{workdir}/sqlite.db"
 
-        storage = RDBStorage(storage_url, skip_compatibility_check=True, skip_table_creation=True)
+        storage = RDBStorage(
+            storage_url,
+            skip_compatibility_check=True,
+            skip_table_creation=True,
+        )
         assert storage.get_current_version() == f"v{optuna_version}"
         head_version = storage.get_head_version()
         with warnings.catch_warnings():
@@ -215,7 +222,10 @@ def test_upgrade_single_objective_optimization(optuna_version: str) -> None:
 
         assert study.user_attrs["d"] == 3
 
+        storage.engine.dispose()  # Be sure to disconnect db
+
 
+@pytest.mark.skipif(platform.system() == "Windows", reason="Skip on Windows")
 @pytest.mark.parametrize(
     "optuna_version", ["2.4.0.a", "2.6.0.a", "3.0.0.a", "3.0.0.b", "3.0.0.c", "3.0.0.d", "3.2.0.a"]
 )
@@ -261,7 +271,10 @@ def test_upgrade_multi_objective_optimization(optuna_version: str) -> None:
             assert 0 <= trial.values[1] <= 150
         assert study.user_attrs["d"] == 3
 
+        storage.engine.dispose()  # Be sure to disconnect db
 
+
+@pytest.mark.skipif(platform.system() == "Windows", reason="Skip on Windows")
 @pytest.mark.parametrize(
     "optuna_version", ["2.4.0.a", "2.6.0.a", "3.0.0.a", "3.0.0.b", "3.0.0.c", "3.0.0.d", "3.2.0.a"]
 )
@@ -295,3 +308,5 @@ def test_upgrade_distributions(optuna_version: str) -> None:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
             new_study.optimize(objective_test_upgrade_distributions, n_trials=1)
+
+        storage.engine.dispose()  # Be sure to disconnect db
diff --git a/tests/storages_tests/test_journal.py b/tests/storages_tests/test_journal.py
--- a/tests/storages_tests/test_journal.py
+++ b/tests/storages_tests/test_journal.py
@@ -2,7 +2,6 @@
 from concurrent.futures import ProcessPoolExecutor
 from concurrent.futures import ThreadPoolExecutor
 import pickle
-import tempfile
 from types import TracebackType
 from typing import Any
 from typing import IO
@@ -21,6 +20,7 @@
 from optuna.storages._journal.file import JournalFileBaseLock
 from optuna.storages._journal.storage import JournalStorageReplayResult
 from optuna.testing.storages import StorageSupplier
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 
 
 LOG_STORAGE = {
@@ -45,7 +45,7 @@ def __init__(self, storage_type: str) -> None:
 
     def __enter__(self) -> optuna.storages.BaseJournalLogStorage:
         if self.storage_type.startswith("file"):
-            self.tempfile = tempfile.NamedTemporaryFile()
+            self.tempfile = NamedTemporaryFilePool().tempfile()
             lock: JournalFileBaseLock
             if self.storage_type == "file_with_open_lock":
                 lock = optuna.storages.JournalFileOpenLock(self.tempfile.name)
@@ -110,7 +110,7 @@ def pop_waiting_trial(file_path: str, study_name: str) -> Optional[int]:
 
 
 def test_pop_waiting_trial_multiprocess_safe() -> None:
-    with tempfile.NamedTemporaryFile() as file:
+    with NamedTemporaryFilePool() as file:
         file_storage = optuna.storages.JournalFileStorage(file.name)
         storage = optuna.storages.JournalStorage(file_storage)
         study = optuna.create_study(storage=storage)
diff --git a/tests/storages_tests/test_storages.py b/tests/storages_tests/test_storages.py
--- a/tests/storages_tests/test_storages.py
+++ b/tests/storages_tests/test_storages.py
@@ -2,6 +2,7 @@
 from datetime import datetime
 import pickle
 import random
+from time import sleep
 from typing import Any
 from typing import Dict
 from typing import List
@@ -289,7 +290,9 @@ def _check_trials(
         n_trial_in_study = 3
         for i in range(n_trial_in_study):
             time_before_creation = datetime.now()
+            sleep(0.001)  # Sleep 1ms to avoid faulty assertion on Windows OS.
             trial_id = storage.create_new_trial(study_id)
+            sleep(0.001)
             time_after_creation = datetime.now()
 
             trials = storage.get_all_trials(study_id)
diff --git a/tests/study_tests/test_study.py b/tests/study_tests/test_study.py
--- a/tests/study_tests/test_study.py
+++ b/tests/study_tests/test_study.py
@@ -3,6 +3,7 @@
 import copy
 import multiprocessing
 import pickle
+import platform
 import threading
 import time
 from typing import Any
@@ -783,7 +784,9 @@ def test_optimize_with_progbar(n_jobs: int, capsys: _pytest.capture.CaptureFixtu
     assert "Best trial: 0" in err
     assert "Best value: 1" in err
     assert "10/10" in err
-    assert "100%" in err
+    if platform.system() != "Windows":
+        # Skip this assertion because the progress bar sometimes stops at 99% on Windows.
+        assert "100%" in err
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
@@ -795,7 +798,9 @@ def test_optimize_without_progbar(n_jobs: int, capsys: _pytest.capture.CaptureFi
     assert "Best trial: 0" not in err
     assert "Best value: 1" not in err
     assert "10/10" not in err
-    assert "100%" not in err
+    if platform.system() != "Windows":
+        # Skip this assertion because the progress bar sometimes stops at 99% on Windows.
+        assert "100%" not in err
 
 
 def test_optimize_with_progbar_timeout(capsys: _pytest.capture.CaptureFixture) -> None:
@@ -806,7 +811,9 @@ def test_optimize_with_progbar_timeout(capsys: _pytest.capture.CaptureFixture) -
     assert "Best trial: 0" in err
     assert "Best value: 1" in err
     assert "00:02/00:02" in err
-    assert "100%" in err
+    if platform.system() != "Windows":
+        # Skip this assertion because the progress bar sometimes stops at 99% on Windows.
+        assert "100%" in err
 
 
 def test_optimize_with_progbar_parallel_timeout(capsys: _pytest.capture.CaptureFixture) -> None:
@@ -851,7 +858,9 @@ def test_optimize_without_progbar_timeout(
     assert "Best trial: 0" not in err
     assert "Best value: 1.0" not in err
     assert "00:02/00:02" not in err
-    assert "100%" not in err
+    if platform.system() != "Windows":
+        # Skip this assertion because the progress bar sometimes stops at 99% on Windows.
+        assert "100%" not in err
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
@@ -865,7 +874,9 @@ def test_optimize_progbar_n_trials_prioritized(
     assert "Best trial: 0" in err
     assert "Best value: 1" in err
     assert "10/10" in err
-    assert "100%" in err
+    if platform.system() != "Windows":
+        # Skip this assertion because the progress bar sometimes stops at 99% on Windows.
+        assert "100%" in err
     assert "it" in err
 
 
diff --git a/tests/test_cli.py b/tests/test_cli.py
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -4,7 +4,6 @@
 import re
 import subprocess
 from subprocess import CalledProcessError
-import tempfile
 from typing import Any
 from typing import Callable
 from typing import Optional
@@ -25,6 +24,7 @@
 from optuna.storages._base import DEFAULT_STUDY_NAME_PREFIX
 from optuna.study import StudyDirection
 from optuna.testing.storages import StorageSupplier
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 from optuna.trial import Trial
 from optuna.trial import TrialState
 
@@ -67,7 +67,7 @@ def _parse_output(output: str, output_format: str) -> Any:
     """
 
     if output_format == "table":
-        rows = output.split("\n")
+        rows = output.split(os.linesep)
         assert all(len(rows[0]) == len(row) for row in rows)
         # Check ruled lines.
         assert rows[0] == rows[2] == rows[-1]
@@ -1006,7 +1006,7 @@ def test_study_optimize_command() -> None:
 
 @pytest.mark.skip_coverage
 def test_study_optimize_command_inconsistent_args() -> None:
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         # --study-name argument is missing.
@@ -1092,7 +1092,7 @@ def test_ask(
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         args = [
@@ -1165,7 +1165,7 @@ def test_ask_flatten(
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         args = [
@@ -1216,7 +1216,7 @@ def test_ask_flatten(
 def test_ask_empty_search_space(output_format: str) -> None:
     study_name = "test_study"
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         args = [
@@ -1249,7 +1249,7 @@ def test_ask_empty_search_space(output_format: str) -> None:
 def test_ask_empty_search_space_flatten(output_format: str) -> None:
     study_name = "test_study"
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         args = [
@@ -1286,7 +1286,7 @@ def test_ask_sampler_kwargs_without_sampler() -> None:
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         args = [
@@ -1330,7 +1330,7 @@ def test_create_study_and_ask(
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         create_study_args = [
@@ -1395,7 +1395,7 @@ def test_create_study_and_ask_with_inconsistent_directions(
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         create_study_args = [
@@ -1441,7 +1441,7 @@ def test_ask_with_both_direction_and_directions() -> None:
         '"y": {"name": "CategoricalDistribution", "attributes": {"choices": ["foo"]}}}'
     )
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         create_study_args = [
@@ -1478,7 +1478,7 @@ def test_ask_with_both_direction_and_directions() -> None:
 def test_tell() -> None:
     study_name = "test_study"
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         output: Any = subprocess.check_output(
@@ -1555,7 +1555,7 @@ def test_tell() -> None:
 def test_tell_with_nan() -> None:
     study_name = "test_study"
 
-    with tempfile.NamedTemporaryFile() as tf:
+    with NamedTemporaryFilePool() as tf:
         db_url = "sqlite:///{}".format(tf.name)
 
         output: Any = subprocess.check_output(
diff --git a/tests/trial_tests/test_trial.py b/tests/trial_tests/test_trial.py
--- a/tests/trial_tests/test_trial.py
+++ b/tests/trial_tests/test_trial.py
@@ -1,6 +1,5 @@
 import datetime
 import math
-import tempfile
 from typing import Any
 from typing import Dict
 from typing import List
@@ -26,6 +25,7 @@
 from optuna.testing.samplers import DeterministicSampler
 from optuna.testing.storages import STORAGE_MODES
 from optuna.testing.storages import StorageSupplier
+from optuna.testing.tempfile_pool import NamedTemporaryFilePool
 from optuna.trial import Trial
 
 
@@ -664,7 +664,7 @@ def objective(trial: Trial) -> Tuple[float, float]:
 def test_persisted_param() -> None:
     study_name = "my_study"
 
-    with tempfile.NamedTemporaryFile() as fp:
+    with NamedTemporaryFilePool() as fp:
         storage = f"sqlite:///{fp.name}"
         study = create_study(storage=storage, study_name=study_name)
         assert isinstance(study._storage, storages._CachedStorage), "Pre-condition."
diff --git a/tests/trial_tests/test_trials.py b/tests/trial_tests/test_trials.py
--- a/tests/trial_tests/test_trials.py
+++ b/tests/trial_tests/test_trials.py
@@ -1,4 +1,5 @@
 import datetime
+import time
 from typing import Any
 from typing import Dict
 from typing import Optional
@@ -215,4 +216,5 @@ def test_datetime_start(trial_type: type) -> None:
     trial = _create_trial(trial_type)
     assert trial.datetime_start is not None
     old_date_time_start = trial.datetime_start
+    time.sleep(0.001)  # Sleep 1ms to avoid faulty assertion on Windows OS.
     assert datetime.datetime.now() != old_date_time_start
diff --git a/tests/visualization_tests/matplotlib_tests/test_optimization_history.py b/tests/visualization_tests/matplotlib_tests/test_optimization_history.py
--- a/tests/visualization_tests/matplotlib_tests/test_optimization_history.py
+++ b/tests/visualization_tests/matplotlib_tests/test_optimization_history.py
@@ -24,3 +24,4 @@ def test_get_optimization_history_plot(
     legends = [legend.get_text() for legend in figure.legend().get_texts()]
     assert sorted(legends) == sorted(expected_legends)
     plt.savefig(BytesIO())
+    plt.close()
diff --git a/tests/visualization_tests/test_edf.py b/tests/visualization_tests/test_edf.py
--- a/tests/visualization_tests/test_edf.py
+++ b/tests/visualization_tests/test_edf.py
@@ -38,6 +38,7 @@ def save_static_image(figure: Union[go.Figure, Axes, np.ndarray]) -> None:
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
 
 
 @parametrized_plot_edf
diff --git a/tests/visualization_tests/test_intermediate_plot.py b/tests/visualization_tests/test_intermediate_plot.py
--- a/tests/visualization_tests/test_intermediate_plot.py
+++ b/tests/visualization_tests/test_intermediate_plot.py
@@ -82,3 +82,4 @@ def test_plot_intermediate_values(
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
diff --git a/tests/visualization_tests/test_parallel_coordinate.py b/tests/visualization_tests/test_parallel_coordinate.py
--- a/tests/visualization_tests/test_parallel_coordinate.py
+++ b/tests/visualization_tests/test_parallel_coordinate.py
@@ -206,6 +206,7 @@ def test_plot_parallel_coordinate(
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
 
 
 def test_get_parallel_coordinate_info() -> None:
diff --git a/tests/visualization_tests/test_param_importances.py b/tests/visualization_tests/test_param_importances.py
--- a/tests/visualization_tests/test_param_importances.py
+++ b/tests/visualization_tests/test_param_importances.py
@@ -87,6 +87,7 @@ def test_plot_param_importances(
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
 
 
 @pytest.mark.parametrize(
diff --git a/tests/visualization_tests/test_pareto_front.py b/tests/visualization_tests/test_pareto_front.py
--- a/tests/visualization_tests/test_pareto_front.py
+++ b/tests/visualization_tests/test_pareto_front.py
@@ -312,6 +312,7 @@ def test_get_pareto_front_plot(
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
 
 
 @pytest.mark.parametrize("direction", ["minimize", "maximize"])
diff --git a/tests/visualization_tests/test_slice.py b/tests/visualization_tests/test_slice.py
--- a/tests/visualization_tests/test_slice.py
+++ b/tests/visualization_tests/test_slice.py
@@ -110,6 +110,7 @@ def test_plot_slice(
         figure.write_image(BytesIO())
     else:
         plt.savefig(BytesIO())
+        plt.close()
 
 
 def test_target_is_none_and_study_is_multi_obj() -> None: