Repository: optuna/optuna

Issue:
CI failed due to the error in AllenNLP integration
### Expected behavior

https://github.com/optuna/optuna/actions/runs/1960157156

### Environment

- Optuna version: 3.0.0b0.dev
- Python version: 3.9.0
- OS: 20.04.3
- (Optional) Other libraries and their versions: AllenNLP 2.9.1


### Error messages, stack traces, or logs

```shell
_ test_allennlp_pruning_callback_with_executor[tests/integration_tests/allennlp_tests/example_with_executor_and_pruner_distributed.jsonnet-HyperbandPruner-pruner_kwargs0] _

pruner_class = <class 'optuna.pruners._hyperband.HyperbandPruner'>
pruner_kwargs = {'max_resource': 10, 'min_resource': 3, 'reduction_factor': 5}
input_config_file = 'tests/integration_tests/allennlp_tests/example_with_executor_and_pruner_distributed.jsonnet'

    @pytest.mark.parametrize(
        "pruner_class,pruner_kwargs",
        [
            (
                optuna.pruners.HyperbandPruner,
                {"min_resource": 3, "max_resource": 10, "reduction_factor": 5},
            ),
            (
                optuna.pruners.MedianPruner,
                {"n_startup_trials": 8, "n_warmup_steps": 1, "interval_steps": 3},
            ),
            (optuna.pruners.NopPruner, {}),
            (
                optuna.pruners.PercentilePruner,
                {"percentile": 50.0, "n_startup_trials": 10, "n_warmup_steps": 1, "interval_steps": 3},
            ),
            (
                optuna.pruners.SuccessiveHalvingPruner,
                {"min_resource": 3, "reduction_factor": 5, "min_early_stopping_rate": 1},
            ),
            (
                optuna.pruners.ThresholdPruner,
                {"lower": 0.0, "upper": 1.0, "n_warmup_steps": 3, "interval_steps": 2},
            ),
        ],
    )
    @pytest.mark.parametrize(
        "input_config_file",
        [
            "tests/integration_tests/allennlp_tests/example_with_executor_and_pruner.jsonnet",
            "tests/integration_tests/allennlp_tests/example_with_executor_and_pruner_distributed.jsonnet",  # noqa: E501
        ],
    )
    def test_allennlp_pruning_callback_with_executor(
        pruner_class: Type[optuna.pruners.BasePruner],
        pruner_kwargs: Dict[str, Union[int, float]],
        input_config_file: str,
    ) -> None:
        def run_allennlp_executor(pruner: optuna.pruners.BasePruner) -> None:
            study = optuna.create_study(direction="maximize", pruner=pruner, storage=storage)
            trial = optuna.trial.Trial(study, study._storage.create_new_trial(study._study_id))
            trial.suggest_float("DROPOUT", 0.0, 0.5)
            executor = optuna.integration.AllenNLPExecutor(trial, input_config_file, serialization_dir)
            executor.run()
    
        with tempfile.TemporaryDirectory() as tmp_dir:
            pruner_name = pruner_class.__name__
            os.mkdir(os.path.join(tmp_dir, pruner_name))
            storage = "sqlite:///" + os.path.join(tmp_dir, pruner_name, "result.db")
            serialization_dir = os.path.join(tmp_dir, pruner_name, "allennlp")
    
            pruner = pruner_class(**pruner_kwargs)  # type: ignore
>           run_allennlp_executor(pruner)

tests/integration_tests/allennlp_tests/test_allennlp.py:391: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/integration_tests/allennlp_tests/test_allennlp.py:382: in run_allennlp_executor
    executor.run()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <optuna.integration.allennlp._executor.AllenNLPExecutor object at 0x7f7d4443ebb0>

    def run(self) -> float:
        """Train a model using AllenNLP."""
        for package_name in self._include_package:
            allennlp.common.util.import_module_and_submodules(package_name)
    
        # Without the following lines, the transformer model construction only takes place in the
        # first trial (which would consume some random numbers), and the cached model will be used
        # in trials afterwards (which would not consume random numbers), leading to inconsistent
        # results between single trial and multiple trials. To make results reproducible in
        # multiple trials, we clear the cache before each trial.
        # TODO(MagiaSN) When AllenNLP has introduced a better API to do this, one should remove
        # these lines and use the new API instead. For example, use the `_clear_caches()` method
        # which will be in the next AllenNLP release after 2.4.0.
        allennlp.common.cached_transformers._model_cache.clear()
        allennlp.common.cached_transformers._tokenizer_cache.clear()
    
        self._set_environment_variables()
        params = allennlp.common.params.Params(self._build_params())
    
        if "distributed" in params:
    
            if OPTUNA_ALLENNLP_DISTRIBUTED_FLAG in os.environ:
                warnings.warn(
                    "Other process may already exists."
                    " If you have trouble, please unset the environment"
                    " variable `OPTUNA_ALLENNLP_USE_DISTRIBUTED`"
                    " and try it again."
                )
    
            os.environ[OPTUNA_ALLENNLP_DISTRIBUTED_FLAG] = "1"
    
        try:
            allennlp.commands.train.train_model(
                params=params,
                serialization_dir=self._serialization_dir,
                file_friendly_logging=self._file_friendly_logging,
                force=self._force,
                include_package=self._include_package,
            )
        except ProcessRaisedException as e:
            if "raise TrialPruned()" in str(e):
                raise TrialPruned()
    
>       metrics = json.load(open(os.path.join(self._serialization_dir, "metrics.json")))
E       FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpawf96wcz/HyperbandPruner/allennlp/metrics.json'

optuna/integration/allennlp/_executor.py:236: FileNotFoundError
```

### Steps to reproduce

See https://github.com/optuna/optuna/actions/runs/1960157156.

### Additional context (optional)

- Maybe related to https://github.com/allenai/allennlp/releases/tag/v2.9.1
- Error happened on distributed configuration


Discussion:
This issue has not seen any recent activity.
This issue has not seen any recent activity.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -85,9 +85,9 @@ def get_extras_require() -> Dict[str, List[str]]:
             "plotly>=4.0.0",  # optuna/visualization.
             "pandas",
             "lightgbm",
-            "torch==1.10.0",
-            "torchvision==0.11.1",
-            "torchaudio==0.10.0",
+            "torch==1.11.0 ; python_version>'3.6'",
+            "torchvision==0.12.0 ; python_version>'3.6'",
+            "torchaudio==0.11.0 ; python_version>'3.6'",
             "thop",
         ],
         "experimental": ["redis"],
@@ -108,23 +108,16 @@ def get_extras_require() -> Dict[str, List[str]]:
             "xgboost",
             "tensorflow ; python_version>'3.6'",
             "tensorflow-datasets",
-            "pytorch-ignite ; python_version<'3.10'",
-            "pytorch-lightning>=1.5.0,<1.6.0 ; python_version<'3.10'",
-            "skorch ; python_version<'3.10'",
-            "catalyst>=21.3 ; python_version<'3.10'",
-            "torch==1.10.0 ; sys_platform=='darwin' and python_version<'3.10'",
-            "torch==1.10.0+cpu ; sys_platform!='darwin' and python_version<'3.10'",
-            "torchvision==0.11.1 ; sys_platform=='darwin' and python_version<'3.10'",
-            "torchvision==0.11.1+cpu ; sys_platform!='darwin' and python_version<'3.10'",
-            "torchaudio==0.10.0; python_version<'3.10'",
-            # TODO(himkt): Remove `nltk` after solving
-            # https://github.com/allenai/allennlp/issues/5521
-            "nltk<3.6.6 ; python_version<'3.10'",
-            # TODO(himkt): Remove upper bound constraint
-            # after solving https://github.com/optuna/optuna/issues/3366
-            "allennlp>=2.2.0,<2.9.1 ; python_version>'3.6' and python_version<'3.10'",
-            "botorch>=0.4.0 ; python_version>'3.6' and python_version<'3.10'",
-            "fastai ; python_version>'3.6' and python_version<'3.10'",
+            "pytorch-ignite ; python_version>'3.6'",
+            "pytorch-lightning>=1.5.0 ; python_version>'3.6'",
+            "skorch ; python_version>'3.6'",
+            "catalyst>=21.3 ; python_version>'3.6'",
+            "torch==1.11.0 ; python_version>'3.6'",
+            "torchvision==0.12.0 ; python_version>'3.6'",
+            "torchaudio==0.11.0 ; python_version>'3.6'",
+            "allennlp>=2.2.0 ; python_version>'3.6'",
+            "botorch>=0.4.0 ; python_version>'3.6'",
+            "fastai ; python_version>'3.6'",
         ],
         "tests": [
             "fakeredis",
@@ -153,23 +146,16 @@ def get_extras_require() -> Dict[str, List[str]]:
             "xgboost",
             "tensorflow ; python_version>'3.6'",
             "tensorflow-datasets",
-            "pytorch-ignite ; python_version<'3.10'",
-            "pytorch-lightning>=1.5.0,<1.6.0 ; python_version<'3.10'",
-            "skorch ; python_version<'3.10'",
-            "catalyst>=21.3 ; python_version<'3.10'",
-            "torch==1.10.0 ; sys_platform=='darwin' and python_version<'3.10'",
-            "torch==1.10.0+cpu ; sys_platform!='darwin' and python_version<'3.10'",
-            "torchvision==0.11.1 ; sys_platform=='darwin' and python_version<'3.10'",
-            "torchvision==0.11.1+cpu ; sys_platform!='darwin' and python_version<'3.10'",
-            "torchaudio==0.10.0; python_version<'3.10'",
-            # TODO(himkt): Remove `nltk` after solving
-            # https://github.com/allenai/allennlp/issues/5521
-            "nltk<3.6.6 ; python_version<'3.10'",
-            # TODO(himkt): Remove upper bound constraint
-            # after solving https://github.com/optuna/optuna/issues/3366
-            "allennlp>=2.2.0,<2.9.1 ; python_version>'3.6' and python_version<'3.10'",
-            "botorch>=0.4.0 ; python_version>'3.6' and python_version<'3.10'",
-            "fastai ; python_version>'3.6' and python_version<'3.10'",
+            "pytorch-ignite ; python_version>'3.6'",
+            "pytorch-lightning>=1.5.0 ; python_version>'3.6'",
+            "skorch ; python_version>'3.6'",
+            "catalyst>=21.3 ; python_version>'3.6'",
+            "torch==1.11.0 ; python_version>'3.6'",
+            "torchvision==0.12.0 ; python_version>'3.6'",
+            "torchaudio==0.11.0 ; python_version>'3.6'",
+            "allennlp>=2.2.0 ; python_version>'3.6'",
+            "botorch>=0.4.0 ; python_version>'3.6'",
+            "fastai ; python_version>'3.6'",
         ],
         "benchmark": [
             "asv>=0.5.0",


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -66,10 +66,10 @@ jobs:
         pip install scikit-optimize
         pip install cma
 
-    # TODO(nzw): Move `run` part to the end of `Install` after PyTorch and BoTorch support Python 3.10.
+    # TODO(nzw): Move `run` part to the end of `Install` after Optuna drops Python 3.6 support.
     - name: Install PyTorch and BoTorch.
-      run: pip install botorch torch==1.10.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
-      if: matrix.python-version != '3.10'
+      run: pip install botorch torch==1.11.0 --extra-index-url https://download.pytorch.org/whl/cpu
+      if: matrix.python-version != '3.6'
 
     - name: Tests
       run: |
diff --git a/tests/integration_tests/test_catboost.py b/tests/integration_tests/test_catboost.py
--- a/tests/integration_tests/test_catboost.py
+++ b/tests/integration_tests/test_catboost.py
@@ -80,6 +80,7 @@ def objective(trial: optuna.trial.Trial) -> float:
     assert study.trials[0].value == 1.0
 
 
+@pytest.mark.skip(reason="Temporally skip due to unknown CatBoost error.")
 @pytest.mark.parametrize(
     "metric, eval_set_index",
     [
diff --git a/tests/samplers_tests/test_samplers.py b/tests/samplers_tests/test_samplers.py
--- a/tests/samplers_tests/test_samplers.py
+++ b/tests/samplers_tests/test_samplers.py
@@ -51,11 +51,10 @@
             lambda: optuna.samplers.QMCSampler(),
         ]
     )
-    # TODO(nzw0301): Remove version constraints if BoTorch supports Python 3.10
-    # or Optuna does not support Python 3.6.
+    # TODO(nzw0301): Update this after the support for Python 3.6 is stopped.
     + (
         []
-        if sys.version_info >= (3, 10, 0) or sys.version_info < (3, 7, 0)
+        if sys.version_info < (3, 7, 0)
         else [lambda: optuna.integration.BoTorchSampler(n_startup_trials=0)]
     ),
 )
@@ -76,11 +75,10 @@
         optuna.samplers.NSGAIISampler,
         lambda: optuna.samplers.MOTPESampler(n_startup_trials=0),
     ]
-    # TODO(nzw0301): Remove version constraints if BoTorch supports Python 3.10
-    # or Optuna does not support Python 3.6.
+    # TODO(nzw0301): Update this after the support for Python 3.6 is stopped.
     + (
         []
-        if sys.version_info >= (3, 10, 0) or sys.version_info < (3, 7, 0)
+        if sys.version_info < (3, 7, 0)
         else [lambda: optuna.integration.BoTorchSampler(n_startup_trials=0)]
     ),
 )