Repository: optuna/optuna

Issue:
Enhance speed benchmark
### Motivation

Currently we don't have speed benchmarks for
- [x] `JournalStorage`
- [x] multi-objective optimization algorithms (`NSGAIISampler` and multi-objective version of `TPESampler`).



### Suggestion

Add those speed benchmarks.

We need to add suitable benchmark script like https://github.com/optuna/optuna/blob/master/benchmarks/asv/optimize.py and possibly update https://github.com/optuna/optuna/blob/master/asv.conf.json

### Additional context (optional)

_No response_


Discussion:

Resolving patch:
diff --git a/benchmarks/asv/optimize.py b/benchmarks/asv/optimize.py
--- a/benchmarks/asv/optimize.py
+++ b/benchmarks/asv/optimize.py
@@ -5,6 +5,7 @@
 import optuna
 from optuna.samplers import BaseSampler
 from optuna.samplers import CmaEsSampler
+from optuna.samplers import NSGAIISampler
 from optuna.samplers import RandomSampler
 from optuna.samplers import TPESampler
 from optuna.testing.storages import StorageSupplier
@@ -34,6 +35,8 @@ def create_sampler(sampler_mode: str) -> BaseSampler:
         return TPESampler()
     elif sampler_mode == "cmaes":
         return CmaEsSampler()
+    elif sampler_mode == "nsgaii":
+        return NSGAIISampler()
     else:
         assert False
 
@@ -44,30 +47,55 @@ def objective(self, trial: optuna.Trial) -> float:
         y = trial.suggest_int("y", -100, 100)
         return x**2 + y**2
 
-    def optimize(self, storage_mode: str, sampler_mode: str, n_trials: int) -> None:
+    def multi_objective(self, trial: optuna.Trial) -> tuple[float, float]:
+        x = trial.suggest_float("x", -100, 100)
+        y = trial.suggest_int("y", -100, 100)
+        return (x**2 + y**2, (x - 2) ** 2 + (y - 2) ** 2)
+
+    def optimize(
+        self, storage_mode: str, sampler_mode: str, n_trials: int, objective_type: str
+    ) -> None:
         with StorageSupplier(storage_mode) as storage:
             sampler = create_sampler(sampler_mode)
-            study = optuna.create_study(storage=storage, sampler=sampler)
-            study.optimize(self.objective, n_trials=n_trials)
+            if objective_type == "single":
+                directions = ["minimize"]
+            elif objective_type == "multi":
+                directions = ["minimize", "minimize"]
+            else:
+                assert False
+            study = optuna.create_study(storage=storage, sampler=sampler, directions=directions)
+            if objective_type == "single":
+                study.optimize(self.objective, n_trials=n_trials)
+            elif objective_type == "multi":
+                study.optimize(self.multi_objective, n_trials=n_trials)
+            else:
+                assert False
 
     def time_optimize(self, args: str) -> None:
-        storage_mode, sampler_mode, n_trials = parse_args(args)
+        storage_mode, sampler_mode, n_trials, objective_type = parse_args(args)
         storage_mode = cast(str, storage_mode)
         sampler_mode = cast(str, sampler_mode)
         n_trials = cast(int, n_trials)
-        self.optimize(storage_mode, sampler_mode, n_trials)
+        objective_type = cast(str, objective_type)
+        self.optimize(storage_mode, sampler_mode, n_trials, objective_type)
 
     params = (
-        "inmemory, random, 1000",
-        "inmemory, random, 10000",
-        "inmemory, tpe, 1000",
-        "inmemory, cmaes, 1000",
-        "sqlite, random, 1000",
-        "sqlite, tpe, 1000",
-        "sqlite, cmaes, 1000",
-        "journal, random, 1000",
-        "journal, tpe, 1000",
-        "journal, cmaes, 1000",
+        "inmemory, random, 1000, single",
+        "inmemory, random, 10000, single",
+        "inmemory, tpe, 1000, single",
+        "inmemory, cmaes, 1000, single",
+        "sqlite, random, 1000, single",
+        "sqlite, tpe, 1000, single",
+        "sqlite, cmaes, 1000, single",
+        "journal, random, 1000, single",
+        "journal, tpe, 1000, single",
+        "journal, cmaes, 1000, single",
+        "inmemory, tpe, 1000, multi",
+        "inmemory, nsgaii, 1000, multi",
+        "sqlite, tpe, 1000, multi",
+        "sqlite, nsgaii, 1000, multi",
+        "journal, tpe, 1000, multi",
+        "journal, nsgaii, 1000, multi",
     )
-    param_names = ["storage, sampler, n_trials"]
+    param_names = ["storage, sampler, n_trials, objective_type"]
     timeout = 600


Test patch: