Repository: optuna/optuna

Issue:
Missing test case for `optuna.visualization.plot_pareto_front`
### Motivation

#3495 added the `target` option to `optuna.visualization.plot_pareto_front`, but the corresponding test case does not check invalid `targets` function.

For example, users may input the `target` functions that returns more than four values (i.e., https://github.com/optuna/optuna/pull/3495#discussion_r857078406). 

### Description

Please add test cases that inputs invalid `targets` functions and check errors.

The expected errors can be seen in the docstring, but it may be updated in https://github.com/optuna/optuna/pull/3429.

For example, 
- `RuntimeError` may be consistent with other component if the study with more than four objective (and `targets=None`).
- `ValueError` may be appropriate if the `targets` functions returns more than three values.

### Alternatives (optional)

_No response_

### Additional context (optional)

_No response_


Discussion:

Resolving patch:
diff --git a/optuna/visualization/_pareto_front.py b/optuna/visualization/_pareto_front.py
--- a/optuna/visualization/_pareto_front.py
+++ b/optuna/visualization/_pareto_front.py
@@ -77,7 +77,7 @@ def objective(trial):
     Args:
         study:
             A :class:`~optuna.study.Study` object whose trials are plotted for their objective
-            values. ``study.n_objectives`` must be eigher 2 or 3.
+            values. ``study.n_objectives`` must be either 2 or 3 when ``targets`` is :obj:`None`.
         target_names:
             Objective name list used as the axis titles. If :obj:`None` is specified,
             "Objective {objective_index}" is used instead. If ``targets`` is specified
@@ -103,7 +103,7 @@ def objective(trial):
             A function that returns targets values to display.
             The argument to this function is :class:`~optuna.trial.FrozenTrial`.
             ``axis_order`` and ``targets`` cannot be used at the same time.
-            If your study has more than 4 objectives, ``targets`` must be specified.
+            If ``study.n_objectives`` is neither 2 nor 3, ``targets`` must be specified.
 
             .. note::
                 Added in v3.0.0 as an experimental feature. The interface may change in newer
@@ -286,6 +286,12 @@ def _infer_n_targets(
                 "If `targets` is specified for empty studies, `target_names` must be specified."
             )
 
+    if n_targets not in (2, 3):
+        raise ValueError(
+            "`plot_pareto_front` function only supports 2 or 3 targets."
+            " you used {} targets now.".format(n_targets)
+        )
+
     if target_names is None:
         target_names = [f"Objective {i}" for i in range(n_targets)]
     elif len(target_names) != n_targets:
@@ -342,7 +348,7 @@ def _make_json_compatible(value: Any) -> Any:
         json.dumps(value)
         return value
     except TypeError:
-        # the value can't be converted to JSON directly, so return a string representation
+        # The value can't be converted to JSON directly, so return a string representation.
         return str(value)
 
 
@@ -357,7 +363,6 @@ def _make_scatter_object(
 ) -> Union["go.Scatter", "go.Scatter3d"]:
     trials_with_values = trials_with_values or []
 
-    assert n_targets in (2, 3)
     marker = _make_marker(
         [trial for trial, _ in trials_with_values],
         include_dominated_trials,
@@ -374,8 +379,7 @@ def _make_scatter_object(
             marker=marker,
             showlegend=False,
         )
-    else:
-        assert n_targets == 3
+    elif n_targets == 3:
         return go.Scatter3d(
             x=[values[axis_order[0]] for _, values in trials_with_values],
             y=[values[axis_order[1]] for _, values in trials_with_values],
@@ -386,6 +390,8 @@ def _make_scatter_object(
             marker=marker,
             showlegend=False,
         )
+    else:
+        assert False, "Must not reach here"
 
 
 def _make_hovertext(trial: FrozenTrial) -> str:
diff --git a/optuna/visualization/matplotlib/_pareto_front.py b/optuna/visualization/matplotlib/_pareto_front.py
--- a/optuna/visualization/matplotlib/_pareto_front.py
+++ b/optuna/visualization/matplotlib/_pareto_front.py
@@ -56,7 +56,7 @@ def objective(trial):
     Args:
         study:
             A :class:`~optuna.study.Study` object whose trials are plotted for their objective
-            values. ``study.n_objectives`` must be eigher 2 or 3.
+            values. ``study.n_objectives`` must be either 2 or 3 when ``targets`` is :obj:`None`.
         target_names:
             Objective name list used as the axis titles. If :obj:`None` is specified,
             "Objective {objective_index}" is used instead. If ``targets`` is specified
@@ -77,7 +77,7 @@ def objective(trial):
             The argument to this function is :class:`~optuna.trial.FrozenTrial`.
             ``targets`` must be :obj:`None` or return 2 or 3 values.
             ``axis_order`` and ``targets`` cannot be used at the same time.
-            If your study has more than 4 objectives, ``targets`` must be specified.
+            If ``study.n_objectives`` is neither 2 nor 3, ``targets`` must be specified.
 
             .. note::
                 Added in v3.0.0 as an experimental feature. The interface may change in newer
@@ -99,10 +99,7 @@ def objective(trial):
     elif info.n_targets == 3:
         return _get_pareto_front_3d(info)
     else:
-        raise ValueError(
-            "`plot_pareto_front` function only supports 2 or 3 targets."
-            " you used {} targets now.".format(info.n_targets)
-        )
+        assert False, "Must not reach here"
 
 
 def _get_pareto_front_2d(info: _ParetoFrontInfo) -> "Axes":


Test patch:
diff --git a/tests/visualization_tests/matplotlib_tests/test_pareto_front.py b/tests/visualization_tests/matplotlib_tests/test_pareto_front.py
--- a/tests/visualization_tests/matplotlib_tests/test_pareto_front.py
+++ b/tests/visualization_tests/matplotlib_tests/test_pareto_front.py
@@ -79,7 +79,9 @@ def test_plot_pareto_front_2d(
     plt.savefig(BytesIO())
 
     # Test with `target_names` argument.
-    with pytest.raises(ValueError):
+    error_message = "The length of `target_names` is supposed to be 2."
+
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=[],
@@ -87,7 +89,7 @@ def test_plot_pareto_front_2d(
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo"],
@@ -95,7 +97,7 @@ def test_plot_pareto_front_2d(
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar", "Baz"],
@@ -197,7 +199,9 @@ def test_plot_pareto_front_3d(
     plt.savefig(BytesIO())
 
     # Test with `target_names` argument.
-    with pytest.raises(ValueError):
+    error_message = "The length of `target_names` is supposed to be 3."
+
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=[],
@@ -206,7 +210,7 @@ def test_plot_pareto_front_3d(
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo"],
@@ -215,7 +219,7 @@ def test_plot_pareto_front_3d(
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar"],
@@ -224,7 +228,7 @@ def test_plot_pareto_front_3d(
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar", "Baz", "Qux"],
@@ -269,19 +273,26 @@ def test_plot_pareto_front_3d(
 @pytest.mark.filterwarnings("ignore::optuna.exceptions.ExperimentalWarning")
 @pytest.mark.parametrize("include_dominated_trials", [False, True])
 def test_plot_pareto_front_unsupported_dimensions(include_dominated_trials: bool) -> None:
+
+    error_message = (
+        "`plot_pareto_front` function only supports 2 or 3 objective"
+        " studies when using `targets` is `None`. Please use `targets`"
+        " if your objective studies have more than 3 objectives."
+    )
+
     # Unsupported: n_objectives == 1.
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(directions=["minimize"])
         study.optimize(lambda t: [0], n_trials=1)
         plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials)
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(direction="minimize")
         study.optimize(lambda t: [0], n_trials=1)
         plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials)
 
     # Unsupported: n_objectives == 4.
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
         study.optimize(lambda t: [0, 0, 0, 0], n_trials=1)
         plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials)
@@ -357,15 +368,7 @@ def test_plot_pareto_front_targets_without_target_names() -> None:
 
 
 @pytest.mark.filterwarnings("ignore::optuna.exceptions.ExperimentalWarning")
-@pytest.mark.parametrize(
-    "targets",
-    [
-        lambda t: (t.values[0]),
-    ],
-)
-def test_plot_pareto_front_invalid_target_values(
-    targets: Optional[Callable[[FrozenTrial], Sequence[float]]]
-) -> None:
+def test_plot_pareto_front_invalid_target_values() -> None:
     study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
     study.optimize(lambda t: [0, 0, 0, 0], n_trials=3)
     with pytest.raises(
@@ -375,7 +378,7 @@ def test_plot_pareto_front_invalid_target_values(
     ):
         plot_pareto_front(
             study=study,
-            targets=targets,
+            targets=lambda t: t.values[0],
         )
 
 
diff --git a/tests/visualization_tests/test_pareto_front.py b/tests/visualization_tests/test_pareto_front.py
--- a/tests/visualization_tests/test_pareto_front.py
+++ b/tests/visualization_tests/test_pareto_front.py
@@ -111,7 +111,9 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
     assert figure.layout.yaxis.title.text == titles[actual_axis_order[1]]
 
     # Test with `target_names` argument.
-    with pytest.raises(ValueError):
+    error_message = "The length of `target_names` is supposed to be 2."
+
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=[],
@@ -119,7 +121,7 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo"],
@@ -127,7 +129,7 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar", "Baz"],
@@ -232,7 +234,9 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
     assert figure.layout.scene.zaxis.title.text == titles[actual_axis_order[2]]
 
     # Test with `target_names` argument.
-    with pytest.raises(ValueError):
+    error_message = "The length of `target_names` is supposed to be 3."
+
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=[],
@@ -241,7 +245,7 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo"],
@@ -250,7 +254,7 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar"],
@@ -259,7 +263,7 @@ def constraints_func(t: FrozenTrial) -> Sequence[float]:
             targets=targets,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         plot_pareto_front(
             study=study,
             target_names=["Foo", "Bar", "Baz", "Qux"],
@@ -282,8 +286,14 @@ def test_plot_pareto_front_unsupported_dimensions(
 ) -> None:
     constraints_func = (lambda x: [-1.0]) if use_constraints_func else None
 
+    error_message = (
+        "`plot_pareto_front` function only supports 2 or 3 objective"
+        " studies when using `targets` is `None`. Please use `targets`"
+        " if your objective studies have more than 3 objectives."
+    )
+
     # Unsupported: n_objectives == 1.
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(directions=["minimize"])
         study.optimize(lambda t: [0], n_trials=1)
         plot_pareto_front(
@@ -292,7 +302,7 @@ def test_plot_pareto_front_unsupported_dimensions(
             constraints_func=constraints_func,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(direction="minimize")
         study.optimize(lambda t: [0], n_trials=1)
         plot_pareto_front(
@@ -302,7 +312,7 @@ def test_plot_pareto_front_unsupported_dimensions(
         )
 
     # Unsupported: n_objectives == 4.
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=error_message):
         study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
         study.optimize(lambda t: [0, 0, 0, 0], n_trials=1)
         plot_pareto_front(
@@ -370,6 +380,71 @@ def test_plot_pareto_front_invalid_axis_order(
         )
 
 
+def test_plot_pareto_front_targets_without_target_names() -> None:
+    study = optuna.create_study(directions=["minimize", "minimize", "minimize"])
+    with pytest.raises(
+        ValueError,
+        match="If `targets` is specified for empty studies, `target_names` must be specified.",
+    ):
+        plot_pareto_front(
+            study=study,
+            target_names=None,
+            targets=lambda t: (t.values[0], t.values[1], t.values[2]),
+        )
+
+
+def test_plot_pareto_front_invalid_target_values() -> None:
+    study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
+    study.optimize(lambda t: [0, 0, 0, 0], n_trials=3)
+    with pytest.raises(
+        ValueError,
+        match="targets` should return a sequence of target values. your `targets`"
+        " returns <class 'float'>",
+    ):
+        plot_pareto_front(
+            study=study,
+            targets=lambda t: t.values[0],
+        )
+
+
+@pytest.mark.parametrize(
+    "targets",
+    [
+        lambda t: (t.values[0],),
+        lambda t: (t.values[0], t.values[1], t.values[2], t.values[3]),
+    ],
+)
+def test_plot_pareto_front_n_targets_unsupported(
+    targets: Callable[[FrozenTrial], Sequence[float]]
+) -> None:
+    study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
+    study.optimize(lambda t: [0, 0, 0, 0], n_trials=3)
+    n_targets = len(targets(study.best_trials[0]))
+    with pytest.raises(
+        ValueError,
+        match="`plot_pareto_front` function only supports 2 or 3 targets."
+        " you used {} targets now.".format(n_targets),
+    ):
+        plot_pareto_front(
+            study=study,
+            targets=targets,
+        )
+
+
+def test_plot_pareto_front_using_axis_order_and_targets() -> None:
+    study = optuna.create_study(directions=["minimize", "minimize", "minimize"])
+    with pytest.raises(
+        ValueError,
+        match="Using both `targets` and `axis_order` is not supported."
+        " Use either `targets` or `axis_order`.",
+    ):
+        plot_pareto_front(
+            study=study,
+            axis_order=[0, 1, 2],
+            targets=lambda t: (t.values[0], t.values[1], t.values[2]),
+        )
+
+
 def test_make_hovertext() -> None:
     trial_no_user_attrs = FrozenTrial(
         number=0,