Repository: optuna/optuna

Issue:
Support best value with constraint
### Motivation

Study's best_value (or best_xxx) will output a value that does not consider the constraint, even if they exist.
I want to output the best value that takes into account the constraints.

### Description

For example, best_trials gets COMPLETE trials and creates pareto front trials, but by making it trials that is COMPLETE and feasible, the best_trials including the constraint condition is obtained.
https://github.com/optuna/optuna/blob/b8ff10bae792d826898500f0e375e6cb2dcecc98/optuna/study/_multi_objective.py#L14

However, since there are multiple codes related to Study.best_xxx, is it possible to implement them?
Let me know if you have any concerns.

### Alternatives (optional)

_No response_

### Additional context (optional)

_No response_


Discussion:
Thanks for the suggestion. I think it is an important feature, but there are a couple of points that need to be discussed.

I have two concerns: first, the loss of backward compatibility with this improvement, and second, the difficulty of efficient implementation at the storage layer.

It is debatable, but since constrained optimization is an experimental feature, backward compatibility may not be too much of a concern. Also, efficiency can be ignored, at least in the current situation.

In my opinion, I am in favor of implementing this feature in the `Study`'s methods with `UserWarning`.
Thanks for the reply.
I will try to implement it and ask again if there are any questions.

Resolving patch:
diff --git a/optuna/study/_constrained_optimization.py b/optuna/study/_constrained_optimization.py
new file mode 100644
--- /dev/null
+++ b/optuna/study/_constrained_optimization.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from optuna.trial import FrozenTrial
+
+
+_CONSTRAINTS_KEY = "constraints"
+
+
+def _get_feasible_trials(trials: Sequence[FrozenTrial]) -> list[FrozenTrial]:
+    """Return feasible trials from given trials.
+
+    This function assumes that the trials were created in constrained optimization.
+    Therefore, if there is no violation value in the trial, it is considered infeasible.
+
+
+    Returns:
+        A list of feasible trials.
+    """
+
+    feasible_trials = []
+    for trial in trials:
+        constraints = trial.system_attrs.get(_CONSTRAINTS_KEY)
+        if constraints is not None and all(x <= 0.0 for x in constraints):
+            feasible_trials.append(trial)
+    return feasible_trials
diff --git a/optuna/study/_multi_objective.py b/optuna/study/_multi_objective.py
--- a/optuna/study/_multi_objective.py
+++ b/optuna/study/_multi_objective.py
@@ -5,23 +5,12 @@
 import numpy as np
 
 import optuna
+from optuna.study._constrained_optimization import _get_feasible_trials
 from optuna.study._study_direction import StudyDirection
 from optuna.trial import FrozenTrial
 from optuna.trial import TrialState
 
 
-_CONSTRAINTS_KEY = "constraints"
-
-
-def _get_feasible_trials(trials: Sequence[FrozenTrial]) -> list[FrozenTrial]:
-    feasible_trials = []
-    for trial in trials:
-        constraints = trial.system_attrs.get(_CONSTRAINTS_KEY)
-        if constraints is None or all([x <= 0.0 for x in constraints]):
-            feasible_trials.append(trial)
-    return feasible_trials
-
-
 def _get_pareto_front_trials_by_trials(
     trials: Sequence[FrozenTrial],
     directions: Sequence[StudyDirection],
diff --git a/optuna/study/study.py b/optuna/study/study.py
--- a/optuna/study/study.py
+++ b/optuna/study/study.py
@@ -30,6 +30,8 @@
 from optuna.distributions import _convert_old_distribution_to_new_distribution
 from optuna.distributions import BaseDistribution
 from optuna.storages._heartbeat import is_heartbeat_enabled
+from optuna.study._constrained_optimization import _CONSTRAINTS_KEY
+from optuna.study._constrained_optimization import _get_feasible_trials
 from optuna.study._multi_objective import _get_pareto_front_trials
 from optuna.study._optimize import _optimize
 from optuna.study._study_direction import StudyDirection
@@ -157,7 +159,23 @@ def best_trial(self) -> FrozenTrial:
                 "using Study.best_trials to retrieve a list containing the best trials."
             )
 
-        return copy.deepcopy(self._storage.get_best_trial(self._study_id))
+        best_trial = self._storage.get_best_trial(self._study_id)
+
+        # If the trial with the best value is infeasible, select the best trial from all feasible
+        # trials. Note that the behavior is undefined when constrained optimization without the
+        # violation value in the best-valued trial.
+        constraints = best_trial.system_attrs.get(_CONSTRAINTS_KEY)
+        if constraints is not None and any([x > 0.0 for x in constraints]):
+            complete_trials = self.get_trials(deepcopy=False, states=[TrialState.COMPLETE])
+            feasible_trials = _get_feasible_trials(complete_trials)
+            if len(feasible_trials) == 0:
+                raise ValueError("No feasible trials are completed yet.")
+            if self.direction == StudyDirection.MAXIMIZE:
+                best_trial = max(feasible_trials, key=lambda t: cast(float, t.value))
+            else:
+                best_trial = min(feasible_trials, key=lambda t: cast(float, t.value))
+
+        return copy.deepcopy(best_trial)
 
     @property
     def best_trials(self) -> list[FrozenTrial]:
@@ -172,7 +190,11 @@ def best_trials(self) -> list[FrozenTrial]:
             A list of :class:`~optuna.trial.FrozenTrial` objects.
         """
 
-        return _get_pareto_front_trials(self)
+        # Check whether the study is constrained optimization.
+        trials = self.get_trials(deepcopy=False)
+        is_constrained = any((_CONSTRAINTS_KEY in trial.system_attrs) for trial in trials)
+
+        return _get_pareto_front_trials(self, consider_constraint=is_constrained)
 
     @property
     def direction(self) -> StudyDirection:


Test patch:
diff --git a/tests/study_tests/test_constrained_optimization.py b/tests/study_tests/test_constrained_optimization.py
new file mode 100644
--- /dev/null
+++ b/tests/study_tests/test_constrained_optimization.py
@@ -0,0 +1,13 @@
+from optuna.study._constrained_optimization import _CONSTRAINTS_KEY
+from optuna.study._constrained_optimization import _get_feasible_trials
+from optuna.trial import create_trial
+
+
+def test_get_feasible_trials() -> None:
+    trials = []
+    trials.append(create_trial(value=0.0, system_attrs={_CONSTRAINTS_KEY: [0.0]}))
+    trials.append(create_trial(value=0.0, system_attrs={_CONSTRAINTS_KEY: [1.0]}))
+    trials.append(create_trial(value=0.0))
+    feasible_trials = _get_feasible_trials(trials)
+    assert len(feasible_trials) == 1
+    assert feasible_trials[0] == trials[0]
diff --git a/tests/study_tests/test_study.py b/tests/study_tests/test_study.py
--- a/tests/study_tests/test_study.py
+++ b/tests/study_tests/test_study.py
@@ -33,6 +33,7 @@
 from optuna.exceptions import DuplicatedStudyError
 from optuna.exceptions import ExperimentalWarning
 from optuna.study import StudyDirection
+from optuna.study._constrained_optimization import _CONSTRAINTS_KEY
 from optuna.study.study import _SYSTEM_ATTR_METRIC_NAMES
 from optuna.testing.objectives import fail_objective
 from optuna.testing.storages import STORAGE_MODES
@@ -1166,6 +1167,38 @@ def objective(trial: Trial) -> list[float]:
         assert len(trial.values) == n_objectives
 
 
+@pytest.mark.parametrize("direction", [StudyDirection.MINIMIZE, StudyDirection.MAXIMIZE])
+def test_best_trial_constrained_optimization(direction: StudyDirection) -> None:
+    study = create_study(direction=direction)
+    storage = study._storage
+
+    with pytest.raises(ValueError):
+        # No trials.
+        study.best_trial
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [1])
+    study.tell(trial, 0)
+    with pytest.raises(ValueError):
+        # No feasible trials.
+        study.best_trial
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [0])
+    study.tell(trial, 0)
+    assert study.best_trial.number == 1
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [1])
+    study.tell(trial, -1 if direction == StudyDirection.MINIMIZE else 1)
+    assert study.best_trial.number == 1
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [0])
+    study.tell(trial, -1 if direction == StudyDirection.MINIMIZE else 1)
+    assert study.best_trial.number == 3
+
+
 def test_best_trials() -> None:
     study = create_study(directions=["minimize", "maximize"])
     study.optimize(lambda t: [2, 2], n_trials=1)
@@ -1174,6 +1207,33 @@ def test_best_trials() -> None:
     assert {tuple(t.values) for t in study.best_trials} == {(1, 1), (2, 2)}
 
 
+def test_best_trials_constrained_optimization() -> None:
+    study = create_study(directions=["minimize", "maximize"])
+    storage = study._storage
+
+    assert study.best_trials == []
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [1])
+    study.tell(trial, [0, 0])
+    assert study.best_trials == []
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [0])
+    study.tell(trial, [0, 0])
+    assert study.best_trials == [study.trials[1]]
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [1])
+    study.tell(trial, [-1, 1])
+    assert study.best_trials == [study.trials[1]]
+
+    trial = study.ask()
+    storage.set_trial_system_attr(trial._trial_id, _CONSTRAINTS_KEY, [0])
+    study.tell(trial, [1, 1])
+    assert {t.number for t in study.best_trials} == {1, 3}
+
+
 def test_wrong_n_objectives() -> None:
     n_objectives = 2
     directions = ["minimize" for _ in range(n_objectives)]