Repository: optuna/optuna

Issue:
Support aliases for learning-to-rank metrics in LightGBM Tuner
In LightGBM, `ndcg` and `map` have many aliases. As for `ndcg`, aliases are `lambdarank`, `rank_xendcg`, `xendcg`, `xe_ndcg`, `xe_ndcg_mart`, and `xendcg_mart`. They all work on the original LightGBM. However, LightGBM Tuner in Optuna doesn't support them.

In addition, the metric `accuracy` coded in LightGBM Tuner doesn't exist in the original LightGBM. This should be removed.

## Expected behavior

Based on @toshihikoyanase's code ( https://colab.research.google.com/drive/1NH3CtNuo7hsZWi4u7E1YOPHKWyP-7x0o#scrollTo=IfrDKfJFRfHv), the following code should work:

```python
# import lightgbm as lgb
import optuna.integration.lightgbm as lgb

dtrain = lgb.Dataset('rank.train')
dval = lgb.Dataset('rank.test')

params = {
    'objective': 'lambdarank',
    'metric': 'lambdarank',  # alias to ndcg.
    'eval_at': 3,
    'boosting_type': 'gbdt',
}

model = lgb.train(params,
                  dtrain,
                  valid_sets=[dtrain, dval],
                  verbose_eval=True,
                  early_stopping_rounds=2,
                )
```

## Environment

- Optuna version: v1.1.0
- Python version: 3.7.6
- OS: Linux 4.4.0-87-generic
- LightGBM version: 2.3.1

## Error messages, stack traces, or logs

```
/home/marzio/gitws/optuna/optuna/_experimental.py:87: ExperimentalWarning: train is experimental (supported from v0.18.0). The interface can change in the future
.
  ExperimentalWarning
tune_feature_fraction, val_score: inf:   0%|                                                                                               | 0/7 [00:00<?, ?it/s]
[1]     training's ndcg@10: 0.812671    training's ndcg@20: 0.865182    valid_1's ndcg@10: 0.653369     valid_1's ndcg@20: 0.738632
Training until validation scores don't improve for 2 rounds
[2]     training's ndcg@10: 0.856561    training's ndcg@20: 0.907013    valid_1's ndcg@10: 0.683942     valid_1's ndcg@20: 0.774721
[3]     training's ndcg@10: 0.876966    training's ndcg@20: 0.925433    valid_1's ndcg@10: 0.700149     valid_1's ndcg@20: 0.782201
[4]     training's ndcg@10: 0.89233     training's ndcg@20: 0.93465     valid_1's ndcg@10: 0.70814      valid_1's ndcg@20: 0.785663
[5]     training's ndcg@10: 0.900897    training's ndcg@20: 0.940685    valid_1's ndcg@10: 0.715149     valid_1's ndcg@20: 0.793221
[6]     training's ndcg@10: 0.910312    training's ndcg@20: 0.946929    valid_1's ndcg@10: 0.723119     valid_1's ndcg@20: 0.799063
[7]     training's ndcg@10: 0.917917    training's ndcg@20: 0.953784    valid_1's ndcg@10: 0.719499     valid_1's ndcg@20: 0.799715
[8]     training's ndcg@10: 0.921102    training's ndcg@20: 0.955849    valid_1's ndcg@10: 0.717558     valid_1's ndcg@20: 0.794649
Early stopping, best iteration is:
[6]     training's ndcg@10: 0.910312    training's ndcg@20: 0.946929    valid_1's ndcg@10: 0.723119     valid_1's ndcg@20: 0.799063
[W 2020-02-26 14:17:40,334] Setting status of trial#0 as TrialState.FAIL because of the following error: KeyError('lambdarank')
Traceback (most recent call last):
  File "/home/marzio/gitws/optuna/optuna/study.py", line 648, in _run_trial
    result = func(trial)
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 269, in __call__
    val_score = self._get_booster_best_score(booster)
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 146, in _get_booster_best_score
    val_score = booster.best_score[valid_name][metric]
KeyError: 'lambdarank'
Traceback (most recent call last):
  File "example_lambdarank.py", line 22, in <module>
    early_stopping_rounds=2,
  File "/home/marzio/gitws/optuna/optuna/_experimental.py", line 90, in new_func
    return func(*args, **kwargs)  # type: ignore
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/__init__.py", line 31, in train
    booster = auto_booster.run()
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 408, in run
    self.tune_feature_fraction()
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 454, in tune_feature_fraction
    self.tune_params([param_name], len(param_values), sampler)
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 516, in tune_params
    study.optimize(objective, n_trials=n_trials, catch=())
  File "/home/marzio/gitws/optuna/optuna/study.py", line 319, in optimize
    gc_after_trial, None)
  File "/home/marzio/gitws/optuna/optuna/study.py", line 597, in _optimize_sequential
    self._run_trial_and_callbacks(func, catch, callbacks, gc_after_trial)
  File "/home/marzio/gitws/optuna/optuna/study.py", line 627, in _run_trial_and_callbacks
    trial = self._run_trial(func, catch, gc_after_trial)
  File "/home/marzio/gitws/optuna/optuna/study.py", line 648, in _run_trial
    result = func(trial)
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 269, in __call__
    val_score = self._get_booster_best_score(booster)
  File "/home/marzio/gitws/optuna/optuna/integration/lightgbm_tuner/optimize.py", line 146, in _get_booster_best_score
    val_score = booster.best_score[valid_name][metric]
KeyError: 'lambdarank'
tune_feature_fraction, val_score: inf:   0%|                                                                                               | 0/7 [00:03<?, ?it/s]
```



Discussion:

Resolving patch:
diff --git a/optuna/integration/lightgbm_tuner/alias.py b/optuna/integration/lightgbm_tuner/alias.py
--- a/optuna/integration/lightgbm_tuner/alias.py
+++ b/optuna/integration/lightgbm_tuner/alias.py
@@ -59,3 +59,33 @@ def _handling_alias_parameters(lgbm_params):
             if alias_name in lgbm_params:
                 lgbm_params[param_name] = lgbm_params[alias_name]
                 del lgbm_params[alias_name]
+
+
+ALIAS_METRIC_LIST = [
+    {
+        'metric_name': 'ndcg',
+        'alias_names': ['lambdarank', 'rank_xendcg', 'xendcg', 'xe_ndcg',
+                        'xe_ndcg_mart', 'xendcg_mart'],
+    },
+    {
+        'metric_name': 'map',
+        'alias_names': ['mean_average_precision'],
+    },
+]  # type: List[Dict[str, Any]]
+
+
+def _handling_alias_metrics(lgbm_params):
+    # type: (Dict[str, Any]) -> None
+    """Handling alias metrics."""
+
+    if 'metric' not in lgbm_params.keys():
+        return
+
+    for metric in ALIAS_METRIC_LIST:
+        metric_name = metric['metric_name']
+        alias_names = metric['alias_names']
+
+        for alias_name in alias_names:
+            if lgbm_params['metric'] == alias_name:
+                lgbm_params['metric'] = metric_name
+                break
diff --git a/optuna/integration/lightgbm_tuner/optimize.py b/optuna/integration/lightgbm_tuner/optimize.py
--- a/optuna/integration/lightgbm_tuner/optimize.py
+++ b/optuna/integration/lightgbm_tuner/optimize.py
@@ -7,6 +7,7 @@
 import tqdm
 
 import optuna
+from optuna.integration.lightgbm_tuner.alias import _handling_alias_metrics
 from optuna.integration.lightgbm_tuner.alias import _handling_alias_parameters
 from optuna import type_checking
 
@@ -105,6 +106,10 @@ def __init__(
     ):
         # type: (Dict[str, Any], Dict[str,Any]) -> None
 
+        # Handling alias metrics.
+        if lgbm_params is not None:
+            _handling_alias_metrics(lgbm_params)
+
         self.lgbm_params = lgbm_params or {}
         self.lgbm_kwargs = lgbm_kwargs or {}
 
@@ -174,7 +179,7 @@ def higher_is_better(self):
         # type: () -> bool
 
         metric_name = self.lgbm_params.get('metric', 'binary_logloss')
-        return metric_name.startswith(('auc', 'ndcg', 'map', 'accuracy'))
+        return metric_name.startswith(('auc', 'ndcg', 'map'))
 
     def compare_validation_metrics(self, val_score, best_score):
         # type: (float, float) -> bool
@@ -317,6 +322,10 @@ def __init__(
             verbosity=1,  # type: Optional[int]
     ):
         params = copy.deepcopy(params)
+
+        # Handling alias metrics.
+        _handling_alias_metrics(params)
+
         args = [params, train_set]
         kwargs = dict(num_boost_round=num_boost_round,
                       valid_sets=valid_sets,


Test patch:
diff --git a/tests/integration_tests/lightgbm_tuner_tests/test_alias.py b/tests/integration_tests/lightgbm_tuner_tests/test_alias.py
--- a/tests/integration_tests/lightgbm_tuner_tests/test_alias.py
+++ b/tests/integration_tests/lightgbm_tuner_tests/test_alias.py
@@ -1,3 +1,4 @@
+from optuna.integration.lightgbm_tuner.alias import _handling_alias_metrics
 from optuna.integration.lightgbm_tuner.alias import _handling_alias_parameters
 
 
@@ -37,3 +38,30 @@ def test_handling_alias_parameter():
     assert 'min_data' not in params
     assert 'min_data_in_leaf' in params
     assert params['min_data_in_leaf'] == 0.2
+
+
+def test_handling_alias_metrics():
+    # type: () -> None
+
+    for alias in ['lambdarank', 'rank_xendcg', 'xendcg', 'xe_ndcg',
+                  'xe_ndcg_mart', 'xendcg_mart', 'ndcg']:
+        lgbm_params = {'metric': alias}
+        _handling_alias_metrics(lgbm_params)
+        assert lgbm_params['metric'] == 'ndcg'
+
+    for alias in ['mean_average_precision', 'map']:
+        lgbm_params = {'metric': alias}
+        _handling_alias_metrics(lgbm_params)
+        assert lgbm_params['metric'] == 'map'
+
+    lgbm_params = {}
+    _handling_alias_metrics(lgbm_params)
+    assert lgbm_params == {}
+
+    lgbm_params = {'metric': 'auc'}
+    _handling_alias_metrics(lgbm_params)
+    assert lgbm_params['metric'] == 'auc'
+
+    lgbm_params = {'metric': 'rmse'}
+    _handling_alias_metrics(lgbm_params)
+    assert lgbm_params['metric'] == 'rmse'
diff --git a/tests/integration_tests/lightgbm_tuner_tests/test_optimize.py b/tests/integration_tests/lightgbm_tuner_tests/test_optimize.py
--- a/tests/integration_tests/lightgbm_tuner_tests/test_optimize.py
+++ b/tests/integration_tests/lightgbm_tuner_tests/test_optimize.py
@@ -129,7 +129,9 @@ def __init__(self):
     def test_higher_is_better(self):
         # type: () -> None
 
-        for metric in ['auc', 'ndcg', 'map', 'accuracy']:
+        for metric in ['auc', 'ndcg', 'lambdarank', 'rank_xendcg', 'xendcg',
+                       'xe_ndcg', 'xe_ndcg_mart', 'xendcg_mart', 'map',
+                       'mean_average_precision']:
             tuner = BaseTuner(lgbm_params={'metric': metric})
             assert tuner.higher_is_better()
 
@@ -189,7 +191,9 @@ def __init__(self):
     def test_compare_validation_metrics(self):
         # type: () -> None
 
-        for metric in ['auc', 'ndcg', 'map', 'accuracy']:
+        for metric in ['auc', 'ndcg', 'lambdarank', 'rank_xendcg', 'xendcg',
+                       'xe_ndcg', 'xe_ndcg_mart', 'xendcg_mart', 'map',
+                       'mean_average_precision']:
             tuner = BaseTuner(lgbm_params={'metric': metric})
             assert tuner.compare_validation_metrics(0.5, 0.1)
             assert not tuner.compare_validation_metrics(0.5, 0.5)