Repository: optuna/optuna

Issue:
[Proposal] Add integration with Tensorboard
<!-- Please write a clear and concise description of the feature proposal. -->

## Motivation
Tensorboard provides [HParams dashboard](https://www.tensorflow.org/tensorboard/hyperparameter_tuning_with_hparams) which is useful for identifying most promising sets of hyperparameters. It would be nice add custom module to optuna for writing experiment results to tensorboard.

<!-- Please write the motivation for the proposal.

If your feature request is related to a problem, please describe a clear and concise description of what the problem is. -->

## Description
We can add a callback, which writes experiment results to tensorboard. Below is an example of tuning number of units in a hidden layer, dropout rate and optimizer for a small network on fashion MNIST.

```
import optuna

import tensorflow as tf
from tensorboard.plugins.hparams import api as hp

fashion_mnist = tf.keras.datasets.fashion_mnist

(x_train, y_train),(x_test, y_test) = fashion_mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([16, 32]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd']))

METRIC_ACCURACY = 'accuracy'

with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
    hp.hparams_config(
        hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER],
        metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
    )

def train_test_model(hparams):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.relu),
        tf.keras.layers.Dropout(hparams[HP_DROPOUT]),
        tf.keras.layers.Dense(10, activation=tf.nn.softmax),
    ])
    model.compile(
        optimizer=hparams[HP_OPTIMIZER],
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy'],
    )

    model.fit(x_train, y_train, epochs=1) # Run with 1 epoch to speed things up for demo purposes
    _, accuracy = model.evaluate(x_test, y_test)
    return accuracy

def run(run_dir, hparams):
    with tf.summary.create_file_writer(run_dir).as_default():
        hp.hparams(hparams)  # record the values used in this trial
        accuracy = train_test_model(hparams)
        tf.summary.scalar(METRIC_ACCURACY, accuracy, step=1)

session_num = 0

def tensorboard_callback(study, trial):
    trial_value = trial.value if trial.value is not None else float("nan")

    num_units = trial.params['NUM_UNITS']
    dropout_rate = trial.params['DROPOUT_RATE']
    optimizer = trial.params['OPTIMIZER']

    hparams = {
        HP_NUM_UNITS: num_units,
        HP_DROPOUT: dropout_rate,
        HP_OPTIMIZER: optimizer,
    }

    global session_num
    run_name = "run-%d" % session_num
    run_dir = 'logs_optuna/hparam_tuning/' + run_name
    #run_dir = 'logs_optuna/hparam_tuning/'

    with tf.summary.create_file_writer(run_dir).as_default():
        hp.hparams(hparams)  # record the values used in this trial
        tf.summary.scalar(METRIC_ACCURACY, trial_value, step=1)
    session_num += 1

def objective(trial):

    num_units = trial.suggest_int('NUM_UNITS', HP_NUM_UNITS.domain.values[0], HP_NUM_UNITS.domain.values[1])
    dropout_rate = trial.suggest_uniform('DROPOUT_RATE', HP_DROPOUT.domain.min_value, HP_DROPOUT.domain.max_value)
    optimizer = trial.suggest_categorical('OPTIMIZER', HP_OPTIMIZER.domain.values)

    hparams = {
        HP_NUM_UNITS: num_units,
        HP_DROPOUT: dropout_rate,
        HP_OPTIMIZER: optimizer,
    }

    accuracy = train_test_model(hparams)
    return accuracy

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=10, timeout=600, callbacks=[tensorboard_callback])

```
Here is the result in Tensorboard HParams dashboard
![image](https://user-images.githubusercontent.com/10694872/77823362-ec7d7000-70fa-11ea-81a3-380c9f55bf35.png)


<!-- Please write a detailed description of the new feature. -->



Discussion:
This issue has not seen any recent activity.

Resolving patch:
diff --git a/examples/tensorboard_simple.py b/examples/tensorboard_simple.py
new file mode 100644
--- /dev/null
+++ b/examples/tensorboard_simple.py
@@ -0,0 +1,42 @@
+import optuna
+from optuna.integration.tensorboard import TensorBoardCallback
+import tensorflow as tf
+
+
+fashion_mnist = tf.keras.datasets.fashion_mnist
+
+(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
+x_train, x_test = x_train / 255.0, x_test / 255.0
+
+
+def train_test_model(num_units: int, dropout_rate: float, optimizer: str) -> float:
+    model = tf.keras.models.Sequential(
+        [
+            tf.keras.layers.Flatten(),
+            tf.keras.layers.Dense(num_units, activation=tf.nn.relu),
+            tf.keras.layers.Dropout(dropout_rate),
+            tf.keras.layers.Dense(10, activation=tf.nn.softmax),
+        ]
+    )
+    model.compile(
+        optimizer=optimizer, loss="sparse_categorical_crossentropy", metrics=["accuracy"],
+    )
+
+    model.fit(x_train, y_train, epochs=1)  # Run with 1 epoch to speed things up for demo purposes
+    _, accuracy = model.evaluate(x_test, y_test)
+    return accuracy
+
+
+def objective(trial: optuna.trial.Trial) -> float:
+    num_units = trial.suggest_int("NUM_UNITS", 16, 32)
+    dropout_rate = trial.suggest_uniform("DROPOUT_RATE", 0.1, 0.2)
+    optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"])
+
+    accuracy = train_test_model(num_units, dropout_rate, optimizer)  # type: ignore
+    return accuracy
+
+
+tensorboard_callback = TensorBoardCallback("logs/", metric_name="accuracy")
+
+study = optuna.create_study(direction="maximize")
+study.optimize(objective, n_trials=10, timeout=600, callbacks=[tensorboard_callback])
diff --git a/optuna/integration/tensorboard.py b/optuna/integration/tensorboard.py
new file mode 100644
--- /dev/null
+++ b/optuna/integration/tensorboard.py
@@ -0,0 +1,92 @@
+import os
+from typing import Dict
+
+import optuna
+from optuna._imports import try_import
+
+with try_import() as _imports:
+    from tensorboard.plugins.hparams import api as hp
+    import tensorflow as tf
+
+
+class TensorBoardCallback(object):
+    """Callback to track Optuna trials with TensorBoard.
+
+    This callback adds relevant information that is tracked by Optuna to TensorBoard.
+    Args:
+        dirname:
+            Directory to store TensorBoard logs.
+        metric_name:
+            Name of the metric. Since the metric itself is just a number,
+            `metric_name` can be used to give it a name. So you know later
+            if it was roc-auc or accuracy.
+    """
+
+    def __init__(self, dirname: str, metric_name: str) -> None:
+        _imports.check()
+        self._dirname = dirname
+        self._metric_name = metric_name
+        self._hp_params = dict()  # type: Dict[str, hp.HParam]
+
+    def __call__(self, study: optuna.study.Study, trial: optuna.trial.FrozenTrial) -> None:
+        if len(self._hp_params) == 0:
+            self._initialization(study)
+        trial_value = trial.value if trial.value is not None else float("nan")
+        hparams = dict()
+        for param_name, param_value in trial.params.items():
+            if param_name not in self._hp_params:
+                self._add_distributions(trial.distributions)
+            hparams[self._hp_params[param_name]] = param_value
+        run_name = "trial-%d" % trial.number
+        run_dir = os.path.join(self._dirname, run_name)
+        with tf.summary.create_file_writer(run_dir).as_default():
+            hp.hparams(hparams)  # record the values used in this trial
+            tf.summary.scalar(self._metric_name, trial_value, step=1)
+
+    def _add_distributions(
+        self, distributions: Dict[str, optuna.distributions.BaseDistribution]
+    ) -> None:
+        for param_name, param_distribution in distributions.items():
+            if isinstance(param_distribution, optuna.distributions.UniformDistribution):
+                self._hp_params[param_name] = hp.HParam(
+                    param_name, hp.RealInterval(param_distribution.low, param_distribution.high)
+                )
+            elif isinstance(param_distribution, optuna.distributions.LogUniformDistribution):
+                self._hp_params[param_name] = hp.HParam(
+                    param_name, hp.RealInterval(param_distribution.low, param_distribution.high)
+                )
+            elif isinstance(param_distribution, optuna.distributions.DiscreteUniformDistribution):
+                self._hp_params[param_name] = hp.HParam(
+                    param_name, hp.Discrete(param_distribution.low, param_distribution.high)
+                )
+            elif isinstance(param_distribution, optuna.distributions.IntUniformDistribution):
+                self._hp_params[param_name] = hp.HParam(
+                    param_name, hp.IntInterval(param_distribution.low, param_distribution.high)
+                )
+            elif isinstance(param_distribution, optuna.distributions.CategoricalDistribution):
+                self._hp_params[param_name] = hp.HParam(
+                    param_name, hp.Discrete(param_distribution.choices)
+                )
+            else:
+                distribution_list = [
+                    optuna.distributions.UniformDistribution.__name__,
+                    optuna.distributions.LogUniformDistribution.__name__,
+                    optuna.distributions.DiscreteUniformDistribution.__name__,
+                    optuna.distributions.IntUniformDistribution.__name__,
+                    optuna.distributions.CategoricalDistribution.__name__,
+                ]
+                raise NotImplementedError(
+                    "The distribution {} is not implemented. "
+                    "The parameter distribution should be one of the {}".format(
+                        param_distribution, distribution_list
+                    )
+                )
+
+    def _initialization(self, study: optuna.Study) -> None:
+        completed_trials = [
+            trial
+            for trial in study.get_trials(deepcopy=False)
+            if trial.state == optuna.trial.TrialState.COMPLETE
+        ]
+        for trial in completed_trials:
+            self._add_distributions(trial.distributions)


Test patch:
diff --git a/tests/integration_tests/test_tensorboard.py b/tests/integration_tests/test_tensorboard.py
new file mode 100644
--- /dev/null
+++ b/tests/integration_tests/test_tensorboard.py
@@ -0,0 +1,44 @@
+import os
+import pytest
+import shutil
+import sys
+import tempfile
+
+import optuna
+from optuna._imports import try_import
+from optuna.integration.tensorboard import TensorBoardCallback
+
+with try_import() as _imports:
+    from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
+
+
+def _objective_func(trial: optuna.trial.Trial) -> float:
+    x = trial.suggest_uniform("x", -1.0, 1.0)
+    y = trial.suggest_loguniform("y", 20.0, 30.0)
+    z = trial.suggest_categorical("z", (-1.0, 1.0))
+    assert isinstance(z, float)
+    trial.set_user_attr("my_user_attr", "my_user_attr_value")
+    return (x - 2) ** 2 + (y - 25) ** 2 + z
+
+
+@pytest.mark.skipif(
+    sys.version_info > (3, 7), reason="Tensorflow is not installed for python 3.8."
+)
+def test_study_name() -> None:
+    dirname = tempfile.mkdtemp()
+    metric_name = "target"
+    study_name = "test_tensorboard_integration"
+
+    tbcallback = TensorBoardCallback(dirname, metric_name)
+    study = optuna.create_study(study_name=study_name)
+    study.optimize(_objective_func, n_trials=1, callbacks=[tbcallback])
+
+    event_acc = EventAccumulator(os.path.join(dirname, "trial-0"))
+    event_acc.Reload()
+
+    try:
+        assert len(event_acc.Tensors("target")) == 1
+    except Exception as e:
+        raise e
+    finally:
+        shutil.rmtree(dirname)