Repository: ray-project/ray

Issue:
[tune] Reporter fails with nested config keys and {'done': True}
### What is the problem?
When a `Trainable` reports `{'done': True}` with some other nested values, the tensorboard logger fails with `NotImplementedError: Got <class 'dict'>, but expected numpy array or torch tensor.`. The same report works for all the preceding logs that don't include `{'done': True}` though.

*Ray version and other system information (Python version, TensorFlow version, OS):* `0.8.2`

### Reproduction (REQUIRED)
```python
from ray import tune


class MyTrainableClass(tune.Trainable):
    def _setup(self, config):
        self.timestep = 0

    def _train(self):
        self.timestep += 1
        result = {"episode_reward_mean": self.timestep}
        result['what'] = {'1': '2', '3': 4, '5': {'6': 4}}
        if self.timestep == 3:
            result['done'] = True
        return result


tune.run(
    MyTrainableClass,
    name="done-test",
    num_samples=1,
    config={
        'b': {'c': {'d': '4'}},
        'a': {
            'b': tune.sample_from(lambda spec: spec['config']['b']['c']),
        },
    })
```

Fails with:
```
$ python ./tests/tune_done_test.py
2020-03-22 12:17:21,455 INFO resource_spec.py:212 -- Starting Ray with 10.3 GiB memory available for workers and up to 5.17 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
2020-03-22 12:17:21,746 INFO services.py:1078 -- View the Ray dashboard at localhost:8265
== Status ==
Memory usage on this node: 17.8/32.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1/16 CPUs, 0/0 GPUs, 0.0/10.3 GiB heap, 0.0/3.56 GiB objects
Result logdir: /Users/hartikainen/ray_results/done-test
Number of trials: 1 (1 RUNNING)
+---------------------------+----------+-------+-------+
| Trial name                | status   | loc   | a/b   |
|---------------------------+----------+-------+-------|
| MyTrainableClass_14fda478 | RUNNING  |       |       |
+---------------------------+----------+-------+-------+


Result for MyTrainableClass_14fda478:
  date: 2020-03-22_12-17-23
  done: false
  episode_reward_mean: 1
  experiment_id: 3549c7334c884d3abc309232da4f6679
  experiment_tag: '0_b={''d'': ''4''}'
  hostname: catz-48fe.stcatz.ox.ac.uk
  iterations_since_restore: 1
  node_ip: 129.67.48.254
  pid: 73820
  time_since_restore: 3.0994415283203125e-06
  time_this_iter_s: 3.0994415283203125e-06
  time_total_s: 3.0994415283203125e-06
  timestamp: 1584879443
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: 14fda478
  what:
    '1': '2'
    '3': 4
    '5':
      '6': 4

Result for MyTrainableClass_14fda478:
  date: 2020-03-22_12-17-23
  done: true
  episode_reward_mean: 3
  experiment_id: 3549c7334c884d3abc309232da4f6679
  experiment_tag: '0_b={''d'': ''4''}'
  hostname: catz-48fe.stcatz.ox.ac.uk
  iterations_since_restore: 3
  node_ip: 129.67.48.254
  pid: 73820
  time_since_restore: 9.775161743164062e-06
  time_this_iter_s: 2.86102294921875e-06
  time_total_s: 9.775161743164062e-06
  timestamp: 1584879443
  timesteps_since_restore: 0
  training_iteration: 3
  trial_id: 14fda478
  what:
    '1': '2'
    '3': 4
    '5':
      '6': 4

2020-03-22 12:17:23,204 ERROR trial_runner.py:513 -- Trial MyTrainableClass_14fda478: Error processing event.
Traceback (most recent call last):
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 511, in _process_trial
    self._execute_action(trial, decision)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 595, in _execute_action
    self.trial_executor.stop_trial(trial)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 263, in stop_trial
    trial, error=error, error_msg=error_msg, stop_logger=stop_logger)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 204, in _stop_trial
    trial.close_logger()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial.py", line 315, in close_logger
    self.result_logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 305, in close
    _logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 233, in close
    self._try_log_hparams(self.last_result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 244, in _try_log_hparams
    hparam_dict=scrubbed_params, metric_dict=result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/summary.py", line 102, in hparams
    v = make_np(v)[0]
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/x2num.py", line 34, in make_np
    'Got {}, but expected numpy array or torch tensor.'.format(type(x)))
NotImplementedError: Got <class 'dict'>, but expected numpy array or torch tensor.
Traceback (most recent call last):
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 511, in _process_trial
    self._execute_action(trial, decision)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 595, in _execute_action
    self.trial_executor.stop_trial(trial)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 263, in stop_trial
    trial, error=error, error_msg=error_msg, stop_logger=stop_logger)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 204, in _stop_trial
    trial.close_logger()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial.py", line 315, in close_logger
    self.result_logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 305, in close
    _logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 233, in close
    self._try_log_hparams(self.last_result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 244, in _try_log_hparams
    hparam_dict=scrubbed_params, metric_dict=result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/summary.py", line 102, in hparams
    v = make_np(v)[0]
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/x2num.py", line 34, in make_np
    'Got {}, but expected numpy array or torch tensor.'.format(type(x)))
NotImplementedError: Got <class 'dict'>, but expected numpy array or torch tensor.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "./tests/tune_done_test.py", line 24, in <module>
    'b': tune.sample_from(lambda spec: spec['config']['b']['c']),
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/tune.py", line 324, in run
    runner.step()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 335, in step
    self._process_events()  # blocking
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 444, in _process_events
    self._process_trial(trial)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 514, in _process_trial
    self._process_trial_failure(trial, traceback.format_exc())
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 580, in _process_trial_failure
    trial, error=True, error_msg=error_msg)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 263, in stop_trial
    trial, error=error, error_msg=error_msg, stop_logger=stop_logger)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 204, in _stop_trial
    trial.close_logger()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial.py", line 315, in close_logger
    self.result_logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 305, in close
    _logger.close()
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 233, in close
    self._try_log_hparams(self.last_result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/logger.py", line 244, in _try_log_hparams
    hparam_dict=scrubbed_params, metric_dict=result)
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/summary.py", line 102, in hparams
    v = make_np(v)[0]
  File "/Users/hartikainen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/tensorboardX/x2num.py", line 34, in make_np
    'Got {}, but expected numpy array or torch tensor.'.format(type(x)))
NotImplementedError: Got <class 'dict'>, but expected numpy array or torch tensor.
```

- [x] I have verified my script runs in a clean environment and reproduces the issue.
- [x] I have verified the issue also occurs with the [latest wheels](https://ray.readthedocs.io/en/latest/installation.html).



Discussion:
Actually, this seems to be related to the nested config. If the above script is run with:
```
'b': tune.sample_from(lambda spec: spec['config']['b']['c']['d'])
```
instead of
```
'b': tune.sample_from(lambda spec: spec['config']['b']['c']),
```
it finishes as expected.

Resolving patch:
diff --git a/python/ray/tune/logger.py b/python/ray/tune/logger.py
--- a/python/ray/tune/logger.py
+++ b/python/ray/tune/logger.py
@@ -239,9 +239,10 @@ def flush(self):
     def close(self):
         if self._file_writer is not None:
             if self.trial and self.trial.evaluated_params and self.last_result:
+                flat_result = flatten_dict(self.last_result, delimiter="/")
                 scrubbed_result = {
                     k: value
-                    for k, value in self.last_result.items()
+                    for k, value in flat_result.items()
                     if type(value) in VALID_SUMMARY_TYPES
                 }
                 self._try_log_hparams(scrubbed_result)
@@ -249,9 +250,10 @@ def close(self):
 
     def _try_log_hparams(self, result):
         # TBX currently errors if the hparams value is None.
+        flat_params = flatten_dict(self.trial.evaluated_params)
         scrubbed_params = {
             k: v
-            for k, v in self.trial.evaluated_params.items() if v is not None
+            for k, v in flat_params.items() if v is not None
         }
         from tensorboardX.summary import hparams
         experiment_tag, session_start_tag, session_end_tag = hparams(
diff --git a/python/ray/tune/trainable.py b/python/ray/tune/trainable.py
--- a/python/ray/tune/trainable.py
+++ b/python/ray/tune/trainable.py
@@ -178,7 +178,7 @@ def __init__(self, config=None, logger_creator=None):
                         "slow to initialize, consider setting "
                         "reuse_actors=True to reduce actor creation "
                         "overheads.".format(setup_time))
-        self._local_ip = ray.services.get_node_ip_address()
+        self._local_ip = self.get_current_ip()
         log_sys_usage = self.config.get("log_sys_usage", False)
         self._monitor = UtilMonitor(start=log_sys_usage)
 
@@ -213,7 +213,7 @@ def resource_help(cls, config):
         """
         return ""
 
-    def current_ip(self):
+    def get_current_ip(self):
         logger.info("Getting current IP.")
         self._local_ip = ray.services.get_node_ip_address()
         return self._local_ip
@@ -419,8 +419,8 @@ def restore(self, checkpoint_path):
         self._timesteps_since_restore = 0
         self._iterations_since_restore = 0
         self._restored = True
-        logger.info("Restored on %s from checkpoint: %s", self.current_ip(),
-                    checkpoint_path)
+        logger.info("Restored on %s from checkpoint: %s",
+                    self.get_current_ip(), checkpoint_path)
         state = {
             "_iteration": self._iteration,
             "_timesteps_total": self._timesteps_total,


Test patch:
diff --git a/python/ray/tune/tests/test_logger.py b/python/ray/tune/tests/test_logger.py
--- a/python/ray/tune/tests/test_logger.py
+++ b/python/ray/tune/tests/test_logger.py
@@ -28,30 +28,30 @@ def tearDown(self):
         shutil.rmtree(self.test_dir, ignore_errors=True)
 
     def testCSV(self):
-        config = {"a": 2, "b": 5}
+        config = {"a": 2, "b": 5, "c": {"c": {"D": 123}, "e": None}}
         t = Trial(evaluated_params=config, trial_id="csv")
         logger = CSVLogger(config=config, logdir=self.test_dir, trial=t)
         logger.on_result(result(2, 4))
         logger.on_result(result(2, 4))
-        logger.on_result(result(2, 4, score=[1, 2, 3]))
+        logger.on_result(result(2, 4, score=[1, 2, 3], hello={"world": 1}))
         logger.close()
 
     def testJSON(self):
-        config = {"a": 2, "b": 5}
+        config = {"a": 2, "b": 5, "c": {"c": {"D": 123}, "e": None}}
         t = Trial(evaluated_params=config, trial_id="json")
         logger = JsonLogger(config=config, logdir=self.test_dir, trial=t)
         logger.on_result(result(0, 4))
         logger.on_result(result(1, 4))
-        logger.on_result(result(2, 4, score=[1, 2, 3]))
+        logger.on_result(result(2, 4, score=[1, 2, 3], hello={"world": 1}))
         logger.close()
 
     def testTBX(self):
-        config = {"a": 2, "b": 5}
+        config = {"a": 2, "b": 5, "c": {"c": {"D": 123}, "e": None}}
         t = Trial(evaluated_params=config, trial_id="tbx")
         logger = TBXLogger(config=config, logdir=self.test_dir, trial=t)
         logger.on_result(result(0, 4))
         logger.on_result(result(1, 4))
-        logger.on_result(result(2, 4, score=[1, 2, 3]))
+        logger.on_result(result(2, 4, score=[1, 2, 3], hello={"world": 1}))
         logger.close()