Repository: ray-project/ray

Issue:
[tune] Working directory not set to logdir in local_mode
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 18.04
- **Ray installed from (source or binary)**: binary (Nightlies)
- **Ray version**: `'0.8.0.dev6'`
- **Python version**: 3.7.3, conda

### Describe the problem

`ray.tune` runs particular trials with working dir set to respective logdir.
This is not the case when script is run in ray `local_mode`.

This is a problem when training function produces artifacts using relative paths.

Although it is possible to get `logdir` in training function and manually
obtain absolute path, it does not appear to be enforced in documentation
(see docstring of `ray.tune.trainable.Trainable.logdir` or `python/ray/tune/tests/tutorial.py#L32`).

### How to reproduce

File `example.py`:

```python
from os import environ

import ray
from ray import tune


def example_train(config, reporter):
    with open('foo.txt', 'w') as fp:
        fp.write(f'bar: {config}')

    reporter(acc=config['x'] / 2)


local_mode = bool(int(environ.get('RAY_LOCAL', '0')))
ray.init(local_mode=local_mode)
tune.run(example_train, config={'x': tune.grid_search([1, 2])})
```

*Run in default mode*

```bash
rm -R ~/ray_results/example_train  # ignore error

python example.py

ls ~/ray_results/example_train/*/foo.txt  # notice presence of foo.txt files in logdirs
```

*Run in local mode*

```bash
rm -R ~/ray_results/example_train  # ignore error

RAY_LOCAL=1 python example.py

ls ~/ray_results/example_train/*/foo.txt  # foo.txt files are *missing* in logdirs
ls foo.txt  # it is in project root
```

Related issue: #2822


Discussion:

Resolving patch:
diff --git a/python/ray/tune/ray_trial_executor.py b/python/ray/tune/ray_trial_executor.py
--- a/python/ray/tune/ray_trial_executor.py
+++ b/python/ray/tune/ray_trial_executor.py
@@ -8,6 +8,7 @@
 import random
 import time
 import traceback
+from contextlib import contextmanager
 
 import ray
 from ray.exceptions import RayTimeoutError
@@ -97,8 +98,9 @@ def _setup_remote_runner(self, trial, reuse_allowed):
         if self._cached_actor:
             logger.debug("Cannot reuse cached runner {} for new trial".format(
                 self._cached_actor))
-            self._cached_actor.stop.remote()
-            self._cached_actor.__ray_terminate__.remote()
+            with self._change_working_directory(trial):
+                self._cached_actor.stop.remote()
+                self._cached_actor.__ray_terminate__.remote()
             self._cached_actor = None
 
         cls = ray.remote(
@@ -128,7 +130,9 @@ def logger_creator(config):
         }
         if issubclass(trial.get_trainable_cls(), DurableTrainable):
             kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir
-        return cls.remote(**kwargs)
+
+        with self._change_working_directory(trial):
+            return cls.remote(**kwargs)
 
     def _train(self, trial):
         """Start one iteration of training and save remote id."""
@@ -148,7 +152,8 @@ def _train(self, trial):
             return
 
         assert trial.status == Trial.RUNNING, trial.status
-        remote = trial.runner.train.remote()
+        with self._change_working_directory(trial):
+            remote = trial.runner.train.remote()
 
         # Local Mode
         if isinstance(remote, dict):
@@ -216,8 +221,9 @@ def _stop_trial(self, trial, error=False, error_msg=None,
                     self._cached_actor = trial.runner
                 else:
                     logger.debug("Trial %s: Destroying actor.", trial)
-                    trial.runner.stop.remote()
-                    trial.runner.__ray_terminate__.remote()
+                    with self._change_working_directory(trial):
+                        trial.runner.stop.remote()
+                        trial.runner.__ray_terminate__.remote()
         except Exception:
             logger.exception("Trial %s: Error stopping runner.", trial)
             self.set_status(trial, Trial.ERROR)
@@ -299,14 +305,16 @@ def reset_trial(self, trial, new_config, new_experiment_tag):
         trial.experiment_tag = new_experiment_tag
         trial.config = new_config
         trainable = trial.runner
-        with warn_if_slow("reset_config"):
-            try:
-                reset_val = ray.get(
-                    trainable.reset_config.remote(new_config),
-                    DEFAULT_GET_TIMEOUT)
-            except RayTimeoutError:
-                logger.exception("Trial %s: reset_config timed out.", trial)
-                return False
+        with self._change_working_directory(trial):
+            with warn_if_slow("reset_config"):
+                try:
+                    reset_val = ray.get(
+                        trainable.reset_config.remote(new_config),
+                        DEFAULT_GET_TIMEOUT)
+                except RayTimeoutError:
+                    logger.exception("Trial %s: reset_config timed out.",
+                                     trial)
+                    return False
         return reset_val
 
     def get_running_trials(self):
@@ -562,14 +570,16 @@ def save(self, trial, storage=Checkpoint.PERSISTENT, result=None):
              Checkpoint future, or None if an Exception occurs.
         """
         result = result or trial.last_result
-        if storage == Checkpoint.MEMORY:
-            value = trial.runner.save_to_object.remote()
-            checkpoint = Checkpoint(storage, value, result)
-        else:
-            with warn_if_slow("save_checkpoint_to_storage"):
-                # TODO(ujvl): Make this asynchronous.
-                value = ray.get(trial.runner.save.remote())
+
+        with self._change_working_directory(trial):
+            if storage == Checkpoint.MEMORY:
+                value = trial.runner.save_to_object.remote()
                 checkpoint = Checkpoint(storage, value, result)
+            else:
+                with warn_if_slow("save_checkpoint_to_storage"):
+                    # TODO(ujvl): Make this asynchronous.
+                    value = ray.get(trial.runner.save.remote())
+                    checkpoint = Checkpoint(storage, value, result)
         with warn_if_slow("on_checkpoint", DEFAULT_GET_TIMEOUT) as profile:
             try:
                 trial.on_checkpoint(checkpoint)
@@ -605,18 +615,21 @@ def restore(self, trial, checkpoint=None):
             logger.debug("Trial %s: Attempting restore from object", trial)
             # Note that we don't store the remote since in-memory checkpoints
             # don't guarantee fault tolerance and don't need to be waited on.
-            trial.runner.restore_from_object.remote(value)
+            with self._change_working_directory(trial):
+                trial.runner.restore_from_object.remote(value)
         else:
             logger.debug("Trial %s: Attempting restore from %s", trial, value)
             if issubclass(trial.get_trainable_cls(), DurableTrainable):
-                remote = trial.runner.restore.remote(value)
+                with self._change_working_directory(trial):
+                    remote = trial.runner.restore.remote(value)
             elif trial.sync_on_checkpoint:
                 # This provides FT backwards compatibility in the
                 # case where a DurableTrainable is not provided.
                 logger.warning("Trial %s: Reading checkpoint into memory.",
                                trial)
                 data_dict = TrainableUtil.pickle_checkpoint(value)
-                remote = trial.runner.restore_from_object.remote(data_dict)
+                with self._change_working_directory(trial):
+                    remote = trial.runner.restore_from_object.remote(data_dict)
             else:
                 raise AbortTrialExecution(
                     "Pass in `sync_on_checkpoint=True` for driver-based trial"
@@ -633,9 +646,10 @@ def export_trial_if_needed(self, trial):
             A dict that maps ExportFormats to successfully exported models.
         """
         if trial.export_formats and len(trial.export_formats) > 0:
-            return ray.get(
-                trial.runner.export_model.remote(trial.export_formats),
-                DEFAULT_GET_TIMEOUT)
+            with self._change_working_directory(trial):
+                return ray.get(
+                    trial.runner.export_model.remote(trial.export_formats),
+                    DEFAULT_GET_TIMEOUT)
         return {}
 
     def has_gpus(self):
@@ -643,6 +657,23 @@ def has_gpus(self):
             self._update_avail_resources()
             return self._avail_resources.gpu > 0
 
+    @contextmanager
+    def _change_working_directory(self, trial):
+        """Context manager changing working directory to trial logdir.
+        Used in local mode.
+
+        For non-local mode it is no-op.
+        """
+        if ray.worker._mode() == ray.worker.LOCAL_MODE:
+            old_dir = os.getcwd()
+            try:
+                os.chdir(trial.logdir)
+                yield
+            finally:
+                os.chdir(old_dir)
+        else:
+            yield
+
 
 def _to_gb(n_bytes):
     return round(n_bytes / (1024**3), 2)


Test patch: