Repository: ray-project/ray

Issue:
[tune] Relative local_dir is not supported.
<!--
General questions should be asked on the mailing list ray-dev@googlegroups.com.
Questions about how to use Ray should be asked on
[StackOverflow](https://stackoverflow.com/questions/tagged/ray).

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: macOS Mojave 
- **Ray installed from (source or binary)**: 
- **Ray version**:0.7.0.dev2
- **Python version**:3.6
- **Exact command to reproduce**:

<!--
You can obtain the Ray version with

python -c "import ray; print(ray.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

If you use

```
        tune.run(
            self.algo,
            name="TuneRelavtieLocalDirTest",
            stop={"training_iteration": 1},
            checkpoint_freq=1,
            local_dir="ray_result", 
            config={
                "env": "CartPole-v0",
            },
        )
```

The path of checkpoint turned out to be: `$CURRENT_DIR/ray_result/TuneRelavtieLocalDirTest/PG_CartPole-v0_0_2019-05-01_15-24-54xo9z1bq8/ray_result/TuneRelavtieLocalDirTest/PG_CartPole-v0_0_2019-05-01_15-24-54xo9z1bq8/checkpoint_1/checkpoint-1`

![image](https://user-images.githubusercontent.com/22206995/57008878-bd7a2800-6c25-11e9-890c-363d0f4facca.png)

You can see that the path is nested. It is ought to be `$CURRENT_DIR/ray_result/TuneRelavtieLocalDirTest/PG_CartPole-v0_0_2019-05-01_15-24-54xo9z1bq8/checkpoint_1/checkpoint-1`

Using `local_dir` with dot identifier gives the same result.

```
        tune.run(
            self.algo,
            name="TuneRelavtieLocalDirTest",
            stop={"training_iteration": 1},
            checkpoint_freq=1,
            local_dir="./ray_result", # This should be ray/tune/tests/ray_result
            config={
                "env": "CartPole-v0",
            },
        )
```

Why this matter? Because in some cases, user needs to take away all the experiments results within their directory of codes, via git or FTP.


------

What about using absolute path?

```
        abs_path = os.path.abspath("ray_result3")
        print("abs_path:", abs_path) # abs_path: /Users/XXX/ray/python/ray/ray_result3

        tune.run(
            self.algo,
            name="TuneRelavtieLocalDirTest",
            stop={"training_iteration": 1},
            checkpoint_freq=1,
            local_dir=abs_path, # This should be ray/tune/tests/ray_result3
            config={
                "env": "CartPole-v0",
            },
        )
```

It's correct!

<img width="422" alt="image" src="https://user-images.githubusercontent.com/22206995/57009099-26ae6b00-6c27-11e9-85e4-80c2ef142a1a.png">

### Discussion


I think maybe we can divide the `local_dir` into three kinds:

1. "./result"
2. "result"
3. "/User/XXX/result"
4. "~/result"

We should consider No.1 & 2 to be the relative path and the last two as absolute path.


Discussion:

Resolving patch:
diff --git a/python/ray/tune/experiment.py b/python/ray/tune/experiment.py
--- a/python/ray/tune/experiment.py
+++ b/python/ray/tune/experiment.py
@@ -96,7 +96,8 @@ def __init__(self,
             "config": config,
             "resources_per_trial": resources_per_trial,
             "num_samples": num_samples,
-            "local_dir": os.path.expanduser(local_dir or DEFAULT_RESULTS_DIR),
+            "local_dir": os.path.abspath(
+                os.path.expanduser(local_dir or DEFAULT_RESULTS_DIR)),
             "upload_dir": upload_dir,
             "trial_name_creator": trial_name_creator,
             "loggers": loggers,
@@ -107,7 +108,8 @@ def __init__(self,
             "checkpoint_score_attr": checkpoint_score_attr,
             "export_formats": export_formats or [],
             "max_failures": max_failures,
-            "restore": restore
+            "restore": os.path.abspath(os.path.expanduser(restore))
+            if restore else None
         }
 
         self.name = name or run_identifier
diff --git a/python/ray/tune/trainable.py b/python/ray/tune/trainable.py
--- a/python/ray/tune/trainable.py
+++ b/python/ray/tune/trainable.py
@@ -350,6 +350,14 @@ def restore(self, checkpoint_path):
         self._timesteps_since_restore = 0
         self._iterations_since_restore = 0
         self._restored = True
+        logger.info("Restored from checkpoint: {}".format(checkpoint_path))
+        state = {
+            "_iteration": self._iteration,
+            "_timesteps_total": self._timesteps_total,
+            "_time_total": self._time_total,
+            "_episodes_total": self._episodes_total,
+        }
+        logger.info("Current state after restoring: {}".format(state))
 
     def restore_from_object(self, obj):
         """Restores training state from a checkpoint object.


Test patch:
diff --git a/python/ray/tune/tests/test_tune_save_restore.py b/python/ray/tune/tests/test_tune_save_restore.py
new file mode 100644
--- /dev/null
+++ b/python/ray/tune/tests/test_tune_save_restore.py
@@ -0,0 +1,159 @@
+# coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import pickle
+import shutil
+import tempfile
+import unittest
+
+import ray
+from ray import tune
+from ray.rllib import _register_all
+from ray.tune import Trainable
+
+
+class SerialTuneRelativeLocalDirTest(unittest.TestCase):
+    local_mode = True
+    prefix = "Serial"
+
+    class MockTrainable(Trainable):
+        _name = "MockTrainable"
+
+        def _setup(self, config):
+            self.state = {"hi": 1}
+
+        def _train(self):
+            return {"timesteps_this_iter": 1, "done": True}
+
+        def _save(self, checkpoint_dir):
+            checkpoint_path = os.path.join(
+                checkpoint_dir, "checkpoint-{}".format(self._iteration))
+            with open(checkpoint_path, "wb") as f:
+                pickle.dump(self.state, f)
+            return checkpoint_path
+
+        def _restore(self, checkpoint_path):
+            with open(checkpoint_path, "rb") as f:
+                extra_data = pickle.load(f)
+            self.state.update(extra_data)
+
+    def setUp(self):
+        ray.init(num_cpus=1, num_gpus=0, local_mode=self.local_mode)
+
+    def tearDown(self):
+        shutil.rmtree(self.absolute_local_dir, ignore_errors=True)
+        self.absolute_local_dir = None
+        ray.shutdown()
+        # Without this line, test_tune_server.testAddTrial would fail.
+        _register_all()
+
+    def _get_trial_dir(self, absoulte_exp_dir):
+        trial_dirname = next(
+            (child_dir for child_dir in os.listdir(absoulte_exp_dir)
+             if (os.path.isdir(os.path.join(absoulte_exp_dir, child_dir))
+                 and child_dir.startswith(self.MockTrainable._name))))
+
+        trial_absolute_dir = os.path.join(absoulte_exp_dir, trial_dirname)
+
+        return trial_dirname, trial_absolute_dir
+
+    def _train(self, exp_name, local_dir, absolute_local_dir):
+        trial, = tune.run(
+            self.MockTrainable,
+            name=exp_name,
+            stop={
+                "training_iteration": 1
+            },
+            checkpoint_freq=1,
+            local_dir=local_dir,
+            config={
+                "env": "CartPole-v0",
+                "log_level": "DEBUG"
+            }).trials
+
+        exp_dir = os.path.join(absolute_local_dir, exp_name)
+        _, abs_trial_dir = self._get_trial_dir(exp_dir)
+
+        self.assertIsNone(trial.error_file)
+        self.assertEqual(trial.local_dir, exp_dir)
+        self.assertEqual(trial.logdir, abs_trial_dir)
+
+        self.assertTrue(os.path.isdir(absolute_local_dir), absolute_local_dir)
+        self.assertTrue(os.path.isdir(exp_dir))
+        self.assertTrue(os.path.isdir(abs_trial_dir))
+        self.assertTrue(
+            os.path.isfile(
+                os.path.join(abs_trial_dir, "checkpoint_1/checkpoint-1")))
+
+    def _restore(self, exp_name, local_dir, absolute_local_dir):
+        trial_name, abs_trial_dir = self._get_trial_dir(
+            os.path.join(absolute_local_dir, exp_name))
+
+        checkpoint_path = os.path.join(
+            local_dir, exp_name, trial_name,
+            "checkpoint_1/checkpoint-1")  # Relative checkpoint path
+
+        # The file tune would find. The absolute checkpoint path.
+        tune_find_file = os.path.abspath(os.path.expanduser(checkpoint_path))
+        self.assertTrue(
+            os.path.isfile(tune_find_file),
+            "{} is not exist!".format(tune_find_file))
+
+        trial, = tune.run(
+            self.MockTrainable,
+            name=exp_name,
+            stop={
+                "training_iteration": 2
+            },  # train one more iteration.
+            restore=checkpoint_path,  # Restore the checkpoint
+            config={
+                "env": "CartPole-v0",
+                "log_level": "DEBUG"
+            }).trials
+        self.assertIsNone(trial.error_file)
+
+    def testDottedRelativePath(self):
+        local_dir = "./test_dotted_relative_local_dir"
+        exp_name = self.prefix + "DottedRelativeLocalDir"
+        absolute_local_dir = os.path.abspath(local_dir)
+        self.absolute_local_dir = absolute_local_dir
+        self.assertFalse(os.path.exists(absolute_local_dir))
+        self._train(exp_name, local_dir, absolute_local_dir)
+        self._restore(exp_name, local_dir, absolute_local_dir)
+
+    def testRelativePath(self):
+        local_dir = "test_relative_local_dir"
+        exp_name = self.prefix + "RelativePath"
+        absolute_local_dir = os.path.abspath(local_dir)
+        self.absolute_local_dir = absolute_local_dir
+        self.assertFalse(os.path.exists(absolute_local_dir))
+        self._train(exp_name, local_dir, absolute_local_dir)
+        self._restore(exp_name, local_dir, absolute_local_dir)
+
+    def testTildeAbsolutePath(self):
+        local_dir = "~/test_tilde_absolute_local_dir"
+        exp_name = self.prefix + "TildeAbsolutePath"
+        absolute_local_dir = os.path.abspath(os.path.expanduser(local_dir))
+        self.absolute_local_dir = absolute_local_dir
+        self.assertFalse(os.path.exists(absolute_local_dir))
+        self._train(exp_name, local_dir, absolute_local_dir)
+        self._restore(exp_name, local_dir, absolute_local_dir)
+
+    def testTempfile(self):
+        local_dir = tempfile.mkdtemp()
+        exp_name = self.prefix + "Tempfile"
+        self.absolute_local_dir = local_dir
+        self._train(exp_name, local_dir, local_dir)
+        self._restore(exp_name, local_dir, local_dir)
+
+
+class ParallelTuneRelativeLocalDirTest(SerialTuneRelativeLocalDirTest):
+    local_mode = False
+    prefix = "Parallel"
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)