Repository: facebookresearch/hydra (python)

Issue:
dir interpolation from hydra into job config is not working
I can set the `second_dir`  from `base_dir`, but `output_subdir` can not in this case.
```
experiment:
  base_dir: ???
  second_dir: ${experiment.base_dir}
hydra:
  output_subdir: ${experiment.base_dir}/.hydra
```
`KeyError: "str interpolation key 'experiment.base_dir' not found"`

Is there any way I can change the hydra config based on current .yaml or some alternately way?  
 Thanks!

Pull request description:
fixed dir interpolation from hydra into job config
closed #240 

Patch:
diff --git a/hydra/plugins/common/utils.py b/hydra/plugins/common/utils.py
index df2cce151f..6be8e22ce3 100644
--- a/hydra/plugins/common/utils.py
+++ b/hydra/plugins/common/utils.py
@@ -73,8 +73,6 @@ def run_job(config, task_function, job_dir_key, job_subdir_key):
         ret = JobReturn()
         ret.working_dir = working_dir
         task_cfg = copy.deepcopy(config)
-        # TODO: update this after https://github.com/omry/omegaconf/issues/42 is resolved
-        hydra_cfg = OmegaConf.create({"hydra": task_cfg["hydra"]})
         del task_cfg["hydra"]
         ret.cfg = task_cfg
         ret.hydra_cfg = copy.deepcopy(HydraConfig())
@@ -82,9 +80,13 @@ def run_job(config, task_function, job_dir_key, job_subdir_key):
         # handle output directories here
         Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
         os.chdir(working_dir)
-        hydra_output = Path(hydra_cfg.hydra.output_subdir)
+        hydra_output = Path(config.hydra.output_subdir)
+
+        configure_log(config.hydra.job_logging, config.hydra.verbose)
+
+        # TODO: update this after https://github.com/omry/omegaconf/issues/42 is resolved
+        hydra_cfg = OmegaConf.create({"hydra": config["hydra"]})
 
-        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
         _save_config(task_cfg, "config.yaml", hydra_output)
         _save_config(hydra_cfg, "hydra.yaml", hydra_output)
         _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)


Test patch:
diff --git a/tests/test_apps/interpolating_dir_hydra_to_app/__init__.py b/tests/test_apps/interpolating_dir_hydra_to_app/__init__.py
new file mode 100644
index 0000000000..168f9979a4
--- /dev/null
+++ b/tests/test_apps/interpolating_dir_hydra_to_app/__init__.py
@@ -0,0 +1,1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_apps/interpolating_dir_hydra_to_app/config.yaml b/tests/test_apps/interpolating_dir_hydra_to_app/config.yaml
new file mode 100644
index 0000000000..edeab509b0
--- /dev/null
+++ b/tests/test_apps/interpolating_dir_hydra_to_app/config.yaml
@@ -0,0 +1,5 @@
+experiment:
+  base_dir: ???
+  second_dir: ${experiment.base_dir}
+hydra:
+  output_subdir: ${experiment.base_dir}/.hydra
\ No newline at end of file
diff --git a/tests/test_apps/interpolating_dir_hydra_to_app/my_app.py b/tests/test_apps/interpolating_dir_hydra_to_app/my_app.py
new file mode 100644
index 0000000000..1a444d7fc1
--- /dev/null
+++ b/tests/test_apps/interpolating_dir_hydra_to_app/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="config.yaml")
+def my_app(_):
+    pass
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
index cb691bf3e6..9b40d7a981 100644
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -1,6 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import os
 import sys
+from hydra._internal.pathlib import Path
 
 import pytest
 import subprocess
@@ -318,3 +319,24 @@ def test_help(tmpdir, script, overrides, expected):
     # normalize newlines on Windows to make testing easier
     result = result.replace("\r\n", "\n")
     assert result == expected
+
+
+@pytest.mark.parametrize(
+    "calling_file, calling_module",
+    [
+        ("tests/test_apps/interpolating_dir_hydra_to_app/my_app.py", None),
+        (None, "tests.test_apps.interpolating_dir_hydra_to_app.my_app"),
+    ],
+)
+def test_interpolating_dir_hydra_to_app(
+    task_runner, calling_file, calling_module  # noqa: F811
+):
+    basedir = "foo"
+    with task_runner(
+        calling_file=calling_file,
+        calling_module=calling_module,
+        config_path="config.yaml",
+        overrides=["experiment.base_dir=" + basedir],
+    ) as task:
+        path = Path(task.temp_dir) / basedir
+        assert path.exists()