internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: facebookresearch/hydra

Issue:
move hydra generated files into an output directory relative to the job dir
automatically generated files like config.yaml, overrides.yaml, and others can go here.
Also additional debug information about the hydra config composition order.


Discussion:

Resolving patch:
diff --git a/hydra/plugins/common/utils.py b/hydra/plugins/common/utils.py
--- a/hydra/plugins/common/utils.py
+++ b/hydra/plugins/common/utils.py
@@ -5,6 +5,7 @@
 import os
 import re
 import sys
+from hydra._internal.pathlib import Path
 from time import strftime, localtime
 
 import six
@@ -39,8 +40,9 @@ def configure_log(log_config, verbose_config):
             logging.getLogger(logger).setLevel(logging.DEBUG)
 
 
-def save_config(cfg, filename):
-    with open(os.path.join(filename), "w") as file:
+def _save_config(cfg, filename, output_dir):
+    Path(str(output_dir)).mkdir(parents=True, exist_ok=True)
+    with open(str(output_dir / filename), "w") as file:
         file.write(cfg.pretty())
 
 
@@ -77,14 +79,15 @@ def run_job(config, task_function, job_dir_key, job_subdir_key):
         ret.cfg = task_cfg
         ret.hydra_cfg = copy.deepcopy(HydraConfig())
         ret.overrides = config.hydra.overrides.task.to_container()
-        if not os.path.exists(working_dir):
-            os.makedirs(working_dir)
+        # handle output directories here
+        Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
         os.chdir(working_dir)
-        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
+        hydra_output = Path(hydra_cfg.hydra.output_subdir)
 
-        save_config(task_cfg, "config.yaml")
-        save_config(hydra_cfg, "hydra.yaml")
-        save_config(config.hydra.overrides.task, "overrides.yaml")
+        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
+        _save_config(task_cfg, "config.yaml", hydra_output)
+        _save_config(hydra_cfg, "hydra.yaml", hydra_output)
+        _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
         ret.return_value = task_function(task_cfg)
         ret.task_name = JobRuntime().get("name")
         return ret


Test patch:
diff --git a/hydra/test_utils/test_utils.py b/hydra/test_utils/test_utils.py
--- a/hydra/test_utils/test_utils.py
+++ b/hydra/test_utils/test_utils.py
@@ -185,10 +185,14 @@ def verify_dir_outputs(job_return, overrides=None):
     assert os.path.exists(
         os.path.join(job_return.working_dir, job_return.task_name + ".log")
     )
-    assert os.path.exists(os.path.join(job_return.working_dir, "config.yaml"))
-    assert os.path.exists(os.path.join(job_return.working_dir, "overrides.yaml"))
+
+    hydra_dir = os.path.join(
+        job_return.working_dir, job_return.hydra_cfg.hydra.output_subdir
+    )
+    assert os.path.exists(os.path.join(hydra_dir, "config.yaml"))
+    assert os.path.exists(os.path.join(hydra_dir, "overrides.yaml"))
     assert OmegaConf.load(
-        os.path.join(job_return.working_dir, "overrides.yaml")
+        os.path.join(hydra_dir, "overrides.yaml")
     ) == OmegaConf.create(overrides or [])
Repository: facebookresearch/hydra

Issue:
Hydra client API
Create Hydra client API to allow jobs to interact with Hydra.
one immediate need is to allow executed code to find out what is the original working directory is was executed from.


Discussion:
is there currently a way to get the original working directory?
Nothing official, but I am adding something now.

Resolving patch:
diff --git a/hydra/utils.py b/hydra/utils.py
--- a/hydra/utils.py
+++ b/hydra/utils.py
@@ -3,6 +3,9 @@
 
 from omegaconf import OmegaConf
 
+from hydra._internal.pathlib import Path
+from hydra.plugins.common.utils import HydraConfig
+
 # pylint: disable=C0103
 log = logging.getLogger(__name__)
 
@@ -51,3 +54,23 @@ def instantiate(config, *args, **kwargs):
     except Exception as e:
         log.error("Error instantiating {} : {}".format(config["class"], e))
         raise e
+
+
+def get_original_cwd():
+    return HydraConfig().hydra.runtime.cwd
+
+
+def to_absolute_path(path):
+    """
+    converts the specified path to be absolute path.
+    if the input path is relative, it's interpreted as relative to the original working directory
+    if it's absolute, it's returned as is
+    :param path:
+    :return:
+    """
+    path = Path(path)
+    if path.is_absolute():
+        ret = path
+    else:
+        ret = Path(get_original_cwd()) / path
+    return str(ret)


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -2,6 +2,8 @@
 import pytest
 from hydra import utils
 from omegaconf import OmegaConf
+from hydra.plugins.common.utils import HydraConfig
+from hydra._internal.pathlib import Path
 
 
 def some_method():
@@ -105,3 +107,28 @@ def test_class_instantiate_passthrough(conf, expected):
     conf = OmegaConf.create(conf)
     obj = utils.instantiate(conf, 10, d=40)
     assert obj == expected
+
+
+def test_get_original_cwd():
+    orig = "/foo/bar"
+    cfg = OmegaConf.create({"hydra": {"runtime": {"cwd": orig}}})
+    HydraConfig().set_config(cfg)
+    assert utils.get_original_cwd() == orig
+
+
+@pytest.mark.parametrize(
+    "orig_cwd, path, expected",
+    [
+        ("/home/omry/hydra", "foo/bar", "/home/omry/hydra/foo/bar"),
+        ("/home/omry/hydra/", "foo/bar", "/home/omry/hydra/foo/bar"),
+        ("/home/omry/hydra/", "/foo/bar", "/foo/bar"),
+    ],
+)
+def test_to_absolute_path(orig_cwd, path, expected):
+    # normalize paths to current OSg
+    orig_cwd = str(Path(orig_cwd))
+    path = str(Path(path))
+    expected = str(Path(expected))
+    cfg = OmegaConf.create({"hydra": {"runtime": {"cwd": orig_cwd}}})
+    HydraConfig().set_config(cfg)
+    assert utils.to_absolute_path(path) == expected
Repository: facebookresearch/hydra

Issue:
[Bug] merging if individual files (not in group) happens in wrong order
something like:
```
defaults:
   - group: opt1
   - file
```

caused the file to be merged in before the group.


Discussion:

Resolving patch:
diff --git a/hydra/_internal/config_loader.py b/hydra/_internal/config_loader.py
--- a/hydra/_internal/config_loader.py
+++ b/hydra/_internal/config_loader.py
@@ -187,9 +187,6 @@ def get_key(d1):
                     idx = key_to_idx[key]
                     primary[idx] = d
                     merged_list.remove(d)
-            else:
-                primary.append(d)
-                merged_list.remove(d)
 
         for d in merged_list:
             primary.append(d)


Test patch:
diff --git a/hydra/test_utils/configs/mixed_compose.yaml b/hydra/test_utils/configs/mixed_compose.yaml
new file mode 100644
--- /dev/null
+++ b/hydra/test_utils/configs/mixed_compose.yaml
@@ -0,0 +1,4 @@
+defaults:
+  - some_config
+  - group1: file1
+  - config
\ No newline at end of file
diff --git a/tests/test_conf_loader.py b/tests/test_conf_loader.py
--- a/tests/test_conf_loader.py
+++ b/tests/test_conf_loader.py
@@ -328,3 +328,30 @@ def test_non_config_group_default():
         ("non_config_group_default.yaml", "hydra/test_utils/configs", "test"),
         ("some_config.yaml", "hydra/test_utils/configs", "test"),
     ]
+
+
+def test_mixed_composition_order():
+    """
+    Tests that the order of mixed composition (defaults contains both config group and non config group
+    items) is correct
+    """
+    config_loader = ConfigLoader(
+        config_search_path=create_search_path(["hydra/test_utils/configs"]),
+        strict_cfg=False,
+        config_file="mixed_compose.yaml",
+    )
+    config_loader.load_configuration()
+    assert config_loader.get_load_history() == [
+        ("hydra.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/hydra_logging/default.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/job_logging/default.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/launcher/basic.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/sweeper/basic.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/output/default.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/help/default.yaml", "pkg://hydra.conf", "hydra"),
+        ("hydra/hydra_help/default.yaml", "pkg://hydra.conf", "hydra"),
+        ("mixed_compose.yaml", "hydra/test_utils/configs", "test"),
+        ("some_config.yaml", "hydra/test_utils/configs", "test"),
+        ("group1/file1.yaml", "hydra/test_utils/configs", "test"),
+        ("config.yaml", "hydra/test_utils/configs", "test"),
+    ]
Repository: facebookresearch/hydra

Issue:
Clean up --cfg
It will be cleaner for --cfg to always take one of job|hydra|all, and not have it default to job.
this will eliminate the problem that occures when --cfg is not the last flag in the command line and some override is associated with --cfg.


Discussion:

Resolving patch:
diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py
--- a/hydra/_internal/utils.py
+++ b/hydra/_internal/utils.py
@@ -92,10 +92,8 @@ def get_args_parser():
     parser.add_argument(
         "--cfg",
         "-c",
-        const="job",
-        nargs="?",
         choices=["job", "hydra", "all"],
-        help="Show config instead of running, optional value indicates which config to show (defaults to job)",
+        help="Show config instead of running [job|hydra|all]",
     )
 
     parser.add_argument("--run", "-r", action="store_true", help="Run a job")


Test patch:
diff --git a/tests/test_completion.py b/tests/test_completion.py
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -125,8 +125,8 @@ def test_shell_integration(
     "line,expected",
     [
         ("-c job", base_completion_list),
-        ("-c job ", base_completion_list),
-        ("-c", base_completion_list),
+        ("-c hydra ", base_completion_list),
+        ("-c all ", base_completion_list),
     ],
 )
 def test_with_flags(line, expected):
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -288,7 +288,7 @@ def test_sweep_complex_defaults(
             """--help,-h : Application's help
 --hydra-help : Hydra's help
 --version : show program's version number and exit
---cfg,-c : Show config instead of running, optional value indicates which config to show (defaults to job)
+--cfg,-c : Show config instead of running [job|hydra|all]
 --run,-r : Run a job
 --multirun,-m : Run multiple jobs with the configured launcher
 --shell_completion,-sc : Install or Uninstall shell completion:
Repository: facebookresearch/hydra

Issue:
[Bug] Documentation inconsistency for `utils.get_original_cwd`
# 🐛 Bug

The tutorial for working directories has a few commands for setting the working directory [see here](https://cli.dev/docs/tutorial/working_directory), but the version of hydra on pip does not have these functions.  Additionally, the install instructions do not include instructions on how to install from source (even if that's fairly trivial).  The simple solution is to update the wheels on pip.  Another alternative would be to put on the installation page that hydra is rapidly developing and suggest that one can install from source directly.

## System information
- 0.10.0 from pip
- python 3.7
- arch linux

## One more thing...
This is very minor but the pip version is `0.10.0` and the github master version is also `0.10.0`, but they not the same as evidenced by this issue.  You should probably bump the version of git master.  Keep up the good work, I think this is a great idea.


Discussion:
Hi David, thanks!
I am aware of this, unfortunately the documentation system I am using (Docusaurus 2) does not yet support multiple documentation versions, and since I like to update the docs as I write new features it results in this inconsistency.

I had quiet a bit of travel this month, so the release of the next version is delayed, you can follow the progress toward 0.11.0 [here](https://github.com/facebookresearch/hydra/milestone/3).

The instructions for installing from source are in the Development/Contributing [section](https://cli.dev/docs/development/contributing) of the docs.

I agree with you that this is not great.
One thing I might do is to make a pre-release (0.11.0-rc*) that will be considered unstable until I 0.11.0 is released).

I think the changing the master to an -rc is a really good idea. It'd give the user a hint that something has changed. 

Resolving patch:
diff --git a/hydra/__init__.py b/hydra/__init__.py
--- a/hydra/__init__.py
+++ b/hydra/__init__.py
@@ -4,6 +4,6 @@
 from .main import main
 
 # Source of truth for Hydra's version
-__version__ = "0.10.0"
+__version__ = "0.11.0-pre1"
 
 __all__ = ["__version__", "MissingConfigException", "main", "utils"]


Test patch:
Repository: facebookresearch/hydra

Issue:
basic ray example with compose API
closes #318 


Discussion:

Resolving patch:
diff --git a/examples/advanced/ray_example/hydra_compose_example.py b/examples/advanced/ray_example/hydra_compose_example.py
new file mode 100644
--- /dev/null
+++ b/examples/advanced/ray_example/hydra_compose_example.py
@@ -0,0 +1,32 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+from hydra.experimental import compose
+import ray
+import time
+
+
+@ray.remote
+def train(overrides, cfg):
+    print(cfg.pretty())
+    time.sleep(5)
+    return overrides, 0.9
+
+
+@hydra.main(config_path="conf/config.yaml")
+def main(cfg):
+    ray.init(**cfg.ray.init)
+
+    results = []
+    for model in ["alexnet", "resnet"]:
+        for dataset in ["cifar10", "imagenet"]:
+            overrides = [f"dataset={dataset}", f"model={model}"]
+            run_cfg = compose(overrides=overrides)
+            ret = train.remote(overrides, run_cfg)
+            results.append(ret)
+
+    for overrides, score in ray.get(results):
+        print(f"Result from {overrides} : {score}")
+
+
+if __name__ == "__main__":
+    main()


Test patch:
Repository: facebookresearch/hydra

Issue:
[Bug] system.exit(code) is not respected in Hydra



Discussion:

Resolving patch:
diff --git a/hydra/main.py b/hydra/main.py
--- a/hydra/main.py
+++ b/hydra/main.py
@@ -1,6 +1,5 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import functools
-import sys
 from typing import Callable, Optional
 
 from ._internal.utils import get_args_parser, run_hydra
@@ -23,17 +22,12 @@ def main(
     def main_decorator(task_function: TaskFunction) -> Callable[[], None]:
         @functools.wraps(task_function)
         def decorated_main() -> None:
-            try:
-                run_hydra(
-                    args_parser=get_args_parser(),
-                    task_function=task_function,
-                    config_path=config_path,
-                    strict=strict,
-                )
-            except KeyboardInterrupt:
-                sys.exit(-1)
-            except SystemExit:
-                pass
+            run_hydra(
+                args_parser=get_args_parser(),
+                task_function=task_function,
+                config_path=config_path,
+                strict=strict,
+            )
 
         return decorated_main
 


Test patch:
diff --git a/tests/test_apps/sys_exit/__init__.py b/tests/test_apps/sys_exit/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sys_exit/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_apps/sys_exit/my_app.py b/tests/test_apps/sys_exit/my_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sys_exit/my_app.py
@@ -0,0 +1,15 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import sys
+
+from omegaconf import DictConfig
+
+import hydra
+
+
+@hydra.main()
+def my_app(_: DictConfig) -> None:
+    sys.exit(42)
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -416,3 +416,12 @@ def test_interpolating_dir_hydra_to_app(
         assert task.temp_dir is not None
         path = Path(task.temp_dir) / basedir
         assert path.exists()
+
+
+def test_sys_exit(tmpdir: Path) -> None:
+    cmd = [
+        sys.executable,
+        "tests/test_apps/sys_exit/my_app.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    assert subprocess.run(cmd).returncode == 42
Repository: facebookresearch/hydra

Issue:
[Bug] setuptools finds and installs tests/ as a top-level package in site-packages/



Discussion:
Hi @bcdarwin, thanks for pointing out. definitely an oversight.
I will fix it for the next version. Is this causing actual issues for you with 0.11.3?
No, it's low priority, just noticed it.  Thanks!

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -102,7 +102,7 @@ def run(self):
         long_description_content_type="text/markdown",
         url="https://github.com/facebookresearch/hydra",
         keywords="command-line configuration yaml tab-completion",
-        packages=find_packages(),
+        packages=find_packages(include=["hydra"]),
         include_package_data=True,
         classifiers=[
             "License :: OSI Approved :: MIT License",


Test patch:
Repository: facebookresearch/hydra

Issue:
Save .hydra/*.yaml before task execution
# 🐛 Bug

Saving the configurations is a really cool idea that helps me check what I actually did. I noticed the save timing is delayed after task execution in the `1.0.0rc1`. I have encountered some scenarios that I think it would be better if the configs are saved earlier.

For example,

1. Tasks may exit accidentally due to bugs. Knowing detailed configurations would be helpful.
2. For time-consuming tasks like ML training which may last for several days, it would be helpful to be able to check the config.yaml. This helps me a lot when I have multiple tasks running but I can't remember the real difference. I do `diff config.yaml`.
3. Some tasks don't need to be finished. The full training may be designed to run 100 epochs but it also happens it's aborted/killed in the middle.

## To reproduce

Default behavior in `1.0.0rc1`.

** Minimal Code/Config snippet to reproduce **
NA

** Stack trace/error message **
NA

## Expected Behavior
NA

## System information
- **Hydra Version** :  1.0.0rc1
- **Python version** : NA
- **Virtual environment type and version** : NA
- **Operating system** : NA




Discussion:
Is this a change in 1.0?
It seems this change was introduced in 98128a4d
Interesting, not sure why I moved it down.
can you send a PR fixing it?

Resolving patch:
diff --git a/hydra/core/utils.py b/hydra/core/utils.py
--- a/hydra/core/utils.py
+++ b/hydra/core/utils.py
@@ -112,6 +112,12 @@ def run_job(
         hydra_cfg = OmegaConf.masked_copy(config, "hydra")
         assert isinstance(hydra_cfg, DictConfig)
 
+        if config.hydra.output_subdir is not None:
+            hydra_output = Path(config.hydra.output_subdir)
+            _save_config(task_cfg, "config.yaml", hydra_output)
+            _save_config(hydra_cfg, "hydra.yaml", hydra_output)
+            _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
+
         with env_override(hydra_cfg.hydra.job.env_set):
             ret.return_value = task_function(task_cfg)
         ret.task_name = JobRuntime.instance().get("name")
@@ -120,12 +126,6 @@ def run_job(
         # If logging is still required after run_job caller is responsible to re-initialize it.
         logging.shutdown()
 
-        if config.hydra.output_subdir is not None:
-            hydra_output = Path(config.hydra.output_subdir)
-            _save_config(task_cfg, "config.yaml", hydra_output)
-            _save_config(hydra_cfg, "hydra.yaml", hydra_output)
-            _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
-
         return ret
     finally:
         os.chdir(old_cwd)


Test patch:
Repository: facebookresearch/hydra

Issue:
Update Example Launcher Plugin mention restoring the singletons
The singletons should be restored when performing multi-process launching.
something 

```python
# In the launcher's launch:
state = Singleton.get_state()

# In the remote process
Singleton.set_state(state)
```


Discussion:

Resolving patch:
diff --git a/plugins/examples/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py b/plugins/examples/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py
--- a/plugins/examples/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py
+++ b/plugins/examples/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py
@@ -6,6 +6,7 @@
 from hydra.core.config_loader import ConfigLoader
 from hydra.core.config_search_path import ConfigSearchPath
 from hydra.core.hydra_config import HydraConfig
+from hydra.core.singleton import Singleton
 from hydra.core.utils import (
     JobReturn,
     configure_log,
@@ -100,6 +101,17 @@ def launch(
                 sweep_config.hydra.job.num = idx
             HydraConfig.instance().set_config(sweep_config)
 
+            # If your launcher is executing code in a different process, it is important to restore
+            # the singleton state in the new process.
+            # To do this, you will likely need to serialize the singleton state along with the other
+            # parameters passed to the child process.
+
+            # happening on launcher process
+            state = Singleton.get_state()
+
+            # happening on the spawned process
+            Singleton.set_state(state)
+
             ret = run_job(
                 config=sweep_config,
                 task_function=self.task_function,


Test patch: