Repository: facebookresearch/hydra

Issue:
Add option to not create .hydra in output directory
In some cases users are not interested in the generation of config.yaml, hydra.yaml etc.
add an option to opt out.


Discussion:
I have my applications in docker containers and don't need that either, also I have to say that the whole output directory feature surprised me (especially changing my working directory), since I was only expecting it to do configuration things. So I'd vote to actually make it opt in. :-)
Hi Jonas,
The working directory feature integrates with other core features like multi-run (where you run your app multiple time with a different configuration with a single command).
I thought hard about this, and while I understand it's surprising it's here to stay :).

Working directory can be fully customized, see [this](https://hydra.cc/docs/configure_hydra/intro).
You can change it to `.` or to `/tmp` for example.
Yeah I already changed it to `.` to keep my app's behavior, `/tmp` would be better to avoid "pollution" of the main dir, but then I'd probably have to change back to the `get_original_cwd` directory or change all the paths in my program, right?
I missed your question, sorry.
See [this](https://hydra.cc/docs/tutorial/working_directory#original-working-directory) for ways to access files in the original working directory.

Resolving patch:
diff --git a/hydra/_internal/config_loader_impl.py b/hydra/_internal/config_loader_impl.py
--- a/hydra/_internal/config_loader_impl.py
+++ b/hydra/_internal/config_loader_impl.py
@@ -61,7 +61,8 @@ def is_package_rename(self) -> bool:
         return self.pkg1 is not None and self.pkg2 is not None
 
     def is_delete(self) -> bool:
-        return self.prefix == "~"
+        legacy_delete = self.value == "null"
+        return self.prefix == "~" or legacy_delete
 
     def is_add(self) -> bool:
         return self.prefix == "+"
@@ -292,6 +293,19 @@ def _apply_overrides_to_defaults(
                 )
         for owl in overrides:
             override = owl.override
+            if override.value == "null":
+                if override.prefix not in (None, "~"):
+                    ConfigLoaderImpl._raise_parse_override_error(owl.input_line)
+                override.prefix = "~"
+                override.value = None
+
+                msg = (
+                    "\nRemoving from the defaults list by assigning 'null' "
+                    "is deprecated and will be removed in Hydra 1.1."
+                    f"\nUse ~{override.key}"
+                )
+                warnings.warn(category=UserWarning, message=msg)
+
             if (
                 not (override.is_delete() or override.is_package_rename())
                 and override.value is None
@@ -453,19 +467,6 @@ def _parse_override(override: str) -> ParsedOverrideWithLine:
             pkg1 = matches.group("pkg1")
             pkg2 = matches.group("pkg2")
             value: Optional[str] = matches.group("value")
-            if value == "null":
-                if prefix not in (None, "~"):
-                    ConfigLoaderImpl._raise_parse_override_error(override)
-                prefix = "~"
-                value = None
-
-                msg = (
-                    "\nRemoving from the defaults list by assigning 'null' "
-                    "is deprecated and will be removed in Hydra 1.1."
-                    f"\nUse ~{key}"
-                )
-                warnings.warn(category=UserWarning, message=msg)
-
             ret = ParsedOverride(prefix, key, pkg1, pkg2, value)
             return ParsedOverrideWithLine(override=ret, input_line=override)
         else:
diff --git a/hydra/conf/__init__.py b/hydra/conf/__init__.py
--- a/hydra/conf/__init__.py
+++ b/hydra/conf/__init__.py
@@ -129,7 +129,8 @@ class HydraConf:
     # Output directory for produced configuration files and overrides.
     # E.g., hydra.yaml, overrides.yaml will go here. Useful for debugging
     # and extra context when looking at past runs.
-    output_subdir: str = ".hydra"
+    # Setting to None will prevent the creation of the output subdir.
+    output_subdir: Optional[str] = ".hydra"
 
     # Those lists will contain runtime overrides
     overrides: OverridesConf = OverridesConf()
diff --git a/hydra/core/utils.py b/hydra/core/utils.py
--- a/hydra/core/utils.py
+++ b/hydra/core/utils.py
@@ -97,16 +97,12 @@ def run_job(
         # handle output directories here
         Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
         os.chdir(working_dir)
-        hydra_output = Path(config.hydra.output_subdir)
 
         configure_log(config.hydra.job_logging, config.hydra.verbose)
 
         hydra_cfg = OmegaConf.masked_copy(config, "hydra")
         assert isinstance(hydra_cfg, DictConfig)
 
-        _save_config(task_cfg, "config.yaml", hydra_output)
-        _save_config(hydra_cfg, "hydra.yaml", hydra_output)
-        _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
         with env_override(hydra_cfg.hydra.job.env_set):
             ret.return_value = task_function(task_cfg)
         ret.task_name = JobRuntime.instance().get("name")
@@ -115,6 +111,12 @@ def run_job(
         # If logging is still required after run_job caller is responsible to re-initialize it.
         logging.shutdown()
 
+        if config.hydra.output_subdir is not None:
+            hydra_output = Path(config.hydra.output_subdir)
+            _save_config(task_cfg, "config.yaml", hydra_output)
+            _save_config(hydra_cfg, "hydra.yaml", hydra_output)
+            _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
+
         return ret
     finally:
         os.chdir(old_cwd)


Test patch:
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -391,6 +391,20 @@ def test_load_config_file_with_schema_validation(
         expected.append(LoadTrace("db/mysql", path, "main", "this_test"))
         assert config_loader.get_load_history() == expected
 
+    def test_assign_null(
+        self, restore_singletons: Any, path: str, recwarn: Any
+    ) -> None:
+        config_loader = ConfigLoaderImpl(
+            config_search_path=create_config_search_path(path)
+        )
+        cfg = config_loader.load_configuration(
+            config_name="config.yaml", overrides=["abc=null"]
+        )
+        with open_dict(cfg):
+            del cfg["hydra"]
+        assert cfg == {"normal_yaml_config": True, "abc": None}
+        assert len(recwarn) == 0
+
 
 @pytest.mark.parametrize(  # type:ignore
     "in_primary,in_merged,expected",
@@ -1034,6 +1048,26 @@ def test_apply_overrides_to_defaults(
             )
 
 
+def test_delete_by_assigning_null_is_deprecated() -> None:
+    msg = (
+        "\nRemoving from the defaults list by assigning 'null' "
+        "is deprecated and will be removed in Hydra 1.1."
+        "\nUse ~db"
+    )
+
+    defaults = ConfigLoaderImpl._parse_defaults(
+        OmegaConf.create({"defaults": [{"db": "mysql"}]})
+    )
+    parsed_overrides = [ConfigLoaderImpl._parse_override("db=null")]
+
+    with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
+        assert parsed_overrides[0].override.is_delete()
+        ConfigLoaderImpl._apply_overrides_to_defaults(
+            overrides=parsed_overrides, defaults=defaults
+        )
+        assert defaults == []
+
+
 @pytest.mark.parametrize(  # type: ignore
     "input_cfg,strict,overrides,expected",
     [
@@ -1064,9 +1098,6 @@ def test_apply_overrides_to_defaults(
             {"user@hostname": "active"},
             id="append@",
         ),
-        # override
-        pytest.param({"x": 20}, False, ["x=10"], {"x": 10}, id="override"),
-        pytest.param({"x": 20}, True, ["x=10"], {"x": 10}, id="override"),
         pytest.param(
             {"x": 20},
             True,
@@ -1077,7 +1108,13 @@ def test_apply_overrides_to_defaults(
                     "Could not append to config. An item is already at 'x'"
                 ),
             ),
-            id="override",
+            id="append",
+        ),
+        # override
+        pytest.param({"x": 20}, False, ["x=10"], {"x": 10}, id="override"),
+        pytest.param({"x": 20}, True, ["x=10"], {"x": 10}, id="override"),
+        pytest.param(
+            {"x": 20}, False, ["x=null"], {"x": None}, id="override_with_null"
         ),
         # delete
         pytest.param({"x": 20}, False, ["~x"], {}, id="delete"),
@@ -1127,17 +1164,3 @@ def test_apply_overrides_to_config(
     else:
         with expected:
             ConfigLoaderImpl._apply_overrides_to_config(overrides=overrides, cfg=cfg)
-
-
-def test_delete_by_assigning_null_is_deprecated() -> None:
-    msg = (
-        "\nRemoving from the defaults list by assigning 'null' "
-        "is deprecated and will be removed in Hydra 1.1."
-        "\nUse ~db"
-    )
-    with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
-        ret = ConfigLoaderImpl._parse_override("db=null")
-        assert ret.override.is_delete()
-        assert ret.override == ParsedOverride(
-            prefix="~", key="db", pkg1=None, pkg2=None, value=None
-        )
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -4,7 +4,7 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any, List
+from typing import Any, List, Set
 
 import pytest
 from omegaconf import DictConfig, OmegaConf
@@ -756,3 +756,39 @@ def test_config_name_and_path_overrides(
     # normalize newlines on Windows to make testing easier
     result = result.replace("\r\n", "\n")
     assert result == f"{config_path}_{config_name}: true"
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "overrides, expected_files",
+    [
+        ([], {"my_app.log", ".hydra"}),
+        (["hydra.output_subdir=foo"], {"my_app.log", "foo"}),
+        (["hydra.output_subdir=null"], {"my_app.log"}),
+    ],
+)
+@pytest.mark.parametrize(  # type: ignore
+    "calling_file, calling_module",
+    [
+        ("tests/test_apps/app_with_cfg/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg.my_app"),
+    ],
+)
+def test_hydra_output_dir(
+    restore_singletons: Any,
+    task_runner: TTaskRunner,
+    calling_file: str,
+    calling_module: str,
+    overrides: List[str],
+    expected_files: Set[str],
+) -> None:
+    with task_runner(
+        calling_file=calling_file,
+        calling_module=calling_module,
+        config_path=None,
+        config_name=None,
+        overrides=overrides,
+    ) as task:
+        assert task.temp_dir is not None
+        path = Path(task.temp_dir)
+        files = set([str(x)[len(task.temp_dir) + 1 :] for x in path.iterdir()])
+        assert files == expected_files