Repository: facebookresearch/hydra

Issue:
Add option to not create .hydra in output directory
In some cases users are not interested in the generation of config.yaml, hydra.yaml etc.
add an option to opt out.

Comments:

Patch:
diff --git a/hydra/_internal/config_loader_impl.py b/hydra/_internal/config_loader_impl.py
index ef18964cef..495f389bcd 100644
--- a/hydra/_internal/config_loader_impl.py
+++ b/hydra/_internal/config_loader_impl.py
@@ -61,7 +61,8 @@ class ParsedOverride:
         return self.pkg1 is not None and self.pkg2 is not None
 
     def is_delete(self) -> bool:
-        return self.prefix == "~"
+        legacy_delete = self.value == "null"
+        return self.prefix == "~" or legacy_delete
 
     def is_add(self) -> bool:
         return self.prefix == "+"
@@ -292,6 +293,19 @@ class ConfigLoaderImpl(ConfigLoader):
                 )
         for owl in overrides:
             override = owl.override
+            if override.value == "null":
+                if override.prefix not in (None, "~"):
+                    ConfigLoaderImpl._raise_parse_override_error(owl.input_line)
+                override.prefix = "~"
+                override.value = None
+
+                msg = (
+                    "\nRemoving from the defaults list by assigning 'null' "
+                    "is deprecated and will be removed in Hydra 1.1."
+                    f"\nUse ~{override.key}"
+                )
+                warnings.warn(category=UserWarning, message=msg)
+
             if (
                 not (override.is_delete() or override.is_package_rename())
                 and override.value is None
@@ -453,19 +467,6 @@ class ConfigLoaderImpl(ConfigLoader):
             pkg1 = matches.group("pkg1")
             pkg2 = matches.group("pkg2")
             value: Optional[str] = matches.group("value")
-            if value == "null":
-                if prefix not in (None, "~"):
-                    ConfigLoaderImpl._raise_parse_override_error(override)
-                prefix = "~"
-                value = None
-
-                msg = (
-                    "\nRemoving from the defaults list by assigning 'null' "
-                    "is deprecated and will be removed in Hydra 1.1."
-                    f"\nUse ~{key}"
-                )
-                warnings.warn(category=UserWarning, message=msg)
-
             ret = ParsedOverride(prefix, key, pkg1, pkg2, value)
             return ParsedOverrideWithLine(override=ret, input_line=override)
         else:
diff --git a/hydra/conf/__init__.py b/hydra/conf/__init__.py
index c7bc9ccbb8..7be98d6180 100644
--- a/hydra/conf/__init__.py
+++ b/hydra/conf/__init__.py
@@ -129,7 +129,8 @@ class HydraConf:
     # Output directory for produced configuration files and overrides.
     # E.g., hydra.yaml, overrides.yaml will go here. Useful for debugging
     # and extra context when looking at past runs.
-    output_subdir: str = ".hydra"
+    # Setting to None will prevent the creation of the output subdir.
+    output_subdir: Optional[str] = ".hydra"
 
     # Those lists will contain runtime overrides
     overrides: OverridesConf = OverridesConf()
diff --git a/hydra/core/utils.py b/hydra/core/utils.py
index 673b421343..e1da729fa3 100644
--- a/hydra/core/utils.py
+++ b/hydra/core/utils.py
@@ -97,16 +97,12 @@ def run_job(
         # handle output directories here
         Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
         os.chdir(working_dir)
-        hydra_output = Path(config.hydra.output_subdir)
 
         configure_log(config.hydra.job_logging, config.hydra.verbose)
 
         hydra_cfg = OmegaConf.masked_copy(config, "hydra")
         assert isinstance(hydra_cfg, DictConfig)
 
-        _save_config(task_cfg, "config.yaml", hydra_output)
-        _save_config(hydra_cfg, "hydra.yaml", hydra_output)
-        _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
         with env_override(hydra_cfg.hydra.job.env_set):
             ret.return_value = task_function(task_cfg)
         ret.task_name = JobRuntime.instance().get("name")
@@ -115,6 +111,12 @@ def run_job(
         # If logging is still required after run_job caller is responsible to re-initialize it.
         logging.shutdown()
 
+        if config.hydra.output_subdir is not None:
+            hydra_output = Path(config.hydra.output_subdir)
+            _save_config(task_cfg, "config.yaml", hydra_output)
+            _save_config(hydra_cfg, "hydra.yaml", hydra_output)
+            _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
+
         return ret
     finally:
         os.chdir(old_cwd)
diff --git a/news/324.feature b/news/324.feature
new file mode 100644
index 0000000000..8f52b743a7
--- /dev/null
+++ b/news/324.feature
@@ -0,0 +1,1 @@
+Support for disabling the creation of the `.hydra` subdirectory by overriding "hydra.output_subdir" to "null"
\ No newline at end of file
diff --git a/website/docs/terminology.md b/website/docs/terminology.md
index e1660dc399..9435b92f98 100644
--- a/website/docs/terminology.md
+++ b/website/docs/terminology.md
@@ -2,15 +2,16 @@
 id: terminology
 title: Terminology
 ---
+## Overview
 This page describes some of the common concepts in Hydra.
 It does not contain a comprehensive description of each item, nor is it a usage guide.  
 Most concepts here are described in much more details throughout the documentation.
 
-## Config
+### Config
 A config is an entity containing user configuration, composed of Config Nodes.
 Configs are always converted to OmegaConf DictConfig or ListConfig objects.
 
-## Primary Config
+### Primary Config
 The config named in `@hydra.main()` or in `hydra.experimental.compose()`.
 
 ### Config Node
@@ -30,7 +31,7 @@ A dataclass or an instance of a dataclass that is used to construct a config.
 The constructed config object is using the underlying type for runtime type validation.  
 Duck typing (The usage of the underlying type for type annotation) enables static type checking of the config.
 
-## Config Search Path:
+### Config Search Path:
 Alternative names: Search Path
    
 The Config Search Path is a list of virtual paths that is searched in order to find configs.  
@@ -38,7 +39,7 @@ Conceptually, this is similar to the Python PYTHONPATH or the Java CLASSPATH.
 When a config searched in the config search path, the first matching config is used.
 Each config search path element has a schema prefix such as file:// or pkg:// that is corresponding to a ConfigSourcePlugin.
 
-## Config Group
+### Config Group
 Alternative names: Group
 
 A config group is a logical directory in the Config Search Path. The Config Group options are the union of all the configs in that
@@ -46,23 +47,23 @@ directory across the Config Search Path.
 Config Groups can be hierarchical and in that case the path elements are separated by a forward slash ('/') 
 regardless of the operating system.
 
-## Config Group Option
+### Config Group Option
 Alternative names: Option
 
 One of the configs in a Config Group.
 
-## Defaults List
+### Defaults List
 Alternative names: Defaults
 
 A special list in the Primary Config. The Defaults List contains composition instructions Hydra is using when creating the 
 final config object.
 The defaults list is removed from the final config object. 
 
-## Package
+### Package
 A package is the parent lineage of a node. You also think of it as the path of the node in the config object.
 The package of a Config can be overridden via the command line or via the defaults list.
 
-# Examples
+## Examples
 
 ```yaml title="foo/oompa/loompa.yaml"
 a:
diff --git a/website/docs/tutorials/basic/running_your_app/logging.md b/website/docs/tutorials/basic/running_your_app/logging.md
index 4e9387bd51..ff997ab696 100644
--- a/website/docs/tutorials/basic/running_your_app/logging.md
+++ b/website/docs/tutorials/basic/running_your_app/logging.md
@@ -51,5 +51,11 @@ $ python my_app.py hydra.verbose=[__main__,hydra]
 [2019-09-29 13:06:00,896][__main__][DEBUG] - Debug level message
 ```
 
+You can disable the logging output using by setting `hydra/job_logging` to `disabled'   
+```commandline
+$ python my_app.py hydra/job_logging=disabled
+<NO OUTPUT>
+```
+
 Logging can be [customized](/configure_hydra/logging.md).
 
diff --git a/website/docs/tutorials/basic/running_your_app/working_directory.md b/website/docs/tutorials/basic/running_your_app/working_directory.md
index 30337df4f4..7532de2c64 100644
--- a/website/docs/tutorials/basic/running_your_app/working_directory.md
+++ b/website/docs/tutorials/basic/running_your_app/working_directory.md
@@ -49,6 +49,11 @@ Inside the configuration output directory we have:
 And in the main output directory:
 * `my_app.log`: A log file created for this run
 
+### Disabling output subdir 
+You can change the `.hydra` subdirectory name by overriding `hydra.output_subdir`.
+You can disable its creation by overriding `hydra.output_subdir` to `null`. 
+
+
 ### Original working directory
 
 You can still access the original working directory if you need to:


Test patch:
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index c6a054519d..c92d0cd17f 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -391,6 +391,20 @@ class TestConfigLoader:
         expected.append(LoadTrace("db/mysql", path, "main", "this_test"))
         assert config_loader.get_load_history() == expected
 
+    def test_assign_null(
+        self, restore_singletons: Any, path: str, recwarn: Any
+    ) -> None:
+        config_loader = ConfigLoaderImpl(
+            config_search_path=create_config_search_path(path)
+        )
+        cfg = config_loader.load_configuration(
+            config_name="config.yaml", overrides=["abc=null"]
+        )
+        with open_dict(cfg):
+            del cfg["hydra"]
+        assert cfg == {"normal_yaml_config": True, "abc": None}
+        assert len(recwarn) == 0
+
 
 @pytest.mark.parametrize(  # type:ignore
     "in_primary,in_merged,expected",
@@ -1034,6 +1048,26 @@ def test_apply_overrides_to_defaults(
             )
 
 
+def test_delete_by_assigning_null_is_deprecated() -> None:
+    msg = (
+        "\nRemoving from the defaults list by assigning 'null' "
+        "is deprecated and will be removed in Hydra 1.1."
+        "\nUse ~db"
+    )
+
+    defaults = ConfigLoaderImpl._parse_defaults(
+        OmegaConf.create({"defaults": [{"db": "mysql"}]})
+    )
+    parsed_overrides = [ConfigLoaderImpl._parse_override("db=null")]
+
+    with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
+        assert parsed_overrides[0].override.is_delete()
+        ConfigLoaderImpl._apply_overrides_to_defaults(
+            overrides=parsed_overrides, defaults=defaults
+        )
+        assert defaults == []
+
+
 @pytest.mark.parametrize(  # type: ignore
     "input_cfg,strict,overrides,expected",
     [
@@ -1064,9 +1098,6 @@ def test_apply_overrides_to_defaults(
             {"user@hostname": "active"},
             id="append@",
         ),
-        # override
-        pytest.param({"x": 20}, False, ["x=10"], {"x": 10}, id="override"),
-        pytest.param({"x": 20}, True, ["x=10"], {"x": 10}, id="override"),
         pytest.param(
             {"x": 20},
             True,
@@ -1077,7 +1108,13 @@ def test_apply_overrides_to_defaults(
                     "Could not append to config. An item is already at 'x'"
                 ),
             ),
-            id="override",
+            id="append",
+        ),
+        # override
+        pytest.param({"x": 20}, False, ["x=10"], {"x": 10}, id="override"),
+        pytest.param({"x": 20}, True, ["x=10"], {"x": 10}, id="override"),
+        pytest.param(
+            {"x": 20}, False, ["x=null"], {"x": None}, id="override_with_null"
         ),
         # delete
         pytest.param({"x": 20}, False, ["~x"], {}, id="delete"),
@@ -1127,17 +1164,3 @@ def test_apply_overrides_to_config(
     else:
         with expected:
             ConfigLoaderImpl._apply_overrides_to_config(overrides=overrides, cfg=cfg)
-
-
-def test_delete_by_assigning_null_is_deprecated() -> None:
-    msg = (
-        "\nRemoving from the defaults list by assigning 'null' "
-        "is deprecated and will be removed in Hydra 1.1."
-        "\nUse ~db"
-    )
-    with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
-        ret = ConfigLoaderImpl._parse_override("db=null")
-        assert ret.override.is_delete()
-        assert ret.override == ParsedOverride(
-            prefix="~", key="db", pkg1=None, pkg2=None, value=None
-        )
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
index a4c617fabe..be8df2b66e 100644
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -4,7 +4,7 @@ import re
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any, List
+from typing import Any, List, Set
 
 import pytest
 from omegaconf import DictConfig, OmegaConf
@@ -756,3 +756,39 @@ def test_config_name_and_path_overrides(
     # normalize newlines on Windows to make testing easier
     result = result.replace("\r\n", "\n")
     assert result == f"{config_path}_{config_name}: true"
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "overrides, expected_files",
+    [
+        ([], {"my_app.log", ".hydra"}),
+        (["hydra.output_subdir=foo"], {"my_app.log", "foo"}),
+        (["hydra.output_subdir=null"], {"my_app.log"}),
+    ],
+)
+@pytest.mark.parametrize(  # type: ignore
+    "calling_file, calling_module",
+    [
+        ("tests/test_apps/app_with_cfg/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg.my_app"),
+    ],
+)
+def test_hydra_output_dir(
+    restore_singletons: Any,
+    task_runner: TTaskRunner,
+    calling_file: str,
+    calling_module: str,
+    overrides: List[str],
+    expected_files: Set[str],
+) -> None:
+    with task_runner(
+        calling_file=calling_file,
+        calling_module=calling_module,
+        config_path=None,
+        config_name=None,
+        overrides=overrides,
+    ) as task:
+        assert task.temp_dir is not None
+        path = Path(task.temp_dir)
+        files = set([str(x)[len(task.temp_dir) + 1 :] for x in path.iterdir()])
+        assert files == expected_files