Repository: facebookresearch/hydra

Issue:
Update the examples in examples/configure_hydra
1. Update them
2. Cover them with tests
3. Mention them in the relevant doc pages.


Discussion:
Hello @omry, I want to take this issue. Can please elaborate what exactly I need to do. Thank You!! 
Awesome, basically this is a cleanup pass for the examples under the examples directory, in particular examples that are mentioned in the documentation.

All of the documented examples should be tested, and most already are. see tests/test_examples in the repo for what is already there.
For example, this covers examples from the tutorials. but there are some gaps.
Another thing is to add a direct link to each example from the relevant documentation page. 
something similar to the "Example application" link in plugins ([Example](https://hydra.cc/docs/next/plugins/ax_sweeper)).

Resolving patch:
diff --git a/examples/configure_hydra/logging/my_app.py b/examples/configure_hydra/logging/my_app.py
--- a/examples/configure_hydra/logging/my_app.py
+++ b/examples/configure_hydra/logging/my_app.py
@@ -1,6 +1,5 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import logging
-import sys
 
 from omegaconf import DictConfig
 
@@ -15,4 +14,4 @@ def my_app(_cfg: DictConfig) -> None:
 
 
 if __name__ == "__main__":
-    sys.exit(my_app())
+    my_app()
diff --git a/examples/configure_hydra/workdir/custom_workdir.py b/examples/configure_hydra/workdir/my_app.py
similarity index 88%
rename from examples/configure_hydra/workdir/custom_workdir.py
rename to examples/configure_hydra/workdir/my_app.py
--- a/examples/configure_hydra/workdir/custom_workdir.py
+++ b/examples/configure_hydra/workdir/my_app.py
@@ -1,6 +1,5 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import os
-import sys
 
 from omegaconf import DictConfig
 
@@ -13,4 +12,4 @@ def experiment(_cfg: DictConfig) -> None:
 
 
 if __name__ == "__main__":
-    sys.exit(experiment())
+    experiment()


Test patch:
diff --git a/hydra/test_utils/test_utils.py b/hydra/test_utils/test_utils.py
--- a/hydra/test_utils/test_utils.py
+++ b/hydra/test_utils/test_utils.py
@@ -11,6 +11,7 @@
 import sys
 import tempfile
 from contextlib import contextmanager
+from difflib import unified_diff
 from pathlib import Path
 from subprocess import PIPE, Popen, check_output
 from typing import Any, Dict, Iterator, List, Optional, Union
@@ -25,6 +26,7 @@
 from hydra.types import TaskFunction
 
 # CircleCI does not have the environment variable USER, breaking the tests.
+
 os.environ["USER"] = "test_user"
 
 log = logging.getLogger(__name__)
@@ -364,8 +366,39 @@ def run_with_error(cmd: Any, env: Any = None) -> str:
 def get_run_output(cmd: Any, env: Any = None) -> str:
     cmd = [sys.executable, "-Werror"] + cmd
     try:
-        return check_output(cmd, env=env).decode().rstrip()
+        ret = check_output(cmd, env=env).decode().rstrip()
+        return normalize_newlines(ret)
     except Exception as e:
         cmd = " ".join(cmd)
         print(f"==Error executing==\n{cmd}\n===================")
         raise e
+
+
+def normalize_newlines(s: str) -> str:
+    """
+    Normalizes new lines such they are comparable across different operating systems
+    :param s:
+    :return:
+    """
+    return s.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def assert_text_same(actual: str, expected: str) -> None:
+    actual = normalize_newlines(actual)
+    expected = normalize_newlines(expected)
+    lines = [
+        line
+        for line in unified_diff(
+            a=actual.splitlines(),
+            b=expected.splitlines(),
+            fromfile="Expected",
+            tofile="Actual",
+        )
+    ]
+
+    diff = "\n".join(lines)
+    if len(diff) > 0:
+        print("------------ DIFF -------------")
+        print(diff)
+        print("-------------------------------")
+        assert False, "Mismatch between expected and actual text"
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -5,15 +5,6 @@
 from hydra.types import TargetConf
 
 
-def normalize_newlines(s: str) -> str:
-    """
-    Normalizes new lines such they are comparable across different operating systems
-    :param s:
-    :return:
-    """
-    return s.replace("\r\n", "\n").replace("\r", "\n")
-
-
 def verify_hydra_pytest_plugin_not_installed() -> None:
     try:
         # This check can be removed in 1.1
diff --git a/tests/test_examples/test_configure_hydra.py b/tests/test_examples/test_configure_hydra.py
new file mode 100644
--- /dev/null
+++ b/tests/test_examples/test_configure_hydra.py
@@ -0,0 +1,95 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from pathlib import Path
+from textwrap import dedent
+from typing import Any
+
+from hydra.test_utils.test_utils import (
+    assert_text_same,
+    chdir_hydra_root,
+    get_run_output,
+)
+
+chdir_hydra_root()
+
+
+def test_custom_help(tmpdir: Path) -> None:
+    result = get_run_output(
+        [
+            "examples/configure_hydra/custom_help/my_app.py",
+            "hydra.run.dir=" + str(tmpdir),
+            "--help",
+        ]
+    )
+    expected = dedent(
+        """\
+            == AwesomeApp ==
+
+            This is AwesomeApp!
+            You can choose a db driver by appending
+            == Configuration groups ==
+            Compose your configuration from those groups (db=mysql)
+
+            db: mysql, postgresql
+
+
+            == Config ==
+            This is the config generated for this run.
+            You can override everything, for example:
+            python my_app.py db.user=foo db.pass=bar
+            -------
+            db:
+              driver: mysql
+              user: omry
+              pass: secret
+
+            -------
+
+            Powered by Hydra (https://hydra.cc)
+            Use --hydra-help to view Hydra specific help
+"""
+    )
+    assert_text_same(actual=result, expected=expected)
+
+
+def test_job_name_no_config_override(tmpdir: Path) -> None:
+    cmd = [
+        "examples/configure_hydra/job_name/no_config_file_override.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = get_run_output(cmd)
+    assert result == "no_config_file_override"
+
+
+def test_job_name_with_config_override(tmpdir: Path) -> None:
+    cmd = [
+        "examples/configure_hydra/job_name/with_config_file_override.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = get_run_output(cmd)
+    assert result == "name_from_config_file"
+
+
+def test_logging(tmpdir: Path) -> None:
+    cmd = [
+        "examples/configure_hydra/logging/my_app.py",
+        "hydra.run.dir=" + str(tmpdir),
+    ]
+    result = get_run_output(cmd)
+    assert result == "[INFO] - Info level message"
+
+
+def test_workdir_config(monkeypatch: Any, tmpdir: Path) -> None:
+    script = str(Path("examples/configure_hydra/workdir/my_app.py").absolute())
+    monkeypatch.chdir(tmpdir)
+    result = get_run_output([script])
+    assert Path(result) == Path(tmpdir) / "run_dir"
+
+    result = get_run_output([script, "--multirun", "hydra/hydra_logging=disabled"])
+    assert Path(result) == Path(tmpdir) / "sweep_dir" / "0"
+
+
+def test_workdir_override(monkeypatch: Any, tmpdir: Path) -> None:
+    script = str(Path("examples/configure_hydra/workdir/my_app.py").absolute())
+    monkeypatch.chdir(tmpdir)
+    result = get_run_output([script, "hydra.run.dir=blah"])
+    assert Path(result) == Path(tmpdir) / "blah"
diff --git a/tests/test_examples/test_patterns.py b/tests/test_examples/test_patterns.py
--- a/tests/test_examples/test_patterns.py
+++ b/tests/test_examples/test_patterns.py
@@ -66,7 +66,7 @@ def test_instantiate_structured_config_example(
     assert result == output
 
 
-def test_frozen(tmpdir: Any) -> None:
+def test_write_protect_config_node(tmpdir: Any) -> None:
     cmd = [
         "examples/patterns/write_protect_config_node/frozen.py",
         "hydra.run.dir=" + str(tmpdir),
diff --git a/tests/test_examples/test_tutorials_basic.py b/tests/test_examples/test_tutorials_basic.py
--- a/tests/test_examples/test_tutorials_basic.py
+++ b/tests/test_examples/test_tutorials_basic.py
@@ -288,33 +288,6 @@ def test_advanced_ad_hoc_composition(
     assert OmegaConf.create(result) == OmegaConf.create(expected)
 
 
-def test_examples_configure_hydra_job_name_no_config_override(tmpdir: Path) -> None:
-    cmd = [
-        "examples/configure_hydra/job_name/no_config_file_override.py",
-        "hydra.run.dir=" + str(tmpdir),
-    ]
-    result = get_run_output(cmd)
-    assert result == "no_config_file_override"
-
-
-def test_examples_configure_hydra_job_name_with_config_override(tmpdir: Path) -> None:
-    cmd = [
-        "examples/configure_hydra/job_name/with_config_file_override.py",
-        "hydra.run.dir=" + str(tmpdir),
-    ]
-    result = get_run_output(cmd)
-    assert result == "name_from_config_file"
-
-
-def test_examples_configure_hydra_logging(tmpdir: Path) -> None:
-    cmd = [
-        "examples/configure_hydra/logging/my_app.py",
-        "hydra.run.dir=" + str(tmpdir),
-    ]
-    result = get_run_output(cmd)
-    assert result == "[INFO] - Info level message"
-
-
 def test_examples_using_the_config_object(tmpdir: Path) -> None:
     cmd = [
         "examples/tutorials/basic/your_first_hydra_app/3_using_config/my_app.py",
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -17,10 +17,10 @@
     chdir_hydra_root,
     get_run_output,
     integration_test,
+    normalize_newlines,
     run_with_error,
     verify_dir_outputs,
 )
-from tests import normalize_newlines
 
 chdir_hydra_root()
 
diff --git a/tests/test_hydra_cli_errors.py b/tests/test_hydra_cli_errors.py
--- a/tests/test_hydra_cli_errors.py
+++ b/tests/test_hydra_cli_errors.py
@@ -5,8 +5,11 @@
 
 import pytest
 
-from hydra.test_utils.test_utils import chdir_hydra_root, run_with_error
-from tests import normalize_newlines
+from hydra.test_utils.test_utils import (
+    chdir_hydra_root,
+    normalize_newlines,
+    run_with_error,
+)
 
 chdir_hydra_root()