Repository: facebookresearch/hydra

Issue:
Clean up test fixtures in Hydra and plugins
Should use conftest.py to simplify


Discussion:

Resolving patch:
diff --git a/noxfile.py b/noxfile.py
--- a/noxfile.py
+++ b/noxfile.py
@@ -63,8 +63,8 @@ def pytest_args(session, *args):
     return ret
 
 
-def run_pytest(session, directory="."):
-    pytest_cmd = pytest_args(session, directory)
+def run_pytest(session, directory=".", *args):
+    pytest_cmd = pytest_args(session, directory, *args)
     session.run(*pytest_cmd, silent=SILENT)
 
 
@@ -194,7 +194,7 @@ def test_core(session, install_cmd):
     run_pytest(session, "tests")
 
     # test discovery_test_plugin
-    run_pytest(session, "tests/test_plugins/discovery_test_plugin")
+    run_pytest(session, "tests/test_plugins/discovery_test_plugin", "--noconftest")
 
     # Install and test example app
     session.run(*install_cmd, "examples/advanced/hydra_app_example", silent=SILENT)


Test patch:
diff --git a/hydra/test_utils/_conftest.py b/hydra/test_utils/_conftest.py
new file mode 100644
--- /dev/null
+++ b/hydra/test_utils/_conftest.py
@@ -0,0 +1,110 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import copy
+from typing import Any, Callable, List, Optional
+
+import pytest
+
+from hydra.core.singleton import Singleton
+from hydra.test_utils.test_utils import (
+    GlobalHydraContext,
+    SweepTaskFunction,
+    TaskTestFunction,
+)
+from hydra.types import TaskFunction
+
+
+@pytest.fixture(scope="function")  # type: ignore
+def restore_singletons() -> Any:
+    """
+    A fixture to restore singletons state after this the function.
+    This is useful for functions that are making a one-off change to singlestons that should not effect
+    other tests
+    """
+    state = copy.deepcopy(Singleton.get_state())
+    yield
+    Singleton.set_state(state)
+
+
+@pytest.fixture(scope="function")  # type: ignore
+def hydra_global_context() -> Callable[
+    [str, Optional[str], Optional[bool]], GlobalHydraContext
+]:
+    def _(
+        task_name: str = "task",
+        config_dir: Optional[str] = None,
+        strict: Optional[bool] = False,
+    ) -> "GlobalHydraContext":
+        ctx = GlobalHydraContext()
+        ctx.task_name = task_name
+        ctx.config_dir = config_dir
+        ctx.strict = strict
+        return ctx
+
+    return _
+
+
+@pytest.fixture(scope="function")  # type: ignore
+def sweep_runner() -> Callable[
+    [
+        Optional[str],
+        Optional[str],
+        Optional[TaskFunction],
+        Optional[str],
+        Optional[str],
+        Optional[List[str]],
+        Optional[bool],
+    ],
+    SweepTaskFunction,
+]:
+    def _(
+        calling_file: Optional[str],
+        calling_module: Optional[str],
+        task_function: Optional[TaskFunction],
+        config_path: Optional[str],
+        config_name: Optional[str],
+        overrides: Optional[List[str]],
+        strict: Optional[bool] = None,
+    ) -> SweepTaskFunction:
+        sweep = SweepTaskFunction()
+        sweep.calling_file = calling_file
+        sweep.calling_module = calling_module
+        sweep.task_function = task_function
+        sweep.config_path = config_path
+        sweep.config_name = config_name
+        sweep.strict = strict
+        sweep.overrides = overrides or []
+        return sweep
+
+    return _
+
+
+@pytest.fixture(scope="function")  # type: ignore
+def task_runner() -> Callable[
+    [
+        Optional[str],
+        Optional[str],
+        Optional[str],
+        Optional[str],
+        Optional[List[str]],
+        Optional[bool],
+    ],
+    TaskTestFunction,
+]:
+    def _(
+        calling_file: Optional[str],
+        calling_module: Optional[str],
+        config_path: Optional[str],
+        config_name: Optional[str],
+        overrides: Optional[List[str]] = None,
+        strict: Optional[bool] = False,
+    ) -> TaskTestFunction:
+        task = TaskTestFunction()
+        task.overrides = overrides or []
+        task.calling_file = calling_file
+        task.config_name = config_name
+        task.calling_module = calling_module
+        task.config_path = config_path
+        task.strict = strict
+        return task
+
+    return _
diff --git a/hydra/test_utils/launcher_common_tests.py b/hydra/test_utils/launcher_common_tests.py
--- a/hydra/test_utils/launcher_common_tests.py
+++ b/hydra/test_utils/launcher_common_tests.py
@@ -28,21 +28,21 @@ def test_sweep_1_job(
 
     def test_sweep_2_jobs(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         sweep_2_jobs(
             sweep_runner, overrides=["hydra/launcher=" + launcher_name] + overrides
         )
 
     def test_not_sweeping_hydra_overrides(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         not_sweeping_hydra_overrides(
             sweep_runner, overrides=["hydra/launcher=" + launcher_name] + overrides
         )
 
     def test_sweep_1_job_strict(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         sweep_1_job(
             sweep_runner,
             strict=True,
@@ -51,7 +51,7 @@ def test_sweep_1_job_strict(
 
     def test_sweep_1_job_strict_and_bad_key(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         # Ideally this would be KeyError, This can't be more specific because some launcher plugins
         # like submitit raises a different exception on job failure and not the underlying exception.
         with pytest.raises(Exception):
@@ -69,10 +69,7 @@ def test_sweep_2_optimizers(
         )
 
     def test_sweep_over_unspecified_mandatory_default(
-        self,
-        sweep_runner: TSweepRunner,  # noqa: F811
-        launcher_name: str,
-        overrides: List[str],
+        self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str],
     ) -> None:
         base_overrides = ["hydra/launcher=" + launcher_name, "group1=file1,file2"]
         sweep = sweep_runner(
@@ -97,7 +94,7 @@ def test_sweep_over_unspecified_mandatory_default(
 
     def test_sweep_and_override(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         """
         Tests that we can override things in the configs merged in only during the sweep config construction
         db.user=someone does not exist db_conf.yaml, and is only appear when we merge in db=mysql or db=postgresql.
@@ -145,7 +142,7 @@ def test_sweep_and_override(
 class BatchedSweeperTestSuite:
     def test_sweep_2_jobs_2_batches(
         self, sweep_runner: TSweepRunner, launcher_name: str, overrides: List[str]
-    ) -> None:  # noqa: F811
+    ) -> None:
         overrides.extend(
             # order sensitive?
             ["hydra/launcher=" + launcher_name, "group1=file1,file2", "bar=100,200,300"]
diff --git a/hydra/test_utils/test_utils.py b/hydra/test_utils/test_utils.py
--- a/hydra/test_utils/test_utils.py
+++ b/hydra/test_utils/test_utils.py
@@ -12,16 +12,14 @@
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Callable, Iterator, List, Optional, Union
+from typing import Any, Iterator, List, Optional, Union
 
-import pytest
 from omegaconf import DictConfig, OmegaConf
 from typing_extensions import Protocol
 
 import hydra.experimental
 from hydra._internal.hydra import Hydra
 from hydra.core.global_hydra import GlobalHydra
-from hydra.core.singleton import Singleton
 from hydra.core.utils import JobReturn, split_config_path
 from hydra.types import TaskFunction
 
@@ -52,24 +50,6 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None:  # type: ignore
         GlobalHydra().clear()
 
 
-@pytest.fixture(scope="function")  # type: ignore
-def hydra_global_context() -> Callable[
-    [str, Optional[str], Optional[bool]], GlobalHydraContext
-]:
-    def _(
-        task_name: str = "task",
-        config_dir: Optional[str] = None,
-        strict: Optional[bool] = False,
-    ) -> "GlobalHydraContext":
-        ctx = GlobalHydraContext()
-        ctx.task_name = task_name
-        ctx.config_dir = config_dir
-        ctx.strict = strict
-        return ctx
-
-    return _
-
-
 class TGlobalHydraContext(Protocol):
     def __call__(
         self,
@@ -138,38 +118,6 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         shutil.rmtree(self.temp_dir)
 
 
-@pytest.fixture(scope="function")  # type: ignore
-def task_runner() -> Callable[
-    [
-        Optional[str],
-        Optional[str],
-        Optional[str],
-        Optional[str],
-        Optional[List[str]],
-        Optional[bool],
-    ],
-    TaskTestFunction,
-]:
-    def _(
-        calling_file: Optional[str],
-        calling_module: Optional[str],
-        config_path: Optional[str],
-        config_name: Optional[str],
-        overrides: Optional[List[str]] = None,
-        strict: Optional[bool] = False,
-    ) -> TaskTestFunction:
-        task = TaskTestFunction()
-        task.overrides = overrides or []
-        task.calling_file = calling_file
-        task.config_name = config_name
-        task.calling_module = calling_module
-        task.config_path = config_path
-        task.strict = strict
-        return task
-
-    return _
-
-
 class TTaskRunner(Protocol):
     def __call__(
         self,
@@ -242,41 +190,6 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         shutil.rmtree(self.temp_dir)
 
 
-@pytest.fixture(scope="function")  # type: ignore
-def sweep_runner() -> Callable[
-    [
-        Optional[str],
-        Optional[str],
-        Optional[TaskFunction],
-        Optional[str],
-        Optional[str],
-        Optional[List[str]],
-        Optional[bool],
-    ],
-    SweepTaskFunction,
-]:
-    def _(
-        calling_file: Optional[str],
-        calling_module: Optional[str],
-        task_function: Optional[TaskFunction],
-        config_path: Optional[str],
-        config_name: Optional[str],
-        overrides: Optional[List[str]],
-        strict: Optional[bool] = None,
-    ) -> SweepTaskFunction:
-        sweep = SweepTaskFunction()
-        sweep.calling_file = calling_file
-        sweep.calling_module = calling_module
-        sweep.task_function = task_function
-        sweep.config_path = config_path
-        sweep.config_name = config_name
-        sweep.strict = strict
-        sweep.overrides = overrides or []
-        return sweep
-
-    return _
-
-
 class TSweepRunner(Protocol):
     returns: List[List[JobReturn]]
 
@@ -411,15 +324,3 @@ def experiment(cfg):
         return file_str
     finally:
         os.chdir(orig_dir)
-
-
-@pytest.fixture(scope="function")  # type: ignore
-def restore_singletons() -> Any:
-    """
-    A fixture to restore singletons state after this the function.
-    This is useful for functions that are making a one-off change to singlestons that should not effect
-    other tests
-    """
-    state = copy.deepcopy(Singleton.get_state())
-    yield
-    Singleton.set_state(state)
diff --git a/plugins/examples/example_configsource_plugin/tests/conftest.py b/plugins/examples/example_configsource_plugin/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/examples/example_configsource_plugin/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/examples/example_generic_plugin/tests/conftest.py b/plugins/examples/example_generic_plugin/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/examples/example_generic_plugin/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/examples/example_launcher_plugin/tests/conftest.py b/plugins/examples/example_launcher_plugin/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/examples/example_launcher_plugin/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/examples/example_launcher_plugin/tests/test_example_launcher_plugin.py b/plugins/examples/example_launcher_plugin/tests/test_example_launcher_plugin.py
--- a/plugins/examples/example_launcher_plugin/tests/test_example_launcher_plugin.py
+++ b/plugins/examples/example_launcher_plugin/tests/test_example_launcher_plugin.py
@@ -7,10 +7,6 @@
     IntegrationTestSuite,
     LauncherTestSuite,
 )
-
-# This has to be included here for the LauncherTestSuite to work.
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import sweep_runner  # noqa: F401
 from hydra_plugins.example_launcher_plugin.example_launcher import ExampleLauncher
 
 
diff --git a/plugins/examples/example_searchpath_plugin/tests/conftest.py b/plugins/examples/example_searchpath_plugin/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/examples/example_searchpath_plugin/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/examples/example_searchpath_plugin/tests/test_example_search_path_plugin.py b/plugins/examples/example_searchpath_plugin/tests/test_example_search_path_plugin.py
--- a/plugins/examples/example_searchpath_plugin/tests/test_example_search_path_plugin.py
+++ b/plugins/examples/example_searchpath_plugin/tests/test_example_search_path_plugin.py
@@ -2,12 +2,7 @@
 from hydra.core.global_hydra import GlobalHydra
 from hydra.core.plugins import Plugins
 from hydra.plugins.search_path_plugin import SearchPathPlugin
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TGlobalHydraContext,
-    hydra_global_context,
-)
+from hydra.test_utils.test_utils import TGlobalHydraContext
 from hydra_plugins.example_searchpath_plugin.example_searchpath_plugin import (
     ExampleSearchPathPlugin,
 )
@@ -20,9 +15,7 @@ def test_discovery() -> None:
     ]
 
 
-def test_config_installed(
-    hydra_global_context: TGlobalHydraContext,  # noqa: F811
-) -> None:
+def test_config_installed(hydra_global_context: TGlobalHydraContext) -> None:
     with hydra_global_context():
         config_loader = GlobalHydra.instance().config_loader()
         assert "my_default_output_dir" in config_loader.get_group_options(
diff --git a/plugins/examples/example_sweeper_plugin/tests/conftest.py b/plugins/examples/example_sweeper_plugin/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/examples/example_sweeper_plugin/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/examples/example_sweeper_plugin/tests/test_example_sweeper_plugin.py b/plugins/examples/example_sweeper_plugin/tests/test_example_sweeper_plugin.py
--- a/plugins/examples/example_sweeper_plugin/tests/test_example_sweeper_plugin.py
+++ b/plugins/examples/example_sweeper_plugin/tests/test_example_sweeper_plugin.py
@@ -8,9 +8,7 @@
     IntegrationTestSuite,
     LauncherTestSuite,
 )
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import TSweepRunner, sweep_runner  # noqa: F401
+from hydra.test_utils.test_utils import TSweepRunner
 from hydra_plugins.example_sweeper_plugin.example_sweeper import ExampleSweeper
 
 
@@ -21,7 +19,7 @@ def test_discovery() -> None:
     ]
 
 
-def test_launched_jobs(sweep_runner: TSweepRunner) -> None:  # noqa: F811 # type: ignore
+def test_launched_jobs(sweep_runner: TSweepRunner) -> None:
     sweep = sweep_runner(
         calling_file=None,
         calling_module="hydra.test_utils.a_module",
diff --git a/plugins/hydra_ax_sweeper/tests/conftest.py b/plugins/hydra_ax_sweeper/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/hydra_ax_sweeper/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py
--- a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py
+++ b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py
@@ -11,13 +11,7 @@
 from hydra.core.hydra_config import HydraConfig
 from hydra.core.plugins import Plugins
 from hydra.plugins.sweeper import Sweeper
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TSweepRunner,
-    chdir_plugin_root,
-    sweep_runner,
-)
+from hydra.test_utils.test_utils import TSweepRunner, chdir_plugin_root
 from hydra_plugins.hydra_ax_sweeper.ax_sweeper import AxSweeper
 
 chdir_plugin_root()
@@ -63,7 +57,7 @@ def quadratic(cfg: DictConfig) -> Any:
 #     ...
 
 
-def test_jobs_configured_via_config(sweep_runner: TSweepRunner,) -> None:  # noqa: F811
+def test_jobs_configured_via_config(sweep_runner: TSweepRunner) -> None:
     sweep = sweep_runner(
         calling_file=os.path.dirname(os.path.abspath(__file__)),
         calling_module=None,
@@ -90,7 +84,7 @@ def test_jobs_configured_via_config(sweep_runner: TSweepRunner,) -> None:  # noq
         assert math.isclose(best_parameters["quadratic.y"], -1.0, abs_tol=1e-4)
 
 
-def test_jobs_configured_via_cmd(sweep_runner: TSweepRunner,) -> None:  # noqa: F811
+def test_jobs_configured_via_cmd(sweep_runner: TSweepRunner,) -> None:
     sweep = sweep_runner(
         calling_file=os.path.dirname(os.path.abspath(__file__)),
         calling_module=None,
@@ -119,9 +113,7 @@ def test_jobs_configured_via_cmd(sweep_runner: TSweepRunner,) -> None:  # noqa:
         assert math.isclose(best_parameters["quadratic.y"], 2.0, abs_tol=1e-4)
 
 
-def test_jobs_configured_via_cmd_and_config(
-    sweep_runner: TSweepRunner,  # noqa: F811
-) -> None:
+def test_jobs_configured_via_cmd_and_config(sweep_runner: TSweepRunner) -> None:
     sweep = sweep_runner(
         calling_file=os.path.dirname(os.path.abspath(__file__)),
         calling_module=None,
@@ -150,7 +142,7 @@ def test_jobs_configured_via_cmd_and_config(
 
 
 def test_configuration_set_via_cmd_and_default_config(
-    sweep_runner: TSweepRunner,  # noqa: F811
+    sweep_runner: TSweepRunner,
 ) -> None:
     sweep = sweep_runner(
         calling_file=os.path.dirname(os.path.abspath(__file__)),
diff --git a/plugins/hydra_colorlog/tests/conftest.py b/plugins/hydra_colorlog/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/hydra_colorlog/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/hydra_colorlog/tests/test_colorlog.py b/plugins/hydra_colorlog/tests/test_colorlog.py
--- a/plugins/hydra_colorlog/tests/test_colorlog.py
+++ b/plugins/hydra_colorlog/tests/test_colorlog.py
@@ -1,19 +1,13 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-from hydra.core.global_hydra import GlobalHydra
+from typing import Any
 
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TGlobalHydraContext,
-    chdir_plugin_root,
-    hydra_global_context,
-)
+from hydra.core.global_hydra import GlobalHydra
+from hydra.test_utils.test_utils import chdir_plugin_root
 
 chdir_plugin_root()
 
 
-def test_config_installed(
-    hydra_global_context: TGlobalHydraContext,  # noqa: F811
-) -> None:
+def test_config_installed(hydra_global_context: Any) -> None:
     """
     Tests that color options are available for both hydra/hydra_logging and hydra/job_logging
     """
diff --git a/plugins/hydra_joblib_launcher/tests/conftest.py b/plugins/hydra_joblib_launcher/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/hydra_joblib_launcher/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/hydra_joblib_launcher/tests/test_joblib_launcher.py b/plugins/hydra_joblib_launcher/tests/test_joblib_launcher.py
--- a/plugins/hydra_joblib_launcher/tests/test_joblib_launcher.py
+++ b/plugins/hydra_joblib_launcher/tests/test_joblib_launcher.py
@@ -5,18 +5,11 @@
 
 from hydra.core.plugins import Plugins
 from hydra.plugins.launcher import Launcher
-from hydra.test_utils.launcher_common_tests import (  # noqa: F401
+from hydra.test_utils.launcher_common_tests import (
     IntegrationTestSuite,
     LauncherTestSuite,
 )
-
-# This has to be included here for the LauncherTestSuite to work.
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TSweepRunner,
-    chdir_plugin_root,
-    sweep_runner,
-)
+from hydra.test_utils.test_utils import TSweepRunner, chdir_plugin_root
 from hydra_plugins.hydra_joblib_launcher.joblib_launcher import JoblibLauncher
 
 chdir_plugin_root()
@@ -68,7 +61,7 @@ class TestJoblibLauncherIntegration(IntegrationTestSuite):
     pass
 
 
-def test_example_app(sweep_runner: TSweepRunner) -> None:  # noqa: F811
+def test_example_app(sweep_runner: TSweepRunner) -> None:
     with sweep_runner(
         calling_file="example/my_app.py",
         calling_module=None,
diff --git a/plugins/hydra_nevergrad_sweeper/tests/conftest.py b/plugins/hydra_nevergrad_sweeper/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/plugins/hydra_nevergrad_sweeper/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/hydra_nevergrad_sweeper/tests/test_nevergrad_sweeper_plugin.py b/plugins/hydra_nevergrad_sweeper/tests/test_nevergrad_sweeper_plugin.py
--- a/plugins/hydra_nevergrad_sweeper/tests/test_nevergrad_sweeper_plugin.py
+++ b/plugins/hydra_nevergrad_sweeper/tests/test_nevergrad_sweeper_plugin.py
@@ -1,7 +1,5 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import subprocess
-
-# noinspection PyUnresolvedReferences
 import sys
 from pathlib import Path
 from typing import Any
@@ -12,11 +10,7 @@
 
 from hydra.core.plugins import Plugins
 from hydra.plugins.sweeper import Sweeper
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TSweepRunner,
-    chdir_plugin_root,
-    sweep_runner,
-)
+from hydra.test_utils.test_utils import TSweepRunner, chdir_plugin_root
 from hydra_plugins.hydra_nevergrad_sweeper import core
 
 chdir_plugin_root()
@@ -54,8 +48,7 @@ def test_make_parameter_from_commandline(
         assert isinstance(param.value, value_cls)
 
 
-# pylint: disable=redefined-outer-name
-def test_launched_jobs(sweep_runner: TSweepRunner) -> None:  # noqa: F811 # type: ignore
+def test_launched_jobs(sweep_runner: TSweepRunner) -> None:
     budget = 8
     sweep = sweep_runner(
         calling_file=None,
@@ -77,9 +70,7 @@ def test_launched_jobs(sweep_runner: TSweepRunner) -> None:  # noqa: F811 # type
         assert sweep.returns is None
 
 
-@pytest.mark.parametrize(  # type: ignore
-    "with_commandline", (True, False)
-)
+@pytest.mark.parametrize("with_commandline", (True, False))  # type: ignore
 def test_nevergrad_example(with_commandline: bool, tmpdir: Path) -> None:
     budget = 32 if with_commandline else 1  # make a full test only once (faster)
     cmd = [
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,2 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/tests/test_basic_launcher.py b/tests/test_basic_launcher.py
--- a/tests/test_basic_launcher.py
+++ b/tests/test_basic_launcher.py
@@ -7,9 +7,6 @@
     LauncherTestSuite,
 )
 
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import sweep_runner  # noqa: F401
-
 
 @pytest.mark.parametrize("launcher_name, overrides", [("basic", [])])
 class TestBasicLauncher(LauncherTestSuite):
diff --git a/tests/test_compose.py b/tests/test_compose.py
--- a/tests/test_compose.py
+++ b/tests/test_compose.py
@@ -7,13 +7,10 @@
 from hydra.core.config_search_path import SearchPathQuery
 from hydra.core.global_hydra import GlobalHydra
 from hydra.experimental import compose, initialize
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
+from hydra.test_utils.test_utils import (
     TGlobalHydraContext,
     chdir_hydra_root,
     does_not_raise,
-    hydra_global_context,
 )
 
 chdir_hydra_root()
@@ -59,7 +56,7 @@ def test_initialize_with_config_dir() -> None:
 class TestCompose:
     def test_compose_decorator(
         self,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_dir: str,
         config_file: str,
         overrides: List[str],
@@ -72,7 +69,7 @@ def test_compose_decorator(
     def test_compose_config(
         self,
         config_dir: str,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_file: str,
         overrides: List[str],
         expected: Any,
@@ -84,7 +81,7 @@ def test_compose_config(
     def test_strict_failure_global_strict(
         self,
         config_dir: str,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_file: str,
         overrides: List[str],
         expected: Any,
@@ -98,7 +95,7 @@ def test_strict_failure_global_strict(
     def test_strict_failure_call_is_strict(
         self,
         config_dir: str,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_file: str,
         overrides: List[str],
         expected: Any,
@@ -117,7 +114,7 @@ def test_strict_failure_call_is_strict(
     def test_strict_failure_disabled_on_call(
         self,
         config_dir: str,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_file: str,
         overrides: List[str],
         expected: Any,
@@ -203,7 +200,7 @@ class TestComposeCloudInfraExample:
     def test_compose(
         self,
         config_dir: str,
-        hydra_global_context: TGlobalHydraContext,  # noqa: F811
+        hydra_global_context: TGlobalHydraContext,
         config_file: str,
         overrides: List[str],
         expected: Any,
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -11,12 +11,7 @@
 from hydra.core.config_loader import LoadTrace
 from hydra.core.config_store import ConfigStore, ConfigStoreWithProvider
 from hydra.errors import MissingConfigException
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    chdir_hydra_root,
-    restore_singletons,
-)
+from hydra.test_utils.test_utils import chdir_hydra_root
 
 chdir_hydra_root()
 
@@ -217,9 +212,7 @@ def test_compose_file_with_dot(self, path: str) -> None:
         del cfg["hydra"]
         assert cfg == {"abc=cde": None, "bar": 100}
 
-    def test_load_config_with_schema(
-        self, restore_singletons: Any, path: str  # noqa: F811
-    ) -> None:
+    def test_load_config_with_schema(self, restore_singletons: Any, path: str) -> None:
 
         ConfigStore.instance().store(
             group="db",
@@ -260,7 +253,7 @@ def test_load_config_with_schema(
             )
 
     def test_load_config_file_with_schema_validation(
-        self, restore_singletons: Any, path: str  # noqa: F811
+        self, restore_singletons: Any, path: str
     ) -> None:
 
         with ConfigStoreWithProvider("test_provider") as config_store:
@@ -500,7 +493,7 @@ def test_mixed_composition_order() -> None:
     assert config_loader.get_load_history() == expected
 
 
-def test_load_schema_as_config(restore_singletons: Any) -> None:  # noqa: F811
+def test_load_schema_as_config(restore_singletons: Any) -> None:
     """
     Load structured config as a configuration
     """
@@ -554,7 +547,7 @@ class Config:
     plugin: Plugin = Plugin()
 
 
-def test_overlapping_schemas(restore_singletons: Any) -> None:  # noqa: F811
+def test_overlapping_schemas(restore_singletons: Any) -> None:
 
     cs = ConfigStore.instance()
     cs.store(name="config", node=Config)
@@ -577,7 +570,7 @@ def test_overlapping_schemas(restore_singletons: Any) -> None:  # noqa: F811
         cfg.plugin = 10
 
 
-def test_invalid_plugin_merge(restore_singletons: Any) -> Any:  # noqa: F811
+def test_invalid_plugin_merge(restore_singletons: Any) -> Any:
     cs = ConfigStore.instance()
     cs.store(name="config", node=Config)
     cs.store(group="plugin", name="invalid", node=InvalidPlugin, path="plugin")
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -9,14 +9,10 @@
 from omegaconf import OmegaConf
 
 from hydra import MissingConfigException
-
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
+from hydra.test_utils.test_utils import (
     TSweepRunner,
     TTaskRunner,
     chdir_hydra_root,
-    sweep_runner,
-    task_runner,
     verify_dir_outputs,
 )
 
@@ -27,7 +23,7 @@
     "calling_file, calling_module", [(".", None), (None, ".")]
 )
 def test_missing_conf_dir(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with pytest.raises(MissingConfigException):
         with task_runner(
@@ -47,7 +43,7 @@ def test_missing_conf_dir(
     ],
 )
 def test_missing_conf_file(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with pytest.raises(MissingConfigException):
         with task_runner(
@@ -67,7 +63,7 @@ def test_missing_conf_file(
     ],
 )
 def test_app_without_config___no_overrides(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -87,7 +83,7 @@ def test_app_without_config___no_overrides(
     ],
 )
 def test_hydra_main_rerun(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -107,7 +103,7 @@ def test_hydra_main_rerun(
     ],
 )
 def test_app_without_config__with_overrides(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -130,7 +126,7 @@ def test_app_without_config__with_overrides(
     ],
 )
 def test_app_with_config_file__no_overrides(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
 
     task = task_runner(
@@ -155,7 +151,7 @@ def test_app_with_config_file__no_overrides(
     ],
 )
 def test_app_with_config_path_backward_compatibility(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     task = task_runner(
         calling_file=calling_file,
@@ -180,7 +176,7 @@ def test_app_with_config_path_backward_compatibility(
     ],
 )
 def test_app_with_config_file__with_overide(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -203,7 +199,7 @@ def test_app_with_config_file__with_overide(
     ],
 )
 def test_app_with_split_config(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -226,7 +222,7 @@ def test_app_with_split_config(
     ],
 )
 def test_app_with_config_groups__override_dataset__wrong(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with pytest.raises(MissingConfigException) as ex:
         with task_runner(
@@ -248,7 +244,7 @@ def test_app_with_config_groups__override_dataset__wrong(
     ],
 )
 def test_app_with_config_groups__override_all_configs(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -271,7 +267,7 @@ def test_app_with_config_groups__override_all_configs(
     ],
 )
 def test_app_with_sweep_cfg__override_to_basic_launcher(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     with task_runner(
         calling_file=calling_file,
@@ -329,7 +325,7 @@ def test_cfg(tmpdir: Path, flag: str, expected_keys: List[str]) -> None:
 )
 @pytest.mark.parametrize("overrides", [["free_group=opt1,opt2"]])  # type: ignore
 def test_multirun_with_free_override(
-    sweep_runner: TSweepRunner,  # noqa: F811
+    sweep_runner: TSweepRunner,
     calling_file: str,
     calling_module: str,
     overrides: List[str],
@@ -359,7 +355,7 @@ def test_multirun_with_free_override(
     ],
 )
 def test_sweep_complex_defaults(
-    sweep_runner: TSweepRunner, calling_file: str, calling_module: str  # noqa: F811
+    sweep_runner: TSweepRunner, calling_file: str, calling_module: str
 ) -> None:
     with sweep_runner(
         calling_file=calling_file,
@@ -468,7 +464,7 @@ def test_help(
     ],
 )
 def test_interpolating_dir_hydra_to_app(
-    task_runner: TTaskRunner, calling_file: str, calling_module: str  # noqa: F811
+    task_runner: TTaskRunner, calling_file: str, calling_module: str
 ) -> None:
     basedir = "foo"
     with task_runner(
diff --git a/tests/test_plugins/discovery_test_plugin/tests/test_discovery.py b/tests/test_plugins/discovery_test_plugin/tests/test_discovery.py
--- a/tests/test_plugins/discovery_test_plugin/tests/test_discovery.py
+++ b/tests/test_plugins/discovery_test_plugin/tests/test_discovery.py
@@ -5,8 +5,6 @@
 from hydra.core.plugins import Plugins
 from hydra.plugins.plugin import Plugin
 
-# TODO: test that launcher and sweeper plugins are imported only once when they are instantiated
-
 
 def test_number_of_imports(tmpdir: Path) -> None:
     os.environ["TMP_FILE"] = str(tmpdir / "import.log")
diff --git a/tests/test_structured_configs_tutorial.py b/tests/test_structured_configs_tutorial.py
--- a/tests/test_structured_configs_tutorial.py
+++ b/tests/test_structured_configs_tutorial.py
@@ -6,16 +6,7 @@
 import pytest
 from omegaconf import OmegaConf
 
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TSweepRunner,
-    TTaskRunner,
-    chdir_hydra_root,
-    does_not_raise,
-    sweep_runner,
-    task_runner,
-    verify_dir_outputs,
-)
+from hydra.test_utils.test_utils import chdir_hydra_root
 
 chdir_hydra_root()
 
diff --git a/tests/test_tutorials.py b/tests/test_tutorials.py
--- a/tests/test_tutorials.py
+++ b/tests/test_tutorials.py
@@ -8,13 +8,11 @@
 import pytest
 from omegaconf import DictConfig, OmegaConf
 
-from hydra.test_utils.test_utils import (  # noqa: F401
+from hydra.test_utils.test_utils import (
     TSweepRunner,
     TTaskRunner,
     chdir_hydra_root,
     does_not_raise,
-    sweep_runner,
-    task_runner,
     verify_dir_outputs,
 )
 
@@ -232,10 +230,7 @@ def test_tutorial_defaults(tmpdir: Path, args: List[str], expected: DictConfig)
     ],
 )
 def test_objects_example(
-    tmpdir: Path,
-    task_runner: TTaskRunner,  # noqa: F811
-    args: List[str],
-    output_conf: DictConfig,
+    tmpdir: Path, task_runner: TTaskRunner, args: List[str], output_conf: DictConfig
 ) -> None:
     with task_runner(
         calling_file="examples/patterns/objects/my_app.py",
@@ -249,7 +244,7 @@ def test_objects_example(
         verify_dir_outputs(task.job_ret, overrides=task.overrides)
 
 
-def test_composition_config_example(task_runner: TTaskRunner) -> None:  # noqa: F811
+def test_composition_config_example(task_runner: TTaskRunner) -> None:
     with task_runner(
         calling_file="examples/tutorial/5_composition/my_app.py",
         calling_module=None,
@@ -282,7 +277,7 @@ def test_composition_config_example(task_runner: TTaskRunner) -> None:  # noqa:
         verify_dir_outputs(task.job_ret, overrides=task.overrides)
 
 
-def test_sweeping_example(sweep_runner: TSweepRunner) -> None:  # noqa: F811
+def test_sweeping_example(sweep_runner: TSweepRunner) -> None:
     with sweep_runner(
         calling_file="examples/tutorial/5_composition/my_app.py",
         calling_module=None,
@@ -303,7 +298,7 @@ def test_sweeping_example(sweep_runner: TSweepRunner) -> None:  # noqa: F811
             assert tuple(ret.overrides) in overrides
 
 
-def test_specializing_config_example(task_runner: TTaskRunner) -> None:  # noqa: F811
+def test_specializing_config_example(task_runner: TTaskRunner) -> None:
     with task_runner(
         calling_file="examples/patterns/specializing_config/example.py",
         calling_module=None,
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -264,7 +264,9 @@ def test_get_original_cwd() -> None:
         ("/home/omry/hydra/", "/foo/bar", "/foo/bar"),
     ],
 )
-def test_to_absolute_path(orig_cwd: str, path: str, expected: str) -> None:
+def test_to_absolute_path(
+    restore_singletons: Any, orig_cwd: str, path: str, expected: str
+) -> None:
     # normalize paths to current OS
     orig_cwd = str(Path(orig_cwd))
     path = str(Path(path))