Repository: facebookresearch/hydra

Issue:
Opaque error message when attempting to instantiate object from a module that has errors
The `hydra.utils.instantiate` function looks up it's target by using Python's `importlib` machinery.
```python
# my_app.py
import hydra
hydra.utils.instantiate({"_target_": "my_module.Foo"})
```
```python
# my_module.py
class Foo:
    print("Hi")
```
```text
$ python3 my_app.py
Hi
```
Suppose the user makes a syntax error in `my_module.py`, forgetting a colon in the class statement:
```python
# my_module.py
class Foo
    print("Hi")
```
```text
python3 my_app.py
Traceback (most recent call last):
...
ValueError: Empty module name

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
...
  File "/home/jasha10/hydra.git/hydra/_internal/utils.py", line 573, in _locate
    raise ImportError(f"Error loading module '{path}'") from e
ImportError: Error loading module 'my_module.Foo'
```
The first part of this traceback, which says "`ValueError: Empty module name`", is not very helpful.
The traceback should instead look like this:
```text
$ python3 my_app.py
Traceback (most recent call last):
...
    class Foo
             ^
SyntaxError: invalid syntax

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
...
  File "/home/jasha10/hydra.git/hydra/_internal/utils.py", line 573, in _locate
    raise ImportError(f"Error loading module '{path}'") from e
ImportError: Error loading module 'my_module.Foo'
```


Discussion:

Resolving patch:
diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py
--- a/hydra/_internal/utils.py
+++ b/hydra/_internal/utils.py
@@ -559,35 +559,37 @@ def _locate(path: str) -> Union[type, Callable[..., Any]]:
     """
     if path == "":
         raise ImportError("Empty path")
-    import builtins
     from importlib import import_module
+    from types import ModuleType
 
     parts = [part for part in path.split(".") if part]
-    module = None
-    for n in reversed(range(len(parts))):
+    for n in reversed(range(1, len(parts) + 1)):
+        mod = ".".join(parts[:n])
         try:
-            mod = ".".join(parts[:n])
-            module = import_module(mod)
-        except Exception as e:
-            if n == 0:
-                raise ImportError(f"Error loading module '{path}'") from e
+            obj = import_module(mod)
+        except Exception as exc_import:
+            if n == 1:
+                raise ImportError(f"Error loading module '{path}'") from exc_import
             continue
-        if module:
-            break
-    if module:
-        obj = module
-    else:
-        obj = builtins
-    for part in parts[n:]:
-        mod = mod + "." + part
-        if not hasattr(obj, part):
-            try:
-                import_module(mod)
-            except Exception as e:
-                raise ImportError(
-                    f"Encountered error: `{e}` when loading module '{path}'"
-                ) from e
-        obj = getattr(obj, part)
+        break
+    for m in range(n, len(parts)):
+        part = parts[m]
+        try:
+            obj = getattr(obj, part)
+        except AttributeError as exc_attr:
+            if isinstance(obj, ModuleType):
+                mod = ".".join(parts[: m + 1])
+                try:
+                    import_module(mod)
+                except ModuleNotFoundError:
+                    pass
+                except Exception as exc_import:
+                    raise ImportError(
+                        f"Error loading '{path}': '{repr(exc_import)}'"
+                    ) from exc_import
+            raise ImportError(
+                f"Encountered AttributeError while loading '{path}': {exc_attr}"
+            ) from exc_attr
     if isinstance(obj, type):
         obj_type: type = obj
         return obj_type
@@ -595,7 +597,7 @@ def _locate(path: str) -> Union[type, Callable[..., Any]]:
         obj_callable: Callable[..., Any] = obj
         return obj_callable
     else:
-        # dummy case
+        # reject if not callable & not a type
         raise ValueError(f"Invalid type ({type(obj)}) found for {path}")
 
 


Test patch:
diff --git a/tests/instantiate/import_error.py b/tests/instantiate/import_error.py
new file mode 100644
--- /dev/null
+++ b/tests/instantiate/import_error.py
@@ -0,0 +1,2 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+assert False
diff --git a/tests/instantiate/test_helpers.py b/tests/instantiate/test_helpers.py
--- a/tests/instantiate/test_helpers.py
+++ b/tests/instantiate/test_helpers.py
@@ -3,7 +3,7 @@
 from typing import Any
 
 from _pytest.python_api import RaisesContext, raises
-from pytest import mark
+from pytest import mark, param
 
 from hydra._internal.utils import _locate
 from hydra.utils import get_class
@@ -20,22 +20,79 @@
 @mark.parametrize(
     "name,expected",
     [
+        param(
+            "int",
+            raises(ImportError, match=re.escape("Error loading module 'int'")),
+            id="int",
+        ),
+        param("builtins.int", int, id="builtins_explicit"),
+        param("builtins.int.from_bytes", int.from_bytes, id="method_of_builtin"),
+        param(
+            "builtins.int.not_found",
+            raises(
+                ImportError,
+                match=re.escape(
+                    "Encountered AttributeError while loading 'builtins.int.not_found':"
+                    + " type object 'int' has no attribute 'not_found'"
+                ),
+            ),
+            id="builtin_attribute_error",
+        ),
+        param(
+            "datetime",
+            raises(
+                ValueError,
+                match=re.escape("Invalid type (<class 'module'>) found for datetime"),
+            ),
+            id="top_level_module",
+        ),
         ("tests.instantiate.Adam", Adam),
         ("tests.instantiate.Parameters", Parameters),
         ("tests.instantiate.AClass", AClass),
+        param(
+            "tests.instantiate.AClass.static_method",
+            AClass.static_method,
+            id="staticmethod",
+        ),
+        param(
+            "tests.instantiate.AClass.not_found",
+            raises(
+                ImportError,
+                match=re.escape(
+                    "Encountered AttributeError while loading 'tests.instantiate.AClass.not_found':"
+                    + " type object 'AClass' has no attribute 'not_found'"
+                ),
+            ),
+            id="class_attribute_error",
+        ),
         ("tests.instantiate.ASubclass", ASubclass),
         ("tests.instantiate.NestingClass", NestingClass),
         ("tests.instantiate.AnotherClass", AnotherClass),
         ("", raises(ImportError, match=re.escape("Empty path"))),
-        [
+        (
             "not_found",
             raises(ImportError, match=re.escape("Error loading module 'not_found'")),
-        ],
-        (
+        ),
+        param(
             "tests.instantiate.b.c.Door",
             raises(
-                ImportError, match=re.escape("No module named 'tests.instantiate.b'")
+                ImportError,
+                match=re.escape(
+                    "Encountered AttributeError while loading 'tests.instantiate.b.c.Door':"
+                    + " module 'tests.instantiate' has no attribute 'b'"
+                ),
+            ),
+            id="nested_not_found",
+        ),
+        param(
+            "tests.instantiate.import_error",
+            raises(
+                ImportError,
+                match=re.escape(
+                    "Error loading 'tests.instantiate.import_error': 'AssertionError()'"
+                ),
             ),
+            id="import_assertion_error",
         ),
     ],
 )