Repository: hylang/hy

Issue:
Macro namespacing is probably too clever for its own good
I was looking into [our one xfailing test](https://github.com/hylang/hy/blob/961e9439156660a8303cac34f46f09e7f30cadeb/tests/native_tests/import.hy#L217) and I suspect this "minimal macro namespacing" feature may not have been for the best. To recap, the motivation, as I understand, was to allow you to do things like this:

```sh
echo >a.hy '
  (defmacro helper [x]
    `[~x ~x])
  (defmacro am [arg]
    `(helper ~arg))'
echo >b.hy '
  (require a [am])
  (print (am 2))'
hy b.hy  # [2, 2]
```

Although the call  to `helper` is quoted, and thus ultimately expands in `b.hy`, it's looked up where `am` is defined, so `a.hy` can use macros it defines in expansions of its own macros, without having to include `(require a …)` in the expansion, and without `b.hy` having to say `(require a [helper])` itself.

But the implementation seems to be pretty fragile. If `` `(helper ~arg) `` above is replaced with `` `(+ [1] (helper ~arg)) ``, the result is `NameError: name 'helper' is not defined`. And the concept itself may be problematic. Consider something like

```sh
echo >a.hy '
  (defmacro helper []
    "helper a (macro)")
  (defmacro am [form]
    form)'
echo >b.hy '
  (require a [am])
  (defn helper []
    "helper b (function)")
  (print (helper))
  (print (am (helper)))'
hy b.hy
  # helper b (function)
  # helper a (macro)
```

The author of `b.hy` is trying to use his function `helper` in a call to `am`, a macro that he required. Unbeknownst to him, `a.hy` defines a macro of the same name, which shadows his function. Rejiggering priorities in which macro names are looked up wouldn't help, because macros always take precedence over functions. To avoid unpredictable collisions like this, we need to ensure that macro lookup in a file is affected only by macros defined or required in that file, and not by macros defined or required in dependencies.

@brandonwillard I think you spearheaded development of this feature (as part of #1682), so perhaps you have some insight here. No pressure, though; it's been a while.


Discussion:

Resolving patch:
diff --git a/hy/macros.py b/hy/macros.py
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -391,14 +391,11 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True):
         else:
             break
 
-        expr_modules = ([] if not hasattr(tree, "module") else [tree.module]) + [module]
-        expr_modules.append(builtins)
-
         # Choose the first namespace with the macro.
         m = next(
             (
                 mod._hy_macros[fn]
-                for mod in expr_modules
+                for mod in (module, builtins)
                 if fn in getattr(mod, "_hy_macros", ())
             ),
             None,
@@ -413,9 +410,6 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True):
             if isinstance(obj, (hy.compiler.Result, AST)):
                 return obj if result_ok else tree
 
-            if isinstance(obj, Expression):
-                obj.module = inspect.getmodule(m)
-
             tree = replace_hy_obj(obj, tree)
 
         if once:
diff --git a/hy/models.py b/hy/models.py
--- a/hy/models.py
+++ b/hy/models.py
@@ -34,7 +34,7 @@ class Object:
     `end_column` 3.
     """
 
-    properties = ["module", "_start_line", "_end_line", "_start_column", "_end_column"]
+    properties = ["_start_line", "_end_line", "_start_column", "_end_column"]
 
     def replace(self, other, recursive=False):
         if isinstance(other, Object):


Test patch:
diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy
--- a/tests/native_tests/import.hy
+++ b/tests/native_tests/import.hy
@@ -214,75 +214,26 @@ in expansions."
   (require-macros))
 
 
-(defn [(pytest.mark.xfail)] test-macro-from-module []
-  "
-  Macros loaded from an external module, which itself `require`s macros, should
-  work without having to `require` the module's macro dependencies (due to
-  [minimal] macro namespace resolution).
-
-  In doing so we also confirm that a module's `_hy_macros` attribute is correctly
-  loaded and used.
-
-  Additionally, we confirm that `require` statements are executed via loaded bytecode.
-  "
-
-  (setv pyc-file (importlib.util.cache-from-source
-                   (os.path.realpath
-                     (os.path.join
-                       "tests" "resources" "macro_with_require.hy"))))
-
-  ;; Remove any cached byte-code, so that this runs from source and
-  ;; gets evaluated in this module.
-  (when (os.path.isfile pyc-file)
-    (os.unlink pyc-file)
-    (.clear sys.path_importer_cache)
-    (when (in  "tests.resources.macro_with_require" sys.modules)
-      (del (get sys.modules "tests.resources.macro_with_require"))
-      (_hy_macros.clear)))
-
-  ;; Ensure that bytecode isn't present when we require this module.
-  (assert (not (os.path.isfile pyc-file)))
-
-  (defn test-requires-and-macros []
-    (require tests.resources.macro-with-require
-             [test-module-macro])
-
-    ;; Make sure that `require` didn't add any of its `require`s
-    (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros)))
-    ;; and that it didn't add its tags.
-    (assert (not (in (hy.mangle "#test-module-tag") _hy_macros)))
-
-    ;; Now, require everything.
-    (require tests.resources.macro-with-require *)
-
-    ;; Again, make sure it didn't add its required macros and/or tags.
-    (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros)))
-
-    ;; Its tag(s) should be here now.
-    (assert (in (hy.mangle "#test-module-tag") _hy_macros))
-
-    ;; The test macro expands to include this symbol.
-    (setv module-name-var "tests.native_tests.native_macros")
-    (assert (= (+ "This macro was created in tests.resources.macros, "
-                  "expanded in tests.native_tests.native_macros "
-                  "and passed the value 1.")
-               (test-module-macro 1))))
-
-  (test-requires-and-macros)
-
-  ;; Now that bytecode is present, reload the module, clear the `require`d
-  ;; macros and tags, and rerun the tests.
-  (assert (os.path.isfile pyc-file))
-
-  ;; Reload the module and clear the local macro context.
-  (.clear sys.path_importer_cache)
-  (del (get sys.modules "tests.resources.macro_with_require"))
-  (.clear _hy_macros)
-
-  ;; There doesn't seem to be a way--via standard import mechanisms--to
-  ;; ensure that an imported module used the cached bytecode.  We'll simply have
-  ;; to trust that the .pyc loading convention was followed.
-  (test-requires-and-macros))
+(defn test-no-surprise-shadow [tmp-path monkeypatch]
+  "Check that an out-of-module macro doesn't shadow a function."
+  ; https://github.com/hylang/hy/issues/2451
+
+  (monkeypatch.syspath-prepend tmp-path)
+  (.write-text (/ tmp-path "wexter_a.hy") #[[
+    (defmacro helper []
+      "helper a (macro)")
+    (defmacro am [form]
+      form)]])
+  (.write-text (/ tmp-path "wexter_b.hy") #[[
+    (require wexter-a [am])
+    (defn helper []
+      "helper b (function)")
+    (setv v1 (helper))
+    (setv v2 (am (helper)))]])
+
+  (import wexter-b)
+  (assert (= wexter-b.v1 "helper b (function)"))
+  (assert (= wexter-b.v2 "helper b (function)")))
 
 
 (defn test-recursive-require-star []