Repository: hylang/hy

Issue:
Support user introspection and modification of the currently defined macros
Help me understand `hy.macros.-hy-macros` initialization.

First run will print `hy.macros.-hy-macros` fine. Second and subsequent runs will print an empty dict.

Deleting the `foo.pyc` will reset the `-hy-macros` dict as expected.

Is this intended behavior? If intended - how do I test code referencing `-hy-macros`?

`pkg/foo.hy`

    (import hy.macros)

    (defn foo []
      (print hy.macros.-hy-macros))

`pkg/test_foo.hy`

    (import [pkg.foo [foo]])

    (defn test-foo []
      (foo)
      (doc ->))  ; Similarly, will print on first runthrough, error on second run



Discussion:
Maybe related to #1324.

It's easy to force a compile at runtime though, just use `hy.eval`, e.g.
```Hy
(hy.eval '(print hy.macros._hy-macros))
```
Ok thanks a simple `(hy.eval '(import hy.macros))` in the module being tested does the trick.
I don't think `hy.macros._hy_macros` is intended for public use.
Is there a better way to access all defined macros? `doc` takes the same approach (and has the same error on a second run). It's for code introspection for Emacs, not sure if that qualifies as public use.
No, I don't think so. We've never worked on introspection of the macro system beyond Hy's own internal needs.
Better support for introspection of the macro system could replace `hy.reserved` and perhaps also `delmacro`.

Resolving patch:
diff --git a/hy/compiler.py b/hy/compiler.py
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -326,7 +326,7 @@ def is_annotate_expression(model):
 class HyASTCompiler:
     """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module, filename=None, source=None):
+    def __init__(self, module, filename=None, source=None, extra_macros=None):
         """
         Args:
             module (Union[str, types.ModuleType]): Module name or object in which the Hy tree is evaluated.
@@ -335,9 +335,12 @@ def __init__(self, module, filename=None, source=None):
                 debugging.
             source (Optional[str]): The source for the file, if any, being compiled.  This is optional
                 information for informative error messages and debugging.
+            extra_macros (Optional[dict]): More macros to use during lookup. They take precedence
+                over macros in `module`.
         """
         self.anon_var_count = 0
         self.temp_if = None
+        self.extra_macros = extra_macros or {}
 
         # Make a list of dictionaries with local compiler settings,
         # such as the definitions of local macros. The last element is
@@ -736,6 +739,7 @@ def hy_eval(
     source=None,
     import_stdlib=True,
     globals=None,
+    extra_macros=None,
 ):
 
     module = get_compiler_module(module, compiler, True)
@@ -752,6 +756,7 @@ def hy_eval(
         filename=filename,
         source=source,
         import_stdlib=import_stdlib,
+        extra_macros=extra_macros,
     )
 
     if globals is None:
@@ -764,14 +769,16 @@ def hy_eval(
     return eval(ast_compile(expr, filename, "eval"), globals, locals)
 
 
-def hy_eval_user(model, globals = None, locals = None, module = None):
+def hy_eval_user(model, globals = None, locals = None, module = None, macros = None):
     # This function is advertised as `hy.eval`.
     """An equivalent of Python's :func:`eval` for evaluating Hy code. The chief difference is that the first argument should be a :ref:`model <models>` rather than source text. If you have a string of source text you want to evaluate, convert it to a model first with :hy:func:`hy.read` or :hy:func:`hy.read-many`::
 
         (hy.eval '(+ 1 1))             ; => 2
         (hy.eval (hy.read "(+ 1 1)"))  ; => 2
 
-    The optional arguments ``globals`` and ``locals`` work as in the case of ``eval``. There's one more optional argument, ``module``, which can be a module object or a string naming a module. The module's ``__dict__`` attribute can fill in for ``globals`` (and hence also for ``locals``) if ``module`` is provided but ``globals`` isn't, but the primary purpose of ``module`` is to control where macro calls are looked up. Without this argument, the calling module of ``hy.eval`` is used instead. ::
+    The optional arguments ``globals`` and ``locals`` work as in the case of :func:`eval`.
+
+    Another optional argument, ``module``, can be a module object or a string naming a module. The module's ``__dict__`` attribute can fill in for ``globals`` (and hence also for ``locals``) if ``module`` is provided but ``globals`` isn't, but the primary purpose of ``module`` is to control where macro calls are looked up. Without this argument, the calling module of ``hy.eval`` is used instead. ::
 
         (defmacro my-test-mac [] 3)
         (hy.eval '(my-test-mac))                 ; => 3
@@ -779,7 +786,15 @@ def hy_eval_user(model, globals = None, locals = None, module = None):
         (hy.eval '(my-test-mac) :module hyrule)  ; NameError
         (hy.eval '(list-n 3 1) :module hyrule)   ; => [1 1 1]
 
-    N.B. Local macros are invisible to ``hy.eval``."""
+    Finally, finer control of macro lookup can be achieved by passing in a dictionary of macros as the ``macros`` argument. The keys of this dictionary should be mangled macro names, and the values should be function objects to implement those macros. This is the same structure as is produced by :hy:func:`local-macros`, and in fact, ``(hy.eval … :macros (local-macros))`` is useful to make local macros visible to ``hy.eval``, which otherwise doesn't see them. ::
+
+        (defn f []
+          (defmacro lmac [] 1)
+          (hy.eval '(lmac))     ; NameError
+          (print (hy.eval '(lmac) :macros (local-macros)))) ; => 1
+        (f)
+
+    In any case, macros provided in this dictionary will shadow macros of the same name that are associated with the provided or implicit module. You can shadow a core macro, too, so be careful: there's no warning for this as there is in the case of :hy:func:`defmacro`."""
 
     if locals is None:
         locals = globals
@@ -793,7 +808,8 @@ def hy_eval_user(model, globals = None, locals = None, module = None):
             locals = (inspect.getargvalues(inspect.stack()[1][0]).locals
                 if locals is None and module is None
                 else locals),
-            module = get_compiler_module(module, None, True))
+            module = get_compiler_module(module, None, True),
+            extra_macros = macros)
     finally:
         if locals is not None:
             if hy_was:
@@ -815,6 +831,7 @@ def hy_compile(
     filename=None,
     source=None,
     import_stdlib=True,
+    extra_macros=None,
 ):
     """Compile a hy.models.Object tree into a Python AST Module.
 
@@ -835,6 +852,7 @@ def hy_compile(
             if `None`, an attempt will be made to obtain it from the module given by
             `module`.  When `compiler` is given, its `source` field value is always
             used.
+        extra_macros (Optional[dict]): Passed through to `HyASTCompiler`, if it's called.
 
     Returns:
         ast.AST: A Python AST tree
@@ -851,7 +869,11 @@ def hy_compile(
             "`tree` must be a hy.models.Object or capable of " "being promoted to one"
         )
 
-    compiler = compiler or HyASTCompiler(module, filename=filename, source=source)
+    compiler = compiler or HyASTCompiler(
+        module,
+        filename = filename,
+        source = source,
+        extra_macros = extra_macros)
 
     with HyReader.using_reader(reader, create=False), compiler.scope:
         result = compiler.compile(tree)
diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py
--- a/hy/core/result_macros.py
+++ b/hy/core/result_macros.py
@@ -18,7 +18,7 @@
 from hy._compat import PY3_11, PY3_12
 from hy.compiler import Result, asty, mkexpr
 from hy.errors import HyEvalError, HyInternalError, HyTypeError
-from hy.macros import pattern_macro, require, require_reader
+from hy.macros import pattern_macro, require, require_reader, local_macro_name
 from hy.model_patterns import (
     FORM,
     KEYWORD,
@@ -1544,21 +1544,26 @@ def E(*x): return Expression(x)
     S = Symbol
 
     compiler.warn_on_core_shadow(name)
-    ret = []
     fn_def = E(
         S("fn"),
         List([S("&compiler"), *expr[2]]),
-        *body)
+        *body).replace(expr)
     if compiler.is_in_local_state():
         # We're in a local scope, so define the new macro locally.
-        compiler.local_state_stack[-1]['macros'][mangle(name)] = (
-            compiler.eval(fn_def.replace(expr)))
-        return Result()
+        state = compiler.local_state_stack[-1]
+        # Produce code that will set the macro to a local variable.
+        ret = compiler.compile(E(
+            S("setv"),
+            S(local_macro_name(name)),
+            fn_def).replace(expr))
+        # Also evaluate the macro definition now, and put it in
+        # state['macros'].
+        state['macros'][mangle(name)] = compiler.eval(fn_def)
+        return ret + ret.expr_as_stmt()
     # Otherwise, define the macro module-wide.
-    ret.append(E(
+    ret = compiler.compile(E(S("eval-and-compile"), E(
         E(dotted("hy.macros.macro"), str(name)),
-        fn_def))
-    ret = compiler.compile(E(S("eval-and-compile"), *ret).replace(expr))
+        fn_def)).replace(expr))
     return ret + ret.expr_as_stmt()
 
 
@@ -1822,12 +1827,22 @@ def compile_require(compiler, expr, root, entries):
         # importing readers but not macros
         # (require a-module :readers ["!"])
         if (rest or not readers) and compiler.is_in_local_state():
-            require(
+            reqs = require(
                 module_name,
                 compiler.local_state_stack[-1]['macros'],
                 assignments = assignments,
                 prefix = prefix,
                 compiler = compiler)
+            ret += compiler.compile(Expression([
+                Symbol("setv"),
+                List([Symbol(local_macro_name(m)) for m, _, _ in reqs]),
+                Expression([
+                    dotted("hy.macros.require_vals"),
+                    String(module_name),
+                    Dict(),
+                    Keyword("assignments"),
+                    List([(String(m), String(m)) for _, m, _ in reqs])])]).replace(expr))
+            ret += ret.expr_as_stmt()
         elif (rest or not readers) and require(
                 module_name,
                 compiler.module,
diff --git a/hy/macros.py b/hy/macros.py
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -184,8 +184,9 @@ def enable_readers(module, reader, names):
 
 
 def require(source_module, target, assignments, prefix="", target_module_name=None, compiler=None):
-    """Load macros from a module. Return a `bool` indicating whether
-    macros were actually transferred.
+    """Load macros from a module. Return a list of (new name, source
+    name, macro object) tuples, including only macros that were
+    actually transferred.
 
     - `target` can be a a string (naming a module), a module object,
       a dictionary, or `None` (meaning the calling module).
@@ -204,7 +205,7 @@ def require(source_module, target, assignments, prefix="", target_module_name=No
         # We use the module's underlying filename for this (when they exist), since
         # it's the most "fixed" attribute.
         if _same_modules(source_module, target_module):
-            return False
+            return []
 
     if not inspect.ismodule(source_module):
         source_module = import_module_from_string(source_module,
@@ -219,27 +220,29 @@ def require(source_module, target, assignments, prefix="", target_module_name=No
 
     if not source_module._hy_macros:
         if assignments in ("ALL", "EXPORTS"):
-            return False
+            return []
+        out = []
         for name, alias in assignments:
             try:
-                require(
+                out.extend(require(
                     f"{source_module.__name__}.{mangle(name)}",
                     target_module or target,
                     "ALL",
                     prefix=alias,
-                )
+                ))
             except HyRequireError as e:
                 raise HyRequireError(
                     f"Cannot import name '{name}'"
                     f" from '{source_module.__name__}'"
                     f" ({source_module.__file__})"
                 )
-        return True
+        return out
 
     target_macros = target_namespace.setdefault("_hy_macros", {}) if target_module else target
 
     if prefix:
         prefix += "."
+    out = []
 
     for name, alias in (
         assignments
@@ -256,12 +259,25 @@ def require(source_module, target, assignments, prefix="", target_module_name=No
         alias = mangle(prefix + alias)
         if _name in source_module._hy_macros:
             target_macros[alias] = source_macros[_name]
+            out.append((alias, _name, source_macros[_name]))
         else:
             raise HyRequireError(
                 "Could not require name {} from {}".format(_name, source_module)
             )
 
-    return True
+    return out
+
+
+def require_vals(*args, **kwargs):
+    return [value for _, _, value in require(*args, **kwargs)]
+
+
+def local_macro_name(original):
+    return '_hy_local_macro__' + (mangle(original)
+      # We have to do a bit of extra mangling beyond `mangle` itself to
+      # handle names with periods.
+        .replace('D', 'DN')
+        .replace('.', 'DD'))
 
 
 def load_macros(module):
@@ -369,9 +385,11 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True):
 
         # Choose the first namespace with the macro.
         m = ((compiler and next(
-                (d['macros'][fn]
-                    for d in reversed(compiler.local_state_stack)
-                    if fn in d['macros']),
+                (d[fn]
+                    for d in [
+                        compiler.extra_macros,
+                        *(s['macros'] for s in reversed(compiler.local_state_stack))]
+                    if fn in d),
                 None)) or
             next(
                 (mod._hy_macros[fn]
@@ -397,12 +415,6 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True):
     return tree
 
 
-def macroexpand_1(tree, module, compiler=None):
-    """Expand the toplevel macro from `tree` once, in the context of
-    `compiler`."""
-    return macroexpand(tree, module, compiler, once=True)
-
-
 def rename_function(f, new_name):
     """Create a copy of a function, but with a new name."""
     f = type(f)(


Test patch:
diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py
--- a/tests/macros/test_macro_processor.py
+++ b/tests/macros/test_macro_processor.py
@@ -2,7 +2,7 @@
 
 from hy.compiler import HyASTCompiler
 from hy.errors import HyMacroExpansionError
-from hy.macros import macro, macroexpand, macroexpand_1
+from hy.macros import macro, macroexpand
 from hy.models import Expression, Float, List, String, Symbol
 from hy.reader import read
 
@@ -58,6 +58,6 @@ def test_macroexpand_source_data():
     ast = Expression([Symbol("when"), String("a")])
     ast.start_line = 3
     ast.start_column = 5
-    bad = macroexpand_1(ast, "hy.core.macros")
+    bad = macroexpand(ast, "hy.core.macros", once = True)
     assert bad.start_line == 3
     assert bad.start_column == 5
diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy
--- a/tests/native_tests/hy_eval.hy
+++ b/tests/native_tests/hy_eval.hy
@@ -206,6 +206,44 @@
   (assert (= (hy.eval '(qplah 1)) [8 1])))
 
 
+(defn test-extra-macros []
+  (setv ab 15)
+
+  (assert (=
+    (hy.eval '(chippy a b) :macros (dict
+      :chippy (fn [&compiler arg1 arg2]
+        (hy.models.Symbol (+ (str arg1) (str arg2))))))
+    15))
+
+  ; By default, `hy.eval` can't see local macros.
+  (defmacro oh-hungee [arg1 arg2]
+    (hy.models.Symbol (+ (str arg1) (str arg2))))
+  (with [(pytest.raises NameError)]
+    (hy.eval '(oh-hungee a b)))
+  ; But you can pass them in with the `macros` argument.
+  (assert (=
+    (hy.eval '(oh-hungee a b) :macros (local-macros))
+    15))
+  (assert (=
+    (hy.eval '(oh-hungee a b) :macros {"oh_hungee" (get-macro oh-hungee)}
+    15)))
+
+  ; You can shadow a global macro.
+  (assert (=
+    (hy.eval '(cheese))
+    "gorgonzola"))
+  (assert (=
+    (hy.eval '(cheese) :macros {"cheese" (fn [&compiler] "cheddar")})
+    "cheddar"))
+
+  ; Or even a core macro, and with no warning.
+  (assert (=
+    (hy.eval '(+ 1 1) :macros
+      {(hy.mangle "+") (fn [&compiler #* args]
+        (.join "" (gfor  x args  (str (int x)))))})
+    "11")))
+
+
 (defn test-filename []
   (setv m (hy.read "(/ 1 0)" :filename "bad_math.hy"))
   (with [e (pytest.raises ZeroDivisionError)]
diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy
--- a/tests/native_tests/hy_misc.hy
+++ b/tests/native_tests/hy_misc.hy
@@ -22,10 +22,30 @@
 
 
 (defn test-macroexpand []
-  (assert (= (hy.macroexpand '(mac (a b) (x y)))
-             '(x y (a b))))
-  (assert (= (hy.macroexpand '(mac (a b) (mac 5)))
-             '(a b 5))))
+  (assert (=
+    (hy.macroexpand '(mac (a b) (x y)))
+    '(x y (a b))))
+  (assert (=
+    (hy.macroexpand '(mac (a b) (mac 5)))
+    '(a b 5)))
+  (assert (=
+    (hy.macroexpand '(qplah "phooey") :module hy.M.tests.resources.tlib)
+    '[8 "phooey"]))
+  (assert (=
+    (hy.macroexpand '(chippy 1) :macros
+      {"chippy" (fn [&compiler x] `[~x ~x])})
+    '[1 1]))
+  ; Non-Expressions just get returned as-is.
+  (defn f [])
+  (assert (is
+    (hy.macroexpand f)
+    f))
+  ; If the macro expands to a `Result`, the user gets the original
+  ; back instead of the `Result`.
+  (setv model '(+ 1 1))
+  (assert (is
+    (hy.macroexpand model)
+    model)))
 
 
 (defmacro m-with-named-import []
diff --git a/tests/native_tests/macros_first_class.hy b/tests/native_tests/macros_first_class.hy
--- a/tests/native_tests/macros_first_class.hy
+++ b/tests/native_tests/macros_first_class.hy
@@ -81,3 +81,23 @@ deleting them, and retrieving their docstrings. We also test `get-macro`
 (defn test-global-shadowing-builtin []
   (assert (= div1 "1/2"))
   (assert (= div2 0.5)))
+
+;; * Local macros
+
+(defn test-local-get []
+  (defmacro local1 [] "local1 doc" 1)
+  (defmacro local2 [] "local2 outer" 2)
+  (require tests.resources.local-req-example :as LRE)
+
+  (assert (= (. (get-macro local1) __doc__) "local1 doc"))
+  (assert (= (. (get-macro local2) __doc__) "local2 outer"))
+  (assert (= (. (get-macro LRE.wiz) __doc__) "remote wiz doc"))
+
+  (defn inner []
+    (defmacro local2 [] "local2 inner" 2)
+    (defmacro local3 [] "local3 doc" 2)
+    (assert (= (. (get-macro local2) __doc__) "local2 inner"))
+    (assert (= (. (get-macro local3) __doc__) "local3 doc"))
+    (assert (= (. (get-macro LRE.wiz) __doc__) "remote wiz doc")))
+
+  (inner))
diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy
--- a/tests/native_tests/macros_local.hy
+++ b/tests/native_tests/macros_local.hy
@@ -95,3 +95,29 @@
 
 (defn test-redefinition-warning []
   (macro-redefinition-warning-tester :local True))
+
+
+(defn test-get-local-macros []
+  "Test the core macro `local-macros`."
+
+  (defn check [ms inner?]
+    (assert (= (set (.keys ms))
+      #{"m1" "m2" "rwiz" #* (if inner? ["m3" "rhelp"] [])}))
+    (assert (= (. ms ["m1"] __doc__) "m1 doc"))
+    (assert (= (. ms ["m2"] __doc__) (if inner? "m2, inner" "m2, outer")))
+    (assert (= (. ms ["rwiz"] __doc__) "remote wiz doc"))
+    (when inner?
+      (assert (= (. ms ["m3"] __doc__) "m3 doc"))
+      (assert (= (. ms ["rhelp"] __doc__) "remote helper doc"))))
+
+  (defmacro m1 [] "m1 doc" 1)
+  (defmacro m2 [] "m2, outer" 2)
+  (require tests.resources.local-req-example [wiz :as rwiz])
+  (check (local-macros) False)
+  (defn inner-f []
+    (defmacro m2 [] "m2, inner" 2)
+    (defmacro m3 [] "m3 doc" 3)
+    (require tests.resources.local-req-example [helper :as rhelp])
+    (check (local-macros) True))
+  (inner-f)
+  (check (local-macros) False))
diff --git a/tests/resources/local_req_example.hy b/tests/resources/local_req_example.hy
--- a/tests/resources/local_req_example.hy
+++ b/tests/resources/local_req_example.hy
@@ -1,6 +1,8 @@
 (defmacro wiz []
+  "remote wiz doc"
   "remote wiz")
 (defmacro get-wiz []
   (wiz))
 (defmacro helper []
+  "remote helper doc"
   "remote helper macro")