Repository: hylang/hy

Issue:
Recursive `hy2py`
We could give `hy2py` a recursive mode that goes through a source tree and saves a matching Python file for each Hy file, if an up-to-date one doesn't already exist. Optionally, it could delete all such Python files and bytecode first, in order to ensure all macro expansions are fully up to date.

I'm not sure how often this would be useful in practice, but it seems like an obvious feature to have.


Discussion:
i [started implementing](https://github.com/dm9pZCAq/hy/tree/hy2py-recursive) this, but i faced a problem:

when i pass actual module name to `hy_compile` it not just compile `hy` to `py` but also evaluates compiled `py` 
hmm, i think this is ok when it is part of module

Resolving patch:
diff --git a/hy/cmdline.py b/hy/cmdline.py
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -7,6 +7,7 @@
 import py_compile
 import runpy
 import sys
+from contextlib import nullcontext
 from pathlib import Path
 
 import hy
@@ -330,6 +331,48 @@ def hyc_main():
     return rv
 
 
+def hy2py_worker(source, options, filename, output_filepath=None):
+    if isinstance(source, Path):
+        source = source.read_text(encoding="UTF-8")
+
+    if not output_filepath and options.output:
+        output_filepath = options.output
+
+    set_path(filename)
+    with (
+        open(output_filepath, "w", encoding="utf-8")
+        if output_filepath
+        else nullcontext()
+    ) as output_file:
+
+        def printing_source(hst):
+            for node in hst:
+                if options.with_source:
+                    print(node, file=output_file)
+                yield node
+
+        hst = hy.models.Lazy(
+            printing_source(read_many(source, filename, skip_shebang=True))
+        )
+        hst.source = source
+        hst.filename = filename
+
+        with filtered_hy_exceptions():
+            _ast = hy_compile(hst, "__main__", filename=filename, source=source)
+
+        if options.with_source:
+            print()
+            print()
+
+        if options.with_ast:
+            print(ast.dump(_ast, **(dict(indent=2) if PY3_9 else {})), file=output_file)
+            print()
+            print()
+
+        if not options.without_python:
+            print(ast.unparse(_ast), file=output_file)
+
+
 # entry point for cmd line script "hy2py"
 def hy2py_main():
     options = dict(
@@ -342,7 +385,8 @@ def hy2py_main():
         "FILE",
         type=str,
         nargs="?",
-        help='Input Hy code (use STDIN if "-" or ' "not provided)",
+        help='Input Hy code (can be file or directory) (use STDIN if "-" or '
+        "not provided)",
     )
     parser.add_argument(
         "--with-source",
@@ -359,46 +403,51 @@ def hy2py_main():
         action="store_true",
         help=("Do not show the Python code generated " "from the AST"),
     )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        nargs="?",
+        help="output file / directory",
+    )
 
     options = parser.parse_args(sys.argv[1:])
 
     if options.FILE is None or options.FILE == "-":
         sys.path.insert(0, "")
         filename = "<stdin>"
-        source = sys.stdin.read()
+        hy2py_worker(sys.stdin.read(), options, filename)
     else:
         filename = options.FILE
-        set_path(filename)
-        with open(options.FILE, "r", encoding="utf-8") as source_file:
-            source = source_file.read()
-
-    def printing_source(hst):
-        for node in hst:
-            if options.with_source:
-                print(node)
-            yield node
-
-    hst = hy.models.Lazy(
-        printing_source(read_many(source, filename, skip_shebang=True))
-    )
-    hst.source = source
-    hst.filename = filename
-
-    with filtered_hy_exceptions():
-        _ast = hy_compile(hst, "__main__", filename=filename, source=source)
-
-    if options.with_source:
-        print()
-        print()
-
-    if options.with_ast:
-        print(ast.dump(_ast, **(dict(indent=2) if PY3_9 else {})))
-        print()
-        print()
-
-    if not options.without_python:
-        print(ast.unparse(_ast))
+        if os.path.isdir(filename):
+            # handle recursively if --output is specified
+            if not options.output:
+                raise ValueError(
+                    f"{filename} is a directory but the output directory is not specified. Use --output or -o in command line arguments to specify the output directory."
+                )
+            os.makedirs(options.output, exist_ok=True)
+            for path, subdirs, files in os.walk(filename):
+                for name in files:
+                    filename_raw, filename_ext = os.path.splitext(name)
+                    if filename_ext == ".hy":
+                        filepath = os.path.join(path, name)
+                        # make sure to follow original file structure
+                        subdirectory = os.path.relpath(path, filename)
+                        output_directory_path = os.path.join(
+                            options.output if options.output else path, subdirectory
+                        )
+                        os.makedirs(output_directory_path, exist_ok=True)
+                        hy2py_worker(
+                            Path(filepath),
+                            options,
+                            filename,
+                            output_filepath=os.path.join(
+                                output_directory_path, filename_raw + ".py"
+                            ),
+                        )
 
+        else:
+            hy2py_worker(Path(options.FILE), options, filename)
     parser.exit(0)
 
 


Test patch:
diff --git a/tests/test_bin.py b/tests/test_bin.py
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -26,7 +26,7 @@ def pyr(s=""):
 
 def run_cmd(
         cmd, stdin_data=None, expect=0, dontwritebytecode=False,
-        stdout=subprocess.PIPE):
+        cwd=None, stdout=subprocess.PIPE):
     env = dict(os.environ)
     if dontwritebytecode:
         env["PYTHONDONTWRITEBYTECODE"] = "1"
@@ -41,6 +41,7 @@ def run_cmd(
         universal_newlines=True,
         shell=False,
         env=env,
+        cwd=cwd,
     )
     output = p.communicate(input=stdin_data)
     assert p.wait() == expect
@@ -717,3 +718,28 @@ def test_assert(tmp_path, monkeypatch):
             show_msg = has_msg and not optim and not test
             assert ("msging" in out) == show_msg
             assert ("bye" in err) == show_msg
+
+
+def test_hy2py_recursive(tmp_path):
+    (tmp_path / 'hy').mkdir()
+    (tmp_path / "hy/first.hy").write_text("""
+        (import folder.second [a b])
+        (print a)
+        (print b)""")
+    (tmp_path / "hy/folder").mkdir()
+    (tmp_path / "hy/folder/second.hy").write_text("""
+        (setv a 1)
+        (setv b "hello world")""")
+
+    _, err = run_cmd(f"hy2py {(tmp_path / 'hy').as_posix()}", expect=1)
+    assert "ValueError" in err
+
+    run_cmd("hy2py " +
+        f"{(tmp_path / 'hy').as_posix()} " +
+        f"--output {(tmp_path / 'py').as_posix()}")
+    assert set((tmp_path / 'py').rglob('*')) == {
+        tmp_path / 'py' / p
+        for p in ('first.py', 'folder', 'folder/second.py')}
+
+    output, _ = run_cmd(f"python3 first.py", cwd = tmp_path / 'py')
+    assert output == "1\nhello world\n"
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -1,8 +1,6 @@
 import asyncio
 import itertools
 import math
-import os
-import platform
 
 import pytest