Repository: ansys/ansys-tools-protoc-helper (language: python)

Issue:
Fix nested directory handling in proto compilation
https://github.com/ansys/ansys-tools-protoc-helper/blob/7a2c2d03f37abc5293ee057b8890a07bc3e8e80c/src/ansys/tools/protoc_helper/_build_helper.py#L26


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a366cc..b0e9467 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,8 +18,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
-          cache: 'pip'
-          cache-dependency-path: 'pyproject.toml'
       - name: Install development requirements
         run: |
           pip install .[pre-commit]
@@ -41,8 +39,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: 'pyproject.toml'
       - name: Install development requirements
         run: |
           pip install .[test]
diff --git a/src/ansys/tools/protoc_helper/__init__.py b/ansys/tools/protoc_helper/__init__.py
similarity index 100%
rename from src/ansys/tools/protoc_helper/__init__.py
rename to ansys/tools/protoc_helper/__init__.py
diff --git a/src/ansys/tools/protoc_helper/_compile_protos.py b/ansys/tools/protoc_helper/_compile_protos.py
similarity index 65%
rename from src/ansys/tools/protoc_helper/_compile_protos.py
rename to ansys/tools/protoc_helper/_compile_protos.py
index 65f8b70..fa1e239 100644
--- a/src/ansys/tools/protoc_helper/_compile_protos.py
+++ b/ansys/tools/protoc_helper/_compile_protos.py
@@ -1,6 +1,5 @@
 import contextlib
 import glob
-import importlib.resources
 import os
 import pathlib
 import shutil
@@ -8,7 +7,10 @@
 import types
 import typing
 
+import importlib_resources  # Replace with importlib.resources once only Py3.9+ is supported
 import pkg_resources
+from importlib_resources.abc import Traversable
+
 
 __all__ = ["compile_proto_files"]
 
@@ -50,10 +52,19 @@ def _dependency_modules() -> typing.Iterator[types.ModuleType]:
 def _proto_directory(module: types.ModuleType) -> typing.Iterator[str]:
     with tempfile.TemporaryDirectory() as tmp_dir:
         relpath = pathlib.Path(tmp_dir, *(module.__name__.split(".")))
-        for filename in importlib.resources.contents(module):
-            if filename.endswith(".proto"):
-                with importlib.resources.path(module, filename) as src_path:
-                    dest_path = relpath / filename
-                    os.makedirs(dest_path.parent, exist_ok=True)
-                    shutil.copyfile(src_path, relpath / filename)
+        _recursive_copy(importlib_resources.files(module), relpath.parent)
         yield tmp_dir
+
+
+def _recursive_copy(src_traversable: Traversable, dest_path: pathlib.Path) -> None:
+    if src_traversable.is_dir():
+        sub_dest_path = dest_path / src_traversable.name
+        for content in src_traversable.iterdir():  # type: ignore
+            _recursive_copy(content, sub_dest_path)
+    else:
+        assert src_traversable.is_file()
+        filename = src_traversable.name
+        if filename.endswith(".proto"):
+            os.makedirs(dest_path, exist_ok=True)
+            with importlib_resources.as_file(src_traversable) as src_path:
+                shutil.copyfile(src_path, dest_path / filename)
diff --git a/src/ansys/tools/protoc_helper/_distutils_overrides.py b/ansys/tools/protoc_helper/_distutils_overrides.py
similarity index 85%
rename from src/ansys/tools/protoc_helper/_distutils_overrides.py
rename to ansys/tools/protoc_helper/_distutils_overrides.py
index 458f81a..64066c7 100644
--- a/src/ansys/tools/protoc_helper/_distutils_overrides.py
+++ b/ansys/tools/protoc_helper/_distutils_overrides.py
@@ -20,7 +20,11 @@
 
 class _CompileProtosMixin:
     def run(self):
-        compile_proto_files(self.distribution.package_dir[""])
+        try:
+            target_dir = self.distribution.package_dir[""]
+        except (KeyError, TypeError):
+            target_dir = "."
+        compile_proto_files(target_dir)
         super().run()
 
 
diff --git a/src/ansys/tools/protoc_helper/py.typed b/ansys/tools/protoc_helper/py.typed
similarity index 100%
rename from src/ansys/tools/protoc_helper/py.typed
rename to ansys/tools/protoc_helper/py.typed
diff --git a/pyproject.toml b/pyproject.toml
index 8be7901..0aaceec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,6 +16,7 @@ dependencies = [
     "grpcio-tools==1.17.0",
     "protobuf==3.19.3",
     "mypy-protobuf==3.1.0",
+    "importlib-resources>=1.3"
 ]
 
 [project.optional-dependencies]


Test patch:
diff --git a/test/test_data/testpkg-greeter-protos/setup.py b/test/test_data/testpkg-greeter-protos/setup.py
index c901df2..037d35b 100644
--- a/test/test_data/testpkg-greeter-protos/setup.py
+++ b/test/test_data/testpkg-greeter-protos/setup.py
@@ -1,4 +1,5 @@
 import setuptools
+
 from ansys.tools.protoc_helper import CMDCLASS_OVERRIDE
 
 
@@ -8,8 +9,7 @@
         author="ANSYS, Inc.",
         python_requires=">=3.7",
         install_requires=["grpcio~=1.0", "protobuf~=3.0", "testpkg-hello-protos"],
-        packages=["testpkg.api.greeter.v0"],
-        package_dir={"": "src"},
+        packages=setuptools.find_namespace_packages(".", include=("testpkg.*",)),
         package_data={
             "": ["*.proto", "*.pyi", "py.typed"],
         },
diff --git a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter.proto b/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter.proto
deleted file mode 100644
index 5e8715b..0000000
--- a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter.proto
+++ /dev/null
@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-import "testpkg/api/hello/v0/hello.proto";
-
-service Greeter {
-    rpc SayHello(Hello) returns (Hello);
-}
diff --git a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter2.proto b/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter2.proto
deleted file mode 100644
index 974a1a6..0000000
--- a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/greeter2.proto
+++ /dev/null
@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-import "testpkg/api/hello/v0/hello2.proto";
-
-service Greeter2 {
-    rpc SayHello(Hello2) returns (Hello2);
-}
diff --git a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/__init__.py b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/__init__.py
similarity index 100%
rename from test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/__init__.py
rename to test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/__init__.py
diff --git a/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter.proto b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter.proto
new file mode 100644
index 0000000..8fe3cfd
--- /dev/null
+++ b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package testpkg.api.greeter.v0.greeter;
+
+import "testpkg/api/hello/v0/hello.proto";
+
+service Greeter {
+    rpc SayHello(testpkg.api.hello.v0.hello.Hello) returns (testpkg.api.hello.v0.hello.Hello);
+}
diff --git a/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter2.proto b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter2.proto
new file mode 100644
index 0000000..2cde782
--- /dev/null
+++ b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter2.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package testpkg.api.greeter.v0.greeter2;
+
+import "testpkg/api/hello/v1/nested/hello.proto";
+
+service Greeter2 {
+    rpc SayHello(testpkg.api.hello.v1.nested.hello.Hello) returns (testpkg.api.hello.v1.nested.hello.Hello);
+}
diff --git a/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter3.proto b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter3.proto
new file mode 100644
index 0000000..2a57d0e
--- /dev/null
+++ b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/greeter3.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package testpkg.api.greeter.v0.greeter3;
+
+import "testpkg/api/hello/hello.proto";
+
+service Greeter3 {
+    rpc SayHello(testpkg.api.hello.hello.Hello) returns (testpkg.api.hello.hello.Hello);
+}
diff --git a/test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/py.typed b/test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/py.typed
similarity index 100%
rename from test/test_data/testpkg-greeter-protos/src/testpkg/api/greeter/v0/py.typed
rename to test/test_data/testpkg-greeter-protos/testpkg/api/greeter/v0/py.typed
diff --git a/test/test_data/testpkg-hello-protos/setup.py b/test/test_data/testpkg-hello-protos/setup.py
index 253625f..52418d9 100644
--- a/test/test_data/testpkg-hello-protos/setup.py
+++ b/test/test_data/testpkg-hello-protos/setup.py
@@ -1,4 +1,5 @@
 import setuptools
+
 from ansys.tools.protoc_helper import CMDCLASS_OVERRIDE
 
 
@@ -8,15 +9,12 @@
         author="ANSYS, Inc.",
         python_requires=">=3.7",
         install_requires=["grpcio~=1.0", "protobuf~=3.0"],
-        packages=["testpkg.api.hello.v0"],
-        package_dir={"": "src"},
+        packages=setuptools.find_namespace_packages(".", include=("testpkg.*",)),
         package_data={
             "": ["*.proto", "*.pyi", "py.typed"],
         },
         entry_points={
-            "ansys.tools.protoc_helper.proto_provider": {
-                "testpkg.api.hello.v0=testpkg.api.hello.v0"
-            },
+            "ansys.tools.protoc_helper.proto_provider": {"testpkg.api.hello=testpkg.api.hello"},
         },
         cmdclass=CMDCLASS_OVERRIDE,
     )
diff --git a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello2.proto b/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello2.proto
deleted file mode 100644
index f0f2ead..0000000
--- a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello2.proto
+++ /dev/null
@@ -1,5 +0,0 @@
-syntax = "proto3";
-
-message Hello2 {
-    string content = 1;
-}
diff --git a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/__init__.py b/test/test_data/testpkg-hello-protos/testpkg/api/hello/__init__.py
similarity index 100%
rename from test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/__init__.py
rename to test/test_data/testpkg-hello-protos/testpkg/api/hello/__init__.py
diff --git a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello.proto b/test/test_data/testpkg-hello-protos/testpkg/api/hello/hello.proto
similarity index 64%
rename from test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello.proto
rename to test/test_data/testpkg-hello-protos/testpkg/api/hello/hello.proto
index 3a189bb..aedf753 100644
--- a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/hello.proto
+++ b/test/test_data/testpkg-hello-protos/testpkg/api/hello/hello.proto
@@ -1,5 +1,7 @@
 syntax = "proto3";
 
+package testpkg.api.hello.hello;
+
 message Hello {
     string content = 1;
 }
diff --git a/test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/py.typed b/test/test_data/testpkg-hello-protos/testpkg/api/hello/py.typed
similarity index 100%
rename from test/test_data/testpkg-hello-protos/src/testpkg/api/hello/v0/py.typed
rename to test/test_data/testpkg-hello-protos/testpkg/api/hello/py.typed
diff --git a/test/test_data/testpkg-hello-protos/testpkg/api/hello/v0/hello.proto b/test/test_data/testpkg-hello-protos/testpkg/api/hello/v0/hello.proto
new file mode 100644
index 0000000..462cd45
--- /dev/null
+++ b/test/test_data/testpkg-hello-protos/testpkg/api/hello/v0/hello.proto
@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package testpkg.api.hello.v0.hello;
+
+message Hello {
+    string content = 1;
+}
diff --git a/test/test_data/testpkg-hello-protos/testpkg/api/hello/v1/nested/hello.proto b/test/test_data/testpkg-hello-protos/testpkg/api/hello/v1/nested/hello.proto
new file mode 100644
index 0000000..5f1a07b
--- /dev/null
+++ b/test/test_data/testpkg-hello-protos/testpkg/api/hello/v1/nested/hello.proto
@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package testpkg.api.hello.v1.nested.hello;
+
+message Hello {
+    string content = 1;
+}
diff --git a/test/test_integration.py b/test/test_integration.py
index c341329..06871d8 100644
--- a/test/test_integration.py
+++ b/test/test_integration.py
@@ -1,6 +1,8 @@
 """Integration tests for end-to-end tests of the protoc-helper."""
+import contextlib
 import os
 import pathlib
+import shutil
 import subprocess
 import tempfile
 import types
@@ -12,63 +14,71 @@
 
 
 @pytest.fixture
-def test_venv():
-    with tempfile.TemporaryDirectory() as tmp_dir:
-        venv_dir = pathlib.Path(tmp_dir) / "venv"
-        venv.main([str(venv_dir)])
-        if os.name == "nt":  # Windows
-            script_dir = venv_dir / "Scripts"
-            python_bin = script_dir / "python.exe"
-            pip_bin = script_dir / "pip.exe"
-        else:  # Linux
-            script_dir = venv_dir / "bin"
-            python_bin = script_dir / "python"
-            pip_bin = script_dir / "pip"
-        test_venv = types.SimpleNamespace(python=python_bin, pip=pip_bin)
-        subprocess.check_call([str(test_venv.python), "--version"])  # for info only
-        subprocess.check_call([str(test_venv.pip), "install", "wheel"])
-        yield test_venv
+def test_venv(tmpdir):
+    venv_dir = tmpdir / "venv"
+    venv.main([str(venv_dir)])
+    if os.name == "nt":  # Windows
+        script_dir = venv_dir / "Scripts"
+        python_bin = script_dir / "python.exe"
+        pip_bin = script_dir / "pip.exe"
+    else:  # Linux
+        script_dir = venv_dir / "bin"
+        python_bin = script_dir / "python"
+        pip_bin = script_dir / "pip"
+    test_venv = types.SimpleNamespace(python=python_bin, pip=pip_bin)
+    subprocess.check_call([str(test_venv.python), "--version"])  # for info only
+    subprocess.check_call([str(test_venv.pip), "install", "wheel"])
+    yield test_venv
 
 
 @pytest.fixture
-def local_pkg_index(test_venv):
+def local_pkg_index(test_venv, tmpdir):
+    pkg_idx_dir = tmpdir / "pkg_idx"
+    extra_deps = ["wheel"]
+    subprocess.check_call(
+        [
+            str(test_venv.pip),
+            "download",
+            "-d",
+            str(pkg_idx_dir),
+        ]
+        + extra_deps
+    )
+    subprocess.check_call(
+        [
+            str(test_venv.pip),
+            "wheel",
+            "-w",
+            str(pkg_idx_dir),
+            f"{PKG_ROOT_DIR}",
+        ]
+    )
+    yield pkg_idx_dir
+
+
+@contextlib.contextmanager
+def copy_test_package(pkgname):
     with tempfile.TemporaryDirectory() as tmp_dir:
-        extra_deps = ["wheel"]
-        subprocess.check_call(
-            [
-                str(test_venv.pip),
-                "download",
-                "-d",
-                tmp_dir,
-            ]
-            + extra_deps
-        )
-        subprocess.check_call(
-            [
-                str(test_venv.pip),
-                "wheel",
-                "-w",
-                f"{tmp_dir}",
-                f"{PKG_ROOT_DIR}",
-            ]
-        )
-        yield tmp_dir
+        dest_dir = pathlib.Path(tmp_dir) / pkgname
+        shutil.copytree(TEST_DATA_DIR / pkgname, dest_dir)
+        yield dest_dir
 
 
 @pytest.fixture
 def create_test_pkg_wheel(local_pkg_index, test_venv):
     def inner(pkgname):
-        subprocess.check_call(
-            [
-                str(test_venv.pip),
-                "wheel",
-                "--no-index",
-                f"--find-links={local_pkg_index}",
-                "-w",
-                f"{local_pkg_index}",
-                f"{TEST_DATA_DIR / pkgname}",
-            ]
-        )
+        with copy_test_package(pkgname) as pkg_dir:
+            subprocess.check_call(
+                [
+                    str(test_venv.pip),
+                    "wheel",
+                    "--no-index",
+                    f"--find-links={local_pkg_index}",
+                    "-w",
+                    str(local_pkg_index),
+                    str(pkg_dir),
+                ]
+            )
 
     return inner
 
@@ -82,7 +92,7 @@ def inner(pkgname):
                 "install",
                 "--no-index",
                 f"--find-links={local_pkg_index}",
-                f"{TEST_DATA_DIR / pkgname}",
+                str(TEST_DATA_DIR / pkgname),
             ]
         )
 
@@ -92,4 +102,13 @@ def inner(pkgname):
 def test_dependency_compile(create_test_pkg_wheel, install_test_pkg, test_venv):
     create_test_pkg_wheel("testpkg-hello-protos")
     install_test_pkg("testpkg-greeter-protos")
-    subprocess.check_call([str(test_venv.python), "-c", "import testpkg.api.greeter.v0"])
+    subprocess.check_call(
+        [
+            str(test_venv.python),
+            "-c",
+            (
+                "from testpkg.api.greeter.v0 import greeter_pb2, greeter_pb2_grpc, "
+                + "greeter2_pb2, greeter2_pb2_grpc, greeter3_pb2, greeter3_pb2_grpc"
+            ),
+        ]
+    )