Repository: canonical/charmcraft

Issue:
The reactive plugin is not correctly checking if `charm` failed
The `charm` tool used by the reactive plugin has a complex return code specification.

While Charmcraft is correctly checking that the return code >= 200 is an error, it's failing to verify that <0 is also an error.

Furthermore, it should log in debug the return code from the execution (no matter which one it is).

All said above applies for both `charm` calls.

Discussion:
fnordahl: Putting this up as a draft as I need to fix the tests.

Patch:
diff --git a/charmcraft/reactive_plugin.py b/charmcraft/reactive_plugin.py
index 5dcc4c3e..350dcb39 100644
--- a/charmcraft/reactive_plugin.py
+++ b/charmcraft/reactive_plugin.py
@@ -21,7 +21,6 @@ import sys
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, cast
 
-from craft_cli import emit
 from craft_parts import plugins
 from craft_parts.errors import PluginEnvironmentValidationError
 
@@ -76,7 +75,7 @@ class ReactivePluginEnvironmentValidator(plugins.validator.PluginEnvironmentVali
                     reason=f"invalid charm tools version {version_data}",
                 )
             tools_version = f"{tool_name} {version_data[tool_name]['version']} ({version_data[tool_name]['git']})"
-            emit.debug(f"found {tools_version}")
+            print(f"found {tools_version}")
         except ValueError as err:
             raise PluginEnvironmentValidationError(
                 part_name=self._part_name,
@@ -140,6 +139,26 @@ class ReactivePlugin(plugins.Plugin):
         return [" ".join(shlex.quote(i) for i in command)]
 
 
+def run_charm_tool(args: List[str]):
+    """Run the charm tool, log and check exit code."""
+    result_classification = "SUCCESS"
+    exc = None
+
+    print(f"charm tool execution command={args}")
+    try:
+        completed_process = subprocess.run(args, check=True)
+    except subprocess.CalledProcessError as call_error:
+        exc = call_error
+        if call_error.returncode < 100 or call_error.returncode >= 200:
+            result_classification = "ERROR"
+            raise
+        result_classification = "WARNING"
+    finally:
+        print(
+            f"charm tool execution {result_classification}: returncode={exc.returncode if exc else completed_process.returncode}"
+        )
+
+
 def build(
     *, charm_name: str, build_dir: Path, install_dir: Path, charm_build_arguments: List[str]
 ):
@@ -161,11 +180,11 @@ def build(
     string produced by `charm` would be misleading.
     """
     # Verify the charm is ok from a charm tool point of view.
+
     try:
-        subprocess.run(["charm", "proof"], check=True)
+        run_charm_tool(["charm", "proof"])
     except subprocess.CalledProcessError as call_error:
-        if call_error.returncode >= 200:
-            return call_error.returncode
+        return call_error.returncode
 
     # Link the installation directory to the place where charm creates
     # the charm.
@@ -179,10 +198,9 @@ def build(
     cmd.extend(["-o", build_dir])
 
     try:
-        subprocess.run(cmd, check=True)
+        run_charm_tool(cmd)
     except subprocess.CalledProcessError as call_error:
-        if call_error.returncode >= 200:
-            return call_error.returncode
+        return call_error.returncode
     finally:
         charm_build_dir.unlink()
 


Test patch:
diff --git a/tests/test_reactive_plugin.py b/tests/test_reactive_plugin.py
index 6322e08d..d2a2d854 100644
--- a/tests/test_reactive_plugin.py
+++ b/tests/test_reactive_plugin.py
@@ -16,7 +16,7 @@
 
 import pathlib
 import sys
-from subprocess import CalledProcessError
+from subprocess import CalledProcessError, CompletedProcess
 from unittest.mock import call, patch
 
 import craft_parts
@@ -173,6 +173,7 @@ def fake_run():
 
 
 def test_build(build_dir, install_dir, fake_run):
+    fake_run.return_value = CompletedProcess(("charm", "build"), 0)
     returncode = reactive_plugin.build(
         charm_name="test-charm",
         build_dir=build_dir,
@@ -248,7 +249,20 @@ def test_build_charm_proof_raises_warning_messages_does_not_raise(
 
 
 def test_build_charm_build_raises_error_messages(build_dir, install_dir, fake_run):
-    fake_run.side_effect = [None, CalledProcessError(200, "E: name missing")]
+    def _run_generator():
+        """Passing an iterable to `side_effect` pivots the mocks return_value,
+        and does not allow us to raise an actual exception.
+
+        Thus we need this helper to accomplish this.
+
+        Ref: https://docs.python.org/3/library/unittest.mock-examples.html#side-effect-functions-and-iterables
+        """
+        yield CompletedProcess(("charm", "proof"), 0)
+        yield CalledProcessError(200, "E: name missing")
+        yield CompletedProcess(("charm", "proof"), 0)
+        yield CalledProcessError(-1, "E: name missing")
+
+    fake_run.side_effect = _run_generator()
 
     returncode = reactive_plugin.build(
         charm_name="test-charm",
@@ -275,11 +289,31 @@ def test_build_charm_build_raises_error_messages(build_dir, install_dir, fake_ru
         ),
     ]
 
+    # Also ensure negative return codes raises error
+    returncode = reactive_plugin.build(
+        charm_name="test-charm",
+        build_dir=build_dir,
+        install_dir=install_dir,
+        charm_build_arguments=["--charm-argument", "--charm-argument-with", "argument"],
+    )
+    assert returncode == -1
+
 
 def test_build_charm_build_raises_warning_messages_does_not_raise(
     build_dir, install_dir, fake_run
 ):
-    fake_run.side_effect = [None, CalledProcessError(100, "W: Description is not pretty")]
+    def _run_generator():
+        """Passing an iterable to `side_effect` pivots the mocks return_value,
+        and does not allow us to raise an actual exception.
+
+        Thus we need this helper to accomplish this.
+
+        Ref: https://docs.python.org/3/library/unittest.mock-examples.html#side-effect-functions-and-iterables
+        """
+        yield CompletedProcess(("charm", "proof"), 0)
+        yield CalledProcessError(100, "W: Description is not pretty")
+
+    fake_run.side_effect = _run_generator()
 
     returncode = reactive_plugin.build(
         charm_name="test-charm",