Repository: facebookresearch/CompilerGym

Issue:
Improve error reporting if CompilerGym service fails to start
## 🚀 Feature

If `gym.make("...")` fails, there is some obtuse "Failed to connect to service after 10.0 seconds" message but it hides any error messages generated by the service, making debugging difficult.



Discussion:

Resolving patch:
diff --git a/compiler_gym/service/connection.py b/compiler_gym/service/connection.py
--- a/compiler_gym/service/connection.py
+++ b/compiler_gym/service/connection.py
@@ -12,7 +12,7 @@
 from datetime import datetime
 from pathlib import Path
 from time import sleep, time
-from typing import List, NamedTuple, Optional, TypeVar, Union
+from typing import Iterable, List, NamedTuple, Optional, TypeVar, Union
 
 import grpc
 
@@ -25,6 +25,8 @@
 )
 from compiler_gym.util.debug_util import get_debug_level
 from compiler_gym.util.runfiles_path import runfiles_path, transient_cache_path
+from compiler_gym.util.shell_format import plural
+from compiler_gym.util.truncate import truncate_lines
 
 GRPC_CHANNEL_OPTIONS = [
     # Raise the default inbound message filter from 4MB.
@@ -189,11 +191,13 @@ def __call__(
                         raise ServiceTransportError(
                             f"{self.url} {e.details()} ({max_retries} retries)"
                         ) from None
+                    remaining = max_retries - attempt
                     self.logger.warning(
-                        "%s %s (%d attempts remaining)",
+                        "%s %s (%d %s remaining)",
                         self.url,
                         e.details(),
-                        max_retries - attempt,
+                        remaining,
+                        plural(remaining, "attempt", "attempts"),
                     )
                     sleep(retry_wait_seconds)
                     retry_wait_seconds *= retry_wait_backoff_exponent
@@ -215,6 +219,13 @@ def __call__(
                         f"RPC call returned status code {e.code()} and error `{e.details()}`"
                     ) from None
 
+    def loglines(self) -> Iterable[str]:
+        """Fetch any available log lines from the service backend.
+
+        :return: An iterator over lines of logs.
+        """
+        yield from ()
+
 
 def make_working_dir():
     """Make a working directory for a service. The calling code is responsible for
@@ -322,20 +333,42 @@ def __init__(
                 channel_ready.result(timeout=wait_secs)
                 break
             except (grpc.FutureTimeoutError, grpc.RpcError) as e:
-                logger.debug("Connection attempt %d = %s", attempts, e)
+                logger.debug(
+                    "Connection attempt %d = %s %s", attempts, type(e).__name__, str(e)
+                )
                 wait_secs *= 1.2
         else:
             self.process.kill()
             self.process.communicate(timeout=process_exit_max_seconds)
+
+            # Include the last few lines of logs generated by the compiler
+            # service, if any.
+            logs = truncate_lines(
+                self.loglines(), max_line_len=100, max_lines=25, tail=True
+            )
+            logs_message = f" Service logs:\n{logs}" if logs else ""
+
             shutil.rmtree(self.working_dir)
             raise TimeoutError(
                 "Failed to connect to RPC service after "
-                f"{process_exit_max_seconds:.1f} seconds"
-                f"({attempts} attempts made)"
+                f"{process_exit_max_seconds:.1f} seconds.{logs_message}"
             )
 
         super().__init__(channel, url, logger)
 
+    def loglines(self) -> Iterable[str]:
+        """Fetch any available log lines from the service backend.
+
+        :return: An iterator over lines of logs.
+        """
+        # Compiler services write log files in the logs directory. Iterate over
+        # them and return their contents.
+        for path in sorted((self.working_dir / "logs").iterdir()):
+            if not path.is_file():
+                continue
+            with open(path) as f:
+                yield from f.readlines()
+
     def close(self):
         """Terminate a local subprocess and close the connection."""
         self.process.kill()
@@ -375,13 +408,14 @@ def __init__(self, url: str, rpc_init_max_seconds: float, logger: logging.Logger
                 channel_ready.result(timeout=wait_secs)
                 break
             except (grpc.FutureTimeoutError, grpc.RpcError) as e:
-                logger.debug("Connection attempt %d = %s", attempts, e)
+                logger.debug(
+                    "Connection attempt %d = %s %s", attempts, type(e).__name__, str(e)
+                )
                 wait_secs *= 1.2
         else:
             raise TimeoutError(
                 f"Failed to connect to {url} after "
-                f"{rpc_init_max_seconds:.1f} seconds "
-                f"({attempts} attempts made)"
+                f"{rpc_init_max_seconds:.1f} seconds"
             )
 
         super().__init__(channel, url, logger)
@@ -531,7 +565,7 @@ def _create_connection(
         raise TimeoutError(
             f"Failed to create connection to {endpoint_name} after "
             f"{time() - start_time:.1f} seconds "
-            f"({attempts}/{opts.init_max_attempts} attempts made)"
+            f"({attempts} {plural(attempts, 'attempt', 'attempts')} made)"
         )
 
     def __repr__(self):


Test patch:
diff --git a/tests/service/connection_test.py b/tests/service/connection_test.py
--- a/tests/service/connection_test.py
+++ b/tests/service/connection_test.py
@@ -56,7 +56,7 @@ def test_create_channel_failed_subprocess(
         )
 
     assert str(ctx.value).startswith("Failed to create connection to localhost:")
-    assert str(ctx.value).endswith(" (2/2 attempts made)")
+    assert str(ctx.value).endswith(" (2 attempts made)")
 
 
 def test_create_channel_failed_subprocess_rpc_timeout(
@@ -76,7 +76,7 @@ def test_create_channel_failed_subprocess_rpc_timeout(
         )
 
     assert str(ctx.value).startswith("Failed to create connection to localhost:")
-    assert str(ctx.value).endswith(" (1/2 attempts made)")
+    assert str(ctx.value).endswith(" (1 attempt made)")
 
 
 def test_call_stub_invalid_type(connection: CompilerGymServiceConnection):