Repository: facebookresearch/CompilerGym

Issue:
CI test failure: test_running_environment_in_background_process
## 🐛 Bug

Test failure in [test-llvm-env-macos](https://github.com/facebookresearch/CompilerGym/runs/3650623731?check_suite_focus=true) job of CI workflow:

```
________________ test_running_environment_in_background_process ________________
[gw2] darwin -- Python 3.9.7 /Users/runner/hostedtoolcache/Python/3.9.7/x64/bin/python

    def test_running_environment_in_background_process():
        """Test launching and running an LLVM environment in a background process."""
        queue = mp.Queue(maxsize=3)
        process = mp.Process(
            target=process_worker,
            args=("llvm-autophase-ic-v0", "cbench-v1/crc32", [0, 0, 0], queue),
        )
        process.start()
        try:
            process.join(timeout=10)
>           result = queue.get(timeout=10)

tests/llvm/multiprocessing_test.py:50: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <multiprocessing.queues.Queue object at 0x114640250>, block = True
timeout = 9.999959355999977

    def get(self, block=True, timeout=None):
        if self._closed:
            raise ValueError(f"Queue {self!r} is closed")
        if block and timeout is None:
            with self._rlock:
                res = self._recv_bytes()
            self._sem.release()
        else:
            if block:
                deadline = time.monotonic() + timeout
            if not self._rlock.acquire(block, timeout):
                raise Empty
            try:
                if block:
                    timeout = deadline - time.monotonic()
                    if not self._poll(timeout):
>                       raise Empty
E                       _queue.Empty

/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/multiprocessing/queues.py:114: Empty
----------------------------- Captured stderr call -----------------------------
WARNING:root:Failed to parse '#include <...>' search paths from echo:

Process Process-1:
Traceback (most recent call last):
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/private/tmp/compiler_gym_runner/install_tests/tests/llvm/multiprocessing_test.py", line 20, in process_worker
    with gym.make(env_name) as env:
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/gym/envs/registration.py", line 200, in make
    return registry.make(id, **kwargs)
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/gym/envs/registration.py", line 105, in make
    env = spec.make(**kwargs)
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/gym/envs/registration.py", line 75, in make
    env = cls(**_kwargs)
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/compiler_gym/envs/llvm/llvm_env.py", line 103, in __init__
    datasets=_get_llvm_datasets(site_data_base=datasets_site_path),
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/compiler_gym/envs/llvm/llvm_env.py", line 70, in _get_llvm_datasets
    _LLVM_DATASETS = list(get_llvm_datasets(site_data_base=site_data_base))
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/compiler_gym/envs/llvm/datasets/__init__.py", line 257, in get_llvm_datasets
    yield CsmithDataset(site_data_base=site_data_base, sort_order=0)
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/compiler_gym/envs/llvm/datasets/csmith.py", line 129, in __init__
    self.clang_compile_command: List[str] = ClangInvocation.from_c_file(
  File "/Users/runner/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/site-packages/compiler_gym/envs/llvm/llvm_benchmark.py", line 143, in command
    for directory in get_system_includes():
TypeError: 'NoneType' object is not iterable
```

## Environment

Please fill in this checklist:

- CompilerGym: v0.1.10
- How you installed CompilerGym (conda, pip, source): source
- OS: macOS
- Python version: 3.9



Discussion:
Another failure on the `test-llvm-env-asan-linux` job:

```
=================================== FAILURES ===================================
________________ test_running_environment_in_background_process ________________
[gw0] linux -- Python 3.9.7 /opt/hostedtoolcache/Python/3.9.7/x64/bin/python

    def test_running_environment_in_background_process():
        """Test launching and running an LLVM environment in a background process."""
        queue = mp.Queue(maxsize=3)
        process = mp.Process(
            target=process_worker,
            args=("llvm-autophase-ic-v0", "cbench-v1/crc32", [0, 0, 0], queue),
        )
        process.start()
        try:
            process.join(timeout=10)
>           result = queue.get(timeout=10)

tests/llvm/multiprocessing_test.py:50: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <multiprocessing.queues.Queue object at 0x7f71e3177520>, block = True
timeout = 9.999984599999948

    def get(self, block=True, timeout=None):
        if self._closed:
            raise ValueError(f"Queue {self!r} is closed")
        if block and timeout is None:
            with self._rlock:
                res = self._recv_bytes()
            self._sem.release()
        else:
            if block:
                deadline = time.monotonic() + timeout
            if not self._rlock.acquire(block, timeout):
                raise Empty
            try:
                if block:
                    timeout = deadline - time.monotonic()
                    if not self._poll(timeout):
>                       raise Empty
E                       _queue.Empty

/opt/hostedtoolcache/Python/3.9.7/x64/lib/python3.9/multiprocessing/queues.py:114: Empty
============================= slowest 5 durations ==============================
242.55s call     tests/llvm/datasets/cbench_validate_test.py::test_validate_benchmark_semantics[benchmark://cbench-v1/tiff2rgba]
236.27s call     tests/llvm/datasets/cbench_validate_test.py::test_validate_benchmark_semantics[benchmark://cbench-v1/tiffmedian]
232.38s call     tests/llvm/datasets/cbench_validate_test.py::test_validate_benchmark_semantics[benchmark://cbench-v1/tiff2bw]
230.25s call     tests/llvm/datasets/cbench_validate_test.py::test_validate_benchmark_semantics[benchmark://cbench-v1/tiffdither]
207.78s call     tests/llvm/datasets/cbench_validate_test.py::test_validate_benchmark_semantics[benchmark://cbench-v1/jpeg-d]
=========================== short test summary info ============================
FAILED tests/llvm/multiprocessing_test.py::test_running_environment_in_background_process
=========== 1 failed, 638 passed, 808 skipped in 2326.49s (0:38:46) ============
```

Resolving patch:
diff --git a/compiler_gym/envs/compiler_env.py b/compiler_gym/envs/compiler_env.py
--- a/compiler_gym/envs/compiler_env.py
+++ b/compiler_gym/envs/compiler_env.py
@@ -894,7 +894,8 @@ def raw_step(
                 self.close()
             except ServiceError as e:
                 # close() can raise ServiceError if the service exists with a
-                # non-zero return code. If so,
+                # non-zero return code. We swallow the error here but propagate
+                # the diagnostic message.
                 info[
                     "error_details"
                 ] += f". Additional error during environment closing: {e}"


Test patch:
diff --git a/tests/bin/BUILD b/tests/bin/BUILD
--- a/tests/bin/BUILD
+++ b/tests/bin/BUILD
@@ -16,7 +16,6 @@ py_test(
 
 py_test(
     name = "manual_env_bin_test",
-    timeout = "short",
     srcs = ["manual_env_bin_test.py"],
     flaky = 1,
     deps = [
diff --git a/tests/llvm/BUILD b/tests/llvm/BUILD
--- a/tests/llvm/BUILD
+++ b/tests/llvm/BUILD
@@ -118,8 +118,9 @@ py_test(
 
 py_test(
     name = "fresh_environment_observation_reward_test",
+    timeout = "long",
     srcs = ["fresh_environment_observation_reward_test.py"],
-    shard_count = 8,
+    shard_count = 12,
     deps = [
         "//compiler_gym/envs",
         "//tests:test_main",
@@ -137,16 +138,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "gvn_sink_test",
-    srcs = ["gvn_sink_test.py"],
-    deps = [
-        "//compiler_gym/envs",
-        "//tests:test_main",
-        "//tests/pytest_plugins:llvm",
-    ],
-)
-
 py_test(
     name = "gym_interface_compatability",
     timeout = "short",
@@ -217,7 +208,6 @@ py_test(
 
 py_test(
     name = "multiprocessing_test",
-    timeout = "short",
     srcs = ["multiprocessing_test.py"],
     flaky = 1,
     deps = [
diff --git a/tests/llvm/all_actions_single_step_test.py b/tests/llvm/all_actions_single_step_test.py
--- a/tests/llvm/all_actions_single_step_test.py
+++ b/tests/llvm/all_actions_single_step_test.py
@@ -7,6 +7,7 @@
 import numpy as np
 
 from compiler_gym.envs import CompilerEnv
+from compiler_gym.service.connection import ServiceError
 from compiler_gym.third_party.autophase import AUTOPHASE_FEATURE_DIM
 from tests.test_main import main
 
@@ -25,6 +26,13 @@ def test_step(env: CompilerEnv, action_name: str):
     assert isinstance(reward, float)
     assert isinstance(done, bool)
 
+    try:
+        env.close()
+    except ServiceError as e:
+        # env.close() will raise an error if the service terminated
+        # ungracefully. In that case, the "done" flag should have been set.
+        assert done, f"Service error was raised when 'done' flag not set: {e}"
+
 
 if __name__ == "__main__":
     main()
diff --git a/tests/llvm/fresh_environment_observation_reward_test.py b/tests/llvm/fresh_environment_observation_reward_test.py
--- a/tests/llvm/fresh_environment_observation_reward_test.py
+++ b/tests/llvm/fresh_environment_observation_reward_test.py
@@ -4,12 +4,15 @@
 # LICENSE file in the root directory of this source tree.
 """Integrations tests for the LLVM CompilerGym environments."""
 
+from flaky import flaky
+
 from compiler_gym.envs import CompilerEnv
 from tests.test_main import main
 
 pytest_plugins = ["tests.pytest_plugins.llvm"]
 
 
+@flaky  # Runtime can timeout
 def test_step(env: CompilerEnv, observation_space: str, reward_space: str):
     """Request every combination of observation and reward in a fresh environment."""
     env.reward_space = None
diff --git a/tests/llvm/gvn_sink_test.py b/tests/llvm/gvn_sink_test.py
deleted file mode 100644
--- a/tests/llvm/gvn_sink_test.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""Tests for action space determinism."""
-import hashlib
-
-import pytest
-
-from compiler_gym.envs import LlvmEnv
-from tests.test_main import main
-
-pytest_plugins = ["tests.pytest_plugins.llvm"]
-
-ACTION_REPTITION_COUNT = 50
-
-
-@pytest.mark.skip(reason="github.com/facebookresearch/CompilerGym/issues/46")
-@pytest.mark.parametrize(
-    "benchmark_name",
-    [
-        "benchmark://cbench-v1/adpcm",
-        "benchmark://cbench-v1/bitcount",
-        "benchmark://cbench-v1/blowfish",
-        "benchmark://cbench-v1/bzip2",
-        "benchmark://cbench-v1/ghostscript",
-        "benchmark://cbench-v1/gsm",
-        "benchmark://cbench-v1/ispell",
-        "benchmark://cbench-v1/jpeg-c",
-        "benchmark://cbench-v1/jpeg-d",
-        "benchmark://cbench-v1/patricia",
-        "benchmark://cbench-v1/rijndael",
-        "benchmark://cbench-v1/stringsearch",
-        "benchmark://cbench-v1/stringsearch2",
-        "benchmark://cbench-v1/susan",
-        "benchmark://cbench-v1/tiff2bw",
-        "benchmark://cbench-v1/tiff2rgba",
-        "benchmark://cbench-v1/tiffdither",
-        "benchmark://cbench-v1/tiffmedian",
-    ],
-)
-def test_gvn_sink_non_determinism(env: LlvmEnv, benchmark_name: str):
-    """Regression test for -gvn-sink non-determinism.
-    See: https://github.com/facebookresearch/CompilerGym/issues/46
-    """
-    env.observation_space = "Ir"
-
-    checksums = set()
-    for i in range(1, ACTION_REPTITION_COUNT + 1):
-        env.reset(benchmark=benchmark_name)
-        ir, _, done, _ = env.step(env.action_space.names.index("-gvn-sink"))
-        assert not done
-        sha1 = hashlib.sha1()
-        sha1.update(ir.encode("utf-8"))
-        checksums.add(sha1.hexdigest())
-
-        if len(checksums) != 1:
-            pytest.fail(
-                f"Repeating the -gvn-sink action {i} times on {benchmark_name} "
-                "produced different states"
-            )
-
-
-if __name__ == "__main__":
-    main()
diff --git a/tests/llvm/multiprocessing_test.py b/tests/llvm/multiprocessing_test.py
--- a/tests/llvm/multiprocessing_test.py
+++ b/tests/llvm/multiprocessing_test.py
@@ -9,6 +9,7 @@
 
 import gym
 import pytest
+from flaky import flaky
 
 from compiler_gym.envs import LlvmEnv
 from tests.pytest_plugins.common import macos_only
@@ -37,6 +38,7 @@ def process_worker_with_env(env: LlvmEnv, actions: List[int], queue: mp.Queue):
     queue.put((env, observation, reward, done, info))
 
 
+@flaky  # Test contains timeouts.
 def test_running_environment_in_background_process():
     """Test launching and running an LLVM environment in a background process."""
     queue = mp.Queue(maxsize=3)
@@ -46,8 +48,8 @@ def test_running_environment_in_background_process():
     )
     process.start()
     try:
-        process.join(timeout=10)
-        result = queue.get(timeout=10)
+        process.join(timeout=60)
+        result = queue.get(timeout=60)
         observation, reward, done, info = result
 
         assert not done
diff --git a/tests/llvm/observation_spaces_test.py b/tests/llvm/observation_spaces_test.py
--- a/tests/llvm/observation_spaces_test.py
+++ b/tests/llvm/observation_spaces_test.py
@@ -10,6 +10,7 @@
 import networkx as nx
 import numpy as np
 import pytest
+from flaky import flaky
 from gym.spaces import Box
 from gym.spaces import Dict as DictSpace
 
@@ -1165,6 +1166,7 @@ def test_object_text_size_observation_spaces(env: LlvmEnv):
     assert value == crc32_code_sizes[sys.platform][2]
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")
     key = "Runtime"
@@ -1188,6 +1190,7 @@ def test_runtime_observation_space(env: LlvmEnv):
     assert len(set(value)) > 1
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     """Test setting a custom observation count for LLVM runtimes."""
     env.reset("cbench-v1/crc32")
@@ -1208,6 +1211,7 @@ def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     assert value.shape == (5,)
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space_invalid_observation_count(env: LlvmEnv):
     """Test setting an invalid custom observation count for LLVM runtimes."""
     env.reset("cbench-v1/crc32")
@@ -1233,6 +1237,7 @@ def test_runtime_observation_space_not_runnable(env: LlvmEnv):
     assert space.space.contains(value)
 
 
+@flaky  # Build can timeout
 def test_buildtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")
     key = "Buildtime"