Repository: facebookresearch/CompilerGym

Issue:
[examples] llvm_rl/tests/training_integration_test.py hangs on CI
## 🐛 Bug

The call to `model.test()` hangs before failing with a TimeoutError.


Discussion:

Resolving patch:
diff --git a/compiler_gym/envs/llvm/llvm_env.py b/compiler_gym/envs/llvm/llvm_env.py
--- a/compiler_gym/envs/llvm/llvm_env.py
+++ b/compiler_gym/envs/llvm/llvm_env.py
@@ -377,6 +377,8 @@ def reset(self, *args, **kwargs):
                 or "Error reading file:" in str(e)
             ):
                 raise BenchmarkInitError(str(e)) from e
+            elif "Failed to parse LLVM bitcode" in str(e):
+                raise BenchmarkInitError(str(e)) from e
             raise
 
     def make_benchmark(
diff --git a/compiler_gym/errors/dataset_errors.py b/compiler_gym/errors/dataset_errors.py
--- a/compiler_gym/errors/dataset_errors.py
+++ b/compiler_gym/errors/dataset_errors.py
@@ -4,7 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 
-class BenchmarkInitError(OSError):
+class BenchmarkInitError(OSError, ValueError):
     """Base class for errors raised if a benchmark fails to initialize."""
 
 


Test patch:
diff --git a/tests/llvm/custom_benchmarks_test.py b/tests/llvm/custom_benchmarks_test.py
--- a/tests/llvm/custom_benchmarks_test.py
+++ b/tests/llvm/custom_benchmarks_test.py
@@ -46,7 +46,7 @@ def test_invalid_benchmark_data(env: LlvmEnv):
     )
 
     with pytest.raises(
-        ValueError, match='Failed to parse LLVM bitcode: "benchmark://new"'
+        BenchmarkInitError, match='Failed to parse LLVM bitcode: "benchmark://new"'
     ):
         env.reset(benchmark=benchmark)
 
@@ -69,7 +69,7 @@ def test_benchmark_path_empty_file(env: LlvmEnv):
 
         benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc")
 
-        with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"):
+        with pytest.raises(BenchmarkInitError, match="Failed to parse LLVM bitcode"):
             env.reset(benchmark=benchmark)
 
 
@@ -81,7 +81,7 @@ def test_invalid_benchmark_path_contents(env: LlvmEnv):
 
         benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc")
 
-        with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"):
+        with pytest.raises(BenchmarkInitError, match="Failed to parse LLVM bitcode"):
             env.reset(benchmark=benchmark)
 
 
diff --git a/tests/llvm/datasets/poj104_test.py b/tests/llvm/datasets/poj104_test.py
--- a/tests/llvm/datasets/poj104_test.py
+++ b/tests/llvm/datasets/poj104_test.py
@@ -13,6 +13,7 @@
 import compiler_gym.envs.llvm  # noqa register environments
 from compiler_gym.envs.llvm import LlvmEnv
 from compiler_gym.envs.llvm.datasets import POJ104Dataset
+from compiler_gym.errors import BenchmarkInitError
 from tests.pytest_plugins.common import skip_on_ci
 from tests.test_main import main
 
@@ -34,7 +35,7 @@ def test_poj104_size(poj104_dataset: POJ104Dataset):
 
 
 @skip_on_ci
-@pytest.mark.parametrize("index", range(250))
+@pytest.mark.parametrize("index", range(100))
 def test_poj104_random_select(
     env: LlvmEnv, poj104_dataset: POJ104Dataset, index: int, tmpwd: Path
 ):
@@ -54,5 +55,20 @@ def test_poj104_random_benchmark(env: LlvmEnv, poj104_dataset: POJ104Dataset):
     assert benchmark.source
 
 
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "benchmark://poj104-v1/1/1486",
+    ],
+)
+def test_poj104_known_bad_bitcodes(env: LlvmEnv, uri: str):
+    # This test is intentionally structured in a way that if the benchmark does
+    # not raise an error, it still passes.
+    try:
+        env.reset(benchmark=uri)
+    except BenchmarkInitError as e:
+        assert "Failed to parse LLVM bitcode" in str(e)
+
+
 if __name__ == "__main__":
     main()
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,3 @@
-coverage<6.0  # Implicit dependency of pytest-cov, fix version < 6.x for Py3.6 support.
 flaky==3.7.0
 psutil==5.8.0  # Implicit dependency of pytest-xdist
 pytest==6.2.5