Repository: facebookresearch/CompilerGym

Issue:
🔥 Flaky CI tests: SSL "wrong version" error
```
2022-04-18T23:25:41.0658299Z E           compiler_gym.util.download.DownloadFailed: HTTPSConnectionPool(host='dl.fbaipublicfiles.com', port=443): Max retries exceeded with url: /compiler_gym/llvm_bitcodes-10.0.0-anghabench-v1-linux-manifest.bz2 (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:852)'),))
...
2022-04-18T23:25:41.0661363Z FAILED tests/bin/manual_env_bin_test.py::test_list_benchmarks - compiler_gym....
```

... and similar errors


Discussion:

Resolving patch:
diff --git a/compiler_gym/envs/llvm/llvm_benchmark.py b/compiler_gym/envs/llvm/llvm_benchmark.py
--- a/compiler_gym/envs/llvm/llvm_benchmark.py
+++ b/compiler_gym/envs/llvm/llvm_benchmark.py
@@ -6,6 +6,7 @@
 import logging
 import os
 import random
+import shlex
 import subprocess
 import sys
 import tempfile
@@ -17,7 +18,7 @@
 
 from compiler_gym.datasets import Benchmark, BenchmarkInitError
 from compiler_gym.third_party import llvm
-from compiler_gym.util.commands import Popen, run_command
+from compiler_gym.util.commands import Popen, communicate, run_command
 from compiler_gym.util.runfiles_path import transient_cache_path
 from compiler_gym.util.thread_pool import get_thread_pool_executor
 
@@ -35,38 +36,50 @@ class UnableToParseHostCompilerOutput(HostCompilerFailure):
 
 def _get_system_library_flags(compiler: str) -> Iterable[str]:
     """Private implementation function."""
-    # Create a temporary directory to write the compiled binary to, since GNU
+    # Create a temporary file to write the compiled binary to, since GNU
     # assembler does not support piping to stdout.
     transient_cache = transient_cache_path(".")
     transient_cache.mkdir(parents=True, exist_ok=True)
     with tempfile.NamedTemporaryFile(dir=transient_cache) as f:
-        try:
-            cmd = [compiler, "-xc++", "-v", "-", "-o", f.name]
-            # On macOS we need to compile a binary to invoke the linker.
-            if sys.platform != "darwin":
-                cmd.append("-c")
-            with Popen(
-                cmd,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.PIPE,
-                stdin=subprocess.PIPE,
-                universal_newlines=True,
-            ) as process:
-                _, stderr = process.communicate(
-                    input="int main(){return 0;}", timeout=30
-                )
-                if process.returncode:
-                    raise HostCompilerFailure(
-                        f"Failed to invoke '{compiler}'. "
-                        f"Is there a working system compiler?\n"
-                        f"Error: {stderr.strip()}"
+        cmd = [compiler, "-xc++", "-v", "-", "-o", f.name]
+        # On macOS we need to compile a binary to invoke the linker.
+        if sys.platform != "darwin":
+            cmd.append("-c")
+
+        # Retry loop to permit timeouts, though unlikely, in case of a
+        # heavily overloaded system (I have observed CI failures because
+        # of this).
+        for _ in range(3):
+            try:
+                with Popen(
+                    cmd,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.PIPE,
+                    stdin=subprocess.PIPE,
+                    universal_newlines=True,
+                ) as process:
+                    _, stderr = communicate(
+                        process=process, input="int main(){return 0;}", timeout=30
                     )
-        except FileNotFoundError as e:
+                    if process.returncode:
+                        raise HostCompilerFailure(
+                            f"Failed to invoke '{compiler}'. "
+                            f"Is there a working system compiler?\n"
+                            f"Error: {stderr.strip()}"
+                        )
+                    break
+            except subprocess.TimeoutExpired:
+                continue
+            except FileNotFoundError as e:
+                raise HostCompilerFailure(
+                    f"Failed to invoke '{compiler}'. "
+                    f"Is there a working system compiler?\n"
+                    f"Error: {e}"
+                ) from e
+        else:
             raise HostCompilerFailure(
-                f"Failed to invoke '{compiler}'. "
-                f"Is there a working system compiler?\n"
-                f"Error: {e}"
-            ) from e
+                f"Compiler invocation '{shlex.join(cmd)}' timed out after 3 attempts."
+            )
 
     # Parse the compiler output that matches the conventional output format
     # used by clang and GCC:


Test patch:
diff --git a/tests/wrappers/llvm_test.py b/tests/wrappers/llvm_test.py
--- a/tests/wrappers/llvm_test.py
+++ b/tests/wrappers/llvm_test.py
@@ -46,6 +46,7 @@ def test_reward_range_not_runnable_benchmark(env: LlvmEnv):
         env.reset(benchmark="benchmark://npb-v0/1")
 
 
+@flaky  # Runtime can fail
 def test_fork(env: LlvmEnv):
     env = RuntimePointEstimateReward(env)
     with env.fork() as fkd: