Repository: facebookresearch/CompilerGym

Issue:
Add a table of accepted file types to llvm.make_benchmark()
## 📚 Documentation

Be more clear about the accepted file types.

https://compilergym.com/llvm/api.html#compiler_gym.envs.llvm.make_benchmark


Discussion:

Resolving patch:
diff --git a/compiler_gym/envs/llvm/llvm_benchmark.py b/compiler_gym/envs/llvm/llvm_benchmark.py
--- a/compiler_gym/envs/llvm/llvm_benchmark.py
+++ b/compiler_gym/envs/llvm/llvm_benchmark.py
@@ -183,10 +183,34 @@ def make_benchmark(
 ) -> Benchmark:
     """Create a benchmark for use by LLVM environments.
 
-    This function takes one or more inputs and uses them to create a benchmark
-    that can be passed to :meth:`compiler_gym.envs.LlvmEnv.reset`.
+    This function takes one or more inputs and uses them to create an LLVM
+    bitcode benchmark that can be passed to
+    :meth:`compiler_gym.envs.LlvmEnv.reset`.
+
+    The following input types are supported:
+
+    +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+    | **File Suffix**                                     | **Treated as**      | **Converted using**                                         |
+    +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+    | :code:`.bc`                                         | LLVM IR bitcode     | No conversion required.                                     |
+    +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+    | :code:`.ll`                                         | LLVM IR text format | Assembled to bitcode using llvm-as.                         |
+    +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+    | :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source      | Compiled to bitcode using clang and the given :code:`copt`. |
+    +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
 
-    For single-source C/C++ programs, you can pass the path of the source file:
+    .. note::
+
+        The LLVM IR format has no compatability guarantees between versions (see
+        `LLVM docs
+        <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_).
+        You must ensure that any :code:`.bc` and :code:`.ll` files are
+        compatible with the LLVM version used by CompilerGym, which can be
+        reported using :func:`env.compiler_version
+        <compiler_gym.envs.CompilerEnv.compiler_version>`.
+
+    E.g. for single-source C/C++ programs, you can pass the path of the source
+    file:
 
         >>> benchmark = make_benchmark('my_app.c')
         >>> env = gym.make("llvm-v0")
@@ -209,7 +233,7 @@ def make_benchmark(
     clang:
 
         >>> benchmark = make_benchmark(
-            ClangInvocation(['/path/to/my_app.c'], timeout=10)
+            ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10)
         )
 
     For multi-file programs, pass a list of inputs that will be compiled
@@ -219,28 +243,9 @@ def make_benchmark(
             'main.c',
             'lib.cpp',
             'lib2.bc',
+            'foo/input.bc'
         ])
 
-    If you already have prepared bitcode files, those can be linked and used
-    directly:
-
-        >>> benchmark = make_benchmark([
-            'bitcode1.bc',
-            'bitcode2.bc',
-        ])
-
-    Text-format LLVM assembly can also be used:
-
-        >>> benchmark = make_benchmark('module.ll')
-
-    .. note::
-
-        LLVM bitcode compatibility is
-        `not guaranteed <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_,
-        so you must ensure that any precompiled bitcodes are compatible with the
-        LLVM version used by CompilerGym, which can be queried using
-        :func:`env.compiler_version <compiler_gym.envs.CompilerEnv.compiler_version>`.
-
     :param inputs: An input, or list of inputs.
 
     :param copt: A list of command line options to pass to clang when compiling
@@ -278,7 +283,7 @@ def _add_path(path: Path):
 
         if path.suffix == ".bc":
             bitcodes.append(path.absolute())
-        elif path.suffix in {".c", ".cxx", ".cpp", ".cc"}:
+        elif path.suffix in {".c", ".cc", ".cpp", ".cxx"}:
             clang_jobs.append(
                 ClangInvocation.from_c_file(
                     path, copt=copt, system_includes=system_includes, timeout=timeout
diff --git a/compiler_gym/envs/llvm/llvm_env.py b/compiler_gym/envs/llvm/llvm_env.py
--- a/compiler_gym/envs/llvm/llvm_env.py
+++ b/compiler_gym/envs/llvm/llvm_env.py
@@ -317,13 +317,36 @@ def make_benchmark(
     ) -> Benchmark:
         """Create a benchmark for use with this environment.
 
-        This function takes one or more inputs and uses them to create a
-        benchmark that can be passed to :meth:`compiler_gym.envs.LlvmEnv.reset`.
+        This function takes one or more inputs and uses them to create an LLVM
+        bitcode benchmark that can be passed to
+        :meth:`compiler_gym.envs.LlvmEnv.reset`.
+
+        The following input types are supported:
+
+        +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+        | **File Suffix**                                     | **Treated as**      | **Converted using**                                         |
+        +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+        | :code:`.bc`                                         | LLVM IR bitcode     | No conversion required.                                     |
+        +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+        | :code:`.ll`                                         | LLVM IR text format | Assembled to bitcode using llvm-as.                         |
+        +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
+        | :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source      | Compiled to bitcode using clang and the given :code:`copt`. |
+        +-----------------------------------------------------+---------------------+-------------------------------------------------------------+
 
-        For single-source C/C++ programs, you can pass the path of the source
+        .. note::
+
+            The LLVM IR format has no compatability guarantees between versions (see
+            `LLVM docs
+            <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_).
+            You must ensure that any :code:`.bc` and :code:`.ll` files are
+            compatible with the LLVM version used by CompilerGym, which can be
+            reported using :func:`env.compiler_version
+            <compiler_gym.envs.CompilerEnv.compiler_version>`.
+
+        E.g. for single-source C/C++ programs, you can pass the path of the source
         file:
 
-            >>> benchmark = make_benchmark('my_app.c')
+            >>> benchmark = env.make_benchmark('my_app.c')
             >>> env = gym.make("llvm-v0")
             >>> env.reset(benchmark=benchmark)
 
@@ -336,42 +359,27 @@ def make_benchmark(
         Additional compile-time arguments to clang can be provided using the
         :code:`copt` argument:
 
-            >>> benchmark = make_benchmark('/path/to/my_app.cpp', copt=['-O2'])
+            >>> benchmark = env.make_benchmark('/path/to/my_app.cpp', copt=['-O2'])
 
         If you need more fine-grained control over the options, you can directly
         construct a :class:`ClangInvocation
         <compiler_gym.envs.llvm.ClangInvocation>` to pass a list of arguments to
         clang:
 
-            >>> benchmark = make_benchmark(
-                ClangInvocation(['/path/to/my_app.c'], timeout=10)
+            >>> benchmark = env.make_benchmark(
+                ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10)
             )
 
         For multi-file programs, pass a list of inputs that will be compiled
         separately and then linked to a single module:
 
-            >>> benchmark = make_benchmark([
+            >>> benchmark = env.make_benchmark([
                 'main.c',
                 'lib.cpp',
                 'lib2.bc',
+                'foo/input.bc'
             ])
 
-        If you already have prepared bitcode files, those can be linked and used
-        directly:
-
-            >>> benchmark = make_benchmark([
-                'bitcode1.bc',
-                'bitcode2.bc',
-            ])
-
-        .. note::
-
-            LLVM bitcode compatibility is
-            `not guaranteed <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_,
-            so you must ensure that any precompiled bitcodes are compatible with the
-            LLVM version used by CompilerGym, which can be queried using
-            :func:`LlvmEnv.compiler_version <compiler_gym.envs.CompilerEnv.compiler_version>`.
-
         :param inputs: An input, or list of inputs.
 
         :param copt: A list of command line options to pass to clang when


Test patch: