Repository: google/fuzzbench

Issue:
make build-$FUZZER-all doesn't build OSS-Fuzz benchmarks
See [these logs](https://pipelines.actions.githubusercontent.com/HfP6ko7vz0HkB2brXZz73OfMQQ34NlRvYVLSJS23HRyGqlGR9r/_apis/pipelines/1/runs/123/signedlogcontent/5?urlExpires=2020-03-16T23%3A13%3A55.5286119Z&urlSigningMethod=HMACV1&urlSignature=mLVmSFZ0NxUO3zZCZHTdwjnNycY4BKQwl%2FsN3%2FuF6Xo%3D) from CI for example which runs make-$FUZZER-all, libpcap becnhmark isn't built.
This is probably why build failures #110 and #107 weren't caught in CI, they were both build failures of OSS-Fuzz benchmarks.


Discussion:

Resolving patch:
diff --git a/.github/workflows/build.py b/.github/workflows/build.py
new file mode 100644
--- /dev/null
+++ b/.github/workflows/build.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script for building fuzzer,benchmark pairs in CI."""
+import sys
+import subprocess
+
+# Don't build php benchmark since it fills up disk in GH actions.
+OSS_FUZZ_BENCHMARKS = [
+    'bloaty_fuzz_target',
+    'curl_curl_fuzzer_http',
+    'irssi_server-fuzz',
+    'jsoncpp_jsoncpp_fuzzer',
+    'libpcap_fuzz_both',
+    'mbedtls_fuzz_dtlsclient',
+    'openssl_x509',
+    'sqlite3_ossfuzz',
+    'systemd_fuzz-link-parser',
+    'zlib_zlib_uncompress_fuzzer',
+]
+
+STANDARD_BENCHMARKS = [
+    'freetype2-2017',
+    'harfbuzz-1.3.2',
+    'lcms-2017-03-21',
+    'libjpeg-turbo-07-2017',
+    'libpng-1.2.56',
+    'libxml2-v2.9.2',
+    'openthread-2019-12-23',
+    'proj4-2017-08-14',
+    're2-2014-12-09',
+    'vorbis-2017-12-11',
+    'woff2-2016-05-06',
+    'wpantund-2018-02-27',
+]
+
+
+def get_make_targets(benchmarks, fuzzer):
+    """Return pull and build targets for |fuzzer| and each benchmark
+    in |benchmarks| to pass to make."""
+    return [('pull-%s-%s' % (fuzzer, benchmark),
+             'build-%s-%s' % (fuzzer, benchmark)) for benchmark in benchmarks]
+
+
+def delete_docker_images():
+    """Delete docker images."""
+    # TODO(metzman): Don't delete base-runner/base-builder so it
+    # doesn't need to be pulled for every target.
+    result = subprocess.run(['docker', 'images', '-q'],
+                            stdout=subprocess.PIPE,
+                            check=True)
+    image_names = result.stdout.splitlines()
+    subprocess.run(['docker', 'rmi', '-f'] + image_names, check=False)
+
+
+def make_builds(benchmarks, fuzzer):
+    """Use make to build each target in |build_targets|."""
+    make_targets = get_make_targets(benchmarks, fuzzer)
+    success = True
+    for pull_target, build_target in make_targets:
+        # Pull target first.
+        subprocess.run(['make', '-j', pull_target], check=False)
+
+        # Then build.
+        result = subprocess.run(['make', build_target], check=False)
+        if not result.returncode == 0:
+            success = False
+        # Delete docker images so disk doesn't fill up.
+        delete_docker_images()
+    return success
+
+
+def do_build(build_type, fuzzer):
+    """Build fuzzer,benchmark pairs for CI."""
+    if build_type == 'oss-fuzz':
+        benchmarks = OSS_FUZZ_BENCHMARKS
+    elif build_type == 'standard':
+        benchmarks = STANDARD_BENCHMARKS
+    else:
+        raise Exception('Invalid build_type: %s' % build_type)
+
+    return make_builds(benchmarks, fuzzer)
+
+
+def main():
+    """Build OSS-Fuzz or standard benchmarks with a fuzzer."""
+    if len(sys.argv) != 3:
+        print('Usage: %s <build_type> <fuzzer>' % sys.argv[0])
+        return 1
+    build_type = sys.argv[1]
+    fuzzer = sys.argv[2]
+    result = do_build(build_type, fuzzer)
+    return 0 if result else 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())


Test patch: