Repository: smarr/ReBench

Issue:
The -f switch should also prevent rebench to fail when perf fails
See https://git.cs.kent.ac.uk/sm951/trufflesom/-/jobs/77668

perf crashes, as previously seen here: https://github.com/smarr/ReBench/issues/252
While rebench now completes, it still indicates a failure.

Discussion:

Patch:
diff --git a/rebench/executor.py b/rebench/executor.py
index fc3383e..bad1e3e 100644
--- a/rebench/executor.py
+++ b/rebench/executor.py
@@ -658,7 +658,7 @@ class Executor(object):
                 run.report_job_completed(self._runs)
                 if run.is_failed:
                     successful = False
-            return successful
+            return successful or self._include_faulty
         finally:
             for run in self._runs:
                 run.close_files()


Test patch:
diff --git a/rebench/tests/perf/issue_166_profiling_test.py b/rebench/tests/perf/issue_166_profiling_test.py
index 20ed1cf..3692863 100644
--- a/rebench/tests/perf/issue_166_profiling_test.py
+++ b/rebench/tests/perf/issue_166_profiling_test.py
@@ -209,3 +209,23 @@ class Issue166ProfilingSupportTest(ReBenchTestCase):
         profiler.command = "./cat-first.sh"
         profiler.record_args = "perf-small.report"
         profiler.report_args = "perf-small.report"
+
+    def test_failing_profiler_and_include_faulty(self):
+        raw_config = self._load_config_and_use_tmp_as_data_file()
+        cnf = Configurator(
+            raw_config, DataStore(self.ui), self.ui, exp_name="profile2", data_file=self._tmp_file)
+
+        runs = cnf.get_runs()
+        run_id = list(cnf.get_runs())[0]
+        self.assertEqual(0, run_id.completed_invocations)
+        self.assertEqual(0, run_id.get_number_of_data_points())
+
+        profilers = run_id.benchmark.suite.executor.profiler
+        profiler = profilers[0]
+        profiler.command = "false"
+
+        executor = Executor(runs, False, self.ui, include_faulty=True)
+        self.assertTrue(executor.execute())
+
+        self.assertEqual(0, run_id.completed_invocations)
+        self.assertEqual(0, run_id.get_number_of_data_points())