Repository: google/fuzzbench

Issue:
Important: Add time feature in report generation
This will make it easy to generate report at 1h,6h,12h,24h


Discussion:
Sorry, I'm not sure I understand what this is for, can you explain please?
Requested by a researcher to ease in generating reports around different times and compare results.
Like how do results at 6h compare to the one at 24h. This can be easily done in generate_report.py with a --end_time or something argument.

Resolving patch:
diff --git a/analysis/data_utils.py b/analysis/data_utils.py
--- a/analysis/data_utils.py
+++ b/analysis/data_utils.py
@@ -62,6 +62,12 @@ def label_fuzzers_by_experiment(experiment_df):
     return experiment_df
 
 
+def filter_max_time(experiment_df, max_time):
+    """Returns table with snapshots that have time less than or equal to
+    |max_time|."""
+    return experiment_df[experiment_df['time'] <= max_time]
+
+
 # Creating "snapshots" (see README.md for definition).
 
 _DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 0.8
diff --git a/analysis/generate_report.py b/analysis/generate_report.py
--- a/analysis/generate_report.py
+++ b/analysis/generate_report.py
@@ -58,6 +58,13 @@ def get_arg_parser():
         '--benchmarks',
         nargs='*',
         help='Names of the benchmarks to include in the report.')
+    parser.add_argument(
+        '-e',
+        '--end-time',
+        default=None,
+        type=int,
+        help=('The last time (in seconds) during an experiment to include in '
+              'the report.'))
     parser.add_argument('-f',
                         '--fuzzers',
                         nargs='*',
@@ -90,7 +97,8 @@ def generate_report(experiment_names,
                     report_type='default',
                     quick=False,
                     from_cached_data=False,
-                    in_progress=False):
+                    in_progress=False,
+                    end_time=None):
     """Generate report helper."""
     report_name = report_name or experiment_names[0]
 
@@ -115,6 +123,9 @@ def generate_report(experiment_names,
     if label_by_experiment:
         experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
 
+    if end_time is not None:
+        experiment_df = data_utils.filter_max_time(experiment_df, end_time)
+
     fuzzer_names = experiment_df.fuzzer.unique()
     plotter = plotting.Plotter(fuzzer_names, quick)
     experiment_ctx = experiment_results.ExperimentResults(
@@ -135,9 +146,16 @@ def main():
     parser = get_arg_parser()
     args = parser.parse_args()
 
-    generate_report(args.experiments, args.report_dir, args.report_name,
-                    args.label_by_experiment, args.benchmarks, args.fuzzers,
-                    args.report_type, args.quick, args.from_cached_data)
+    generate_report(experiment_names=args.experiments,
+                    report_directory=args.report_dir,
+                    report_name=args.report_name,
+                    label_by_experiment=args.label_by_experiment,
+                    benchmarks=args.benchmarks,
+                    fuzzers=args.fuzzers,
+                    report_type=args.report_type,
+                    quick=args.quick,
+                    from_cached_data=args.from_cached_data,
+                    end_time=args.end_time)
 
 
 if __name__ == '__main__':


Test patch:
diff --git a/analysis/test_data_utils.py b/analysis/test_data_utils.py
--- a/analysis/test_data_utils.py
+++ b/analysis/test_data_utils.py
@@ -94,6 +94,14 @@ def test_label_fuzzers_by_experiment():
     assert labeled_df.fuzzer.unique().tolist() == expected_fuzzer_names
 
 
+def test_filter_max_time():
+    experiment_df = create_experiment_data()
+    max_time = 5
+    filtered_df = data_utils.filter_max_time(experiment_df, max_time)
+    expected_times = range(max_time + 1)
+    assert filtered_df.time.unique().tolist() == list(expected_times)
+
+
 def test_benchmark_snapshot():
     """Tests that the snapshot data contains only the latest timestamp for all
     trials, in case all trials have the same lengths."""