Repository: firecracker-microvm/firecracker
File: tools/ab_test.py (Python)

Code before:
                f"\033[0;32m[Firecracker A/B-Test Runner]\033[0m A/B-testing shows a change of "
                f"{format_with_reduced_unit(result.statistic, unit)}, or {result.statistic / old_mean:.2%}, "
                f"(from {format_with_reduced_unit(old_mean, unit)} to {format_with_reduced_unit(new_mean, unit)}) "
                f"for metric \033[1m{metric}\033[0m with \033[0;31m\033[1mp={result.pvalue}\033[0m. "
                f"This means that observing a change of this magnitude or worse, assuming that performance "
                f"characteristics did not change across the tested commits, has a probability of {result.pvalue:.2%}. "
                f"Tested Dimensions:\n{json.dumps({k: v for k, v in dimension_set if k not in do_not_print_list}, indent=2, sort_keys=True)}"
            )
            messages.append(msg)

    assert not messages, "\n" + "\n".join(messages)
    print("No regressions detected!")


def ab_performance_test(
    a_revision: Path,
    b_revision: Path,
    pytest_opts,
    p_thresh,
    strength_abs_thresh,
    noise_threshold,
):
    """Does an A/B-test of the specified test with the given firecracker/jailer binaries"""

    return binary_ab_test(
        lambda tag, bin_dir, _: collect_data(tag, bin_dir, pytest_opts),
        lambda ah, be: analyze_data(
            ah,
            be,
            p_thresh,
            strength_abs_thresh,
            noise_threshold,
            n_resamples=int(100 / p_thresh),
        ),
        a_directory=a_revision,
        b_directory=b_revision,
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Executes Firecracker's A/B testsuite across the specified commits"
    )
    subparsers = parser.add_subparsers(help="commands", dest="command", required=True)
    run_parser = subparsers.add_parser(
        "run",
        help="Run an specific test of our test suite as an A/B-test across two specified commits",
    )
    run_parser.add_argument(
        "a_revision",
        help="Directory containing firecracker and jailer binaries to be considered the performance baseline",

Review comment (suggestion):
```suggestion
        lambda bin_dir, is_a: collect_data(is_a and 'A' or 'B', bin_dir, pytest_opts),
```

Code after:
                f"\033[0;32m[Firecracker A/B-Test Runner]\033[0m A/B-testing shows a change of "
                f"{format_with_reduced_unit(result.statistic, unit)}, or {result.statistic / old_mean:.2%}, "
                f"(from {format_with_reduced_unit(old_mean, unit)} to {format_with_reduced_unit(new_mean, unit)}) "
                f"for metric \033[1m{metric}\033[0m with \033[0;31m\033[1mp={result.pvalue}\033[0m. "
                f"This means that observing a change of this magnitude or worse, assuming that performance "
                f"characteristics did not change across the tested commits, has a probability of {result.pvalue:.2%}. "
                f"Tested Dimensions:\n{json.dumps({k: v for k, v in dimension_set if k not in do_not_print_list}, indent=2, sort_keys=True)}"
            )
            messages.append(msg)

    assert not messages, "\n" + "\n".join(messages)
    print("No regressions detected!")


def ab_performance_test(
    a_revision: Path,
    b_revision: Path,
    pytest_opts,
    p_thresh,
    strength_abs_thresh,
    noise_threshold,
):
    """Does an A/B-test of the specified test with the given firecracker/jailer binaries"""

    return binary_ab_test(
        lambda bin_dir, is_a: collect_data(is_a and "A" or "B", bin_dir, pytest_opts),
        lambda ah, be: analyze_data(
            ah,
            be,
            p_thresh,
            strength_abs_thresh,
            noise_threshold,
            n_resamples=int(100 / p_thresh),
        ),
        a_directory=a_revision,
        b_directory=b_revision,
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Executes Firecracker's A/B testsuite across the specified commits"
    )
    subparsers = parser.add_subparsers(help="commands", dest="command", required=True)
    run_parser = subparsers.add_parser(
        "run",
        help="Run an specific test of our test suite as an A/B-test across two specified commits",
    )
    run_parser.add_argument(
        "a_revision",
        help="Directory containing firecracker and jailer binaries to be considered the performance baseline",