Repository: firecracker-microvm/firecracker
File: tools/ab_test.py (Python)

Code before:

    return processed_emf


def uninteresting_dimensions(processed_emf):
    """
    Computes the set of cloudwatch dimensions that only ever take on a
    single value across the entire dataset.
    """
    values_per_dimension = defaultdict(set)

    for dimension_set in processed_emf:
        for dimension, value in dimension_set:
            values_per_dimension[dimension].add(value)

    uninteresting = set()

    for dimension, distinct_values in values_per_dimension.items():
        if len(distinct_values) == 1:
            uninteresting.add(dimension)

    return uninteresting


def collect_data(tag: str, binary_dir: Path, pytest_opts: str):
    """Executes the specified test using the provided firecracker binaries"""
    binary_dir = binary_dir.resolve()

    print(f"Collecting samples with {binary_dir}")
    subprocess.run(
        f"./tools/test.sh --binary-dir={binary_dir} {pytest_opts} -m ''",
        env=os.environ
        | {
            "AWS_EMF_ENVIRONMENT": "local",
            "AWS_EMF_NAMESPACE": "local",
        },
        check=True,
        shell=True,
    )

    # move results into a tag directory
    os.mkdir(tag)
    os.rename("test_results", f"{tag}")

    return load_data_series(Path(f"{tag}/test-report.json"), binary_dir, reemit=True)


def analyze_data(
    processed_emf_a,
    processed_emf_b,
    p_thresh,

Review comment (suggestion):
iirc, the location of the json files can be modified via pytest's `--json-report-file` option. I think it'll be cleaner to use this, and modify our test framework to dynamically determine `TEST_ARTIFACTS_DIR` to be the parent of the file passed via --json-report-file.

Code after:
                        values.extend(result[metric][0])

    return processed_emf


def uninteresting_dimensions(processed_emf):
    """
    Computes the set of cloudwatch dimensions that only ever take on a
    single value across the entire dataset.
    """
    values_per_dimension = defaultdict(set)

    for dimension_set in processed_emf:
        for dimension, value in dimension_set:
            values_per_dimension[dimension].add(value)

    uninteresting = set()

    for dimension, distinct_values in values_per_dimension.items():
        if len(distinct_values) == 1:
            uninteresting.add(dimension)

    return uninteresting


def collect_data(tag: str, binary_dir: Path, pytest_opts: str):
    """
    Executes the specified test using the provided firecracker binaries and
    stores results into the `test_results/tag` directory
    """
    binary_dir = binary_dir.resolve()

    print(f"Collecting samples with {binary_dir}")
    test_report_path = f"test_results/{tag}/test-report.json"
    subprocess.run(
        f"./tools/test.sh --binary-dir={binary_dir} {pytest_opts} -m '' --json-report-file=../{test_report_path}",
        env=os.environ
        | {
            "AWS_EMF_ENVIRONMENT": "local",
            "AWS_EMF_NAMESPACE": "local",
        },
        check=True,
        shell=True,
    )

    return load_data_series(Path(test_report_path), binary_dir, reemit=True)


def analyze_data(
    processed_emf_a,
    processed_emf_b,