Repository: firecracker-microvm/firecracker
File: tests/integration_tests/performance/test_block_ab.py (Python)

Code before:
            value = int(value.strip())

            match direction.strip():
                case "0":
                    read_values.append(value)
                case "1":
                    write_values.append(value)
                case _:
                    assert False

        yield read_values, write_values


def emit_fio_metrics(logs_dir, metrics):
    """Parses the fio logs in `{logs_dir}/*_[bw|lat].*.log and emits their contents as CloudWatch metrics"""

    for bw_read, bw_write in process_fio_log_files(f"{logs_dir}/*_bw.*.log"):
        if bw_read:
            metrics.put_metric("bw_read", sum(bw_read), "Kilobytes/Second")
        if bw_write:
            metrics.put_metric("bw_write", sum(bw_write), "Kilobytes/Second")

    for lat_read, lat_write in process_fio_log_files(f"{logs_dir}/*_clat.*.log"):
        # latency values in fio logs are in nanosecons, but cloudwatch only supports
        # microseconds as the more granular unit, so need to divide by 1000.
        for value in lat_read:
            metrics.put_metric("clat_read", value / 1000, "Microseconds")
        for value in lat_write:
            metrics.put_metric("clat_write", value / 1000, "Microseconds")


@pytest.mark.timeout(120)
@pytest.mark.nonci
@pytest.mark.parametrize("vcpus", [1, 2], ids=["1vcpu", "2vcpu"])
@pytest.mark.parametrize("fio_mode", ["randread", "randwrite"])
@pytest.mark.parametrize("fio_block_size", [4096], ids=["bs4096"])
def test_block_performance(
    microvm_factory,
    guest_kernel_acpi,
    rootfs,
    vcpus,
    fio_mode,
    fio_block_size,
    io_engine,
    metrics,
):
    """
    Execute block device emulation benchmarking scenarios.
    """
    vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
    vm.spawn(log_level="Info", emit_metrics=True)

Review comment (performance):
What I've noticed when looking into PCI is that, when using the async fio engine, the latency is much higher than the actual device latency (as we're pushing many requests at the same time asynchronously), and it's also quite unstable, with one fio thread having much higher latency than the other (if you look at bandwidth, one accounts for 80% of the throughput, while the other just 20% as it also has to handle the driver processing and interrupts).
If the goal is to measure the actual device latency, it's best to use the sync fio engine, which is also more representative of the performance of real world readers/writers.

Code after:
            value = int(value.strip())

            match direction.strip():
                case "0":
                    read_values.append(value)
                case "1":
                    write_values.append(value)
                case _:
                    assert False

        yield read_values, write_values


def emit_fio_metrics(logs_dir, metrics):
    """Parses the fio logs in `{logs_dir}/*_[clat|bw].*.log and emits their contents as CloudWatch metrics"""
    for bw_read, bw_write in process_fio_log_files(f"{logs_dir}/*_bw.*.log"):
        if bw_read:
            metrics.put_metric("bw_read", sum(bw_read), "Kilobytes/Second")
        if bw_write:
            metrics.put_metric("bw_write", sum(bw_write), "Kilobytes/Second")

    for lat_read, lat_write in process_fio_log_files(f"{logs_dir}/*_clat.*.log"):
        # latency values in fio logs are in nanoseconds, but cloudwatch only supports
        # microseconds as the more granular unit, so need to divide by 1000.
        for value in lat_read:
            metrics.put_metric("clat_read", value / 1000, "Microseconds")
        for value in lat_write:
            metrics.put_metric("clat_write", value / 1000, "Microseconds")


@pytest.mark.nonci
@pytest.mark.parametrize("vcpus", [1, 2], ids=["1vcpu", "2vcpu"])
@pytest.mark.parametrize("fio_mode", ["randread", "randwrite"])
@pytest.mark.parametrize("fio_block_size", [4096], ids=["bs4096"])
@pytest.mark.parametrize("fio_engine", ["libaio", "psync"])
def test_block_performance(
    microvm_factory,
    guest_kernel_acpi,
    rootfs,
    vcpus,
    fio_mode,
    fio_block_size,
    fio_engine,
    io_engine,
    metrics,
):
    """
    Execute block device emulation benchmarking scenarios.
    """
    vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
    vm.spawn(log_level="Info", emit_metrics=True)