Repository: firecracker-microvm/firecracker-containerd
Pull request #268 (2019-09-25 20:28:58)

Add performance test case comparing tc w/ bridge.
The intention of this test case is to add basic support for running performance
tests of various network configurations and to get a comparison of the CPU usage
differences between a TC-based VM network and a bridge-based VM network.

The test case added tries to make as fair a comparison as possible between TC
and a bridge. The TC approach uses the tc-redirect-tap CNI plugin, which results
in the following network layout for each VM:
[iperf server] <- vethA <- vethB <- [tc mirror] <- tap0 <- [iperf client]

In order to get a fair comparison, we want to run a bridge-based test with the
following layout:
[iperf server] <- vethA <- vethB <- [bridge] <- tap0 <- [iperf client]

Unfortunately, existing bridge based plugins usable in a general purpose test
like this cannot setup a bridge with a tap device like that, so this change
includes a barebones CNI plugin (under internal/cmd/test-bridged-tap/) that
sets up the network layout above w/ a bridge.

The test case then works by spinning up a configurable number of VMs, each of
which has a single container running an iperf client. The iperf client sends
data to a iperf server running on the host at a configurable bandwidth for a
configurable amount of time. The final output prints the average CPU cycles
consumed per second for the duration of the test case.

Signed-off-by: Erik Sipsma <sipsma@amazon.com>

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.



Patch:
diff --git a/go.mod b/go.mod
index a7500f6..588c01d 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,8 @@ require (
 	github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 // indirect
 	github.com/containerd/ttrpc v0.0.0-20190613183316-1fb3814edf44
 	github.com/containerd/typeurl v0.0.0-20181015155603-461401dc8f19
+	github.com/containernetworking/cni v0.7.2-0.20190807151350-8c6c47d1c7fc
+	github.com/containernetworking/plugins v0.8.2
 	github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect
@@ -39,6 +41,7 @@ require (
 	github.com/stretchr/testify v1.3.0
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
 	github.com/urfave/cli v1.20.0 // indirect
+	github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf
 	go.etcd.io/bbolt v1.3.1-etcd.8
 	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
 	golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f
diff --git a/go.sum b/go.sum
index f392e4c..ed9c2b5 100644
--- a/go.sum
+++ b/go.sum
@@ -119,6 +119,7 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
diff --git a/internal/Makefile b/internal/Makefile
index 9ab0537..8c60bd9 100644
--- a/internal/Makefile
+++ b/internal/Makefile
@@ -20,6 +20,9 @@ DOCKER_IMAGE_TAG?=latest
 
 all:
 
+test-bridged-tap: cmd/test-bridged-tap $(GOMOD) $(GOSUM)
+	go build -o test-bridged-tap $(CURDIR)/cmd/test-bridged-tap
+
 test:
 	go test ./... $(EXTRAGOARGS)
 
diff --git a/internal/psutil.go b/internal/psutil.go
index 556fe36..0bc0315 100644
--- a/internal/psutil.go
+++ b/internal/psutil.go
@@ -17,6 +17,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/pkg/errors"
+	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/process"
 )
 
@@ -79,3 +81,97 @@ func WaitForPidToExit(ctx context.Context, queryInterval time.Duration, pid int3
 
 	panic("unreachable code in WaitForPidToExit")
 }
+
+// AverageCPUDeltas returns the average CPU cycles consumed per sampleInterval, as calculated
+// between the time this function is invoked and the time the provided ctx is cancelled.
+func AverageCPUDeltas(ctx context.Context, sampleInterval time.Duration) (*cpu.TimesStat, error) {
+	sampleCount := 0
+	sampleCh := sampleCPUTimes(ctx, sampleInterval)
+
+	firstSample := <-sampleCh
+	if firstSample == nil {
+		return nil, errors.New("sample channel closed before first data point")
+	}
+	if firstSample.Err != nil {
+		return nil, firstSample.Err
+	}
+
+	var lastSample *cpuTimesSample
+	for lastSample = range sampleCh {
+		sampleCount++
+		if lastSample.Err != nil {
+			return nil, lastSample.Err
+		}
+	}
+
+	if lastSample == nil {
+		return nil, errors.New("only got one data point, cannot calculate average")
+	}
+
+	avg := func(first float64, last float64) float64 {
+		return (last - first) / float64(sampleCount)
+	}
+
+	return &cpu.TimesStat{
+		User:      avg(firstSample.User, lastSample.User),
+		System:    avg(firstSample.System, lastSample.System),
+		Idle:      avg(firstSample.Idle, lastSample.Idle),
+		Nice:      avg(firstSample.Nice, lastSample.Nice),
+		Iowait:    avg(firstSample.Iowait, lastSample.Iowait),
+		Irq:       avg(firstSample.Irq, lastSample.Irq),
+		Softirq:   avg(firstSample.Softirq, lastSample.Softirq),
+		Steal:     avg(firstSample.Steal, lastSample.Steal),
+		Guest:     avg(firstSample.Guest, lastSample.Guest),
+		GuestNice: avg(firstSample.GuestNice, lastSample.GuestNice),
+	}, nil
+}
+
+type cpuTimesSample struct {
+	*cpu.TimesStat
+	Index int
+	Err   error
+}
+
+func sampleCPUTimes(ctx context.Context, sampleInterval time.Duration) <-chan *cpuTimesSample {
+	returnCh := make(chan *cpuTimesSample)
+	go func() {
+		defer close(returnCh)
+
+		var index int
+		for range time.NewTicker(sampleInterval).C {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				index++
+			}
+
+			sample := &cpuTimesSample{
+				Index: index,
+			}
+
+			const percpu = false // just give us the aggregate numbers, not each cpu's numbers
+			curTimesList, err := cpu.Times(percpu)
+			if err != nil {
+				sample.Err = err
+				returnCh <- sample
+				return
+			}
+
+			// we set percpu to false, so there should be only one stat
+			if len(curTimesList) != 1 {
+				sample.Err = errors.Errorf("unexpected number of cpu times in sample %d", len(curTimesList))
+				returnCh <- sample
+				return
+			}
+			sample.TimesStat = &curTimesList[0]
+
+			select {
+			case returnCh <- sample:
+			default:
+				// just skip the sample if there's no room for it
+			}
+		}
+	}()
+	return returnCh
+}
diff --git a/runtime/Makefile b/runtime/Makefile
index d2cf7d8..1b37541 100644
--- a/runtime/Makefile
+++ b/runtime/Makefile
@@ -57,6 +57,40 @@ integ-test:
 			"go test $(EXTRAGOARGS) -run \"^$(TESTNAME)$$\"" || exit 1; \
 	)
 
+PERF_TESTNAME?=TestCNIPlugin_Performance
+PERF_VMCOUNT?=25
+PERF_RUNTIME_SECONDS?=600
+PERF_VM_MEMSIZE_MB?=1024
+PERF_TARGET_BANDWIDTH?=1G
+
+tc-redirect-tap-perf:
+	$(MAKE) PERF_PLUGIN_NAME=tc-redirect-tap perf-test
+
+test-bridged-tap-perf:
+	$(MAKE) PERF_PLUGIN_NAME=test-bridged-tap perf-test
+
+perf-test:
+	$(if $(PERF_PLUGIN_NAME),true,$(error PERF_PLUGIN_NAME must be set for perf-test target))
+	mkdir -p $(CURDIR)/logs
+
+	docker run --rm -it \
+		--privileged \
+		--ipc=host \
+		--network=none \
+		--volume /dev:/dev \
+		--volume /run/udev/control:/run/udev/control \
+		--volume $(CURDIR)/logs:/var/log/firecracker-containerd-test \
+		--env ENABLE_ISOLATED_TESTS=1 \
+		--env PERF_PLUGIN_NAME=$(PERF_PLUGIN_NAME) \
+		--env PERF_VMCOUNT=$(PERF_VMCOUNT) \
+		--env PERF_RUNTIME_SECONDS=$(PERF_RUNTIME_SECONDS) \
+		--env PERF_VM_MEMSIZE_MB=$(PERF_VM_MEMSIZE_MB) \
+		--env PERF_TARGET_BANDWIDTH=$(PERF_TARGET_BANDWIDTH) \
+		--workdir="/firecracker-containerd/runtime" \
+		--init \
+		localhost/firecracker-containerd-naive-integ-test:$(DOCKER_IMAGE_TAG) \
+		"go test -timeout 0 -v -count=1 -run \"^$(PERF_TESTNAME)$$\""
+
 clean:
 	- rm -f containerd-shim-aws-firecracker
 
diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile
index dba193d..0a68f13 100644
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@@ -100,6 +100,7 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \
 		curl \
 		git \
 		iptables \
+		iperf3 \
 		libdevmapper-dev \
 		libseccomp-dev
 
@@ -154,10 +155,15 @@ COPY tools/docker/firecracker-runtime.json /etc/containerd/firecracker-runtime.j
 # access during the tests themselves
 COPY tools/docker/naive-snapshotter/config.toml /etc/containerd/config.toml
 RUN containerd 2>/dev/null & \
-	ctr content fetch docker.io/library/alpine:3.10.1 >/dev/null
+	ctr content fetch docker.io/library/alpine:3.10.1 >/dev/null && \
+	ctr content fetch docker.io/mlabbe/iperf3:3.6-r0 >/dev/null
 
 RUN mkdir -p /var/lib/firecracker-containerd/naive
 RUN make -C /firecracker-containerd demo-network
+RUN make -C /firecracker-containerd/internal test-bridged-tap && \
+	cp /firecracker-containerd/internal/test-bridged-tap /opt/cni/bin/ && \
+	chmod a+x /firecracker-containerd/internal/test-bridged-tap
+
 COPY tools/docker/firecracker-runtime.json /etc/containerd/firecracker-runtime.json
 COPY tools/docker/naive-snapshotter/entrypoint.sh /entrypoint
 


Test patch:
diff --git a/internal/cmd/test-bridged-tap/main.go b/internal/cmd/test-bridged-tap/main.go
new file mode 100644
index 0000000..47c5698
--- /dev/null
+++ b/internal/cmd/test-bridged-tap/main.go
@@ -0,0 +1,198 @@
+// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+/*
+This is a bare-bones CNI plugin (with only ADD actually implemented, other
+methods are no-ops) used to setup a network bridge w/ tap in order to
+compare performance of a bridge-based VM network w/ a TC-based approach,
+such as that created by the tc-redirect-tap plugin. It expects to be chained
+after the ptp plugin and sets up a layout like the following:
+[host] <- vethA <- vethB <- [bridge] <- tap0 <- [vm guest]
+
+with "vethA <- vethB" crossing a network namespace boundary.
+*/
+package main
+
+import (
+	"encoding/json"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/cni/pkg/version"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/utils/buildversion"
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+)
+
+func main() {
+	skel.PluginMain(add, check, del,
+		// support CNI versions that support plugin chaining
+		version.PluginSupports("0.3.0", "0.3.1", version.Current()),
+		buildversion.BuildString("test-bridged-tap"),
+	)
+}
+
+func add(args *skel.CmdArgs) error {
+	const mtu = 1500
+	vethName := args.IfName
+
+	return ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
+		currentResult, err := getCurrentResult(args)
+		if err != nil {
+			return err
+		}
+
+		vethLink, err := netlink.LinkByName(vethName)
+		if err != nil {
+			return err
+		}
+
+		var vethIPConf *current.IPConfig
+		for _, ipConfig := range currentResult.IPs {
+			if ifaceIndex := ipConfig.Interface; ifaceIndex != nil && *ifaceIndex == 1 {
+				vethIPConf = ipConfig
+				break
+			}
+		}
+		if vethIPConf == nil {
+			return errors.Errorf("did not find veth ip: %+v", currentResult)
+		}
+
+		err = netlink.AddrDel(vethLink, &netlink.Addr{
+			IPNet: &vethIPConf.Address,
+		})
+		if err != nil {
+			return err
+		}
+
+		tapName := "tap0"
+		err = netlink.LinkAdd(&netlink.Tuntap{
+			Mode:   netlink.TUNTAP_MODE_TAP,
+			Queues: 1,
+			Flags:  netlink.TUNTAP_ONE_QUEUE | netlink.TUNTAP_VNET_HDR,
+			LinkAttrs: netlink.LinkAttrs{
+				Name: tapName,
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		tapLink, err := netlink.LinkByName(tapName)
+		if err != nil {
+			return err
+		}
+
+		err = netlink.LinkSetMTU(tapLink, mtu)
+		if err != nil {
+			return err
+		}
+
+		err = netlink.LinkSetUp(tapLink)
+		if err != nil {
+			return err
+		}
+
+		bridgeName := "br0"
+		err = netlink.LinkAdd(&netlink.Bridge{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         bridgeName,
+				MTU:          mtu,
+				HardwareAddr: vethLink.Attrs().HardwareAddr,
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		bridgeLink, err := netlink.LinkByName(bridgeName)
+		if err != nil {
+			return err
+		}
+
+		err = netlink.LinkSetMaster(vethLink, bridgeLink.(*netlink.Bridge))
+		if err != nil {
+			return err
+		}
+
+		err = netlink.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge))
+		if err != nil {
+			return err
+		}
+
+		err = netlink.LinkSetUp(bridgeLink)
+		if err != nil {
+			return err
+		}
+
+		currentResult.Interfaces = append(currentResult.Interfaces, &current.Interface{
+			Name:    tapLink.Attrs().Name,
+			Sandbox: args.Netns,
+			Mac:     tapLink.Attrs().HardwareAddr.String(),
+		})
+
+		tapMac := tapLink.Attrs().HardwareAddr
+		tapMac[1], tapMac[2] = tapMac[2], tapMac[1] // just need a different addr from the tap
+		currentResult.Interfaces = append(currentResult.Interfaces, &current.Interface{
+			Name:    tapLink.Attrs().Name,
+			Sandbox: args.ContainerID,
+			Mac:     tapMac.String(),
+		})
+		vmIfaceIndex := len(currentResult.Interfaces) - 1
+
+		currentResult.IPs = append(currentResult.IPs, &current.IPConfig{
+			Version:   "4",
+			Address:   vethIPConf.Address,
+			Gateway:   vethIPConf.Gateway,
+			Interface: &vmIfaceIndex,
+		})
+
+		return types.PrintResult(currentResult, currentResult.CNIVersion)
+	})
+}
+
+func getCurrentResult(args *skel.CmdArgs) (*current.Result, error) {
+	// parse the previous CNI result (or throw an error if there wasn't one)
+	cniConf := types.NetConf{}
+	err := json.Unmarshal(args.StdinData, &cniConf)
+	if err != nil {
+		return nil, errors.Wrap(err, "failure checking for previous result output")
+	}
+
+	err = version.ParsePrevResult(&cniConf)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse previous CNI result")
+	}
+
+	if cniConf.PrevResult == nil {
+		return nil, errors.New("no previous result")
+	}
+
+	currentResult, err := current.NewResultFromResult(cniConf.PrevResult)
+	if err != nil {
+		return nil, errors.Wrap(err,
+			"failed to generate current result from previous CNI result")
+	}
+
+	return currentResult, nil
+}
+
+func del(args *skel.CmdArgs) error {
+	return nil
+}
+
+func check(args *skel.CmdArgs) error {
+	return nil
+}
diff --git a/runtime/cni_integ_test.go b/runtime/cni_integ_test.go
index 2ee5da1..7887d01 100644
--- a/runtime/cni_integ_test.go
+++ b/runtime/cni_integ_test.go
@@ -17,6 +17,9 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
@@ -25,6 +28,7 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/pkg/ttrpcutil"
+	"github.com/shirou/gopsutil/cpu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -64,7 +68,8 @@ func TestCNISupport_Isolated(t *testing.T) {
 	require.NoError(t, err, "failed to create local network test services")
 
 	cniNetworkName := "fcnet-test"
-	err = writeCNIConf("/etc/cni/conf.d/fcnet-test.conflist", cniNetworkName, localServices.DNSServerIP())
+	err = writeCNIConf("/etc/cni/conf.d/fcnet-test.conflist",
+		"tc-redirect-tap", cniNetworkName, localServices.DNSServerIP())
 	require.NoError(t, err, "failed to write test cni conf")
 
 	go func() {
@@ -155,7 +160,8 @@ func TestAutomaticCNISupport_Isolated(t *testing.T) {
 	require.NoError(t, err, "failed to create local network test services")
 
 	cniNetworkName := "fcnet"
-	err = writeCNIConf("/etc/cni/conf.d/fcnet.conflist", cniNetworkName, localServices.DNSServerIP())
+	err = writeCNIConf("/etc/cni/conf.d/fcnet.conflist",
+		"tc-redirect-tap", cniNetworkName, localServices.DNSServerIP())
 	require.NoError(t, err, "failed to write test cni conf")
 
 	go func() {
@@ -193,7 +199,141 @@ func TestAutomaticCNISupport_Isolated(t *testing.T) {
 	taskGroup.Wait()
 }
 
-func writeCNIConf(path string, networkName string, nameserver string) error {
+func TestCNIPlugin_Performance(t *testing.T) {
+	internal.RequiresIsolation(t)
+
+	numVMs := perfTestVMCount(t)
+	runtimeDuration := perfTestRuntime(t)
+	vmMemSizeMB := perfTestVMMemsizeMB(t)
+	targetBandwidth := perfTestTargetBandwidth(t)
+	pluginName := perfTestChainedPluginName(t)
+
+	testTimeout := runtimeDuration + 20*time.Minute
+	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), defaultNamespace), testTimeout)
+	defer cancel()
+
+	client, err := containerd.New(containerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime))
+	require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
+	defer client.Close()
+
+	pluginClient, err := ttrpcutil.NewClient(containerdSockPath + ".ttrpc")
+	require.NoError(t, err, "failed to create ttrpc client")
+
+	fcClient := fccontrol.NewFirecrackerClient(pluginClient.Client())
+
+	image, err := iperf3Image(ctx, client, naiveSnapshotterName)
+	require.NoError(t, err, "failed to get iperf3 image")
+
+	cniNetworkName := "fcnet"
+	err = writeCNIConf("/etc/cni/conf.d/fcnet.conflist",
+		pluginName, cniNetworkName, "")
+	require.NoError(t, err, "failed to write test cni conf")
+
+	// create an endpoint in the host netns for the iperf servers to bind/listen to
+	testDevName := "testdev0"
+	ipAddr := "10.0.0.1"
+	ipCidr := fmt.Sprintf("%s/32", ipAddr)
+	runCommand(ctx, t, "ip", "tuntap", "add", testDevName, "mode", "tun")
+	runCommand(ctx, t, "ip", "addr", "add", ipCidr, "dev", testDevName)
+	runCommand(ctx, t, "ip", "link", "set", "dev", testDevName, "up")
+
+	vmID := func(vmIndex int) string {
+		return fmt.Sprintf("vm-%d", vmIndex)
+	}
+
+	var vmGroup sync.WaitGroup
+	containers := make(chan containerd.Container, numVMs)
+	for i := 0; i < numVMs; i++ {
+		vmGroup.Add(1)
+		go func(vmIndex int) {
+			defer vmGroup.Done()
+
+			// use a unique port for each VM
+			iperfPort := fmt.Sprintf("34%03d", vmIndex)
+			ifName := fmt.Sprintf("veth%d", vmIndex)
+
+			go func() {
+				runCommand(ctx, t, "iperf3",
+					"--bind", ipAddr,
+					"--port", iperfPort,
+					"--interval", "60",
+					"--server",
+				)
+			}()
+
+			_, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
+				VMID: vmID(vmIndex),
+				MachineCfg: &proto.FirecrackerMachineConfiguration{
+					MemSizeMib: uint32(vmMemSizeMB),
+				},
+				RootDrive: &proto.FirecrackerDrive{
+					PathOnHost:   defaultVMRootfsPath,
+					IsReadOnly:   false,
+					IsRootDevice: true,
+				},
+				NetworkInterfaces: []*proto.FirecrackerNetworkInterface{{
+					CNIConfig: &proto.CNIConfiguration{
+						NetworkName:   cniNetworkName,
+						InterfaceName: ifName,
+					},
+				}},
+			})
+			require.NoError(t, err, "failed to create vm")
+
+			containerName := fmt.Sprintf("%s-container", vmID(vmIndex))
+			snapshotName := fmt.Sprintf("%s-snapshot", vmID(vmIndex))
+
+			newContainer, err := client.NewContainer(ctx,
+				containerName,
+				containerd.WithSnapshotter(naiveSnapshotterName),
+				containerd.WithNewSnapshot(snapshotName, image),
+				containerd.WithNewSpec(
+					oci.WithProcessArgs("/usr/bin/iperf3",
+						"--port", iperfPort,
+						"--time", strconv.Itoa(int(runtimeDuration/time.Second)),
+						"--bitrate", targetBandwidth,
+						"--interval", "60",
+						"--client", ipAddr,
+					),
+					firecrackeroci.WithVMID(vmID(vmIndex)),
+					firecrackeroci.WithVMNetwork,
+				),
+			)
+			require.NoError(t, err, "failed to create container %s", containerName)
+
+			containers <- newContainer
+		}(i)
+	}
+
+	vmGroup.Wait()
+	close(containers)
+
+	avgCPUDeltas := make(chan cpu.TimesStat)
+	cpuCtx, cpuCtxCancel := context.WithCancel(ctx)
+	go func() {
+		defer close(avgCPUDeltas)
+		result, err := internal.AverageCPUDeltas(cpuCtx, 1*time.Second)
+		require.NoError(t, err, "failed collecting cpu times")
+		avgCPUDeltas <- *result
+	}()
+
+	var taskGroup sync.WaitGroup
+	for container := range containers {
+		taskGroup.Add(1)
+		go func(container containerd.Container) {
+			defer taskGroup.Done()
+			stdout := startAndWaitTask(ctx, t, container)
+			t.Logf("stdout output from task %q: %s", container.ID(), stdout)
+		}(container)
+	}
+
+	taskGroup.Wait()
+	cpuCtxCancel()
+
+	t.Logf("%+v", <-avgCPUDeltas)
+}
+
+func writeCNIConf(path, chainedPluginName, networkName, nameserver string) error {
 	return ioutil.WriteFile(path, []byte(fmt.Sprintf(`{
   "cniVersion": "0.3.1",
   "name": "%s",
@@ -209,8 +349,53 @@ func writeCNIConf(path string, networkName string, nameserver string) error {
       "dns": {"nameservers": ["%s"]}
     },
     {
-      "type": "tc-redirect-tap"
+      "type": "%s"
     }
   ]
-}`, networkName, nameserver)), 0644)
+}`, networkName, nameserver, chainedPluginName)), 0644)
+}
+
+func runCommand(ctx context.Context, t *testing.T, name string, args ...string) {
+	t.Helper()
+	output, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
+	require.NoErrorf(t, err, "command %+v failed, output: %s", append([]string{name}, args...), string(output))
+}
+
+func perfTestVMCount(t *testing.T) int {
+	t.Helper()
+	return requiredEnvInt(t, "PERF_VMCOUNT")
+}
+
+func perfTestRuntime(t *testing.T) time.Duration {
+	t.Helper()
+	return time.Duration(requiredEnvInt(t, "PERF_RUNTIME_SECONDS")) * time.Second
+}
+
+func perfTestVMMemsizeMB(t *testing.T) int {
+	t.Helper()
+	return requiredEnvInt(t, "PERF_VM_MEMSIZE_MB")
+}
+
+func perfTestTargetBandwidth(t *testing.T) string {
+	t.Helper()
+	return requiredEnv(t, "PERF_TARGET_BANDWIDTH")
+}
+
+func perfTestChainedPluginName(t *testing.T) string {
+	t.Helper()
+	return requiredEnv(t, "PERF_PLUGIN_NAME")
+}
+
+func requiredEnvInt(t *testing.T, key string) int {
+	t.Helper()
+	val, err := strconv.Atoi(requiredEnv(t, key))
+	require.NoErrorf(t, err, "%s env is not an int: %q", key, val)
+	return val
+}
+
+func requiredEnv(t *testing.T, key string) string {
+	t.Helper()
+	envVal := os.Getenv(key)
+	require.NotEmpty(t, envVal, "%s env is not set", key)
+	return envVal
 }
diff --git a/runtime/service_integ_test.go b/runtime/service_integ_test.go
index ed6b3ee..4df9cc0 100644
--- a/runtime/service_integ_test.go
+++ b/runtime/service_integ_test.go
@@ -85,6 +85,10 @@ func alpineImage(ctx context.Context, client *containerd.Client, snapshotterName
 	return unpackImage(ctx, client, snapshotterName, "docker.io/library/alpine:3.10.1")
 }
 
+func iperf3Image(ctx context.Context, client *containerd.Client, snapshotterName string) (containerd.Image, error) {
+	return unpackImage(ctx, client, snapshotterName, "docker.io/mlabbe/iperf3:3.6-r0")
+}
+
 func TestShimExitsUponContainerDelete_Isolated(t *testing.T) {
 	internal.RequiresIsolation(t)
 
@@ -585,12 +589,14 @@ func startAndWaitTask(ctx context.Context, t *testing.T, c containerd.Container)
 
 	select {
 	case exitStatus := <-exitCh:
+		assert.NoError(t, exitStatus.Error(), "failed to retrieve exitStatus")
 		assert.Equal(t, uint32(0), exitStatus.ExitCode())
 
 		status, err := task.Delete(ctx)
 		assert.NoErrorf(t, err, "failed to delete task %q after exit", c.ID())
-		assert.NoError(t, status.Error())
-		assert.NoError(t, exitStatus.Error(), "failed to retrieve exitStatus")
+		if status != nil {
+			assert.NoError(t, status.Error())
+		}
 
 		assert.Equal(t, "", stderr.String())
 	case <-ctx.Done():