Repository: firecracker-microvm/firecracker-containerd
Pull request #160 (2019-04-09 16:41:54)

Group containers into different VMs by VMID
1. This is the first half of the shim updates needed for running multiple containers per-VM. It consists of the host-side changes needed to group containers up into separate VMs by VMID. The second part of the updates are going to be VM-side in `agent`; they will be a separate PR from this one.
   * Right now, multiple VMs can be spun up in parallel each with it's own container, but the above mentioned changes in `agent` will be needed to support multiple containers within each VM.
1. Many of the TODOs added will be removed by integrating with Max's fc-control plugin (#156).
1. There's a few small updates that will be needed to the shim design doc once this is approved; namely
   * the use of just the `address` file instead of that plus the pid file (this is also what containerd has done in their newer runc v2 shim)
   * the "start shim" routine will fork/exec a child when it creates a new shim for handling a new VM (it turns out containerd forces you to do this; you can't just let the existing process take over even though it's the same binary...)
1. The original strategy for the e2e test was to use MMDS. This worked but when it came time to productionizing it, there were a lot of issues to be sorted out with getting a container in the VM w/ curl installed, making the timing of the MMDS update request and container query of it work, etc. I realized a different approach of just mapping the VMID to a mac address and having the container print that was much simpler but just as robust a test, so I switched to that.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.



Patch:
diff --git a/go.mod b/go.mod
index 7ada0a5..a089281 100644
--- a/go.mod
+++ b/go.mod
@@ -2,17 +2,16 @@ module github.com/firecracker-microvm/firecracker-containerd
 
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/Microsoft/go-winio v0.4.11 // indirect
+	github.com/Microsoft/go-winio v0.4.12 // indirect
 	github.com/Microsoft/hcsshim v0.8.1 // indirect
 	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
-	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/containerd/cgroups v0.0.0-20181105182409-82cb49fc1779 // indirect
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect
 	github.com/containerd/containerd v1.2.0
 	github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac
 	github.com/containerd/cri v1.11.1 // indirect
 	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3 // indirect
+	github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 // indirect
 	github.com/containerd/ttrpc v0.0.0-20181001154009-f51df4475b76
 	github.com/containerd/typeurl v0.0.0-20181015155603-461401dc8f19
 	github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect
@@ -24,6 +23,7 @@ require (
 	github.com/firecracker-microvm/firecracker-go-sdk v0.15.2-0.20190321195537-b704cc3feb1e
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gogo/googleapis v1.1.0 // indirect
 	github.com/gogo/protobuf v1.2.1
 	github.com/golang/mock v1.1.1
@@ -44,7 +44,7 @@ require (
 	github.com/stretchr/testify v1.2.2
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
 	github.com/urfave/cli v1.20.0 // indirect
-	go.etcd.io/bbolt v1.3.0
+	go.etcd.io/bbolt v1.3.1-etcd.8
 	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
 	golang.org/x/net v0.0.0-20190327214358-63eda1eb0650
 	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
diff --git a/go.sum b/go.sum
index 115e985..da001ef 100644
--- a/go.sum
+++ b/go.sum
@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
-github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc=
+github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/hcsshim v0.8.1 h1:0RKPd1pQB/4YRjdw0jFwq3A5nWFN4n1ojNzcm4B+8ZI=
 github.com/Microsoft/hcsshim v0.8.1/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
@@ -15,8 +15,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzs
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/cgroups v0.0.0-20181105182409-82cb49fc1779 h1:j1IsLW6/hNZPIyBH1v0hhEeB1WXE2ffhHaqSuXhgknY=
 github.com/containerd/cgroups v0.0.0-20181105182409-82cb49fc1779/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
@@ -30,8 +28,8 @@ github.com/containerd/cri v1.11.1 h1:mR8+eNW4zEcbWGTGEpmDd7GzMmK7IMxMSVAZ2aIDKA4
 github.com/containerd/cri v1.11.1/go.mod h1:DavH5Qa8+6jOmeOMO3dhWoqksucZDe06LfuhBz/xPZs=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 h1:XGyg7oTtD0DoRFhbpV6x1WfV0flKC4UxXU7ab1zC08U=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3 h1:esQOJREg8nw8aXj6uCN5dfW5cKUBiEJ/+nni1Q/D/sw=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 h1:biYGul6kwz5/Lh6UxkYQyRKPeYA/ZL5n2pdCAcYUpLA=
+github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/ttrpc v0.0.0-20181001154009-f51df4475b76 h1:vUPO9S35+FvukX2L/1mmPWgdv0y6FEKJzeWzmgVe9IA=
 github.com/containerd/ttrpc v0.0.0-20181001154009-f51df4475b76/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/typeurl v0.0.0-20181015155603-461401dc8f19 h1:gzdItdct+4eLnZxiZi1YcIXx3uo5QWa/xXKnsldEqY8=
@@ -50,8 +48,6 @@ github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zF
 github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/firecracker-microvm/firecracker-go-sdk v0.15.0 h1:KKbgkZFjtrvMfrv2XyxJW+YQAe2cHHsSTFSi6VTRzuE=
-github.com/firecracker-microvm/firecracker-go-sdk v0.15.0/go.mod h1:UOsA6IaI52MQntB9PrarJFReY1nA9iZg2DI04kxPNMY=
 github.com/firecracker-microvm/firecracker-go-sdk v0.15.2-0.20190321195537-b704cc3feb1e h1:LQKqzqwiUYtucykGP3WDaH8wfULY8v86HFSW95T+k+E=
 github.com/firecracker-microvm/firecracker-go-sdk v0.15.2-0.20190321195537-b704cc3feb1e/go.mod h1:QcNsz2gYkcTI/zAQl3dYeLwf7KxtjKnt7aGklhn9yYk=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb h1:D4uzjWwKYQ5XnAvUbuvHW93esHg7F8N/OYeBBcJoTr0=
@@ -86,6 +82,8 @@ github.com/go-openapi/validate v0.17.1 h1:RfQTLHm/gEu0oSUmbTOy0PMufjkE5/pPfnqYpo
 github.com/go-openapi/validate v0.17.1/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384 h1:xNwo3yd3PZYRDAr/Dz0sBfDWY6El2xPCKJrwJVfMFjY=
 github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
@@ -156,8 +154,9 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-go.etcd.io/bbolt v1.3.0 h1:oY10fI923Q5pVCVt1GBTZMn8LHo5M+RCInFpeMnV4QI=
-go.etcd.io/bbolt v1.3.0/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.1-etcd.8 h1:6J7QAKqfFBGnU80KRnuQxfjjeE5xAGE/qB810I3FQHQ=
+go.etcd.io/bbolt v1.3.1-etcd.8/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
@@ -181,6 +180,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g=
 golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/runtime/config.go b/runtime/config.go
index bee8102..929bfe8 100644
--- a/runtime/config.go
+++ b/runtime/config.go
@@ -24,7 +24,6 @@ import (
 const (
 	configPathEnvName = "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH"
 	defaultConfigPath = "/etc/containerd/firecracker-runtime.json"
-	defaultSocketPath = "./firecracker.sock"
 	defaultKernelArgs = "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw"
 	defaultFilesPath  = "/var/lib/firecracker-containerd/runtime/"
 	defaultKernelPath = defaultFilesPath + "default-vmlinux.bin"
@@ -34,18 +33,15 @@ const (
 
 // Config represents runtime configuration parameters
 type Config struct {
-	FirecrackerBinaryPath string            `json:"firecracker_binary_path"`
-	KernelImagePath       string            `json:"kernel_image_path"`
-	KernelArgs            string            `json:"kernel_args"`
-	RootDrive             string            `json:"root_drive"`
-	CPUCount              int               `json:"cpu_count"`
-	CPUTemplate           string            `json:"cpu_template"`
-	AdditionalDrives      map[string]string `json:"additional_drives"`
-	LogFifo               string            `json:"log_fifo"`
-	LogLevel              string            `json:"log_level"`
-	MetricsFifo           string            `json:"metrics_fifo"`
-	HtEnabled             bool              `json:"ht_enabled"`
-	Debug                 bool              `json:"debug"`
+	FirecrackerBinaryPath string `json:"firecracker_binary_path"`
+	KernelImagePath       string `json:"kernel_image_path"`
+	KernelArgs            string `json:"kernel_args"`
+	RootDrive             string `json:"root_drive"`
+	CPUCount              int    `json:"cpu_count"`
+	CPUTemplate           string `json:"cpu_template"`
+	LogLevel              string `json:"log_level"`
+	HtEnabled             bool   `json:"ht_enabled"`
+	Debug                 bool   `json:"debug"`
 }
 
 // LoadConfig loads configuration from JSON file at 'path'
diff --git a/runtime/config.json.example b/runtime/config.json.example
index 3c89732..1395ad3 100644
--- a/runtime/config.json.example
+++ b/runtime/config.json.example
@@ -5,11 +5,6 @@
   "root_drive": "./vsock.img",
   "cpu_count": 1,
   "cpu_template": "T2",
-  "additional_drives": {
-    "./shim.img": "rw"
-  },
-  "log_fifo": "/tmp/fc-logs.fifo",
   "log_level": "Debug",
-  "metrics_fifo": "/tmp/fc-metrics.fifo",
   "ht_enabled": false
 }
\ No newline at end of file
diff --git a/runtime/firecrackeroci/annotation.go b/runtime/firecrackeroci/annotation.go
new file mode 100644
index 0000000..0f87e4f
--- /dev/null
+++ b/runtime/firecrackeroci/annotation.go
@@ -0,0 +1,39 @@
+// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package firecrackeroci
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+)
+
+const (
+	// VMIDAnnotationKey is the key specified in an OCI-runtime config annotation section
+	// specifying the ID of the VM in which the container should be spun up.
+	VMIDAnnotationKey = "aws.firecracker.vm.id"
+)
+
+// WithVMID annotates a containerd client's container object with a given firecracker VMID.
+func WithVMID(vmID string) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		if s.Annotations == nil {
+			s.Annotations = make(map[string]string)
+		}
+
+		s.Annotations[VMIDAnnotationKey] = vmID
+		return nil
+	}
+}
diff --git a/runtime/service.go b/runtime/service.go
index ea847dd..d18de56 100644
--- a/runtime/service.go
+++ b/runtime/service.go
@@ -15,14 +15,23 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
+	"strings"
+
+	// disable gosec check for math/rand. We just need a random starting
+	// place to start looking for CIDs; no need for cryptographically
+	// secure randomness
+	"math/rand" // #nosec
+
 	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sync"
 	"syscall"
 	"time"
 	"unsafe"
@@ -39,6 +48,7 @@ import (
 	"github.com/containerd/typeurl"
 	"github.com/firecracker-microvm/firecracker-go-sdk"
 	models "github.com/firecracker-microvm/firecracker-go-sdk/client/models"
+	"github.com/gofrs/uuid"
 	ptypes "github.com/gogo/protobuf/types"
 	"github.com/mdlayher/vsock"
 	"github.com/pkg/errors"
@@ -48,20 +58,41 @@ import (
 	"github.com/firecracker-microvm/firecracker-containerd/eventbridge"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
+	"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
 const (
 	defaultVsockPort     = 10789
 	supportedMountFSType = "ext4"
+
+	vmIDEnvVarKey = "FIRECRACKER_VM_ID"
+
+	varRunDir                  = "/var/run/firecracker-containerd"
+	firecrackerSockName        = "firecracker.sock"
+	firecrackerLogFifoName     = "fc-logs.fifo"
+	firecrackerMetricsFifoName = "fc-metrics.fifo"
+
+	// TODO these strings are hardcoded throughout the containerd codebase, it may
+	// be worth sending them a PR to define them as constants accessible to shim
+	// implementations like our own
+	shimAddrFileName = "address"
+	shimLogFifoName  = "log"
+	ociConfigName    = "config.json"
 )
 
 // implements shimapi
 type service struct {
 	server        *ttrpc.Server
 	id            string
+	vmID          string
 	publish       events.Publisher
 	eventExchange *exchange.Exchange
 
+	startVMMutex sync.Mutex
 	agentStarted bool
 	agentClient  taskAPI.TaskService
 	config       *Config
@@ -76,8 +107,78 @@ var (
 	sysCall = syscall.Syscall
 )
 
+func shimOpts(ctx context.Context) (*shim.Opts, error) {
+	opts, ok := ctx.Value(shim.OptsKey{}).(shim.Opts)
+	if !ok {
+		return nil, errors.New("failed to parse containerd shim opts from context")
+	}
+
+	return &opts, nil
+}
+
+// The bundle dir for the container can be provided by a containerd-defined flag, or, if that's
+// not set, it's assumed to be the current working directory of this process (as set by the
+// containerd parent process).
+func resolveBundleDir(ctx context.Context) (string, error) {
+	// Check to see if it was set by command line flag
+	opts, err := shimOpts(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	if opts.BundlePath != "" {
+		return opts.BundlePath, nil
+	}
+
+	// If no command line flag, this must be a shim start routine, so we can assume it's the
+	// current working directory
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read bundle dir path")
+	}
+
+	return cwd, nil
+}
+
+// If the VMID was provided by the client, we'll find it in the OCI config Annotations section.
+// If, however, no VMID was provided in the OCI config, we it may have been set by a "shim start"
+// parent process as an env var. resolveVMID checks those two locations and returns the VMID
+// if found in either.
+func resolveVMID(ctx context.Context) (string, error) {
+	// If the vmID is provided via env var, use that
+	if vmID := os.Getenv(vmIDEnvVarKey); vmID != "" {
+		return vmID, nil
+	}
+
+	// if no env var, check the OCI config annotation section
+	bundleDir, err := resolveBundleDir(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	specPath := filepath.Join(bundleDir, ociConfigName)
+	ociConfigFile, err := os.Open(specPath)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to open OCI config file %s", specPath)
+	}
+
+	defer ociConfigFile.Close()
+	var ociConfig struct {
+		Annotations map[string]string `json:"annotations,omitempty"`
+	}
+
+	if err := json.NewDecoder(ociConfigFile).Decode(&ociConfig); err != nil {
+		return "", errors.Wrapf(err, "failed to parse Annotations section of OCI config file %s", specPath)
+	}
+
+	// This will return empty string if the key is not present in the OCI config, which the caller can decided
+	// how to deal with
+	return ociConfig.Annotations[firecrackeroci.VMIDAnnotationKey], nil
+}
+
+var _ shim.Init = NewService
+
 // NewService creates new runtime shim.
-// Matches type Init func(..).. defined https://github.com/containerd/containerd/blob/master/runtime/v2/shim/shim.go#L47
 func NewService(ctx context.Context, id string, remotePublisher events.Publisher) (shim.Shim, error) {
 	server, err := newServer()
 	if err != nil {
@@ -90,8 +191,11 @@ func NewService(ctx context.Context, id string, remotePublisher events.Publisher
 	}
 
 	if !config.Debug {
-		// Check if containerd is running in debug mode
-		opts := ctx.Value(shim.OptsKey{}).(shim.Opts)
+		opts, err := shimOpts(ctx)
+		if err != nil {
+			return nil, err
+		}
+
 		config.Debug = opts.Debug
 	}
 
@@ -107,9 +211,25 @@ func NewService(ctx context.Context, id string, remotePublisher events.Publisher
 		}
 	}()
 
+	vmID, err := resolveVMID(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// If no VMID was provided by the client or already auto-generated by a parent "shim start", generate a new random one.
+	// This results in a default behavior of running each container in its own VM.
+	if vmID == "" {
+		uuid, err := uuid.NewV4()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to generate UUID for VMID")
+		}
+		vmID = uuid.String()
+	}
+
 	s := &service{
 		server:        server,
 		id:            id,
+		vmID:          vmID,
 		publish:       remotePublisher,
 		eventExchange: eventExchange,
 
@@ -119,41 +239,161 @@ func NewService(ctx context.Context, id string, remotePublisher events.Publisher
 	return s, nil
 }
 
-func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
-	log.G(ctx).WithField("id", id).Debug("StartShim")
-	cmd, err := s.newCommand(ctx, containerdBinary, containerdAddress)
+// vmDir holds files, sockets and FIFOs scoped to a single given VM
+func (s *service) vmDir() string {
+	return filepath.Join(varRunDir, s.vmID)
+}
+
+// shimAddrFilePath is the path to the shim address file as found in the vmDir. The
+// address file holds the abstract unix socket path at which the shim serves its API.
+// It's read by containerd to find the shim.
+func (s *service) shimAddrFilePath() string {
+	return filepath.Join(s.vmDir(), shimAddrFileName)
+}
+
+// bundleAddrFilePath is the path to the address file as found in the bundleDir.
+// Even though the shim address is set per-VM, not per-container, containerd expects
+// to find the shim addr file in the bundle dir, so we still have to create it or
+// symlink it to the shimAddrFilePath
+func (s *service) bundleAddrFilePath(ctx context.Context) (string, error) {
+	bundleDir, err := resolveBundleDir(ctx)
 	if err != nil {
 		return "", err
 	}
 
-	address, err := shim.SocketAddress(ctx, id)
+	return filepath.Join(bundleDir, shimAddrFileName), nil
+}
+
+// shimLogFifoPath is a path to a FIFO for writing shim logs as found in the vmDir.
+func (s *service) shimLogFifoPath() string {
+	return filepath.Join(s.vmDir(), shimLogFifoName)
+}
+
+// bundleLogFifoPath is a path to a FIFO for writing shim logs as found in the bundleDir.
+// It is the path created by containerd for us, the shimLogFifoPath is just a symlink to one.
+func (s *service) bundleLogFifoPath(ctx context.Context) (string, error) {
+	bundleDir, err := resolveBundleDir(ctx)
 	if err != nil {
 		return "", err
 	}
 
-	socket, err := shim.NewSocket(address)
+	return filepath.Join(bundleDir, shimLogFifoName), nil
+}
+
+// shimSocketAddress is the abstract unix socket path at which our shim serves its API
+// It is unique per-VM.
+func (s *service) shimSocketAddress(ctx context.Context) (string, error) {
+	return shim.SocketAddress(ctx, s.vmID)
+}
+
+// firecrackerSockPath is the path to the firecracker VMM's API socket
+func (s *service) firecrackerSockPath() string {
+	return filepath.Join(s.vmDir(), firecrackerSockName)
+}
+
+// firecrackerLogFifoPath is the path to the firecracker VMM's log fifo
+func (s *service) firecrackerLogFifoPath() string {
+	return filepath.Join(s.vmDir(), firecrackerLogFifoName)
+}
+
+// firecrackerMetricsFifoPath is the path to the firecracker VMM's metrics fifo
+func (s *service) firecrackerMetricsFifoPath() string {
+	return filepath.Join(s.vmDir(), firecrackerMetricsFifoName)
+}
+
+// containerd expects there to be a file named "address" in the container bundle directory (it contains the
+// unix abstract address at which the shim managing that container can be found). We create that file in the
+// shim's VM directory and then symlink to it from each container's bundle dir.
+func (s *service) createAddressSymlink(ctx context.Context) error {
+	bundleAddrFilePath, err := s.bundleAddrFilePath(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = os.Symlink(s.shimAddrFilePath(), bundleAddrFilePath)
+	if err != nil {
+		return errors.Wrapf(err, `failed to create shim address file symlink from "%s"->"%s"`, bundleAddrFilePath, s.shimAddrFilePath())
+	}
+
+	return nil
+}
+
+// containerd creates a fifo for writing shim logs every time it spins up a container in the container's bundle dir.
+// Code defined as part containerd packages also assume there to be a fifo name "log" in the current working directory
+// of the shim. Our shim's working directory after initialization is the vmID dir, so we create a symlink from
+// <vmdir>/log to the log fifo of the first container spun up for the shim.
+func (s *service) createShimLogFifoSymlink(ctx context.Context) error {
+	bundleLogFifoPath, err := s.bundleLogFifoPath(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = os.Symlink(bundleLogFifoPath, s.shimLogFifoPath())
+	if err != nil {
+		return errors.Wrapf(err, `failed to create shim log fifo symlink from "%s"->"%s"`, s.shimLogFifoPath(), bundleLogFifoPath)
+	}
+
+	return nil
+}
+
+func isEADDRINUSE(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "address already in use")
+}
+
+func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
+	log.G(ctx).WithField("id", id).Debug("StartShim")
+
+	shimSocketAddress, err := s.shimSocketAddress(ctx)
 	if err != nil {
 		return "", err
 	}
 
+	// We determine if there is already a shim managing a VM with the current VMID by attempting
+	// to listen on the abstract socket address (which is parameterized by VMID). If we get
+	// EADDRINUSE, then we assume there is already a shim for the VM and just hand off to that
+	// one. This is in line with the approach used by containerd's reference runC shim v2
+	// implementation (which is also designed to manage multiple containers from a single shim
+	// process)
+	shimSocket, err := shim.NewSocket(shimSocketAddress)
+	if isEADDRINUSE(err) {
+		// There's already a shim for this VMID, so just hand off to it
+		err = s.createAddressSymlink(ctx)
+		if err != nil {
+			return "", err
+		}
+
+		return shimSocketAddress, nil
+	} else if err != nil {
+		return "", errors.Wrapf(err, "failed to create new shim socket at address \"%s\"", shimSocketAddress)
+	}
+
+	// If we're here, there is no pre-existing shim for this VMID, so we spawn a new one
 	defer func() {
 		log.G(ctx).WithField("id", id).Debug("closing shim socket")
-		socket.Close()
+		shimSocket.Close()
 	}()
 
-	f, err := socket.File()
+	err = os.MkdirAll(s.vmDir(), 0700)
 	if err != nil {
 		return "", err
 	}
 
+	shimSocketFile, err := shimSocket.File()
+	if err != nil {
+		return "", err
+	}
 	defer func() {
 		log.G(ctx).WithField("id", id).Debug("closing shim socket file")
-		f.Close()
+		shimSocketFile.Close()
 	}()
 
-	cmd.ExtraFiles = append(cmd.ExtraFiles, f)
+	cmd, err := s.newCommand(ctx, id, containerdBinary, containerdAddress, shimSocketFile)
+	if err != nil {
+		return "", err
+	}
 
-	if err := cmd.Start(); err != nil {
+	err = cmd.Start()
+	if err != nil {
 		log.G(ctx).WithError(err).WithField("id", id).Error("Failed to Start()")
 		return "", err
 	}
@@ -171,22 +411,30 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container
 		waitErr := cmd.Wait()
 		log.G(ctx).WithError(waitErr).WithField("id", id).Debug("completed waiting on shim process")
 	}()
-	if err := shim.WritePidFile("shim.pid", cmd.Process.Pid); err != nil {
-		log.G(ctx).WithError(err).WithField("id", id).Error("failed to write pid file")
+
+	err = shim.WriteAddress(s.shimAddrFilePath(), shimSocketAddress)
+	if err != nil {
+		log.G(ctx).WithError(err).WithField("id", id).Error("failed to write address")
 		return "", err
 	}
 
-	if err := shim.WriteAddress("address", address); err != nil {
-		log.G(ctx).WithError(err).WithField("id", id).Error("failed to write address")
+	err = s.createAddressSymlink(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	err = s.createShimLogFifoSymlink(ctx)
+	if err != nil {
 		return "", err
 	}
 
-	if err := shim.SetScore(cmd.Process.Pid); err != nil {
+	err = shim.SetScore(cmd.Process.Pid)
+	if err != nil {
 		log.G(ctx).WithError(err).WithField("id", id).Error("failed to set OOM score on shim")
 		return "", errors.Wrap(err, "failed to set OOM Score on shim")
 	}
 
-	return address, nil
+	return shimSocketAddress, nil
 }
 
 func (s *service) Create(ctx context.Context, request *taskAPI.CreateTaskRequest) (*taskAPI.CreateTaskResponse, error) {
@@ -215,11 +463,14 @@ func (s *service) Create(ctx context.Context, request *taskAPI.CreateTaskRequest
 			return nil, errors.Wrapf(err, "unmarshaling task create request options")
 		}
 	}
-	// TODO: should there be a lock here
+
+	// TODO: handle case where VM is pre-created
+	s.startVMMutex.Lock()
 	if !s.agentStarted {
 		log.G(ctx).WithField("id", request.ID).Debug("calling startVM")
 		client, err := s.startVM(ctx, request, firecrackerConfig)
 		if err != nil {
+			s.startVMMutex.Unlock()
 			log.G(ctx).WithError(err).WithField("id", request.ID).Error("failed to start VM")
 			return nil, err
 		}
@@ -227,6 +478,7 @@ func (s *service) Create(ctx context.Context, request *taskAPI.CreateTaskRequest
 		s.agentClient = client
 		s.agentStarted = true
 	}
+	s.startVMMutex.Unlock()
 
 	log.G(ctx).WithField("id", request.ID).Info("creating task")
 
@@ -425,11 +677,18 @@ func (s *service) Shutdown(ctx context.Context, req *taskAPI.ShutdownRequest) (*
 	if _, err := s.agentClient.Shutdown(ctx, req); err != nil {
 		log.G(ctx).WithError(err).Error("failed to shutdown agent")
 	}
+
 	log.G(ctx).Debug("stopping VM")
 	if err := s.stopVM(); err != nil {
 		log.G(ctx).WithError(err).Error("failed to stop VM")
 		return nil, err
 	}
+
+	if err := os.RemoveAll(s.vmDir()); err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to remove VM dir %s", s.vmDir())
+		return nil, err
+	}
+
 	s.cancel()
 	// Exit to avoid 'zombie' shim processes
 	defer os.Exit(0)
@@ -479,7 +738,7 @@ func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error)
 	}, nil
 }
 
-func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
+func (s *service) newCommand(ctx context.Context, id, containerdBinary, containerdAddress string, shimSocketFile *os.File) (*exec.Cmd, error) {
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return nil, err
@@ -490,13 +749,15 @@ func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAd
 		return nil, err
 	}
 
-	cwd, err := os.Getwd()
+	bundleDir, err := resolveBundleDir(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	args := []string{
 		"-namespace", ns,
+		"-bundle", bundleDir,
+		"-id", id,
 		"-address", containerdAddress,
 		"-publish-binary", containerdBinary,
 	}
@@ -506,8 +767,11 @@ func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAd
 	}
 
 	cmd := exec.Command(self, args...)
-	cmd.Dir = cwd
-	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
+	cmd.Dir = s.vmDir()
+	cmd.Env = append(os.Environ(),
+		"GOMAXPROCS=2",
+		fmt.Sprintf("%s=%s", vmIDEnvVarKey, s.vmID))
+	cmd.ExtraFiles = append(cmd.ExtraFiles, shimSocketFile)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
@@ -556,7 +820,7 @@ func findNextAvailableVsockCID(ctx context.Context) (uint32, error) {
 		// Corresponds to VHOST_VSOCK_SET_GUEST_CID in vhost.h
 		ioctlVsockSetGuestCID = uintptr(0x4008AF60)
 		// 0, 1 and 2 are reserved CIDs, see http://man7.org/linux/man-pages/man7/vsock.7.html
-		startCID        = 3
+		minCID          = 3
 		maxCID          = math.MaxUint32
 		vsockDevicePath = "/dev/vhost-vsock"
 	)
@@ -565,15 +829,18 @@ func findNextAvailableVsockCID(ctx context.Context) (uint32, error) {
 	if err != nil {
 		return 0, errors.Wrap(err, "failed to open vsock device")
 	}
-
 	defer file.Close()
 
-	for contextID := startCID; contextID < maxCID; contextID++ {
+	// Start at a random ID to minimize chances of conflicts when shims are racing each other here
+	// TODO the actual fix to this problem will come when the shim is updated to just use the FC-control plugin,
+	// this current implementation is a very short-term bandaid.
+	start := rand.Intn(maxCID - minCID)
+	for n := 0; n < maxCID-minCID; n++ {
+		cid := minCID + ((start + n) % (maxCID - minCID))
 		select {
 		case <-ctx.Done():
 			return 0, ctx.Err()
 		default:
-			cid := contextID
 			_, _, err = sysCall(
 				unix.SYS_IOCTL,
 				file.Fd(),
@@ -582,7 +849,7 @@ func findNextAvailableVsockCID(ctx context.Context) (uint32, error) {
 
 			switch err {
 			case unix.Errno(0):
-				return uint32(contextID), nil
+				return uint32(cid), nil
 			case unix.EADDRINUSE:
 				// ID is already taken, try next one
 				continue
@@ -614,6 +881,7 @@ func parseCreateTaskOpts(ctx context.Context, opts *ptypes.Any) (*proto.Firecrac
 	return nil, opts, nil
 }
 
+// TODO: replace startVM with calls to the FC-control plugin
 func (s *service) startVM(ctx context.Context,
 	request *taskAPI.CreateTaskRequest,
 	vmConfig *proto.FirecrackerConfig,
@@ -626,7 +894,7 @@ func (s *service) startVM(ctx context.Context,
 	}
 
 	cfg := firecracker.Config{
-		SocketPath:      defaultSocketPath,
+		SocketPath:      s.firecrackerSockPath(),
 		VsockDevices:    []firecracker.VsockDevice{{Path: "root", CID: cid}},
 		KernelImagePath: s.config.KernelImagePath,
 		KernelArgs:      s.config.KernelArgs,
@@ -635,9 +903,9 @@ func (s *service) startVM(ctx context.Context,
 			CPUTemplate: models.CPUTemplate(s.config.CPUTemplate),
 			MemSizeMib:  256,
 		},
-		LogFifo:     s.config.LogFifo,
+		LogFifo:     s.firecrackerLogFifoPath(),
 		LogLevel:    s.config.LogLevel,
-		MetricsFifo: s.config.MetricsFifo,
+		MetricsFifo: s.firecrackerMetricsFifoPath(),
 		Debug:       s.config.Debug,
 	}
 
@@ -659,10 +927,11 @@ func (s *service) startVM(ctx context.Context,
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to build VM options")
 	}
+
 	cfg.Drives = driveBuilder.Build()
 	cmd := firecracker.VMCommandBuilder{}.
 		WithBin(s.config.FirecrackerBinaryPath).
-		WithSocketPath(defaultSocketPath).
+		WithSocketPath(s.firecrackerSockPath()).
 		Build(ctx)
 	machineOpts := []firecracker.Opt{
 		firecracker.WithProcessRunner(cmd),
diff --git a/tools/docker/naive-snapshotter/entrypoint.sh b/tools/docker/naive-snapshotter/entrypoint.sh
index 32472d2..9f1bf07 100755
--- a/tools/docker/naive-snapshotter/entrypoint.sh
+++ b/tools/docker/naive-snapshotter/entrypoint.sh
@@ -12,6 +12,6 @@ chmod a+rw ${FICD_SNAPSHOTTER_OUTFILE}
 
 touch ${FICD_CONTAINERD_OUTFILE}
 chmod a+rw ${FICD_CONTAINERD_OUTFILE}
-/usr/local/bin/containerd &>> ${FICD_CONTAINERD_OUTFILE} &
+/usr/local/bin/containerd --log-level debug &>> ${FICD_CONTAINERD_OUTFILE} &
 
 exec /bin/bash -c "$@"


Test patch:
diff --git a/runtime/service_integ_test.go b/runtime/service_integ_test.go
index 9882774..39e7683 100644
--- a/runtime/service_integ_test.go
+++ b/runtime/service_integ_test.go
@@ -14,9 +14,16 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -28,10 +35,14 @@ import (
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/runtime"
 	"github.com/containerd/typeurl"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
 	"github.com/shirou/gopsutil/process"
 	"github.com/stretchr/testify/require"
 
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
+	"github.com/firecracker-microvm/firecracker-containerd/proto"
+	"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
 )
 
 const (
@@ -41,6 +52,9 @@ const (
 	firecrackerRuntime   = "aws.firecracker"
 	naiveSnapshotterName = "firecracker-naive"
 	shimProcessName      = "containerd-shim-aws-firecracker"
+
+	defaultVMRootfsPath = "/var/lib/firecracker-containerd/runtime/hello-rootfs.ext4"
+	defaultVMNetDevName = "eth0"
 )
 
 func TestShimExitsUponContainerKill_Isolated(t *testing.T) {
@@ -115,9 +129,162 @@ func TestShimExitsUponContainerKill_Isolated(t *testing.T) {
 
 		err = internal.WaitForPidToExit(testCtx, time.Second, shimProcess.Pid)
 		require.NoError(t, err, "failed waiting for shim process \"%s\" to exit", shimProcessName)
+
+		varRunFCContents, err := ioutil.ReadDir(varRunDir)
+		require.NoError(t, err, `failed to list directory "%s"`, varRunDir)
+		// there should just be one file left in the directory for the naive snapshotter socket
+		require.Len(t, varRunFCContents, 1, "expect %s to be cleared after shims shutdown", varRunDir)
 	case err = <-exitEventErrCh:
 		require.Fail(t, "unexpected error", "unexpectedly received on task exit error channel: %s", err.Error())
 	case <-testCtx.Done():
 		require.Fail(t, "context canceled", "context canceled while waiting for container \"%s\" exit: %s", containerName, testCtx.Err())
 	}
 }
+
+// vmIDtoMacAddr converts a provided VMID to a unique Mac Address. This is a convenient way of providing the VMID to clients within
+// the VM without the extra complication of alternative approaches like MMDS.
+func vmIDtoMacAddr(vmID uint) string {
+	var addrParts []string
+
+	// mac addresses have 6 hex components separate by ":", i.e. "11:22:33:44:55:66"
+	numMacAddrComponents := uint(6)
+
+	for n := uint(0); n < numMacAddrComponents; n++ {
+		// To isolate the value of the nth component, right bit shift the vmID by 8*n (there are 8 bits per component) and
+		// mask out any upper bits leftover (bitwise AND with 255)
+		addrComponent := (vmID >> (8 * n)) & 255
+
+		// format the component as a two-digit hex string
+		addrParts = append(addrParts, fmt.Sprintf("%02x", addrComponent))
+	}
+
+	return strings.Join(addrParts, ":")
+}
+
+func createTapDevice(ctx context.Context, tapName string) error {
+	err := exec.CommandContext(ctx, "ip", "tuntap", "add", tapName, "mode", "tap").Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to create tap device %s", tapName)
+	}
+
+	err = exec.CommandContext(ctx, "ip", "link", "set", tapName, "up").Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to up tap device %s", tapName)
+	}
+
+	return nil
+}
+
+func TestMultipleVMs_Isolated(t *testing.T) {
+	internal.RequiresIsolation(t)
+
+	const (
+		numVMs = 5
+		// TODO update with multiple containers per-VM once that PR is merged
+		containersPerVM = 1
+	)
+
+	ctx := namespaces.WithNamespace(context.Background(), "default")
+
+	client, err := containerd.New(containerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime))
+	require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
+	defer client.Close()
+
+	image, err := client.Pull(ctx, debianDockerImage, containerd.WithPullUnpack, containerd.WithPullSnapshotter(naiveSnapshotterName))
+	require.NoError(t, err, "failed to pull image %s, is the the %s snapshotter running?", debianDockerImage, naiveSnapshotterName)
+
+	rootfsBytes, err := ioutil.ReadFile(defaultVMRootfsPath)
+	require.NoError(t, err, "failed to read rootfs file")
+
+	// This test spawns separate VMs in parallel and ensures containers are spawned within each expected VM. It asserts each
+	// container ends up in the right VM by assigning each VM a network device with a unique mac address and having each container
+	// print the mac address it sees inside its VM.
+	var vmWg sync.WaitGroup
+	for vmID := 0; vmID < numVMs; vmID++ {
+		vmWg.Add(1)
+		go func(vmID int) {
+			defer vmWg.Done()
+
+			tapName := fmt.Sprintf("tap%d", vmID)
+			err = createTapDevice(ctx, tapName)
+			require.NoError(t, err, "failed to create tap device for vm %d", vmID)
+
+			// TODO once Noah's immutable rootfs change is merged, we can use that as our rootfs for all VMs instead of copying
+			// one per-VM
+			rootfsPath := fmt.Sprintf("%s.%d", defaultVMRootfsPath, vmID)
+			err = ioutil.WriteFile(rootfsPath, rootfsBytes, 0600)
+			require.NoError(t, err, "failed to copy vm rootfs to %s", rootfsPath)
+
+			var containerWg sync.WaitGroup
+			for containerID := 0; containerID < containersPerVM; containerID++ {
+				containerWg.Add(1)
+				go func(containerID int) {
+					defer containerWg.Done()
+					containerName := fmt.Sprintf("container-%d-%d", vmID, containerID)
+					snapshotName := fmt.Sprintf("snapshot-%d-%d", vmID, containerID)
+
+					// spawn a container that just prints the VM's eth0 mac address (which we have set uniquely per VM)
+					newContainer, err := client.NewContainer(ctx,
+						containerName,
+						containerd.WithSnapshotter(naiveSnapshotterName),
+						containerd.WithNewSnapshot(snapshotName, image),
+						containerd.WithNewSpec(
+							oci.WithProcessArgs("cat", fmt.Sprintf("/sys/class/net/%s/address", defaultVMNetDevName)),
+							oci.WithHostNamespace(specs.NetworkNamespace),
+							firecrackeroci.WithVMID(strconv.Itoa(vmID)),
+						),
+					)
+					require.NoError(t, err, "failed to create container %s", containerName)
+
+					var stdout bytes.Buffer
+					var stderr bytes.Buffer
+
+					newTask, err := newContainer.NewTask(ctx,
+						cio.NewCreator(cio.WithStreams(nil, bufio.NewWriter(&stdout), bufio.NewWriter(&stderr))),
+						func(ctx context.Context, _ *containerd.Client, ti *containerd.TaskInfo) error {
+							ti.Options = &proto.FirecrackerConfig{
+								RootDrive: &proto.FirecrackerDrive{
+									PathOnHost:   rootfsPath,
+									IsReadOnly:   false,
+									IsRootDevice: true,
+								},
+								NetworkInterfaces: []*proto.FirecrackerNetworkInterface{
+									{
+										HostDevName: tapName,
+										MacAddress:  vmIDtoMacAddr(uint(vmID)),
+									},
+								},
+							}
+							return nil
+						})
+					require.NoError(t, err, "failed to create task for container %s", containerName)
+
+					err = newTask.Start(ctx)
+					require.NoError(t, err, "failed to start task for container %s", containerName)
+
+					exitCh, err := newTask.Wait(ctx)
+					require.NoError(t, err, "failed to wait on task for container %s", containerName)
+
+					select {
+					case exitStatus := <-exitCh:
+						// if there was anything on stderr, print it to assist debugging
+						stderrOutput := stderr.String()
+						if len(stderrOutput) != 0 {
+							fmt.Printf("stderr output from vm %d, container %d: %s", vmID, containerID, stderrOutput)
+						}
+
+						require.Equal(t, uint32(0), exitStatus.ExitCode())
+						require.Equal(t, vmIDtoMacAddr(uint(vmID)), strings.TrimSpace(stdout.String()))
+					case <-ctx.Done():
+						require.Fail(t, "context cancelled",
+							"context cancelled while waiting for container %s to exit, err: %v", containerName, ctx.Err())
+					}
+				}(containerID)
+			}
+
+			containerWg.Wait()
+		}(vmID)
+	}
+
+	vmWg.Wait()
+}
diff --git a/runtime/service_test.go b/runtime/service_test.go
index 65669f6..6731e17 100644
--- a/runtime/service_test.go
+++ b/runtime/service_test.go
@@ -38,11 +38,11 @@ func TestFindNextAvailableVsockCID(t *testing.T) {
 		sysCall = syscall.Syscall
 	}()
 
-	cid, err := findNextAvailableVsockCID(context.Background())
+	_, err := findNextAvailableVsockCID(context.Background())
 	require.NoError(t, err,
 		"Do you have permission to interact with /dev/vhost-vsock?\n"+
 			"Grant yourself permission with `sudo setfacl -m u:${USER}:rw /dev/vhost-vsock`")
-	require.EqualValues(t, uint32(3), cid)
+	// we generate a random CID, so it's not possible to make assertions on its value
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()