Repository: firecracker-microvm/firecracker-containerd (go)

Issue:
jailed VM creation fails on Amazon Linux 2
firecracker-containerd currently uses jail locations under `/run`. On AL2 (and presumably CentOS and other RHEL variants) this filesystem is mounted with the `nodev` flag, so the device nodes created by runc are not usable.

firecracker-containerd debug logs show:

```
INFO[2019-11-26T22:15:06.264606168Z] Successfully ran jailer handler               jailer=runc ociBundlePath=/run/firecracker-containerd/firecracker-containerd-example/fc-example runcBinaryPath=/usr/local/bin/runc runtime=aws.firecracker vmID=fc-example
INFO[2019-11-26T22:15:06.264668138Z] Called startVMM(), setting up a VMM on /run/firecracker-containerd/firecracker-containerd-example/fc-example/rootfs/api.socket  runtime=aws.firecracker vmID=fc-example
WARN[2019-11-26T22:15:06.381824962Z] firecracker exited: exit status 1             runtime=aws.firecracker vmID=fc-example
WARN[2019-11-26T22:15:06.381911059Z] Failed handler "fcinit.StartVMM": Firecracker did not create API socket /run/firecracker-containerd/firecracker-containerd-example/fc-example/rootfs/api.socket: 1 error occurred:
        * exit status 1
  runtime=aws.firecracker vmID=fc-example
ERRO[2019-11-26T22:15:06.381939624Z]                                               error="failed to create VM: failed to start the VM: Firecracker did not create API socket /run/firecracker-containerd/firecracker-containerd-example/fc-example/rootfs/api.socket: 1 error occurred:\n\t* exit status 1\n\n" runtime=aws.firecracker vmID=fc-example
ERRO[2019-11-26T22:15:06.382144578Z]                                               error="shim CreateVM returned error: rpc error: code = Unknown desc = failed to create VM: failed to start the VM: Firecracker did not create API socket /run/firecracker-containerd/firecracker-containerd-example/fc-example/rootfs/api.socket: 1 error occurred:\n\t* exit status 1\n\n"
DEBU[2019-11-26T22:15:06.383322943Z] shim has been terminated                      error="signal: killed" vmID=fc-example
```

...which is not the most helpful error.

The location used for VM jailing should be configurable.  Appropriate locations will vary from site to site.

Pull request:
Adds configurable jail path
Due to the issue of Amazon Linux 2 having restriction on /run, this was causing
no devices to be able to be ran. To fix this we added a configurable jail path
which should allow for differences in environments that have restrictions on
the default jail path. Also changes the default from being the location of the
shim dir to /srv/jailer.

This also bind mounts instead of hard linking the fifo files as there were issues cross linking since
/run, which is where the shim dir lives, being a tmpfs

Fixes #345 

Signed-off-by: xibz <impactbchang@gmail.com>

*Issue #, if available:*

*Description of changes:*


By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.


Patch:
diff --git a/runtime/config.go b/config/config.go
similarity index 88%
rename from runtime/config.go
rename to config/config.go
index b371980..b5bc667 100644
--- a/runtime/config.go
+++ b/config/config.go
@@ -11,7 +11,7 @@
 // express or implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package main
+package config
 
 import (
 	"encoding/json"
@@ -33,7 +33,7 @@ const (
 	defaultRootfsPath  = defaultFilesPath + "default-rootfs.img"
 	defaultCPUCount    = 1
 	defaultCPUTemplate = models.CPUTemplateT2
-	defaultMemSizeMb   = 128
+	defaultShimBaseDir = "/var/lib/firecracker-containerd/shim-base"
 )
 
 // Config represents runtime configuration parameters
@@ -51,7 +51,11 @@ type Config struct {
 	// the VM will default to using the network interfaces as specified here. This is especially
 	// useful when a CNI-based network interface is provided in DefaultNetworkInterfaces.
 	DefaultNetworkInterfaces []proto.FirecrackerNetworkInterface `json:"default_network_interfaces"`
-	JailerConfig             JailerConfig                        `json:"jailer"`
+	// ShimBaseDir is the base directory which shim dirs will be created for each
+	// VM. In addition to this if jailing is enabled the jail will also use this
+	// directory.
+	ShimBaseDir  string       `json:"shim_base_dir"`
+	JailerConfig JailerConfig `json:"jailer"`
 }
 
 // JailerConfig houses a set of configurable values for jailing
@@ -81,6 +85,7 @@ func LoadConfig(path string) (*Config, error) {
 		RootDrive:       defaultRootfsPath,
 		CPUCount:        defaultCPUCount,
 		CPUTemplate:     string(defaultCPUTemplate),
+		ShimBaseDir:     defaultShimBaseDir,
 	}
 
 	if err := json.Unmarshal(data, cfg); err != nil {
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 0f5d12f..365145f 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -241,6 +241,8 @@ configuration file has the following fields:
   a VM with if no list of network interfaces is provided with a CreateVM call. Defaults
   to an empty list. The structure of the items in the list is the same as the Go API
   FirecrackerNetworkInterface defined [in protobuf here](../proto/types.proto).
+* `shim_base_dir` - (optional) Set the path to which Firecracker will run the
+  shim from. Defaults to /var/lib/firecracker-containerd/shim-base
 
 <details>
 <summary>A reasonable example configuration</summary>
@@ -261,6 +263,12 @@ configuration file has the following fields:
 
 ## Usage
 
+Ensure that /var/lib/firecracker-containerd exists as the default shim base
+directory used is /var/lib/firecracker-containerd/shim-base
+```bash
+mkdir -p /var/lib/firecracker-containerd
+```
+
 Start containerd
 
 ```bash
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 914b014..0f525bc 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -86,7 +86,9 @@ curl -fsSL -o hello-vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/hello
 # separate storage from the default containerd binary
 sudo mkdir -p /etc/firecracker-containerd
 sudo mkdir -p /var/lib/firecracker-containerd/containerd
-sudo mkdir -p /run/firecracker-containerd
+# Create the shim base directory for which firecracker-containerd will run the
+# shim from
+sudo mkdir -p /var/lib/firecracker-containerd
 sudo tee /etc/firecracker-containerd/config.toml <<EOF
 disabled_plugins = ["cri"]
 root = "/var/lib/firecracker-containerd/containerd"
@@ -96,7 +98,7 @@ state = "/run/firecracker-containerd"
 [proxy_plugins]
   [proxy_plugins.firecracker-naive]
     type = "snapshot"
-    address = "/var/run/firecracker-containerd/naive-snapshotter.sock"
+    address = "/var/lib/firecracker-containerd/naive-snapshotter.sock"
 
 [debug]
   level = "debug"
@@ -134,9 +136,9 @@ EOF
    foreground
 
 ```bash
-sudo mkdir -p /var/run/firecracker-containerd /var/lib/firecracker-containerd/naive
+sudo mkdir -p /var/lib/firecracker-containerd /var/lib/firecracker-containerd/naive
 sudo naive_snapshotter \
-     -address /var/run/firecracker-containerd/naive-snapshotter.sock \
+     -address /var/lib/firecracker-containerd/naive-snapshotter.sock \
      -path /var/lib/firecracker-containerd/naive \
      -debug
 ```
diff --git a/docs/shim-design.md b/docs/shim-design.md
index 837416b..c0145e4 100644
--- a/docs/shim-design.md
+++ b/docs/shim-design.md
@@ -140,7 +140,7 @@ How many Host Shims will we run? One per-VM? One for all VMs? How do we map a gi
 
 In this option, a single Host Shim process will handle all microVMs. Similar to our snapshotter implementations, it can just be started as a daemon alongside containerd.
 
-Every “shim start” routine will just return the pid/address of the single Host Shim process, which will be stored at a known location `/var/run/firecracker-containerd/{shim.pid, shim.sock}`.
+Every “shim start” routine will just return the pid/address of the single Host Shim process, which will be stored at a known location `/var/lib/firecracker-containerd/shim-base/{shim.pid, shim.sock}`.
 
 * **Pros**
     * **Lifecycle Complexity** - Can just start Host Shim when the EC2 host starts; do not have to spin up/down as workloads come in/out
@@ -154,7 +154,7 @@ Every “shim start” routine will just return the pid/address of the single Ho
 
 In this option, each VM will have a single Host Shim process dedicated to managing all the containers within it; there is a 1-to-1 mapping between `vm_id` and Host Shim process.
 
-Every “shim start” routine will return the pid/address of a Host Shim process for the given `vm_id`, which will be stored at a known location like `/var/run/firecracker-containerd/<vm_id>/{shim.pid, shim.sock}`. Race conditions related to creation of `<vm_id>/shim.pid` can be dealt with by using [open(2) with the O_EXCL flag.](http://man7.org/linux/man-pages/man2/open.2.html)
+Every “shim start” routine will return the pid/address of a Host Shim process for the given `vm_id`, which will be stored at a known location like `/var/lib/firecracker-containerd/shim-base/<vm_id>/{shim.pid, shim.sock}`. Race conditions related to creation of `<vm_id>/shim.pid` can be dealt with by using [open(2) with the O_EXCL flag.](http://man7.org/linux/man-pages/man2/open.2.html)
 
 * **Pros**
     * **Security/Resilience** - A single runtime shim manages a single customer's workload; if one runtime shim crashes (for whatever reason), others are unaffected
@@ -169,7 +169,7 @@ Every “shim start” routine will return the pid/address of a Host Shim proces
 
 In this option each container will have a single Host Shim process dedicated to managing it; there is a 1-to-1 mapping between `container_id` and Host Shim process.
 
-Every “shim start” routine will just continue running, returning to containerd a new shim.pid/shim.sock, which will be stored at a known location like `/var/run/firecracker-containerd/<container_id>/{shim.pid, shim.sock}`.
+Every “shim start” routine will just continue running, returning to containerd a new shim.pid/shim.sock, which will be stored at a known location like `/var/lib/firecracker-containerd/shim-base/<container_id>/{shim.pid, shim.sock}`.
 
 * **Pros**
     * **Security/Resilience** - A single Host Shim manages not just a single customer's workload but also a single container within that workload; if one Host Shim crashes (for whatever reason), other customers and other containers in the VM are unaffected
@@ -246,7 +246,7 @@ The proposed solution is to
 ![firecracker-containerd shim flow diagram](img/firecracker-containerd-shim-flow.png)
 
 1. The orchestrator sends a CreateVM request with a given `vm_id` to our containerd GRPC plugin
-2. Our plugin creates the VM, placing the firecracker socket at path `/var/run/firecracker-containerd/<vm_id>/firecracker.sock`
+2. Our plugin creates the VM, placing the firecracker socket at path `/var/lib/firecracker-containerd/shim-base/<vm_id>/firecracker.sock`
     1. The single Guest Shim is started with the VM, it either listens on a vsock addr or begins trying to connect to the host (depending on what our final vsock implementation ends up looking like; out of scope for this doc)
 3. The plugin returns the CreateVM response back to the orchestrator
 4. The orchestrator creates the Container definition, including both the `container_id` and `vm_id` in the request
diff --git a/firecracker-control/local.go b/firecracker-control/local.go
index 02aed82..d50b8e8 100644
--- a/firecracker-control/local.go
+++ b/firecracker-control/local.go
@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	fcclient "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	fcShim "github.com/firecracker-microvm/firecracker-containerd/internal/shim"
@@ -62,6 +63,7 @@ func init() {
 type local struct {
 	containerdAddress string
 	logger            *logrus.Entry
+	config            *config.Config
 }
 
 func newLocal(ic *plugin.InitContext) (*local, error) {
@@ -69,9 +71,15 @@ func newLocal(ic *plugin.InitContext) (*local, error) {
 		return nil, errors.Wrapf(err, "failed to create root directory: %s", ic.Root)
 	}
 
+	cfg, err := config.LoadConfig("")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load config")
+	}
+
 	return &local{
 		containerdAddress: ic.Address,
 		logger:            log.G(ic.Context),
+		config:            cfg,
 	}, nil
 }
 
@@ -117,8 +125,12 @@ func (s *local) CreateVM(requestCtx context.Context, req *proto.CreateVMRequest)
 
 	// If we're here, there is no pre-existing shim for this VMID, so we spawn a new one
 	defer shimSocket.Close()
+	if err := os.Mkdir(s.config.ShimBaseDir, 0700); err != nil && !os.IsExist(err) {
+		s.logger.WithError(err).Error()
+		return nil, errors.Wrapf(err, "failed to make shim base directory: %s", s.config.ShimBaseDir)
+	}
 
-	shimDir, err := vm.ShimDir(ns, id)
+	shimDir, err := vm.ShimDir(s.config.ShimBaseDir, ns, id)
 	if err != nil {
 		err = errors.Wrapf(err, "failed to build shim path")
 		s.logger.WithError(err).Error()
@@ -310,7 +322,7 @@ func (s *local) newShim(ns, vmID, containerdAddress string, shimSocket *net.Unix
 
 	cmd := exec.Command(internal.ShimBinaryName, args...)
 
-	shimDir, err := vm.ShimDir(ns, vmID)
+	shimDir, err := vm.ShimDir(s.config.ShimBaseDir, ns, vmID)
 	if err != nil {
 		err = errors.Wrap(err, "failed to create shim dir")
 		logger.WithError(err).Error()
diff --git a/internal/vm/dir.go b/internal/vm/dir.go
index e8c3c7d..d964381 100644
--- a/internal/vm/dir.go
+++ b/internal/vm/dir.go
@@ -22,16 +22,17 @@ import (
 	"github.com/containerd/containerd/identifiers"
 	"github.com/containerd/containerd/runtime/v2/shim"
 	"github.com/containerd/fifo"
-	"github.com/firecracker-microvm/firecracker-containerd/internal"
-	"github.com/firecracker-microvm/firecracker-containerd/internal/bundle"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
+
+	"github.com/firecracker-microvm/firecracker-containerd/internal"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/bundle"
 )
 
 // ShimDir holds files, sockets and FIFOs scoped to a single shim managing the
 // VM with the given VMID. It is unique per-VM and containerd namespace.
-func ShimDir(namespace, vmID string) (Dir, error) {
-	return shimDir("/var/run/firecracker-containerd/", namespace, vmID)
+func ShimDir(shimBaseDir, namespace, vmID string) (Dir, error) {
+	return shimDir(shimBaseDir, namespace, vmID)
 }
 
 func shimDir(varRunDir, namespace, vmID string) (Dir, error) {
diff --git a/runtime/helpers.go b/runtime/helpers.go
index 63de86d..fbae9bc 100644
--- a/runtime/helpers.go
+++ b/runtime/helpers.go
@@ -22,10 +22,15 @@ import (
 	"github.com/firecracker-microvm/firecracker-go-sdk"
 	models "github.com/firecracker-microvm/firecracker-go-sdk/client/models"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 )
 
-func machineConfigurationFromProto(cfg *Config, req *proto.FirecrackerMachineConfiguration) models.MachineConfiguration {
+const (
+	defaultMemSizeMb = 128
+)
+
+func machineConfigurationFromProto(cfg *config.Config, req *proto.FirecrackerMachineConfiguration) models.MachineConfiguration {
 	config := models.MachineConfiguration{
 		CPUTemplate: models.CPUTemplate(cfg.CPUTemplate),
 		VcpuCount:   firecracker.Int64(int64(cfg.CPUCount)),
diff --git a/runtime/jailer.go b/runtime/jailer.go
index ba4c83f..05f55c5 100644
--- a/runtime/jailer.go
+++ b/runtime/jailer.go
@@ -18,8 +18,10 @@ import (
 	"os"
 
 	"github.com/firecracker-microvm/firecracker-go-sdk"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 )
@@ -47,7 +49,7 @@ type jailer interface {
 	// BuildJailedMachine will modify the firecracker.Config and provide
 	// firecracker.Opt to be passed into firecracker.NewMachine which will allow
 	// for the VM to be jailed.
-	BuildJailedMachine(cfg *Config, machineCfg *firecracker.Config, vmID string) ([]firecracker.Opt, error)
+	BuildJailedMachine(cfg *config.Config, machineCfg *firecracker.Config, vmID string) ([]firecracker.Opt, error)
 	// ExposeFileToJail will expose the given file to the jailed filesystem, including
 	// regular files and block devices. An error is returned if provided a path to a file
 	// with type that is not supported.
@@ -81,6 +83,10 @@ func newJailer(
 		return newNoopJailer(ctx, l, service.shimDir), nil
 	}
 
+	if err := os.MkdirAll(ociBundlePath, 0700); err != nil {
+		return nil, errors.Wrapf(err, "failed to create oci bundle path: %s", ociBundlePath)
+	}
+
 	l := logger.WithField("jailer", "runc")
 	config := runcJailerConfig{
 		OCIBundlePath: ociBundlePath,
diff --git a/runtime/noop_jailer.go b/runtime/noop_jailer.go
index 6513e0a..56a2cab 100644
--- a/runtime/noop_jailer.go
+++ b/runtime/noop_jailer.go
@@ -16,9 +16,11 @@ package main
 import (
 	"context"
 
-	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
 	"github.com/firecracker-microvm/firecracker-go-sdk"
 	"github.com/sirupsen/logrus"
+
+	"github.com/firecracker-microvm/firecracker-containerd/config"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
 )
 
 // noopJailer is a jailer that returns only successful responses and performs
@@ -37,7 +39,7 @@ func newNoopJailer(ctx context.Context, logger *logrus.Entry, shimDir vm.Dir) no
 	}
 }
 
-func (j noopJailer) BuildJailedMachine(cfg *Config, machineConfig *firecracker.Config, vmID string) ([]firecracker.Opt, error) {
+func (j noopJailer) BuildJailedMachine(cfg *config.Config, machineConfig *firecracker.Config, vmID string) ([]firecracker.Opt, error) {
 	if len(cfg.FirecrackerBinaryPath) == 0 {
 		return []firecracker.Opt{}, nil
 	}
diff --git a/runtime/runc_jailer.go b/runtime/runc_jailer.go
index 22c092c..296d000 100644
--- a/runtime/runc_jailer.go
+++ b/runtime/runc_jailer.go
@@ -31,6 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
 )
@@ -114,7 +115,7 @@ func (j *runcJailer) JailPath() vm.Dir {
 // BuildJailedMachine will return the needed options for a jailed Firecracker
 // instance. In addition, some configuration values will be overwritten to the
 // jailed values, like SocketPath in the machineConfig.
-func (j *runcJailer) BuildJailedMachine(cfg *Config, machineConfig *firecracker.Config, vmID string) ([]firecracker.Opt, error) {
+func (j *runcJailer) BuildJailedMachine(cfg *config.Config, machineConfig *firecracker.Config, vmID string) ([]firecracker.Opt, error) {
 	handler := j.BuildJailedRootHandler(cfg, machineConfig, vmID)
 	fifoHandler := j.BuildLinkFifoHandler()
 	// Build a new client since BuildJailedRootHandler modifies the socket path value.
@@ -146,7 +147,7 @@ func (j *runcJailer) BuildJailedMachine(cfg *Config, machineConfig *firecracker.
 
 // BuildJailedRootHandler will populate the jail with the necessary files, which may be
 // device nodes, hard links, and/or bind-mount targets
-func (j *runcJailer) BuildJailedRootHandler(cfg *Config, machineConfig *firecracker.Config, vmID string) firecracker.Handler {
+func (j *runcJailer) BuildJailedRootHandler(cfg *config.Config, machineConfig *firecracker.Config, vmID string) firecracker.Handler {
 	ociBundlePath := j.OCIBundlePath()
 	rootPath := j.RootPath()
 	machineConfig.SocketPath = filepath.Join(rootPath, "api.socket")
@@ -379,7 +380,7 @@ func (j *runcJailer) jailerCommand(containerName string, isDebug bool) *exec.Cmd
 }
 
 // overwriteConfig will set the proper default values if a field had not been set.
-func (j *runcJailer) overwriteConfig(cfg *Config, machineConfig *firecracker.Config, socketPath, configPath string) error {
+func (j *runcJailer) overwriteConfig(cfg *config.Config, machineConfig *firecracker.Config, socketPath, configPath string) error {
 	spec := j.configSpec
 	if spec.Process.User.UID != 0 ||
 		spec.Process.User.GID != 0 {
@@ -435,7 +436,7 @@ func (j runcJailer) CgroupPath() string {
 
 // setDefaultConfigValues will process the spec file provided and allow any
 // empty/zero values to be replaced with default values.
-func (j *runcJailer) setDefaultConfigValues(cfg *Config, socketPath string, spec specs.Spec) specs.Spec {
+func (j *runcJailer) setDefaultConfigValues(cfg *config.Config, socketPath string, spec specs.Spec) specs.Spec {
 	if spec.Process == nil {
 		spec.Process = &specs.Process{}
 	}
diff --git a/runtime/service.go b/runtime/service.go
index 2b207ca..7ba86c4 100644
--- a/runtime/service.go
+++ b/runtime/service.go
@@ -20,6 +20,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime/debug"
 	"strconv"
 	"strings"
@@ -53,6 +54,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/eventbridge"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	"github.com/firecracker-microvm/firecracker-containerd/internal/bundle"
@@ -117,7 +119,7 @@ type service struct {
 	vmID    string
 	shimDir vm.Dir
 
-	config *Config
+	config *config.Config
 
 	// vmReady is closed once CreateVM has been successfully called
 	vmReady                  chan struct{}
@@ -147,18 +149,18 @@ func shimOpts(shimCtx context.Context) (*shim.Opts, error) {
 
 // NewService creates new runtime shim.
 func NewService(shimCtx context.Context, id string, remotePublisher shim.Publisher, shimCancel func()) (shim.Shim, error) {
-	config, err := LoadConfig("")
+	cfg, err := config.LoadConfig("")
 	if err != nil {
 		return nil, err
 	}
 
-	if !config.Debug {
+	if !cfg.Debug {
 		opts, err := shimOpts(shimCtx)
 		if err != nil {
 			return nil, err
 		}
 
-		config.Debug = opts.Debug
+		cfg.Debug = opts.Debug
 	}
 
 	namespace, ok := namespaces.Namespace(shimCtx)
@@ -172,13 +174,13 @@ func NewService(shimCtx context.Context, id string, remotePublisher shim.Publish
 	if vmID != "" {
 		logger = logger.WithField("vmID", vmID)
 
-		shimDir, err = vm.ShimDir(namespace, vmID)
+		shimDir, err = vm.ShimDir(cfg.ShimBaseDir, namespace, vmID)
 		if err != nil {
 			return nil, errors.Wrap(err, "invalid shim directory")
 		}
 	}
 
-	if config.Debug {
+	if cfg.Debug {
 		logrus.SetLevel(logrus.DebugLevel)
 		logger.Logger.SetLevel(logrus.DebugLevel)
 	}
@@ -195,7 +197,7 @@ func NewService(shimCtx context.Context, id string, remotePublisher shim.Publish
 		vmID:    vmID,
 		shimDir: shimDir,
 
-		config: config,
+		config: cfg,
 
 		vmReady: make(chan struct{}),
 		jailer:  newNoopJailer(shimCtx, logger, shimDir),
@@ -471,7 +473,7 @@ func (s *service) createVM(requestCtx context.Context, request *proto.CreateVMRe
 	}()
 
 	s.logger.Info("creating new VM")
-	s.jailer, err = newJailer(s.shimCtx, s.logger, string(s.shimDir), s, request)
+	s.jailer, err = newJailer(s.shimCtx, s.logger, filepath.Join(s.config.ShimBaseDir, s.vmID), s, request)
 	if err != nil {
 		return errors.Wrap(err, "failed to create jailer")
 	}
diff --git a/tools/docker/do_not_edit_for_firecracker-control.config.json b/tools/docker/do_not_edit_for_firecracker-control.config.json
new file mode 100644
index 0000000..d4fb320
--- /dev/null
+++ b/tools/docker/do_not_edit_for_firecracker-control.config.json
@@ -0,0 +1,3 @@
+{
+  "shim_base_dir": "/srv/firecracker_containerd_tests"
+}


Test patch:
diff --git a/runtime/config.json.example b/config/config.json.example
similarity index 100%
rename from runtime/config.json.example
rename to config/config.json.example
diff --git a/runtime/config_test.go b/config/config_test.go
similarity index 99%
rename from runtime/config_test.go
rename to config/config_test.go
index a6d8209..bf577a1 100644
--- a/runtime/config_test.go
+++ b/config/config_test.go
@@ -11,7 +11,7 @@
 // express or implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package main
+package config
 
 import (
 	"fmt"
diff --git a/runtime/cni_integ_test.go b/runtime/cni_integ_test.go
index 0029889..095b6eb 100644
--- a/runtime/cni_integ_test.go
+++ b/runtime/cni_integ_test.go
@@ -33,6 +33,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 	fccontrol "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc"
@@ -360,8 +361,8 @@ func writeCNIConf(path, chainedPluginName, networkName, nameserver string) error
 }`, networkName, nameserver, chainedPluginName)), 0644)
 }
 
-func withDefaultNetwork() func(c *Config) {
-	return func(c *Config) {
+func withDefaultNetwork() func(c *config.Config) {
+	return func(c *config.Config) {
 		c.DefaultNetworkInterfaces = []proto.FirecrackerNetworkInterface{
 			{
 				CNIConfig: &proto.CNIConfiguration{
diff --git a/runtime/helpers_test.go b/runtime/helpers_test.go
index 32dc890..73b4617 100644
--- a/runtime/helpers_test.go
+++ b/runtime/helpers_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 )
 
@@ -35,13 +36,13 @@ const (
 func TestMachineConfigurationFromProto(t *testing.T) {
 	testcases := []struct {
 		name                  string
-		config                *Config
+		config                *config.Config
 		proto                 *proto.FirecrackerMachineConfiguration
 		expectedMachineConfig models.MachineConfiguration
 	}{
 		{
 			name:   "ProtoOnly",
-			config: &Config{},
+			config: &config.Config{},
 			proto: &proto.FirecrackerMachineConfiguration{
 				CPUTemplate: string(models.CPUTemplateC3),
 				VcpuCount:   vcpuCount,
@@ -57,7 +58,7 @@ func TestMachineConfigurationFromProto(t *testing.T) {
 		},
 		{
 			name: "ConfigOnly",
-			config: &Config{
+			config: &config.Config{
 				CPUTemplate: "C3",
 				CPUCount:    vcpuCount,
 			},
@@ -71,7 +72,7 @@ func TestMachineConfigurationFromProto(t *testing.T) {
 		},
 		{
 			name: "NilProto",
-			config: &Config{
+			config: &config.Config{
 				CPUTemplate: "C3",
 				CPUCount:    vcpuCount,
 			},
@@ -84,7 +85,7 @@ func TestMachineConfigurationFromProto(t *testing.T) {
 		},
 		{
 			name: "Overrides",
-			config: &Config{
+			config: &config.Config{
 				CPUTemplate: "T2",
 				CPUCount:    vcpuCount + 1,
 			},
diff --git a/runtime/integ_test.go b/runtime/integ_test.go
index 8d3748d..5c931db 100644
--- a/runtime/integ_test.go
+++ b/runtime/integ_test.go
@@ -22,13 +22,16 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
-	"github.com/firecracker-microvm/firecracker-containerd/internal"
 	"github.com/pkg/errors"
+
+	"github.com/firecracker-microvm/firecracker-containerd/config"
+	"github.com/firecracker-microvm/firecracker-containerd/internal"
 )
 
 const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
+const shimBaseDir = "/srv/firecracker_containerd_tests"
 
-var defaultRuntimeConfig = Config{
+var defaultRuntimeConfig = config.Config{
 	FirecrackerBinaryPath: "/usr/local/bin/firecracker",
 	KernelImagePath:       "/var/lib/firecracker-containerd/runtime/default-vmlinux.bin",
 	KernelArgs:            "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init",
@@ -37,6 +40,7 @@ var defaultRuntimeConfig = Config{
 	CPUTemplate:           "T2",
 	LogLevel:              "Debug",
 	Debug:                 true,
+	ShimBaseDir:           shimBaseDir,
 }
 
 func defaultSnapshotterName() string {
@@ -48,7 +52,7 @@ func defaultSnapshotterName() string {
 	return name
 }
 
-func prepareIntegTest(t *testing.T, options ...func(*Config)) {
+func prepareIntegTest(t *testing.T, options ...func(*config.Config)) {
 	t.Helper()
 
 	internal.RequiresIsolation(t)
@@ -59,13 +63,13 @@ func prepareIntegTest(t *testing.T, options ...func(*Config)) {
 	}
 }
 
-func writeRuntimeConfig(options ...func(*Config)) error {
+func writeRuntimeConfig(options ...func(*config.Config)) error {
 	config := defaultRuntimeConfig
 	for _, option := range options {
 		option(&config)
 	}
 
-	file, err := os.Create(runtimeConfigPath)
+	file, err := os.OpenFile(runtimeConfigPath, os.O_CREATE|os.O_WRONLY, 0600)
 	if err != nil {
 		return err
 	}
@@ -84,8 +88,8 @@ func writeRuntimeConfig(options ...func(*Config)) error {
 	return nil
 }
 
-func withJailer() func(*Config) {
-	return func(c *Config) {
+func withJailer() func(*config.Config) {
+	return func(c *config.Config) {
 		c.JailerConfig.RuncBinaryPath = "/usr/local/bin/runc"
 	}
 }
diff --git a/runtime/runc_jailer_test.go b/runtime/runc_jailer_test.go
index a879c68..b36ff48 100644
--- a/runtime/runc_jailer_test.go
+++ b/runtime/runc_jailer_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
 )
 
@@ -53,17 +54,17 @@ func TestBuildJailedRootHandler_Isolated(t *testing.T) {
 	defer firecrackerFd.Close()
 
 	l := logrus.NewEntry(logrus.New())
-	config := runcJailerConfig{
+	runcConfig := runcJailerConfig{
 		OCIBundlePath: dir,
 		RuncBinPath:   "bin-path",
 		UID:           123,
 		GID:           456,
 	}
 	vmID := "foo"
-	jailer, err := newRuncJailer(context.Background(), l, vmID, config)
+	jailer, err := newRuncJailer(context.Background(), l, vmID, runcConfig)
 	require.NoError(t, err, "failed to create runc jailer")
 
-	cfg := Config{
+	cfg := config.Config{
 		FirecrackerBinaryPath: firecrackerPath,
 		KernelImagePath:       kernelImagePath,
 		RootDrive:             rootDrivePath,
diff --git a/runtime/service_integ_test.go b/runtime/service_integ_test.go
index f6af8d8..7e956d6 100644
--- a/runtime/service_integ_test.go
+++ b/runtime/service_integ_test.go
@@ -43,6 +43,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/firecracker-microvm/firecracker-containerd/config"
 	_ "github.com/firecracker-microvm/firecracker-containerd/firecracker-control"
 	fcClient "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
@@ -65,10 +66,8 @@ const (
 
 	defaultVMRootfsPath = "/var/lib/firecracker-containerd/runtime/default-rootfs.img"
 	defaultVMNetDevName = "eth0"
-	varRunDir           = "/run/firecracker-containerd"
-
-	numberOfVmsEnvName = "NUMBER_OF_VMS"
-	defaultNumberOfVms = 5
+	numberOfVmsEnvName  = "NUMBER_OF_VMS"
+	defaultNumberOfVms  = 5
 )
 
 // Images are presumed by the isolated tests to have already been pulled
@@ -172,10 +171,12 @@ func TestShimExitsUponContainerDelete_Isolated(t *testing.T) {
 		err = internal.WaitForPidToExit(testCtx, time.Second, shimProcess.Pid)
 		require.NoError(t, err, "failed waiting for shim process \"%s\" to exit", shimProcessName)
 
-		namespaceVarRunDir := filepath.Join(varRunDir, namespaces.Default)
-		varRunFCContents, err := ioutil.ReadDir(namespaceVarRunDir)
-		require.NoError(t, err, `failed to list directory "%s"`, namespaceVarRunDir)
-		require.Len(t, varRunFCContents, 0, "expect %s to be cleared after shims shutdown", namespaceVarRunDir)
+		cfg, err := config.LoadConfig("")
+		require.NoError(t, err, "failed to load config")
+		namespaceShimBaseDir := filepath.Join(cfg.ShimBaseDir, namespaces.Default)
+		varRunFCContents, err := ioutil.ReadDir(namespaceShimBaseDir)
+		require.NoError(t, err, `failed to list directory "%s"`, namespaceShimBaseDir)
+		require.Len(t, varRunFCContents, 0, "expect %s to be cleared after shims shutdown", namespaceShimBaseDir)
 	case err = <-exitEventErrCh:
 		require.Fail(t, "unexpected error", "unexpectedly received on task exit error channel: %s", err.Error())
 	case <-testCtx.Done():
@@ -311,7 +312,15 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 				containerWg.Add(1)
 				go func(containerID int) {
 					defer containerWg.Done()
-					testMultipleExecs(ctx, t, vmID, containerID, client, image, jailerConfig, resp.CgroupPath)
+					testMultipleExecs(
+						ctx,
+						t,
+						vmID,
+						containerID,
+						client, image,
+						jailerConfig,
+						resp.CgroupPath,
+					)
 				}(containerID)
 			}
 
@@ -323,9 +332,12 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 			vmInfoResp, err := fcClient.GetVMInfo(ctx, &proto.GetVMInfoRequest{VMID: strconv.Itoa(vmID)})
 			require.NoError(t, err, "failed to get VM Info for VM %d", vmID)
 			require.Equal(t, vmInfoResp.VMID, strconv.Itoa(vmID))
-			require.Equal(t, vmInfoResp.SocketPath, filepath.Join(varRunDir, defaultNamespace, strconv.Itoa(vmID), "firecracker.sock"))
-			require.Equal(t, vmInfoResp.LogFifoPath, filepath.Join(varRunDir, defaultNamespace, strconv.Itoa(vmID), "fc-logs.fifo"))
-			require.Equal(t, vmInfoResp.MetricsFifoPath, filepath.Join(varRunDir, defaultNamespace, strconv.Itoa(vmID), "fc-metrics.fifo"))
+
+			cfg, err := config.LoadConfig("")
+			require.NoError(t, err, "failed to load config")
+			require.Equal(t, vmInfoResp.SocketPath, filepath.Join(cfg.ShimBaseDir, defaultNamespace, strconv.Itoa(vmID), "firecracker.sock"))
+			require.Equal(t, vmInfoResp.LogFifoPath, filepath.Join(cfg.ShimBaseDir, defaultNamespace, strconv.Itoa(vmID), "fc-logs.fifo"))
+			require.Equal(t, vmInfoResp.MetricsFifoPath, filepath.Join(cfg.ShimBaseDir, defaultNamespace, strconv.Itoa(vmID), "fc-metrics.fifo"))
 
 			// just verify that updating the metadata doesn't return an error, a separate test case is needed
 			// to very the MMDS update propagates to the container correctly
@@ -345,7 +357,16 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 	vmWg.Wait()
 }
 
-func testMultipleExecs(ctx context.Context, t *testing.T, vmID int, containerID int, client *containerd.Client, image containerd.Image, jailerConfig *proto.JailerConfig, cgroupPath string) {
+func testMultipleExecs(
+	ctx context.Context,
+	t *testing.T,
+	vmID int,
+	containerID int,
+	client *containerd.Client,
+	image containerd.Image,
+	jailerConfig *proto.JailerConfig,
+	cgroupPath string,
+) {
 	vmIDStr := strconv.Itoa(vmID)
 	testTimeout := 600 * time.Second
 
@@ -406,12 +427,9 @@ func testMultipleExecs(ctx context.Context, t *testing.T, vmID int, containerID
 	close(execStdouts)
 
 	if jailerConfig != nil {
-		shimDir, err := vm.ShimDir("default", strconv.Itoa(vmID))
-		require.NoError(t, err, "failed to get shim dir")
-
 		jailer := &runcJailer{
 			Config: runcJailerConfig{
-				OCIBundlePath: string(shimDir),
+				OCIBundlePath: filepath.Join(shimBaseDir, vmIDStr),
 			},
 			vmID: vmIDStr,
 		}
@@ -509,6 +527,9 @@ func getMountNamespace(ctx context.Context, t *testing.T, client *containerd.Cli
 func TestLongUnixSocketPath_Isolated(t *testing.T) {
 	prepareIntegTest(t)
 
+	cfg, err := config.LoadConfig("")
+	require.NoError(t, err, "failed to load config")
+
 	// Verify that if the absolute path of the Firecracker unix sockets are longer
 	// than the max length enforced by the kernel (UNIX_PATH_MAX, usually 108), we
 	// don't fail (due to the internal implementation using relative paths).
@@ -532,7 +553,7 @@ func TestLongUnixSocketPath_Isolated(t *testing.T) {
 
 	// double-check that the sockets are at the expected path and that their absolute
 	// length exceeds 108 bytes
-	shimDir, err := vm.ShimDir("default", vmID)
+	shimDir, err := vm.ShimDir(cfg.ShimBaseDir, "default", vmID)
 	require.NoError(t, err, "failed to get shim dir")
 
 	_, err = os.Stat(shimDir.FirecrackerSockPath())
@@ -745,8 +766,11 @@ func testCreateContainerWithSameName(t *testing.T, vmID string) {
 	_, err = os.Stat(containerPath)
 	require.True(t, os.IsNotExist(err))
 
+	cfg, err := config.LoadConfig("")
+	require.NoError(t, err, "failed to load config")
+
 	if len(vmID) != 0 {
-		shimPath := fmt.Sprintf("%s/default/%s/%s", varRunDir, vmID, containerName)
+		shimPath := fmt.Sprintf("%s/default/%s/%s", cfg.ShimBaseDir, vmID, containerName)
 		_, err = os.Stat(shimPath)
 		require.True(t, os.IsNotExist(err))
 	}
@@ -768,7 +792,7 @@ func testCreateContainerWithSameName(t *testing.T, vmID string) {
 	require.True(t, os.IsNotExist(err))
 
 	if len(vmID) != 0 {
-		shimPath := fmt.Sprintf("%s/default/%s/%s", varRunDir, vmID, containerName)
+		shimPath := fmt.Sprintf("%s/default/%s/%s", cfg.ShimBaseDir, vmID, containerName)
 		_, err = os.Stat(shimPath)
 		require.True(t, os.IsNotExist(err))
 	}
@@ -832,7 +856,7 @@ func TestStubDriveReserveAndReleaseByContainers_Isolated(t *testing.T) {
 }
 
 func TestDriveMount_Isolated(t *testing.T) {
-	prepareIntegTest(t, func(cfg *Config) {
+	prepareIntegTest(t, func(cfg *config.Config) {
 		cfg.JailerConfig.RuncBinaryPath = "/usr/local/bin/runc"
 	})
 
diff --git a/runtime/service_test.go b/runtime/service_test.go
index 48c5c84..a322526 100644
--- a/runtime/service_test.go
+++ b/runtime/service_test.go
@@ -21,13 +21,15 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
-	"github.com/firecracker-microvm/firecracker-containerd/proto"
 	"github.com/firecracker-microvm/firecracker-go-sdk"
 	models "github.com/firecracker-microvm/firecracker-go-sdk/client/models"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/firecracker-microvm/firecracker-containerd/config"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
+	"github.com/firecracker-microvm/firecracker-containerd/proto"
 )
 
 const (
@@ -40,14 +42,14 @@ func TestBuildVMConfiguration(t *testing.T) {
 	testcases := []struct {
 		name                   string
 		request                *proto.CreateVMRequest
-		config                 *Config
+		config                 *config.Config
 		expectedCfg            *firecracker.Config
 		expectedStubDriveCount int
 	}{
 		{
 			name:    "Only Config",
 			request: &proto.CreateVMRequest{},
-			config: &Config{
+			config: &config.Config{
 				KernelArgs:      "KERNEL ARGS",
 				KernelImagePath: "KERNEL IMAGE",
 				RootDrive:       "ROOT DRIVE",
@@ -88,7 +90,7 @@ func TestBuildVMConfiguration(t *testing.T) {
 					VcpuCount:   2,
 				},
 			},
-			config: &Config{},
+			config: &config.Config{},
 			expectedCfg: &firecracker.Config{
 				KernelArgs:      "REQUEST KERNEL ARGS",
 				KernelImagePath: "REQUEST KERNEL IMAGE",
@@ -123,7 +125,7 @@ func TestBuildVMConfiguration(t *testing.T) {
 					VcpuCount:   3,
 				},
 			},
-			config: &Config{
+			config: &config.Config{
 				KernelArgs:      "KERNEL ARGS",
 				KernelImagePath: "KERNEL IMAGE",
 				CPUTemplate:     "C3",
@@ -159,7 +161,7 @@ func TestBuildVMConfiguration(t *testing.T) {
 				},
 				MachineCfg: &proto.FirecrackerMachineConfiguration{},
 			},
-			config: &Config{
+			config: &config.Config{
 				KernelArgs:      "KERNEL ARGS",
 				KernelImagePath: "KERNEL IMAGE",
 				CPUTemplate:     "C3",
@@ -188,7 +190,7 @@ func TestBuildVMConfiguration(t *testing.T) {
 		{
 			name:    "Container Count affects StubDriveCount",
 			request: &proto.CreateVMRequest{ContainerCount: 2},
-			config: &Config{
+			config: &config.Config{
 				KernelArgs:      "KERNEL ARGS",
 				KernelImagePath: "KERNEL IMAGE",
 				RootDrive:       "ROOT DRIVE",
@@ -274,14 +276,14 @@ func TestDebugConfig(t *testing.T) {
 			name: "empty",
 			service: &service{
 				logger: logrus.NewEntry(logrus.New()),
-				config: &Config{},
+				config: &config.Config{},
 			},
 		},
 		{
 			name: "LogLevel set",
 			service: &service{
 				logger: logrus.NewEntry(logrus.New()),
-				config: &Config{
+				config: &config.Config{
 					LogLevel: "foo",
 				},
 			},
@@ -290,7 +292,7 @@ func TestDebugConfig(t *testing.T) {
 			name: "Debug set",
 			service: &service{
 				logger: logrus.NewEntry(logrus.New()),
-				config: &Config{
+				config: &config.Config{
 					Debug: true,
 				},
 			},
@@ -299,7 +301,7 @@ func TestDebugConfig(t *testing.T) {
 			name: "Both set",
 			service: &service{
 				logger: logrus.NewEntry(logrus.New()),
-				config: &Config{
+				config: &config.Config{
 					LogLevel: "foo",
 					Debug:    true,
 				},
diff --git a/tools/docker/Dockerfile.integ-test b/tools/docker/Dockerfile.integ-test
index a3a0311..ee22010 100644
--- a/tools/docker/Dockerfile.integ-test
+++ b/tools/docker/Dockerfile.integ-test
@@ -27,12 +27,14 @@ RUN mkdir -p \
     /var/run/firecracker-containerd \
     /src \
     /var/lib/firecracker-containerd/naive \
+    /srv/firecracker_containerd_tests \
     ${FICD_LOG_DIR}
 
 # Pull the images the tests need into the content store so we don't need internet
 # access during the tests themselves. This runs as a seperate step before the other
 # installs so we can minimize re-runs of the time-expensive downloading of images.
 COPY tools/docker/config.toml /etc/containerd/config.toml
+COPY tools/docker/do_not_edit_for_firecracker-control.config.json /etc/containerd/firecracker-runtime.json
 RUN --mount=type=bind,source=firecracker-control/cmd/containerd,target=/src \
   make -C /src install && \
   ln -sv /usr/local/bin/firecracker-containerd /usr/local/bin/containerd && \