Repository: opencontainers/runc
Pull request #2951 (2021-05-12 14:39:37)

cgroup2: devices filtering cleanup
This is my attempt at solving both #2797 and #2366. I'm not entirely sure if the behaviour I've implemented if a cgroup has more than one filter installed (detach all old filters, so that only the new one is effective) is quite correct, but it's necessary to fix the program leak problem (outside of switching to BPF pins entirely).

NOTE: I wanted to add some unit tests for our devicefilter code, but unfortunately `BPF_PROG_TEST_RUN` doesn't support cgroup programs at the moment. I would have to push some code upstream for that to happen (though I'll look into that next week).

<hr>

Changelog entry:

```
cgroup2: devices: rework the filter generation to produce consistent results with cgroupv1, and always clobber any existing eBPF program(s) to fix `runc update` and avoid leaking eBPF programs (resulting in errors when managing containers). 
```

Signed-off-by: Aleksa Sarai <cyphar@cyphar.com>

kolyshkin: A typo in the second commit message: `s/speciifc/specific/`.
cyphar: We also can't test the `runc update` behaviour because it turns out `runc update` doesn't support cgroup device limits... I tried adding support for it, but the behaviour is a bit weird because of the default allowed devices... Hmmm....
kolyshkin: > We also can't test the runc update behaviour because it turns out runc update doesn't support cgroup device limits... I tried adding support for it, but the behaviour is a bit weird because of the default allowed devices... Hmmm....

Yeah, `update.go` basically repeats specconv, a year ago I tried to reuse the specconv in update but got nowhere...

Maybe we can at least have a simple test case added to libcontainer/integration, which removes and re-adds a device allow rule say 100 times (and check that device access works as expected). Without this PR we should fail on 64th iteration due to eBPF progs limit reached.

Same for runc update -- we could have a stupid test calling `runc update` 100 times (without actually changing anything) but I am not sure if such test makes sense if we implement what's described in the previous paragraph.
cyphar: Yeah looking at it I think we would need to rethink the "default allow" device list behaviour if we ever want to add devices support to `runc update` (otherwise users will accidentally brick their containers when using it). I'll cook up a unit test on Monday.
cyphar: Seems as though the Microsoft PPA is down?
kolyshkin: CI restarted (looks like it's working today).
kolyshkin: CI restarted; GHA still has some issues :(

> E: Failed to fetch https://packages.microsoft.com/ubuntu/20.04/prod/dists/focal/main/binary-amd64/Packages.bz2  File has unexpected size (73402 != 73160). Mirror sync in progress? [IP: 104.42.185.173 443]
cyphar: CI passes again now. :smile_cat: 
kolyshkin: @AkihiroSuda @mrunalp PTAL

Patch:
diff --git a/libcontainer/cgroups/devices/devices_emulator.go b/libcontainer/cgroups/devices/devices_emulator.go
index 3572a5ea..6f29ef03 100644
--- a/libcontainer/cgroups/devices/devices_emulator.go
+++ b/libcontainer/cgroups/devices/devices_emulator.go
@@ -371,3 +371,12 @@ func (source *Emulator) Transition(target *Emulator) ([]*devices.Rule, error) {
 	}
 	return transitionRules, nil
 }
+
+// Rules returns the minimum set of rules necessary to convert a *deny-all*
+// cgroup to the emulated filter state (note that this is not the same as a
+// default cgroupv1 cgroup -- which is allow-all). This is effectively just a
+// wrapper around Transition() with the source emulator being an empty cgroup.
+func (e *Emulator) Rules() ([]*devices.Rule, error) {
+	defaultCgroup := &Emulator{defaultAllow: false}
+	return defaultCgroup.Transition(e)
+}
diff --git a/libcontainer/cgroups/ebpf/devicefilter/devicefilter.go b/libcontainer/cgroups/ebpf/devicefilter/devicefilter.go
index fcd3746e..10e575d2 100644
--- a/libcontainer/cgroups/ebpf/devicefilter/devicefilter.go
+++ b/libcontainer/cgroups/ebpf/devicefilter/devicefilter.go
@@ -11,6 +11,7 @@ import (
 	"strconv"
 
 	"github.com/cilium/ebpf/asm"
+	devicesemulator "github.com/opencontainers/runc/libcontainer/cgroups/devices"
 	"github.com/opencontainers/runc/libcontainer/devices"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
@@ -22,11 +23,44 @@ const (
 )
 
 // DeviceFilter returns eBPF device filter program and its license string
-func DeviceFilter(devices []*devices.Rule) (asm.Instructions, string, error) {
-	p := &program{}
+func DeviceFilter(rules []*devices.Rule) (asm.Instructions, string, error) {
+	// Generate the minimum ruleset for the device rules we are given. While we
+	// don't care about minimum transitions in cgroupv2, using the emulator
+	// gives us a guarantee that the behaviour of devices filtering is the same
+	// as cgroupv1, including security hardenings to avoid misconfiguration
+	// (such as punching holes in wildcard rules).
+	emu := new(devicesemulator.Emulator)
+	for _, rule := range rules {
+		if err := emu.Apply(*rule); err != nil {
+			return nil, "", err
+		}
+	}
+	cleanRules, err := emu.Rules()
+	if err != nil {
+		return nil, "", err
+	}
+
+	p := &program{
+		defaultAllow: emu.IsBlacklist(),
+	}
 	p.init()
-	for i := len(devices) - 1; i >= 0; i-- {
-		if err := p.appendDevice(devices[i]); err != nil {
+
+	for idx, rule := range cleanRules {
+		if rule.Type == devices.WildcardDevice {
+			// We can safely skip over wildcard entries because there should
+			// only be one (at most) at the very start to instruct cgroupv1 to
+			// go into allow-list mode. However we do double-check this here.
+			if idx != 0 || rule.Allow != emu.IsBlacklist() {
+				return nil, "", errors.Errorf("[internal error] emulated cgroupv2 devices ruleset had bad wildcard at idx %v (%s)", idx, rule.CgroupString())
+			}
+			continue
+		}
+		if rule.Allow == p.defaultAllow {
+			// There should be no rules which have an action equal to the
+			// default action, the emulator removes those.
+			return nil, "", errors.Errorf("[internal error] emulated cgroupv2 devices ruleset had no-op rule at idx %v (%s)", idx, rule.CgroupString())
+		}
+		if err := p.appendRule(rule); err != nil {
 			return nil, "", err
 		}
 	}
@@ -35,9 +69,9 @@ func DeviceFilter(devices []*devices.Rule) (asm.Instructions, string, error) {
 }
 
 type program struct {
-	insts       asm.Instructions
-	hasWildCard bool
-	blockID     int
+	insts        asm.Instructions
+	defaultAllow bool
+	blockID      int
 }
 
 func (p *program) init() {
@@ -67,39 +101,36 @@ func (p *program) init() {
 		asm.LoadMem(asm.R5, asm.R1, 8, asm.Word))
 }
 
-// appendDevice needs to be called from the last element of OCI linux.resources.devices to the head element.
-func (p *program) appendDevice(dev *devices.Rule) error {
+// appendRule rule converts an OCI rule to the relevant eBPF block and adds it
+// to the in-progress filter program. In order to operate properly, it must be
+// called with a "clean" rule list (generated by devices.Emulator.Rules() --
+// with any "a" rules removed).
+func (p *program) appendRule(rule *devices.Rule) error {
 	if p.blockID < 0 {
 		return errors.New("the program is finalized")
 	}
-	if p.hasWildCard {
-		// All entries after wildcard entry are ignored
-		return nil
-	}
 
 	bpfType := int32(-1)
 	hasType := true
-	switch dev.Type {
-	case 'c':
+	switch rule.Type {
+	case devices.CharDevice:
 		bpfType = int32(unix.BPF_DEVCG_DEV_CHAR)
-	case 'b':
+	case devices.BlockDevice:
 		bpfType = int32(unix.BPF_DEVCG_DEV_BLOCK)
-	case 'a':
-		hasType = false
 	default:
-		// if not specified in OCI json, typ is set to DeviceTypeAll
-		return errors.Errorf("invalid Type %q", string(dev.Type))
+		// We do not permit 'a', nor any other types we don't know about.
+		return errors.Errorf("invalid Type %q", string(rule.Type))
 	}
-	if dev.Major > math.MaxUint32 {
-		return errors.Errorf("invalid major %d", dev.Major)
+	if rule.Major > math.MaxUint32 {
+		return errors.Errorf("invalid major %d", rule.Major)
 	}
-	if dev.Minor > math.MaxUint32 {
-		return errors.Errorf("invalid minor %d", dev.Major)
+	if rule.Minor > math.MaxUint32 {
+		return errors.Errorf("invalid minor %d", rule.Major)
 	}
-	hasMajor := dev.Major >= 0 // if not specified in OCI json, major is set to -1
-	hasMinor := dev.Minor >= 0
+	hasMajor := rule.Major >= 0 // if not specified in OCI json, major is set to -1
+	hasMinor := rule.Minor >= 0
 	bpfAccess := int32(0)
-	for _, r := range dev.Permissions {
+	for _, r := range rule.Permissions {
 		switch r {
 		case 'r':
 			bpfAccess |= unix.BPF_DEVCG_ACC_READ
@@ -136,19 +167,16 @@ func (p *program) appendDevice(dev *devices.Rule) error {
 	if hasMajor {
 		p.insts = append(p.insts,
 			// if (R4 != major) goto next
-			asm.JNE.Imm(asm.R4, int32(dev.Major), nextBlockSym),
+			asm.JNE.Imm(asm.R4, int32(rule.Major), nextBlockSym),
 		)
 	}
 	if hasMinor {
 		p.insts = append(p.insts,
 			// if (R5 != minor) goto next
-			asm.JNE.Imm(asm.R5, int32(dev.Minor), nextBlockSym),
+			asm.JNE.Imm(asm.R5, int32(rule.Minor), nextBlockSym),
 		)
 	}
-	if !hasType && !hasAccess && !hasMajor && !hasMinor {
-		p.hasWildCard = true
-	}
-	p.insts = append(p.insts, acceptBlock(dev.Allow)...)
+	p.insts = append(p.insts, acceptBlock(rule.Allow)...)
 	// set blockSym to the first instruction we added in this iteration
 	p.insts[prevBlockLastIdx+1] = p.insts[prevBlockLastIdx+1].Sym(blockSym)
 	p.blockID++
@@ -156,14 +184,14 @@ func (p *program) appendDevice(dev *devices.Rule) error {
 }
 
 func (p *program) finalize() (asm.Instructions, error) {
-	if p.hasWildCard {
-		// acceptBlock with asm.Return() is already inserted
-		return p.insts, nil
+	var v int32
+	if p.defaultAllow {
+		v = 1
 	}
 	blockSym := "block-" + strconv.Itoa(p.blockID)
 	p.insts = append(p.insts,
-		// R0 <- 0
-		asm.Mov.Imm32(asm.R0, 0).Sym(blockSym),
+		// R0 <- v
+		asm.Mov.Imm32(asm.R0, v).Sym(blockSym),
 		asm.Return(),
 	)
 	p.blockID = -1
@@ -171,7 +199,7 @@ func (p *program) finalize() (asm.Instructions, error) {
 }
 
 func acceptBlock(accept bool) asm.Instructions {
-	v := int32(0)
+	var v int32
 	if accept {
 		v = 1
 	}
diff --git a/libcontainer/cgroups/ebpf/ebpf.go b/libcontainer/cgroups/ebpf/ebpf.go
index 3bee21cb..6066d08c 100644
--- a/libcontainer/cgroups/ebpf/ebpf.go
+++ b/libcontainer/cgroups/ebpf/ebpf.go
@@ -1,22 +1,83 @@
 package ebpf
 
 import (
+	"fmt"
+	"runtime"
+	"unsafe"
+
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/asm"
 	"github.com/cilium/ebpf/link"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
+func nilCloser() error {
+	return nil
+}
+
+func findAttachedCgroupDeviceFilters(dirFd int) ([]*ebpf.Program, error) {
+	type bpfAttrQuery struct {
+		TargetFd    uint32
+		AttachType  uint32
+		QueryType   uint32
+		AttachFlags uint32
+		ProgIds     uint64 // __aligned_u64
+		ProgCnt     uint32
+	}
+
+	// Currently you can only have 64 eBPF programs attached to a cgroup.
+	size := 64
+	retries := 0
+	for retries < 10 {
+		progIds := make([]uint32, size)
+		query := bpfAttrQuery{
+			TargetFd:   uint32(dirFd),
+			AttachType: uint32(unix.BPF_CGROUP_DEVICE),
+			ProgIds:    uint64(uintptr(unsafe.Pointer(&progIds[0]))),
+			ProgCnt:    uint32(len(progIds)),
+		}
+
+		// Fetch the list of program ids.
+		_, _, errno := unix.Syscall(unix.SYS_BPF,
+			uintptr(unix.BPF_PROG_QUERY),
+			uintptr(unsafe.Pointer(&query)),
+			unsafe.Sizeof(query))
+		size = int(query.ProgCnt)
+		runtime.KeepAlive(query)
+		if errno != 0 {
+			// On ENOSPC we get the correct number of programs.
+			if errno == unix.ENOSPC {
+				retries++
+				continue
+			}
+			return nil, fmt.Errorf("bpf_prog_query(BPF_CGROUP_DEVICE) failed: %w", errno)
+		}
+
+		// Convert the ids to program handles.
+		progIds = progIds[:size]
+		programs := make([]*ebpf.Program, len(progIds))
+		for idx, progId := range progIds {
+			program, err := ebpf.NewProgramFromID(ebpf.ProgramID(progId))
+			if err != nil {
+				return nil, fmt.Errorf("cannot fetch program from id: %w", err)
+			}
+			programs[idx] = program
+		}
+		runtime.KeepAlive(progIds)
+		return programs, nil
+	}
+
+	return nil, errors.New("could not get complete list of CGROUP_DEVICE programs")
+}
+
 // LoadAttachCgroupDeviceFilter installs eBPF device filter program to /sys/fs/cgroup/<foo> directory.
 //
 // Requires the system to be running in cgroup2 unified-mode with kernel >= 4.15 .
 //
 // https://github.com/torvalds/linux/commit/ebc614f687369f9df99828572b1d85a7c2de3d92
-func LoadAttachCgroupDeviceFilter(insts asm.Instructions, license string, dirFD int) (func() error, error) {
-	nilCloser := func() error {
-		return nil
-	}
+func LoadAttachCgroupDeviceFilter(insts asm.Instructions, license string, dirFd int) (func() error, error) {
 	// Increase `ulimit -l` limit to avoid BPF_PROG_LOAD error (#2167).
 	// This limit is not inherited into the container.
 	memlockLimit := &unix.Rlimit{
@@ -24,6 +85,11 @@ func LoadAttachCgroupDeviceFilter(insts asm.Instructions, license string, dirFD
 		Max: unix.RLIM_INFINITY,
 	}
 	_ = unix.Setrlimit(unix.RLIMIT_MEMLOCK, memlockLimit)
+	// Get the list of existing programs.
+	oldProgs, err := findAttachedCgroupDeviceFilters(dirFd)
+	if err != nil {
+		return nilCloser, err
+	}
 	spec := &ebpf.ProgramSpec{
 		Type:         ebpf.CGroupDevice,
 		Instructions: insts,
@@ -33,25 +99,49 @@ func LoadAttachCgroupDeviceFilter(insts asm.Instructions, license string, dirFD
 	if err != nil {
 		return nilCloser, err
 	}
+	// If there is only one old program, we can just replace it directly.
+	var replaceProg *ebpf.Program
+	if len(oldProgs) == 1 {
+		replaceProg = oldProgs[0]
+	}
 	err = link.RawAttachProgram(link.RawAttachProgramOptions{
-		Target:  dirFD,
+		Target:  dirFd,
 		Program: prog,
+		Replace: replaceProg,
 		Attach:  ebpf.AttachCGroupDevice,
 		Flags:   unix.BPF_F_ALLOW_MULTI,
 	})
 	if err != nil {
-		return nilCloser, errors.Wrap(err, "failed to call BPF_PROG_ATTACH (BPF_CGROUP_DEVICE, BPF_F_ALLOW_MULTI)")
+		return nilCloser, fmt.Errorf("failed to call BPF_PROG_ATTACH (BPF_CGROUP_DEVICE, BPF_F_ALLOW_MULTI): %w", err)
 	}
 	closer := func() error {
 		err = link.RawDetachProgram(link.RawDetachProgramOptions{
-			Target:  dirFD,
+			Target:  dirFd,
 			Program: prog,
 			Attach:  ebpf.AttachCGroupDevice,
 		})
 		if err != nil {
-			return errors.Wrap(err, "failed to call BPF_PROG_DETACH (BPF_CGROUP_DEVICE)")
+			return fmt.Errorf("failed to call BPF_PROG_DETACH (BPF_CGROUP_DEVICE): %w", err)
 		}
+		// TODO: Should we attach the old filters back in this case? Otherwise
+		//       we fail-open on a security feature, which is a bit scary.
 		return nil
 	}
+	// If there was more than one old program, give a warning (since this
+	// really shouldn't happen with runc-managed cgroups) and then detach all
+	// the old programs.
+	if len(oldProgs) > 1 {
+		logrus.Warnf("found more than one filter (%d) attached to a cgroup -- removing extra filters!", len(oldProgs))
+		for _, oldProg := range oldProgs {
+			err = link.RawDetachProgram(link.RawDetachProgramOptions{
+				Target:  dirFd,
+				Program: oldProg,
+				Attach:  ebpf.AttachCGroupDevice,
+			})
+			if err != nil {
+				return closer, fmt.Errorf("failed to call BPF_PROG_DETACH (BPF_CGROUP_DEVICE) on old filter program: %w", err)
+			}
+		}
+	}
 	return closer, nil
 }
diff --git a/libcontainer/cgroups/fs2/devices.go b/libcontainer/cgroups/fs2/devices.go
index fa9194b5..6b4b73a1 100644
--- a/libcontainer/cgroups/fs2/devices.go
+++ b/libcontainer/cgroups/fs2/devices.go
@@ -58,9 +58,6 @@ func setDevices(dirPath string, r *configs.Resources) error {
 	if r.SkipDevices {
 		return nil
 	}
-	// XXX: This is currently a white-list (but all callers pass a blacklist of
-	//      devices). This is bad for a whole variety of reasons, but will need
-	//      to be fixed with co-ordinated effort with downstreams.
 	insts, license, err := devicefilter.DeviceFilter(r.Devices)
 	if err != nil {
 		return err
@@ -70,17 +67,6 @@ func setDevices(dirPath string, r *configs.Resources) error {
 		return errors.Errorf("cannot get dir FD for %s", dirPath)
 	}
 	defer unix.Close(dirFD)
-	// XXX: This code is currently incorrect when it comes to updating an
-	//      existing cgroup with new rules (new rulesets are just appended to
-	//      the program list because this uses BPF_F_ALLOW_MULTI). If we didn't
-	//      use BPF_F_ALLOW_MULTI we could actually atomically swap the
-	//      programs.
-	//
-	//      The real issue is that BPF_F_ALLOW_MULTI makes it hard to have a
-	//      race-free blacklist because it acts as a whitelist by default, and
-	//      having a deny-everything program cannot be overridden by other
-	//      programs. You could temporarily insert a deny-everything program
-	//      but that would result in spurrious failures during updates.
 	if _, err := ebpf.LoadAttachCgroupDeviceFilter(insts, license, dirFD); err != nil {
 		if !canSkipEBPFError(r) {
 			return err
diff --git a/libcontainer/cgroups/systemd/common.go b/libcontainer/cgroups/systemd/common.go
index de69617e..e1733945 100644
--- a/libcontainer/cgroups/systemd/common.go
+++ b/libcontainer/cgroups/systemd/common.go
@@ -203,8 +203,7 @@ func generateDeviceProperties(rules []*devices.Rule) ([]systemdDbus.Property, er
 	// Now generate the set of rules we actually need to apply. Unlike the
 	// normal devices cgroup, in "strict" mode systemd defaults to a deny-all
 	// whitelist which is the default for devices.Emulator.
-	baseEmu := &cgroupdevices.Emulator{}
-	finalRules, err := baseEmu.Transition(configEmu)
+	finalRules, err := configEmu.Rules()
 	if err != nil {
 		return nil, errors.Wrap(err, "get simplified rules for systemd")
 	}


Test patch:
diff --git a/libcontainer/cgroups/ebpf/devicefilter/devicefilter_test.go b/libcontainer/cgroups/ebpf/devicefilter/devicefilter_test.go
index cfcaa203..a8fc562d 100644
--- a/libcontainer/cgroups/ebpf/devicefilter/devicefilter_test.go
+++ b/libcontainer/cgroups/ebpf/devicefilter/devicefilter_test.go
@@ -55,81 +55,81 @@ block-0:
 func TestDeviceFilter_BuiltInAllowList(t *testing.T) {
 	expected := `
 // load parameters into registers
-         0: LdXMemW dst: r2 src: r1 off: 0 imm: 0
-         1: And32Imm dst: r2 imm: 65535
-         2: LdXMemW dst: r3 src: r1 off: 0 imm: 0
-         3: RSh32Imm dst: r3 imm: 16
-         4: LdXMemW dst: r4 src: r1 off: 4 imm: 0
-         5: LdXMemW dst: r5 src: r1 off: 8 imm: 0
+        0: LdXMemW dst: r2 src: r1 off: 0 imm: 0
+        1: And32Imm dst: r2 imm: 65535
+        2: LdXMemW dst: r3 src: r1 off: 0 imm: 0
+        3: RSh32Imm dst: r3 imm: 16
+        4: LdXMemW dst: r4 src: r1 off: 4 imm: 0
+        5: LdXMemW dst: r5 src: r1 off: 8 imm: 0
 block-0:
-// tuntap (c, 10, 200, rwm, allow)
-         6: JNEImm dst: r2 off: -1 imm: 2 <block-1>
-         7: JNEImm dst: r4 off: -1 imm: 10 <block-1>
-         8: JNEImm dst: r5 off: -1 imm: 200 <block-1>
-         9: Mov32Imm dst: r0 imm: 1
-        10: Exit
+// (b, wildcard, wildcard, m, true)
+        6: JNEImm dst: r2 off: -1 imm: 1 <block-1>
+        7: Mov32Reg dst: r1 src: r3
+        8: And32Imm dst: r1 imm: 1
+        9: JNEReg dst: r1 off: -1 src: r3 <block-1>
+        10: Mov32Imm dst: r0 imm: 1
+        11: Exit
 block-1:
-        11: JNEImm dst: r2 off: -1 imm: 2 <block-2>
-        12: JNEImm dst: r4 off: -1 imm: 5 <block-2>
-        13: JNEImm dst: r5 off: -1 imm: 2 <block-2>
-        14: Mov32Imm dst: r0 imm: 1
-        15: Exit
+// (c, wildcard, wildcard, m, true)
+        12: JNEImm dst: r2 off: -1 imm: 2 <block-2>
+        13: Mov32Reg dst: r1 src: r3
+        14: And32Imm dst: r1 imm: 1
+        15: JNEReg dst: r1 off: -1 src: r3 <block-2>
+        16: Mov32Imm dst: r0 imm: 1
+        17: Exit
 block-2:
-// /dev/pts (c, 136, wildcard, rwm, true)
-        16: JNEImm dst: r2 off: -1 imm: 2 <block-3>
-        17: JNEImm dst: r4 off: -1 imm: 136 <block-3>
-        18: Mov32Imm dst: r0 imm: 1
-        19: Exit
+        18: JNEImm dst: r2 off: -1 imm: 2 <block-3>
+        19: JNEImm dst: r4 off: -1 imm: 1 <block-3>
+        20: JNEImm dst: r5 off: -1 imm: 3 <block-3>
+        21: Mov32Imm dst: r0 imm: 1
+        22: Exit
 block-3:
-        20: JNEImm dst: r2 off: -1 imm: 2 <block-4>
-        21: JNEImm dst: r4 off: -1 imm: 1 <block-4>
-        22: JNEImm dst: r5 off: -1 imm: 9 <block-4>
-        23: Mov32Imm dst: r0 imm: 1
-        24: Exit
+        23: JNEImm dst: r2 off: -1 imm: 2 <block-4>
+        24: JNEImm dst: r4 off: -1 imm: 1 <block-4>
+        25: JNEImm dst: r5 off: -1 imm: 5 <block-4>
+        26: Mov32Imm dst: r0 imm: 1
+        27: Exit
 block-4:
-        25: JNEImm dst: r2 off: -1 imm: 2 <block-5>
-        26: JNEImm dst: r4 off: -1 imm: 1 <block-5>
-        27: JNEImm dst: r5 off: -1 imm: 5 <block-5>
-        28: Mov32Imm dst: r0 imm: 1
-        29: Exit
+        28: JNEImm dst: r2 off: -1 imm: 2 <block-5>
+        29: JNEImm dst: r4 off: -1 imm: 1 <block-5>
+        30: JNEImm dst: r5 off: -1 imm: 7 <block-5>
+        31: Mov32Imm dst: r0 imm: 1
+        32: Exit
 block-5:
-        30: JNEImm dst: r2 off: -1 imm: 2 <block-6>
-        31: JNEImm dst: r4 off: -1 imm: 5 <block-6>
-        32: JNEImm dst: r5 off: -1 imm: 0 <block-6>
-        33: Mov32Imm dst: r0 imm: 1
-        34: Exit
+        33: JNEImm dst: r2 off: -1 imm: 2 <block-6>
+        34: JNEImm dst: r4 off: -1 imm: 1 <block-6>
+        35: JNEImm dst: r5 off: -1 imm: 8 <block-6>
+        36: Mov32Imm dst: r0 imm: 1
+        37: Exit
 block-6:
-        35: JNEImm dst: r2 off: -1 imm: 2 <block-7>
-        36: JNEImm dst: r4 off: -1 imm: 1 <block-7>
-        37: JNEImm dst: r5 off: -1 imm: 7 <block-7>
-        38: Mov32Imm dst: r0 imm: 1
-        39: Exit
+        38: JNEImm dst: r2 off: -1 imm: 2 <block-7>
+        39: JNEImm dst: r4 off: -1 imm: 1 <block-7>
+        40: JNEImm dst: r5 off: -1 imm: 9 <block-7>
+        41: Mov32Imm dst: r0 imm: 1
+        42: Exit
 block-7:
-        40: JNEImm dst: r2 off: -1 imm: 2 <block-8>
-        41: JNEImm dst: r4 off: -1 imm: 1 <block-8>
-        42: JNEImm dst: r5 off: -1 imm: 8 <block-8>
-        43: Mov32Imm dst: r0 imm: 1
-        44: Exit
+        43: JNEImm dst: r2 off: -1 imm: 2 <block-8>
+        44: JNEImm dst: r4 off: -1 imm: 5 <block-8>
+        45: JNEImm dst: r5 off: -1 imm: 0 <block-8>
+        46: Mov32Imm dst: r0 imm: 1
+        47: Exit
 block-8:
-        45: JNEImm dst: r2 off: -1 imm: 2 <block-9>
-        46: JNEImm dst: r4 off: -1 imm: 1 <block-9>
-        47: JNEImm dst: r5 off: -1 imm: 3 <block-9>
-        48: Mov32Imm dst: r0 imm: 1
-        49: Exit
+        48: JNEImm dst: r2 off: -1 imm: 2 <block-9>
+        49: JNEImm dst: r4 off: -1 imm: 5 <block-9>
+        50: JNEImm dst: r5 off: -1 imm: 2 <block-9>
+        51: Mov32Imm dst: r0 imm: 1
+        52: Exit
 block-9:
-// (b, wildcard, wildcard, m, true)
-        50: JNEImm dst: r2 off: -1 imm: 1 <block-10>
-        51: Mov32Reg dst: r1 src: r3
-        52: And32Imm dst: r1 imm: 1
-        53: JNEReg dst: r1 off: -1 src: r3 <block-10>
-        54: Mov32Imm dst: r0 imm: 1
-        55: Exit
+// tuntap (c, 10, 200, rwm, allow)
+        53: JNEImm dst: r2 off: -1 imm: 2 <block-10>
+        54: JNEImm dst: r4 off: -1 imm: 10 <block-10>
+        55: JNEImm dst: r5 off: -1 imm: 200 <block-10>
+        56: Mov32Imm dst: r0 imm: 1
+        57: Exit
 block-10:
-// (c, wildcard, wildcard, m, true)
-        56: JNEImm dst: r2 off: -1 imm: 2 <block-11>
-        57: Mov32Reg dst: r1 src: r3
-        58: And32Imm dst: r1 imm: 1
-        59: JNEReg dst: r1 off: -1 src: r3 <block-11>
+// /dev/pts (c, 136, wildcard, rwm, true)
+        58: JNEImm dst: r2 off: -1 imm: 2 <block-11>
+        59: JNEImm dst: r4 off: -1 imm: 136 <block-11>
         60: Mov32Imm dst: r0 imm: 1
         61: Exit
 block-11:
diff --git a/libcontainer/configs/validate/validator_test.go b/libcontainer/configs/validate/validator_test.go
index 327776b5..7d1ef0f6 100644
--- a/libcontainer/configs/validate/validator_test.go
+++ b/libcontainer/configs/validate/validator_test.go
@@ -154,7 +154,7 @@ func TestValidateSecurityWithoutNEWNS(t *testing.T) {
 
 func TestValidateUsernamespace(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 	config := &configs.Config{
 		Rootfs: "/var",
diff --git a/libcontainer/integration/checkpoint_test.go b/libcontainer/integration/checkpoint_test.go
index 9ec03b92..398a214b 100644
--- a/libcontainer/integration/checkpoint_test.go
+++ b/libcontainer/integration/checkpoint_test.go
@@ -41,7 +41,7 @@ func showFile(t *testing.T, fname string) error {
 
 func TestUsernsCheckpoint(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 	cmd := exec.Command("criu", "check", "--feature", "userns")
 	if err := cmd.Run(); err != nil {
diff --git a/libcontainer/integration/exec_test.go b/libcontainer/integration/exec_test.go
index 505006a8..26de5b64 100644
--- a/libcontainer/integration/exec_test.go
+++ b/libcontainer/integration/exec_test.go
@@ -29,7 +29,7 @@ func TestExecPS(t *testing.T) {
 
 func TestUsernsExecPS(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 	testExecPS(t, true)
 }
@@ -166,7 +166,7 @@ func TestRlimit(t *testing.T) {
 
 func TestUsernsRlimit(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 
 	testRlimit(t, true)
@@ -498,7 +498,7 @@ func TestFreeze(t *testing.T) {
 
 func TestSystemdFreeze(t *testing.T) {
 	if !systemd.IsRunningSystemd() {
-		t.Skip("Systemd is unsupported")
+		t.Skip("Test requires systemd.")
 	}
 	testFreeze(t, true)
 }
@@ -555,7 +555,7 @@ func TestCpuShares(t *testing.T) {
 
 func TestCpuSharesSystemd(t *testing.T) {
 	if !systemd.IsRunningSystemd() {
-		t.Skip("Systemd is unsupported")
+		t.Skip("Test requires systemd.")
 	}
 	testCpuShares(t, true)
 }
@@ -590,7 +590,7 @@ func TestPids(t *testing.T) {
 
 func TestPidsSystemd(t *testing.T) {
 	if !systemd.IsRunningSystemd() {
-		t.Skip("Systemd is unsupported")
+		t.Skip("Test requires systemd.")
 	}
 	testPids(t, true)
 }
@@ -664,7 +664,7 @@ func TestCgroupResourcesUnifiedErrorOnV1(t *testing.T) {
 
 func TestCgroupResourcesUnifiedErrorOnV1Systemd(t *testing.T) {
 	if !systemd.IsRunningSystemd() {
-		t.Skip("Systemd is unsupported")
+		t.Skip("Test requires systemd.")
 	}
 	testCgroupResourcesUnifiedErrorOnV1(t, true)
 }
@@ -699,7 +699,7 @@ func TestCgroupResourcesUnified(t *testing.T) {
 
 func TestCgroupResourcesUnifiedSystemd(t *testing.T) {
 	if !systemd.IsRunningSystemd() {
-		t.Skip("Systemd is unsupported")
+		t.Skip("Test requires systemd.")
 	}
 	testCgroupResourcesUnified(t, true)
 }
@@ -1720,7 +1720,7 @@ func TestInitJoinPID(t *testing.T) {
 
 func TestInitJoinNetworkAndUser(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 	if testing.Short() {
 		return
@@ -1861,7 +1861,7 @@ func TestTmpfsCopyUp(t *testing.T) {
 
 func TestCGROUPPrivate(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) {
-		t.Skip("cgroupns is unsupported")
+		t.Skip("Test requires cgroupns.")
 	}
 	if testing.Short() {
 		return
@@ -1890,7 +1890,7 @@ func TestCGROUPPrivate(t *testing.T) {
 
 func TestCGROUPHost(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) {
-		t.Skip("cgroupns is unsupported")
+		t.Skip("Test requires cgroupns.")
 	}
 	if testing.Short() {
 		return
@@ -1921,6 +1921,9 @@ func TestFdLeaks(t *testing.T) {
 }
 
 func TestFdLeaksSystemd(t *testing.T) {
+	if !systemd.IsRunningSystemd() {
+		t.Skip("Test requires systemd.")
+	}
 	testFdLeaks(t, true)
 }
 
diff --git a/libcontainer/integration/execin_test.go b/libcontainer/integration/execin_test.go
index 894e54f3..f4e58c01 100644
--- a/libcontainer/integration/execin_test.go
+++ b/libcontainer/integration/execin_test.go
@@ -72,7 +72,7 @@ func TestExecIn(t *testing.T) {
 
 func TestExecInUsernsRlimit(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 
 	testExecInRlimit(t, true)
@@ -528,7 +528,7 @@ func TestExecInOomScoreAdj(t *testing.T) {
 
 func TestExecInUserns(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 	if testing.Short() {
 		return
diff --git a/libcontainer/specconv/spec_linux_test.go b/libcontainer/specconv/spec_linux_test.go
index e8deabae..1a0aae9a 100644
--- a/libcontainer/specconv/spec_linux_test.go
+++ b/libcontainer/specconv/spec_linux_test.go
@@ -470,7 +470,7 @@ func TestDupNamespaces(t *testing.T) {
 
 func TestNonZeroEUIDCompatibleSpecconvValidate(t *testing.T) {
 	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
-		t.Skip("userns is unsupported")
+		t.Skip("Test requires userns.")
 	}
 
 	spec := Example()
diff --git a/tests/integration/update.bats b/tests/integration/update.bats
index e204ea49..41344a9d 100644
--- a/tests/integration/update.bats
+++ b/tests/integration/update.bats
@@ -648,3 +648,29 @@ EOF
 	runc resume test_update
 	[ "$status" -eq 0 ]
 }
+
+@test "runc update replaces devices cgroup program" {
+	[[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup
+
+	# Unfortunately we can't update device rules directly with runc ("runc
+	# update" doesn't support it, and adding support would require ironing out
+	# some long-standing design issues with device configuration). So instead
+	# we just run "runc update" many times, relying on the fact that runc will
+	# re-apply devices cgroup rules on each runc update.
+	#
+	# In the past runc would not delete old cgroupv2 eBPF programs, so this
+	# test ensures that once we go past the program limit (64 stacked programs
+	# at time of writing) you can still run "runc" update.
+
+	# Run the container in the background.
+	runc run -d --console-socket "$CONSOLE_SOCKET" test_update
+	[ "$status" -eq 0 ]
+
+	for new_limit in $(seq 300); do
+		runc update --pids-limit "$((2 * new_limit))" test_update
+		[ "$status" -eq 0 ]
+	done
+
+	# The container should still be running.
+	testcontainer test_update running
+}