Repository: opencontainers/runc
Pull request #3900 (2023-06-13 02:15:01)

libct/cg/stats: support PSI for cgroup v2
This is a carry over of #3679, adding support for PSI stats for cgroup v2 to cgroup manager' Stats(), as well as `runc events'. Mostly written by @dqminh, amended by @szuecs and @kolyshkin.

Closes: #3679
Closes: #3358
Closes: #3627

kolyshkin: The full diff from PR #3679 is here: https://github.com/opencontainers/runc/pull/3679#issuecomment-1588413825
Dragoncell: Thanks for the work. The change looks pretty good to me !
szuecs: Thanks @kolyshkin !

Patch:
diff --git a/events.go b/events.go
index 6cdc01cd..47aa1abe 100644
--- a/events.go
+++ b/events.go
@@ -129,6 +129,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	s.CPU.Throttling.Periods = cg.CpuStats.ThrottlingData.Periods
 	s.CPU.Throttling.ThrottledPeriods = cg.CpuStats.ThrottlingData.ThrottledPeriods
 	s.CPU.Throttling.ThrottledTime = cg.CpuStats.ThrottlingData.ThrottledTime
+	s.CPU.PSI = cg.CpuStats.PSI
 
 	s.CPUSet = types.CPUSet(cg.CPUSetStats)
 
@@ -138,6 +139,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	s.Memory.Swap = convertMemoryEntry(cg.MemoryStats.SwapUsage)
 	s.Memory.Usage = convertMemoryEntry(cg.MemoryStats.Usage)
 	s.Memory.Raw = cg.MemoryStats.Stats
+	s.Memory.PSI = cg.MemoryStats.PSI
 
 	s.Blkio.IoServiceBytesRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceBytesRecursive)
 	s.Blkio.IoServicedRecursive = convertBlkioEntry(cg.BlkioStats.IoServicedRecursive)
@@ -147,6 +149,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	s.Blkio.IoMergedRecursive = convertBlkioEntry(cg.BlkioStats.IoMergedRecursive)
 	s.Blkio.IoTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoTimeRecursive)
 	s.Blkio.SectorsRecursive = convertBlkioEntry(cg.BlkioStats.SectorsRecursive)
+	s.Blkio.PSI = cg.BlkioStats.PSI
 
 	s.Hugetlb = make(map[string]types.Hugetlb)
 	for k, v := range cg.HugetlbStats {
diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go
index 42b5bcb6..47b67afc 100644
--- a/libcontainer/cgroups/fs2/fs2.go
+++ b/libcontainer/cgroups/fs2/fs2.go
@@ -114,6 +114,17 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) {
 	if err := statCpu(m.dirPath, st); err != nil && !os.IsNotExist(err) {
 		errs = append(errs, err)
 	}
+	// PSI (since kernel 4.20).
+	var err error
+	if st.CpuStats.PSI, err = statPSI(m.dirPath, "cpu.pressure"); err != nil {
+		errs = append(errs, err)
+	}
+	if st.MemoryStats.PSI, err = statPSI(m.dirPath, "memory.pressure"); err != nil {
+		errs = append(errs, err)
+	}
+	if st.BlkioStats.PSI, err = statPSI(m.dirPath, "io.pressure"); err != nil {
+		errs = append(errs, err)
+	}
 	// hugetlb (since kernel 5.6)
 	if err := statHugeTlb(m.dirPath, st); err != nil && !os.IsNotExist(err) {
 		errs = append(errs, err)
diff --git a/libcontainer/cgroups/fs2/psi.go b/libcontainer/cgroups/fs2/psi.go
new file mode 100644
index 00000000..09f34888
--- /dev/null
+++ b/libcontainer/cgroups/fs2/psi.go
@@ -0,0 +1,89 @@
+package fs2
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func statPSI(dirPath string, file string) (*cgroups.PSIStats, error) {
+	f, err := cgroups.OpenFile(dirPath, file, os.O_RDONLY)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			// Kernel < 4.20, or CONFIG_PSI is not set,
+			// or PSI stats are turned off for the cgroup
+			// ("echo 0 > cgroup.pressure", kernel >= 6.1).
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	var psistats cgroups.PSIStats
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		parts := strings.Fields(sc.Text())
+		var pv *cgroups.PSIData
+		switch parts[0] {
+		case "some":
+			pv = &psistats.Some
+		case "full":
+			pv = &psistats.Full
+		}
+		if pv != nil {
+			*pv, err = parsePSIData(parts[1:])
+			if err != nil {
+				return nil, &parseError{Path: dirPath, File: file, Err: err}
+			}
+		}
+	}
+	if err := sc.Err(); err != nil {
+		if errors.Is(err, unix.ENOTSUP) {
+			// Some kernels (e.g. CS9) may return ENOTSUP on read
+			// if psi=1 kernel cmdline parameter is required.
+			return nil, nil
+		}
+		return nil, &parseError{Path: dirPath, File: file, Err: err}
+	}
+	return &psistats, nil
+}
+
+func parsePSIData(psi []string) (cgroups.PSIData, error) {
+	data := cgroups.PSIData{}
+	for _, f := range psi {
+		kv := strings.SplitN(f, "=", 2)
+		if len(kv) != 2 {
+			return data, fmt.Errorf("invalid psi data: %q", f)
+		}
+		var pv *float64
+		switch kv[0] {
+		case "avg10":
+			pv = &data.Avg10
+		case "avg60":
+			pv = &data.Avg60
+		case "avg300":
+			pv = &data.Avg300
+		case "total":
+			v, err := strconv.ParseUint(kv[1], 10, 64)
+			if err != nil {
+				return data, fmt.Errorf("invalid %s PSI value: %w", kv[0], err)
+			}
+			data.Total = v
+		}
+		if pv != nil {
+			v, err := strconv.ParseFloat(kv[1], 64)
+			if err != nil {
+				return data, fmt.Errorf("invalid %s PSI value: %w", kv[0], err)
+			}
+			*pv = v
+		}
+	}
+	return data, nil
+}
diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go
index 40a81dd5..8ff1fbb5 100644
--- a/libcontainer/cgroups/stats.go
+++ b/libcontainer/cgroups/stats.go
@@ -32,9 +32,22 @@ type CpuUsage struct {
 	UsageInUsermode uint64 `json:"usage_in_usermode"`
 }
 
+type PSIData struct {
+	Avg10  float64 `json:"avg10"`
+	Avg60  float64 `json:"avg60"`
+	Avg300 float64 `json:"avg300"`
+	Total  uint64  `json:"total"`
+}
+
+type PSIStats struct {
+	Some PSIData `json:"some,omitempty"`
+	Full PSIData `json:"full,omitempty"`
+}
+
 type CpuStats struct {
 	CpuUsage       CpuUsage       `json:"cpu_usage,omitempty"`
 	ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
+	PSI            *PSIStats      `json:"psi,omitempty"`
 }
 
 type CPUSetStats struct {
@@ -89,6 +102,7 @@ type MemoryStats struct {
 	UseHierarchy bool `json:"use_hierarchy"`
 
 	Stats map[string]uint64 `json:"stats,omitempty"`
+	PSI   *PSIStats         `json:"psi,omitempty"`
 }
 
 type PageUsageByNUMA struct {
@@ -133,6 +147,7 @@ type BlkioStats struct {
 	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive,omitempty"`
 	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive,omitempty"`
 	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive,omitempty"`
+	PSI                     *PSIStats        `json:"psi,omitempty"`
 }
 
 type HugetlbStats struct {
diff --git a/types/events.go b/types/events.go
index 81bde829..e28ac8c3 100644
--- a/types/events.go
+++ b/types/events.go
@@ -1,6 +1,9 @@
 package types
 
-import "github.com/opencontainers/runc/libcontainer/intelrdt"
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/intelrdt"
+)
 
 // Event struct for encoding the event data to json.
 type Event struct {
@@ -21,6 +24,10 @@ type Stats struct {
 	NetworkInterfaces []*NetworkInterface `json:"network_interfaces"`
 }
 
+type PSIData = cgroups.PSIData
+
+type PSIStats = cgroups.PSIStats
+
 type Hugetlb struct {
 	Usage   uint64 `json:"usage,omitempty"`
 	Max     uint64 `json:"max,omitempty"`
@@ -43,6 +50,7 @@ type Blkio struct {
 	IoMergedRecursive       []BlkioEntry `json:"ioMergedRecursive,omitempty"`
 	IoTimeRecursive         []BlkioEntry `json:"ioTimeRecursive,omitempty"`
 	SectorsRecursive        []BlkioEntry `json:"sectorsRecursive,omitempty"`
+	PSI                     *PSIStats    `json:"psi,omitempty"`
 }
 
 type Pids struct {
@@ -69,6 +77,7 @@ type CpuUsage struct {
 type Cpu struct {
 	Usage      CpuUsage   `json:"usage,omitempty"`
 	Throttling Throttling `json:"throttling,omitempty"`
+	PSI        *PSIStats  `json:"psi,omitempty"`
 }
 
 type CPUSet struct {
@@ -99,6 +108,7 @@ type Memory struct {
 	Kernel    MemoryEntry       `json:"kernel,omitempty"`
 	KernelTCP MemoryEntry       `json:"kernelTCP,omitempty"`
 	Raw       map[string]uint64 `json:"raw,omitempty"`
+	PSI       *PSIStats         `json:"psi,omitempty"`
 }
 
 type L3CacheInfo struct {


Test patch:
diff --git a/libcontainer/cgroups/fs2/psi_test.go b/libcontainer/cgroups/fs2/psi_test.go
new file mode 100644
index 00000000..f70c19f7
--- /dev/null
+++ b/libcontainer/cgroups/fs2/psi_test.go
@@ -0,0 +1,47 @@
+package fs2
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func TestStatCPUPSI(t *testing.T) {
+	const examplePSIData = `some avg10=1.71 avg60=2.36 avg300=2.57 total=230548833
+full avg10=1.00 avg60=1.01 avg300=1.00 total=157622356`
+
+	// We're using a fake cgroupfs.
+	cgroups.TestMode = true
+
+	fakeCgroupDir := t.TempDir()
+	statPath := filepath.Join(fakeCgroupDir, "cpu.pressure")
+
+	if err := os.WriteFile(statPath, []byte(examplePSIData), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	st, err := statPSI(fakeCgroupDir, "cpu.pressure")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(*st, cgroups.PSIStats{
+		Some: cgroups.PSIData{
+			Avg10:  1.71,
+			Avg60:  2.36,
+			Avg300: 2.57,
+			Total:  230548833,
+		},
+		Full: cgroups.PSIData{
+			Avg10:  1.00,
+			Avg60:  1.01,
+			Avg300: 1.00,
+			Total:  157622356,
+		},
+	}) {
+		t.Errorf("unexpected PSI result: %+v", st)
+	}
+}
diff --git a/tests/integration/events.bats b/tests/integration/events.bats
index 051c81e4..707fe4e7 100644
--- a/tests/integration/events.bats
+++ b/tests/integration/events.bats
@@ -27,6 +27,35 @@ function teardown() {
 	[[ "${lines[0]}" == *"data"* ]]
 }
 
+# shellcheck disable=SC2030
+@test "events --stats with psi data" {
+	requires root cgroups_v2 psi
+	init_cgroup_paths
+
+	update_config '.linux.resources.cpu |= { "quota": 1000 }'
+
+	runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	[ "$status" -eq 0 ]
+
+	# Stress the CPU a bit. Need something that runs for more than 10s.
+	runc exec test_busybox dd if=/dev/zero bs=1 count=128K of=/dev/null
+	[ "$status" -eq 0 ]
+
+	runc exec test_busybox sh -c 'tail /sys/fs/cgroup/*.pressure'
+
+	runc events --stats test_busybox
+	[ "$status" -eq 0 ]
+
+	# Check PSI metrics.
+	jq '.data.cpu.psi' <<<"${lines[0]}"
+	for psi_type in some full; do
+		for psi_metric in avg10 avg60 avg300 total; do
+			echo -n "checking .data.cpu.psi.$psi_type.$psi_metric != 0: "
+			jq -e '.data.cpu.psi.'$psi_type.$psi_metric' != 0' <<<"${lines[0]}"
+		done
+	done
+}
+
 function test_events() {
 	# XXX: currently cgroups require root containers.
 	requires root
diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash
old mode 100644
new mode 100755
index 7b1cc4a0..fb3d0e50
--- a/tests/integration/helpers.bash
+++ b/tests/integration/helpers.bash
@@ -477,6 +477,13 @@ function requires() {
 				skip_me=1
 			fi
 			;;
+		psi)
+			# If PSI is not compiled in the kernel, the file will not exist.
+			# If PSI is compiled, but not enabled, read will fail with ENOTSUPP.
+			if ! cat /sys/fs/cgroup/cpu.pressure &>/dev/null; then
+				skip_me=1
+			fi
+			;;
 		*)
 			fail "BUG: Invalid requires $var."
 			;;