Repository: opencontainers/runc
Pull request #2271 (2020-03-26 14:17:24)

Add reading of information from cpuacct.usage_all
This pull request introduces reading of cpuacct.usage_all with unit tests for reading stats from cpuacct cgroup.

AkihiroSuda: LGTM, but please squash the commits

katarzyna-z: @AkihiroSuda I squashed commits
AkihiroSuda: CI failing
katarzyna-z: I think that CI will pass when #2268 is merged. 
katarzyna-z: CI passed thanks to small fix in .travis.yaml (related to #2268) 
AkihiroSuda: needs rebase
AkihiroSuda: LGTM
katarzyna-z:  @caniszczyk, @crosbymichael, @cyphar, @dqminh, @hqhq, @mrunalp, @rjnagal, @vmarmol
Could you take a look on this pull request?
katarzyna-z: @cyphar I applied your suggestions.
katarzyna-z: @AkihiroSuda @cyphar Could you make the 2nd review? I applied all suggestions.
katarzyna-z: @caniszczyk, @crosbymichael, @cyphar, @dqminh, @hqhq, @mrunalp, @rjnagal, @vmarmol
Could you make the 2nd review of this pull request?
I want to extend metrics in cAdvisor by information from cpuacct.usage_all and this blocks me because cAdvisor uses packages from runc to read cgroups.
AkihiroSuda: LGTM
katarzyna-z: @kolyshkin I addressed your suggestions, please check if I understood you correctly.
katarzyna-z: @kolyshkin Could you take a look at this once again?
katarzyna-z: @akihirosuda, @caniszczyk, @crosbymichael, @cyphar, @dqminh, @hqhq, @kolyshkin, @mrunalp
please take another look
kolyshkin: LGTM

Patch:
diff --git a/events.go b/events.go
index fb3f6302..f0f74ef0 100644
--- a/events.go
+++ b/events.go
@@ -125,6 +125,8 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	s.CPU.Usage.User = cg.CpuStats.CpuUsage.UsageInUsermode
 	s.CPU.Usage.Total = cg.CpuStats.CpuUsage.TotalUsage
 	s.CPU.Usage.Percpu = cg.CpuStats.CpuUsage.PercpuUsage
+	s.CPU.Usage.PercpuKernel = cg.CpuStats.CpuUsage.PercpuUsageInKernelmode
+	s.CPU.Usage.PercpuUser = cg.CpuStats.CpuUsage.PercpuUsageInUsermode
 	s.CPU.Throttling.Periods = cg.CpuStats.ThrottlingData.Periods
 	s.CPU.Throttling.ThrottledPeriods = cg.CpuStats.ThrottlingData.ThrottledPeriods
 	s.CPU.Throttling.ThrottledTime = cg.CpuStats.ThrottlingData.ThrottledTime
diff --git a/libcontainer/cgroups/fs/cpuacct.go b/libcontainer/cgroups/fs/cpuacct.go
index 0b334b23..008bc529 100644
--- a/libcontainer/cgroups/fs/cpuacct.go
+++ b/libcontainer/cgroups/fs/cpuacct.go
@@ -3,8 +3,10 @@
 package fs
 
 import (
+	"bufio"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -15,9 +17,15 @@ import (
 )
 
 const (
-	cgroupCpuacctStat   = "cpuacct.stat"
+	cgroupCpuacctStat     = "cpuacct.stat"
+	cgroupCpuacctUsageAll = "cpuacct.usage_all"
+
 	nanosecondsInSecond = 1000000000
 
+	userModeColumn              = 1
+	kernelModeColumn            = 2
+	cuacctUsageAllColumnsNumber = 3
+
 	// The value comes from `C.sysconf(C._SC_CLK_TCK)`, and
 	// on Linux it's a constant which is safe to be hard coded,
 	// so we can avoid using cgo here. For details, see:
@@ -65,8 +73,15 @@ func (s *CpuacctGroup) GetStats(path string, stats *cgroups.Stats) error {
 		return err
 	}
 
+	percpuUsageInKernelmode, percpuUsageInUsermode, err := getPercpuUsageInModes(path)
+	if err != nil {
+		return err
+	}
+
 	stats.CpuStats.CpuUsage.TotalUsage = totalUsage
 	stats.CpuStats.CpuUsage.PercpuUsage = percpuUsage
+	stats.CpuStats.CpuUsage.PercpuUsageInKernelmode = percpuUsageInKernelmode
+	stats.CpuStats.CpuUsage.PercpuUsageInUsermode = percpuUsageInUsermode
 	stats.CpuStats.CpuUsage.UsageInUsermode = userModeUsage
 	stats.CpuStats.CpuUsage.UsageInKernelmode = kernelModeUsage
 	return nil
@@ -123,3 +138,42 @@ func getPercpuUsage(path string) ([]uint64, error) {
 	}
 	return percpuUsage, nil
 }
+
+func getPercpuUsageInModes(path string) ([]uint64, []uint64, error) {
+	usageKernelMode := []uint64{}
+	usageUserMode := []uint64{}
+
+	file, err := os.Open(filepath.Join(path, cgroupCpuacctUsageAll))
+	if err != nil {
+		return nil, nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	scanner.Scan() //skipping header line
+
+	for scanner.Scan() {
+		lineFields := strings.SplitN(scanner.Text(), " ", cuacctUsageAllColumnsNumber+1)
+		if len(lineFields) != cuacctUsageAllColumnsNumber {
+			continue
+		}
+
+		usageInKernelMode, err := strconv.ParseUint(lineFields[kernelModeColumn], 10, 64)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Unable to convert CPU usage in kernel mode to uint64: %s", err)
+		}
+		usageKernelMode = append(usageKernelMode, usageInKernelMode)
+
+		usageInUserMode, err := strconv.ParseUint(lineFields[userModeColumn], 10, 64)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Unable to convert CPU usage in user mode to uint64: %s", err)
+		}
+		usageUserMode = append(usageUserMode, usageInUserMode)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, nil, fmt.Errorf("Problem in reading %s line by line, %s", cgroupCpuacctUsageAll, err)
+	}
+
+	return usageKernelMode, usageUserMode, nil
+}
diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go
index eaf8f234..7ac81660 100644
--- a/libcontainer/cgroups/stats.go
+++ b/libcontainer/cgroups/stats.go
@@ -20,6 +20,12 @@ type CpuUsage struct {
 	// Total CPU time consumed per core.
 	// Units: nanoseconds.
 	PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
+	// CPU time consumed per core in kernel mode
+	// Units: nanoseconds.
+	PercpuUsageInKernelmode []uint64 `json:"percpu_usage_in_kernelmode"`
+	// CPU time consumed per core in user mode
+	// Units: nanoseconds.
+	PercpuUsageInUsermode []uint64 `json:"percpu_usage_in_usermode"`
 	// Time spent by tasks of the cgroup in kernel mode.
 	// Units: nanoseconds.
 	UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
diff --git a/types/events.go b/types/events.go
index c6f0e97a..366c0a34 100644
--- a/types/events.go
+++ b/types/events.go
@@ -55,10 +55,12 @@ type Throttling struct {
 
 type CpuUsage struct {
 	// Units: nanoseconds.
-	Total  uint64   `json:"total,omitempty"`
-	Percpu []uint64 `json:"percpu,omitempty"`
-	Kernel uint64   `json:"kernel"`
-	User   uint64   `json:"user"`
+	Total        uint64   `json:"total,omitempty"`
+	Percpu       []uint64 `json:"percpu,omitempty"`
+	PercpuKernel []uint64 `json:"percpu_kernel,omitempty"`
+	PercpuUser   []uint64 `json:"percpu_user,omitempty"`
+	Kernel       uint64   `json:"kernel"`
+	User         uint64   `json:"user"`
 }
 
 type Cpu struct {


Test patch:
diff --git a/libcontainer/cgroups/fs/cpuacct_test.go b/libcontainer/cgroups/fs/cpuacct_test.go
new file mode 100644
index 00000000..73623311
--- /dev/null
+++ b/libcontainer/cgroups/fs/cpuacct_test.go
@@ -0,0 +1,61 @@
+// +build linux
+
+package fs
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+const (
+	cpuAcctUsageContents       = "12262454190222160"
+	cpuAcctUsagePerCPUContents = "1564936537989058 1583937096487821 1604195415465681 1596445226820187 1481069084155629 1478735613864327 1477610593414743 1476362015778086"
+	cpuAcctStatContents        = "user 452278264\nsystem 291429664"
+	cpuAcctUsageAll            = `cpu user system
+	0 962250696038415 637727786389114
+	1 981956408513304 638197595421064
+	2 1002658817529022 638956774598358
+	3 994937703492523 637985531181620
+	4 874843781648690 638837766495476
+	5 872544369885276 638763309884944
+	6 870104915696359 640081778921247
+	7 870202363887496 638716766259495
+	`
+)
+
+func TestCpuacctStats(t *testing.T) {
+	helper := NewCgroupTestUtil("cpuacct.", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"cpuacct.usage":        cpuAcctUsageContents,
+		"cpuacct.usage_percpu": cpuAcctUsagePerCPUContents,
+		"cpuacct.stat":         cpuAcctStatContents,
+		"cpuacct.usage_all":    cpuAcctUsageAll,
+	})
+
+	cpuacct := &CpuacctGroup{}
+	actualStats := *cgroups.NewStats()
+	err := cpuacct.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedStats := cgroups.CpuUsage{
+		TotalUsage: uint64(12262454190222160),
+		PercpuUsage: []uint64{1564936537989058, 1583937096487821, 1604195415465681, 1596445226820187,
+			1481069084155629, 1478735613864327, 1477610593414743, 1476362015778086},
+		PercpuUsageInKernelmode: []uint64{637727786389114, 638197595421064, 638956774598358, 637985531181620,
+			638837766495476, 638763309884944, 640081778921247, 638716766259495},
+		PercpuUsageInUsermode: []uint64{962250696038415, 981956408513304, 1002658817529022, 994937703492523,
+			874843781648690, 872544369885276, 870104915696359, 870202363887496},
+		UsageInKernelmode: (uint64(291429664) * nanosecondsInSecond) / clockTicks,
+		UsageInUsermode:   (uint64(452278264) * nanosecondsInSecond) / clockTicks,
+	}
+
+	if !reflect.DeepEqual(expectedStats, actualStats.CpuStats.CpuUsage) {
+		t.Errorf("Expected CPU usage %#v but found %#v\n",
+			expectedStats, actualStats.CpuStats.CpuUsage)
+	}
+}