Repository: prometheus/procfs
Pull request #186 (2019-06-25 17:05:07)

Parse task queue stats from /proc/schedstat
These are useful as an indication of CPU contention and scheduler
latency.

bitglue: @discordianfish @pgier ready for review.

I'll update https://github.com/prometheus/node_exporter/pull/1389 to use this after review.
bitglue: @pgier Everything addressed I think, please re-review
bitglue: @discordianfish @pgier bump, review please

Patch:
diff --git a/fixtures.ttar b/fixtures.ttar
index 8ebd5b9..b743744 100644
--- a/fixtures.ttar
+++ b/fixtures.ttar
@@ -168,6 +168,11 @@ SymlinkTo: net:[4026531993]
 Path: fixtures/proc/26231/root
 SymlinkTo: /
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/proc/26231/schedstat
+Lines: 1
+411605849 93680043 79
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Path: fixtures/proc/26231/stat
 Lines: 1
 26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0
@@ -300,6 +305,18 @@ Lines: 1
 com.github.uiautomatorNULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTEEOF
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/proc/26233/schedstat
+Lines: 8
+ ____________________________________
+< this is a malformed schedstat file >
+ ------------------------------------
+        \   ^__^
+         \  (oo)\_______
+            (__)\       )\/\
+                ||----w |
+                ||     ||
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: fixtures/proc/584
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -609,6 +626,16 @@ some avg10=0.10 avg60=2.00 avg300=3.85 total=15
 full avg10=0.20 avg60=3.00 avg300=4.95 total=25
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/proc/schedstat
+Lines: 6
+version 15
+timestamp 15819019232
+cpu0 498494191 0 3533438552 2553969831 3853684107 2465731542 2045936778163039 343796328169361 4767485306
+domain0 00000000,00000003 212499247 210112015 1861015 1860405436 536440 369895 32599 210079416 25368550 24241256 384652 927363878 807233 6366 1647 24239609 2122447165 1886868564 121112060 2848625533 125678146 241025 1032026 1885836538 2545 12 2533 0 0 0 0 0 0 1387952561 21076581 0
+cpu1 518377256 0 4155211005 2778589869 10466382 2867629021 1904686152592476 364107263788241 5145567945
+domain0 00000000,00000003 217653037 215526982 1577949 1580427380 557469 393576 28538 215498444 28721913 27662819 371153 870843407 745912 5523 1639 27661180 2331056874 2107732788 111442342 652402556 123615235 196159 1045245 2106687543 2400 3 2397 0 0 0 0 0 0 1437804657 26220076 0
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Path: fixtures/proc/self
 SymlinkTo: 26231
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/proc.go b/proc.go
index a0aee59..b7c79cf 100644
--- a/proc.go
+++ b/proc.go
@@ -300,3 +300,12 @@ func (p Proc) FileDescriptorsInfo() (ProcFDInfos, error) {
 
 	return fdinfos, nil
 }
+
+// Schedstat returns task scheduling information for the process.
+func (p Proc) Schedstat() (ProcSchedstat, error) {
+	contents, err := ioutil.ReadFile(p.path("schedstat"))
+	if err != nil {
+		return ProcSchedstat{}, err
+	}
+	return parseProcSchedstat(string(contents))
+}
diff --git a/schedstat.go b/schedstat.go
new file mode 100644
index 0000000..86396c0
--- /dev/null
+++ b/schedstat.go
@@ -0,0 +1,128 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package procfs
+
+import (
+	"bufio"
+	"errors"
+	"os"
+	"regexp"
+	"strconv"
+)
+
+var (
+	cpuLineRE  = regexp.MustCompile(`cpu(\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+)`)
+	procLineRE = regexp.MustCompile(`(\d+) (\d+) (\d+)`)
+)
+
+// Schedstat contains scheduler statistics from /proc/schedstats
+//
+// See
+// https://www.kernel.org/doc/Documentation/scheduler/sched-stats.txt
+// for a detailed description of what these numbers mean.
+type Schedstat struct {
+	CPUs []*SchedstatCPU
+}
+
+// SchedstatCPU contains the values from one "cpu<N>" line
+type SchedstatCPU struct {
+	CPUNum string
+
+	RunningJiffies uint64
+	WaitingJiffies uint64
+	RunTimeslices  uint64
+}
+
+// ProcSchedstat contains the values from /proc/<pid>/schedstat
+type ProcSchedstat struct {
+	RunningJiffies uint64
+	WaitingJiffies uint64
+	RunTimeslices  uint64
+}
+
+func (fs FS) Schedstat() (*Schedstat, error) {
+	file, err := os.Open(fs.proc.Path("schedstat"))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	stats := &Schedstat{}
+	scanner := bufio.NewScanner(file)
+
+	for scanner.Scan() {
+		match := cpuLineRE.FindStringSubmatch(scanner.Text())
+		if match != nil {
+			cpu := &SchedstatCPU{}
+			cpu.CPUNum = match[1]
+
+			cpu.RunningJiffies, err = strconv.ParseUint(match[8], 10, 64)
+			if err != nil {
+				continue
+			}
+
+			cpu.WaitingJiffies, err = strconv.ParseUint(match[9], 10, 64)
+			if err != nil {
+				continue
+			}
+
+			cpu.RunTimeslices, err = strconv.ParseUint(match[10], 10, 64)
+			if err != nil {
+				continue
+			}
+
+			stats.CPUs = append(stats.CPUs, cpu)
+		}
+	}
+
+	return stats, nil
+}
+
+func parseProcSchedstat(contents string) (stats ProcSchedstat, err error) {
+	match := procLineRE.FindStringSubmatch(contents)
+
+	if match != nil {
+		stats.RunningJiffies, err = strconv.ParseUint(match[1], 10, 64)
+		if err != nil {
+			return
+		}
+
+		stats.WaitingJiffies, err = strconv.ParseUint(match[2], 10, 64)
+		if err != nil {
+			return
+		}
+
+		stats.RunTimeslices, err = strconv.ParseUint(match[3], 10, 64)
+		return
+	}
+
+	err = errors.New("could not parse schedstat")
+	return
+}
+
+func (stat *SchedstatCPU) RunningSeconds() float64 {
+	return float64(stat.RunningJiffies) / userHZ
+}
+
+func (stat *SchedstatCPU) WaitingSeconds() float64 {
+	return float64(stat.WaitingJiffies) / userHZ
+}
+
+func (stat *ProcSchedstat) RunningSeconds() float64 {
+	return float64(stat.RunningJiffies) / userHZ
+}
+
+func (stat *ProcSchedstat) WaitingSeconds() float64 {
+	return float64(stat.WaitingJiffies) / userHZ
+}


Test patch:
diff --git a/schedstat_test.go b/schedstat_test.go
new file mode 100644
index 0000000..42e43f1
--- /dev/null
+++ b/schedstat_test.go
@@ -0,0 +1,139 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package procfs
+
+import "testing"
+
+func TestSchedstat(t *testing.T) {
+	stats, err := getProcFixtures(t).Schedstat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(stats.CPUs) != 2 {
+		t.Errorf("expected 2 CPUs, got %v", len(stats.CPUs))
+	}
+
+	var cpu *SchedstatCPU
+	for _, cpu = range stats.CPUs {
+		if cpu.CPUNum == "0" {
+			break
+		}
+	}
+
+	if cpu == nil || cpu.CPUNum != "0" {
+		t.Error("could not find cpu0")
+	}
+
+	if want, have := uint64(2045936778163039), cpu.RunningJiffies; want != have {
+		t.Errorf("want RunningJiffies %v, have %v", want, have)
+	}
+	if want, have := float64(2045936778163039)/userHZ, cpu.RunningSeconds(); want != have {
+		t.Errorf("want RunningSeconds() %v, have %v", want, have)
+	}
+
+	if want, have := uint64(343796328169361), cpu.WaitingJiffies; want != have {
+		t.Errorf("want WaitingJiffies %v, have %v", want, have)
+	}
+	if want, have := float64(343796328169361)/userHZ, cpu.WaitingSeconds(); want != have {
+		t.Errorf("want WaitingSeconds() %v, have %v", want, have)
+	}
+
+	if want, have := uint64(4767485306), cpu.RunTimeslices; want != have {
+		t.Errorf("want RunTimeslices %v, have %v", want, have)
+	}
+}
+
+func TestProcSchedstat(t *testing.T) {
+	p1, err := getProcFixtures(t).Proc(26231)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	schedstat, err := p1.Schedstat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if want, have := uint64(411605849), schedstat.RunningJiffies; want != have {
+		t.Errorf("want RunningJiffies %v, have %v", want, have)
+	}
+	if want, have := float64(411605849)/userHZ, schedstat.RunningSeconds(); want != have {
+		t.Errorf("want RunningSeconds() %v, have %v", want, have)
+	}
+
+	if want, have := uint64(93680043), schedstat.WaitingJiffies; want != have {
+		t.Errorf("want WaitingJiffies %v, have %v", want, have)
+	}
+	if want, have := float64(93680043)/userHZ, schedstat.WaitingSeconds(); want != have {
+		t.Errorf("want WaitingSeconds() %v, have %v", want, have)
+	}
+
+	if want, have := uint64(79), schedstat.RunTimeslices; want != have {
+		t.Errorf("want RunTimeslices %v, have %v", want, have)
+	}
+}
+
+func TestProcSchedstatErrors(t *testing.T) {
+	p1, err := getProcFixtures(t).Proc(26232)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = p1.Schedstat()
+	if err == nil {
+		t.Error("proc 26232 doesn't have schedstat -- should have gotten an error")
+	}
+
+	p2, err := getProcFixtures(t).Proc(26233)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = p2.Schedstat()
+	if err == nil {
+		t.Error("proc 26233 has malformed schedstat -- should have gotten an error")
+	}
+}
+
+// schedstat can have a 2nd line: it should be ignored
+func TestProcSchedstatMultipleLines(t *testing.T) {
+	schedstat, err := parseProcSchedstat("123 456 789\n10 11\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want, have := uint64(123), schedstat.RunningJiffies; want != have {
+		t.Errorf("want RunningJiffies %v, have %v", want, have)
+	}
+	if want, have := uint64(456), schedstat.WaitingJiffies; want != have {
+		t.Errorf("want WaitingJiffies %v, have %v", want, have)
+	}
+	if want, have := uint64(789), schedstat.RunTimeslices; want != have {
+		t.Errorf("want RunTimeslices %v, have %v", want, have)
+	}
+}
+
+func TestProcSchedstatUnparsableInt(t *testing.T) {
+	if _, err := parseProcSchedstat("abc 456 789\n"); err == nil {
+		t.Error("schedstat should have been unparsable\n")
+	}
+
+	if _, err := parseProcSchedstat("123 abc 789\n"); err == nil {
+		t.Error("schedstat should have been unparsable\n")
+	}
+
+	if _, err := parseProcSchedstat("123 456 abc\n"); err == nil {
+		t.Error("schedstat should have been unparsable\n")
+	}
+}