Repository: prometheus/procfs
Pull request #286 (2020-05-01 17:52:26)

Add smaps information from /proc/pid/smaps_rollup
Follow-up of #281 

Unlike #281, this one only return the sum of value, either from smaps_rollup or (if file does not exists) from smaps.

The information that interest me here is the "Pss" and "SwapPss" which is the process proportional usage of resident memory and swap respectively (which is I think the pages size divided by the number of process that share those pages).

I need this to expose a more significant memory usage for a group of process for prometheus process-exporter (ncabatoff/process-exporter#140)

Signed-off-by: Pierre Fersing <pierre.fersing@bleemeo.com>


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index ca2082c..ffd28ca 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -42,22 +42,22 @@ workflows:
     jobs:
     - test:
         command: default
-        name: linux-1-13
+        name: linux-1-11
         os: linux
-        go_version: "1.13"
+        go_version: "1.11"
     - test:
         command: default
-        name: linux-1-14
+        name: linux-1-12
         os: linux
-        go_version: "1.14"
+        go_version: "1.12"
     - test:
         command: no-test
-        name: windows-1-13
+        name: windows-1-11
         os: windows
-        go_version: "1.13"
+        go_version: "1.11"
     - test:
         command: no-test
-        name: windows-1-14
+        name: windows-1-12
         os: windows
-        go_version: "1.14"
+        go_version: "1.12"
     - codespell
diff --git a/fixtures.ttar b/fixtures.ttar
index 8ea9cff..371de89 100644
--- a/fixtures.ttar
+++ b/fixtures.ttar
@@ -173,6 +173,283 @@ Lines: 1
 411605849 93680043 79
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/proc/26231/smaps
+Lines: 252
+00400000-00cb1000 r-xp 00000000 fd:01 952273                             /bin/alertmanager
+Size:               8900 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                2952 kB
+Pss:                2952 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:      2952 kB
+Private_Dirty:         0 kB
+Referenced:         2864 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd ex mr mw me dw sd 
+00cb1000-016b0000 r--p 008b1000 fd:01 952273                             /bin/alertmanager
+Size:              10236 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                6152 kB
+Pss:                6152 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:      6152 kB
+Private_Dirty:         0 kB
+Referenced:         5308 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd mr mw me dw sd 
+016b0000-0171a000 rw-p 012b0000 fd:01 952273                             /bin/alertmanager
+Size:                424 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                 176 kB
+Pss:                 176 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:        84 kB
+Private_Dirty:        92 kB
+Referenced:          176 kB
+Anonymous:            92 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                 12 kB
+SwapPss:              12 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me dw ac sd 
+0171a000-0173f000 rw-p 00000000 00:00 0 
+Size:                148 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                  76 kB
+Pss:                  76 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:        76 kB
+Referenced:           76 kB
+Anonymous:            76 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me ac sd 
+c000000000-c000400000 rw-p 00000000 00:00 0 
+Size:               4096 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                2564 kB
+Pss:                2564 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:        20 kB
+Private_Dirty:      2544 kB
+Referenced:         2544 kB
+Anonymous:          2564 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:               1100 kB
+SwapPss:            1100 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me ac sd 
+c000400000-c001600000 rw-p 00000000 00:00 0 
+Size:              18432 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:               16024 kB
+Pss:               16024 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:      5864 kB
+Private_Dirty:     10160 kB
+Referenced:        11944 kB
+Anonymous:         16024 kB
+LazyFree:           5848 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                440 kB
+SwapPss:             440 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me ac sd nh 
+c001600000-c004000000 rw-p 00000000 00:00 0 
+Size:              43008 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                   0 kB
+Pss:                   0 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:         0 kB
+Referenced:            0 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me ac sd 
+7f0ab95ca000-7f0abbb7b000 rw-p 00000000 00:00 0 
+Size:              38596 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                1992 kB
+Pss:                1992 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:       476 kB
+Private_Dirty:      1516 kB
+Referenced:         1828 kB
+Anonymous:          1992 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                384 kB
+SwapPss:             384 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me ac sd 
+7ffc07ecf000-7ffc07ef0000 rw-p 00000000 00:00 0                          [stack]
+Size:                132 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                   8 kB
+Pss:                   8 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:         8 kB
+Referenced:            8 kB
+Anonymous:             8 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  4 kB
+SwapPss:               4 kB
+Locked:                0 kB
+VmFlags: rd wr mr mw me gd ac 
+7ffc07f9e000-7ffc07fa1000 r--p 00000000 00:00 0                          [vvar]
+Size:                 12 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                   0 kB
+Pss:                   0 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:         0 kB
+Referenced:            0 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd mr pf io de dd sd 
+7ffc07fa1000-7ffc07fa3000 r-xp 00000000 00:00 0                          [vdso]
+Size:                  8 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                   4 kB
+Pss:                   0 kB
+Shared_Clean:          4 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:         0 kB
+Referenced:            4 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd ex mr mw me de sd 
+ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
+Size:                  4 kB
+KernelPageSize:        4 kB
+MMUPageSize:           4 kB
+Rss:                   0 kB
+Pss:                   0 kB
+Shared_Clean:          0 kB
+Shared_Dirty:          0 kB
+Private_Clean:         0 kB
+Private_Dirty:         0 kB
+Referenced:            0 kB
+Anonymous:             0 kB
+LazyFree:              0 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:                  0 kB
+SwapPss:               0 kB
+Locked:                0 kB
+VmFlags: rd ex 
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/proc/26231/smaps_rollup
+Lines: 17
+00400000-ffffffffff601000 ---p 00000000 00:00 0                          [rollup]
+Rss:               29948 kB
+Pss:               29944 kB
+Shared_Clean:          4 kB
+Shared_Dirty:          0 kB
+Private_Clean:     15548 kB
+Private_Dirty:     14396 kB
+Referenced:        24752 kB
+Anonymous:         20756 kB
+LazyFree:           5848 kB
+AnonHugePages:         0 kB
+ShmemPmdMapped:        0 kB
+Shared_Hugetlb:        0 kB
+Private_Hugetlb:       0 kB
+Swap:               1940 kB
+SwapPss:            1940 kB
+Locked:                0 kB
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Path: fixtures/proc/26231/stat
 Lines: 1
 26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0
diff --git a/proc_smaps.go b/proc_smaps.go
new file mode 100644
index 0000000..a576a72
--- /dev/null
+++ b/proc_smaps.go
@@ -0,0 +1,165 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package procfs
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/procfs/internal/util"
+)
+
+var (
+	// match the header line before each mapped zone in /proc/pid/smaps
+	procSMapsHeaderLine = regexp.MustCompile(`^[a-f0-9].*$`)
+)
+
+type ProcSMapsRollup struct {
+	// Amount of the mapping that is currently resident in RAM
+	Rss uint64
+	// Process's proportional share of this mapping
+	Pss uint64
+	// Size in bytes of clean shared pages
+	SharedClean uint64
+	// Size in bytes of dirty shared pages
+	SharedDirty uint64
+	// Size in bytes of clean private pages
+	PrivateClean uint64
+	// Size in bytes of dirty private pages
+	PrivateDirty uint64
+	// Amount of memory currently marked as referenced or accessed
+	Referenced uint64
+	// Amount of memory that does not belong to any file
+	Anonymous uint64
+	// Amount would-be-anonymous memory currently on swap
+	Swap uint64
+	// Process's proportional memory on swap
+	SwapPss uint64
+}
+
+// ProcSMapsRollup reads from /proc/[pid]/smaps_rollup to get summed memory information of the
+// process.
+//
+// If smaps_rollup does not exists (require kernel >= 4.15), the content of /proc/pid/smaps will
+// we read and summed.
+func (p Proc) ProcSMapsRollup() (ProcSMapsRollup, error) {
+	data, err := util.ReadFileNoStat(p.path("smaps_rollup"))
+	if err != nil && os.IsNotExist(err) {
+		return p.procSMapsRollupManual()
+	}
+	if err != nil {
+		return ProcSMapsRollup{}, err
+	}
+
+	lines := strings.Split(string(data), "\n")
+	smaps := ProcSMapsRollup{}
+
+	// skip first line which don't contains information we need
+	lines = lines[1:]
+	for _, line := range lines {
+		if line == "" {
+			continue
+		}
+
+		if err := smaps.parseLine(line); err != nil {
+			return ProcSMapsRollup{}, err
+		}
+	}
+
+	return smaps, nil
+}
+
+// Read /proc/pid/smaps and do the roll-up in Go code.
+func (p Proc) procSMapsRollupManual() (ProcSMapsRollup, error) {
+	file, err := os.Open(p.path("smaps"))
+	if err != nil {
+		return ProcSMapsRollup{}, err
+	}
+	defer file.Close()
+
+	smaps := ProcSMapsRollup{}
+	scan := bufio.NewScanner(file)
+
+	for scan.Scan() {
+		line := scan.Text()
+
+		if procSMapsHeaderLine.MatchString(line) {
+			continue
+		}
+
+		if err := smaps.parseLine(line); err != nil {
+			return ProcSMapsRollup{}, err
+		}
+	}
+
+	return smaps, nil
+}
+
+func (s *ProcSMapsRollup) parseLine(line string) error {
+	kv := strings.SplitN(line, ":", 2)
+	if len(kv) != 2 {
+		fmt.Println(line)
+		return errors.New("invalid net/dev line, missing colon")
+	}
+
+	k := kv[0]
+	if k == "VmFlags" {
+		return nil
+	}
+
+	v := strings.TrimSpace(kv[1])
+	v = strings.TrimRight(v, " kB")
+
+	vKBytes, err := strconv.ParseUint(v, 10, 64)
+	if err != nil {
+		return err
+	}
+	vBytes := vKBytes * 1024
+
+	s.addValue(k, v, vKBytes, vBytes)
+
+	return nil
+}
+
+func (s *ProcSMapsRollup) addValue(k string, vString string, vUint uint64, vUintBytes uint64) {
+	switch k {
+	case "Rss":
+		s.Rss += vUintBytes
+	case "Pss":
+		s.Pss += vUintBytes
+	case "Shared_Clean":
+		s.SharedClean += vUintBytes
+	case "Shared_Dirty":
+		s.SharedDirty += vUintBytes
+	case "Private_Clean":
+		s.PrivateClean += vUintBytes
+	case "Private_Dirty":
+		s.PrivateDirty += vUintBytes
+	case "Referenced":
+		s.Referenced += vUintBytes
+	case "Anonymous":
+		s.Anonymous += vUintBytes
+	case "Swap":
+		s.Swap += vUintBytes
+	case "SwapPss":
+		s.SwapPss += vUintBytes
+	}
+}


Test patch:
diff --git a/proc_smaps_test.go b/proc_smaps_test.go
new file mode 100644
index 0000000..bedb1cf
--- /dev/null
+++ b/proc_smaps_test.go
@@ -0,0 +1,91 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package procfs
+
+import (
+	"testing"
+)
+
+func BenchmarkProcSMapsRollup(b *testing.B) {
+	fs, err := NewFS(procTestFixtures)
+	if err != nil {
+		b.Fatalf("Creating pseudo fs from getProcFixtures failed at fixtures/proc with error: %s", err)
+	}
+
+	p, err := fs.Proc(26231)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		_, _ = p.ProcSMapsRollup()
+	}
+}
+
+func TestProcSmapsRollup(t *testing.T) {
+	p, err := getProcFixtures(t).Proc(26231)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s1, err := p.ProcSMapsRollup()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s2, err := p.procSMapsRollupManual()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []struct {
+		name  string
+		smaps ProcSMapsRollup
+	}{
+		{
+			name:  "ProcSMapsRollup",
+			smaps: s1,
+		},
+		{
+			name:  "procSMapsRollupManual",
+			smaps: s2,
+		},
+	}
+
+	for _, c := range cases {
+		for _, test := range []struct {
+			name string
+			want uint64
+			have uint64
+		}{
+			{name: "Rss", want: 29948 * 1024, have: c.smaps.Rss},
+			{name: "Pss", want: 29944 * 1024, have: c.smaps.Pss},
+			{name: "SharedClean", want: 4 * 1024, have: c.smaps.SharedClean},
+			{name: "SharedDirty", want: 0 * 1024, have: c.smaps.SharedDirty},
+			{name: "PrivateClean", want: 15548 * 1024, have: c.smaps.PrivateClean},
+			{name: "PrivateDirty", want: 14396 * 1024, have: c.smaps.PrivateDirty},
+			{name: "Referenced", want: 24752 * 1024, have: c.smaps.Referenced},
+			{name: "Anonymous", want: 20756 * 1024, have: c.smaps.Anonymous},
+			{name: "Swap", want: 1940 * 1024, have: c.smaps.Swap},
+			{name: "SwapPss", want: 1940 * 1024, have: c.smaps.SwapPss},
+		} {
+			if test.want != test.have {
+				t.Errorf("want %s %s %d, have %d", c.name, test.name, test.want, test.have)
+			}
+		}
+	}
+}