Repository: prometheus/procfs
Pull request #121 (2018-12-04 20:29:32)

Add diskstats
Added parsing of /proc/diskstats and /sys/block/\<device\>/stat

pgier: Related to https://github.com/prometheus/node_exporter/issues/1179
pgier: Re-factored to use a single struct shared between procfs and sysfs since the fields should be identical.

Patch:
diff --git a/diskstats.go b/diskstats.go
new file mode 100644
index 0000000..a116cf3
--- /dev/null
+++ b/diskstats.go
@@ -0,0 +1,81 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package procfs
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/prometheus/procfs/iostats"
+)
+
+const (
+	diskstatsFilename = "diskstats"
+	statFormat        = "%d %d %s %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d"
+)
+
+// NewDiskstats reads the diskstats file and returns
+// an array of Diskstats (one per line/device)
+func NewDiskstats() ([]iostats.IODeviceStats, error) {
+	fs, err := NewFS(DefaultMountPoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs.NewDiskstats()
+}
+
+// NewDiskstats reads the diskstats file and returns
+// an array of Diskstats (one per line/device)
+func (fs FS) NewDiskstats() ([]iostats.IODeviceStats, error) {
+	file, err := os.Open(fs.Path(diskstatsFilename))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	diskstats := []iostats.IODeviceStats{}
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		d := &iostats.IODeviceStats{}
+		count, err := fmt.Sscanf(scanner.Text(), statFormat,
+			&d.MajorNumber,
+			&d.MinorNumber,
+			&d.DeviceName,
+			&d.ReadIOs,
+			&d.ReadMerges,
+			&d.ReadSectors,
+			&d.ReadTicks,
+			&d.WriteIOs,
+			&d.WriteMerges,
+			&d.WriteSectors,
+			&d.WriteTicks,
+			&d.IOsInProgress,
+			&d.IOsTotalTicks,
+			&d.WeightedIOTicks,
+			&d.DiscardIOs,
+			&d.DiscardMerges,
+			&d.DiscardSectors,
+			&d.DiscardTicks)
+		if err != nil && err != io.EOF {
+			return diskstats, err
+		}
+		if count == 14 || count == 18 {
+			diskstats = append(diskstats, *d)
+		}
+	}
+	return diskstats, nil
+}
diff --git a/fixtures.ttar b/fixtures.ttar
index 13c831e..6e603a0 100644
--- a/fixtures.ttar
+++ b/fixtures.ttar
@@ -234,6 +234,59 @@ Node 0, zone    DMA32    759    572    791    475    194     45     12      0
 Node 0, zone   Normal   4381   1093    185   1530    567    102      4      0      0      0      0 
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/diskstats
+Lines: 49
+   1       0 ram0 0 0 0 0 0 0 0 0 0 0 0
+   1       1 ram1 0 0 0 0 0 0 0 0 0 0 0
+   1       2 ram2 0 0 0 0 0 0 0 0 0 0 0
+   1       3 ram3 0 0 0 0 0 0 0 0 0 0 0
+   1       4 ram4 0 0 0 0 0 0 0 0 0 0 0
+   1       5 ram5 0 0 0 0 0 0 0 0 0 0 0
+   1       6 ram6 0 0 0 0 0 0 0 0 0 0 0
+   1       7 ram7 0 0 0 0 0 0 0 0 0 0 0
+   1       8 ram8 0 0 0 0 0 0 0 0 0 0 0
+   1       9 ram9 0 0 0 0 0 0 0 0 0 0 0
+   1      10 ram10 0 0 0 0 0 0 0 0 0 0 0
+   1      11 ram11 0 0 0 0 0 0 0 0 0 0 0
+   1      12 ram12 0 0 0 0 0 0 0 0 0 0 0
+   1      13 ram13 0 0 0 0 0 0 0 0 0 0 0
+   1      14 ram14 0 0 0 0 0 0 0 0 0 0 0
+   1      15 ram15 0 0 0 0 0 0 0 0 0 0 0
+   7       0 loop0 0 0 0 0 0 0 0 0 0 0 0
+   7       1 loop1 0 0 0 0 0 0 0 0 0 0 0
+   7       2 loop2 0 0 0 0 0 0 0 0 0 0 0
+   7       3 loop3 0 0 0 0 0 0 0 0 0 0 0
+   7       4 loop4 0 0 0 0 0 0 0 0 0 0 0
+   7       5 loop5 0 0 0 0 0 0 0 0 0 0 0
+   7       6 loop6 0 0 0 0 0 0 0 0 0 0 0
+   7       7 loop7 0 0 0 0 0 0 0 0 0 0 0
+   8       0 sda 25354637 34367663 1003346126 18492372 28444756 11134226 505697032 63877960 0 9653880 82621804
+   8       1 sda1 250 0 2000 36 0 0 0 0 0 36 36
+   8       2 sda2 246 0 1968 32 0 0 0 0 0 32 32
+   8       3 sda3 340 13 2818 52 11 8 152 8 0 56 60
+   8       4 sda4 25353629 34367650 1003337964 18492232 27448755 11134218 505696880 61593380 0 7576432 80332428
+ 252       0 dm-0 59910002 0 1003337218 46229572 39231014 0 505696880 1158557800 0 11325968 1206301256
+ 252       1 dm-1 388 0 3104 84 74 0 592 0 0 76 84
+ 252       2 dm-2 11571 0 308350 6536 153522 0 5093416 122884 0 65400 129416
+ 252       3 dm-3 3870 0 3870 104 0 0 0 0 0 16 104
+ 252       4 dm-4 392 0 1034 28 38 0 137 16 0 24 44
+ 252       5 dm-5 3729 0 84279 924 98918 0 1151688 104684 0 58848 105632
+ 179       0 mmcblk0 192 3 1560 156 0 0 0 0 0 136 156
+ 179       1 mmcblk0p1 17 3 160 24 0 0 0 0 0 24 24
+ 179       2 mmcblk0p2 95 0 760 68 0 0 0 0 0 68 68
+   2       0 fd0 2 0 16 80 0 0 0 0 0 80 80
+ 254       0 vda 1775784 15386 32670882 8655768 6038856 20711856 213637440 2069221364 0 41614592 2077872228
+ 254       1 vda1 668 85 5984 956 207 4266 35784 32772 0 8808 33720
+ 254       2 vda2 1774936 15266 32663262 8654692 5991028 20707590 213601656 2069152216 0 41607628 2077801992
+  11       0 sr0 0 0 0 0 0 0 0 0 0 0 0
+ 259       0 nvme0n1 47114 4 4643973 21650 1078320 43950 39451633 1011053 0 222766 1032546
+ 259       1 nvme0n1p1 1140 0 9370 16 1 0 1 0 0 16 16
+ 259       2 nvme0n1p2 45914 4 4631243 21626 1036885 43950 39451632 919480 0 131580 940970
+   8       0 sdb 326552 841 9657779 84 41822 2895 1972905 5007 0 60730 67070 68851 0 1925173784 11130
+   8       1 sdb1 231 3 34466 4 24 23 106 0 0 64 64 0 0 0 0
+   8       2 sdb2 326310 838 9622281 67 40726 2872 1972799 4924 0 58250 64567 68851 0 1925173784 11130
+Mode: 664
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: fixtures/fs
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/iostats/iostats.go b/iostats/iostats.go
new file mode 100644
index 0000000..97e8314
--- /dev/null
+++ b/iostats/iostats.go
@@ -0,0 +1,67 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iostats
+
+// IODevice contains identifying information for an I/O device
+type IODevice struct {
+	MajorNumber uint32
+	MinorNumber uint32
+	DeviceName  string
+}
+
+// IOStats models the iostats data described in the kernel documentation
+// https://www.kernel.org/doc/Documentation/iostats.txt,
+// https://www.kernel.org/doc/Documentation/block/stat.txt,
+// and https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
+type IOStats struct {
+	// ReadIOs is the number of reads completed successfully.
+	ReadIOs uint64
+	// ReadMerges is the number of reads merged.  Reads and writes
+	// which are adjacent to each other may be merged for efficiency.
+	ReadMerges uint64
+	// ReadSectors is the total number of sectors read successfully.
+	ReadSectors uint64
+	// ReadTicks is the total number of milliseconds spent by all reads.
+	ReadTicks uint64
+	// WriteIOs is the total number of writes completed successfully.
+	WriteIOs uint64
+	// WriteMerges is the number of reads merged.
+	WriteMerges uint64
+	// WriteSectors is the total number of sectors written successfully.
+	WriteSectors uint64
+	// WriteTicks is the total number of milliseconds spent by all writes.
+	WriteTicks uint64
+	// IOsInProgress is number of I/Os currently in progress.
+	IOsInProgress uint64
+	// IOsTotalTicks is the number of milliseconds spent doing I/Os.
+	// This field increases so long as IosInProgress is nonzero.
+	IOsTotalTicks uint64
+	// WeightedIOTicks is the weighted number of milliseconds spent doing I/Os.
+	// This can also be used to estimate average queue wait time for requests.
+	WeightedIOTicks uint64
+	// DiscardIOs is the total number of discards completed successfully.
+	DiscardIOs uint64
+	// DiscardMerges is the number of discards merged.
+	DiscardMerges uint64
+	// DiscardSectors is the total number of sectors discarded successfully.
+	DiscardSectors uint64
+	// DiscardTicks is the total number of milliseconds spent by all discards.
+	DiscardTicks uint64
+}
+
+// IODeviceStats combines IODevice and IOStats
+type IODeviceStats struct {
+	IODevice
+	IOStats
+}
diff --git a/sysfs/block_device.go b/sysfs/block_device.go
new file mode 100644
index 0000000..6939529
--- /dev/null
+++ b/sysfs/block_device.go
@@ -0,0 +1,81 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package sysfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/prometheus/procfs/iostats"
+)
+
+const (
+	blockPath             = "block"
+	blockDeviceStatFormat = "%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d"
+)
+
+// BlockDevice represents block device in the sys filesystem
+// Docs here: https://www.kernel.org/doc/Documentation/block/
+type BlockDevice struct {
+	DeviceName string
+	fs         FS
+}
+
+// AllBlockDevices gets the list of block devices from the sys file system
+func (fs FS) AllBlockDevices() ([]BlockDevice, error) {
+	deviceDirs, err := ioutil.ReadDir(fs.Path(blockPath))
+	if err != nil {
+		return nil, err
+	}
+	devices := []BlockDevice{}
+	for _, deviceDir := range deviceDirs {
+		if deviceDir.IsDir() {
+			devices = append(devices, BlockDevice{deviceDir.Name(), fs})
+		}
+	}
+	return devices, nil
+}
+
+// NewBlockDeviceStat returns stats for the block device read from /sys/block/<device>/stat.
+func (d BlockDevice) NewBlockDeviceStat() (iostats.IOStats, error) {
+	stat := iostats.IOStats{}
+	bytes, err := ioutil.ReadFile(d.fs.Path(blockPath, d.DeviceName, "stat"))
+	if err != nil {
+		return stat, err
+	}
+	count, err := fmt.Sscanf(strings.TrimSpace(string(bytes)), blockDeviceStatFormat,
+		&stat.ReadIOs,
+		&stat.ReadMerges,
+		&stat.ReadSectors,
+		&stat.ReadTicks,
+		&stat.WriteIOs,
+		&stat.WriteMerges,
+		&stat.WriteSectors,
+		&stat.WriteTicks,
+		&stat.IOsInProgress,
+		&stat.IOsTotalTicks,
+		&stat.WeightedIOTicks,
+		&stat.DiscardIOs,
+		&stat.DiscardMerges,
+		&stat.DiscardSectors,
+		&stat.DiscardTicks,
+	)
+	if count == 11 || count == 15 {
+		return stat, nil
+	}
+	return iostats.IOStats{}, err
+}
diff --git a/sysfs/fixtures.ttar b/sysfs/fixtures.ttar
index c0ac683..2d2c6c5 100644
--- a/sysfs/fixtures.ttar
+++ b/sysfs/fixtures.ttar
@@ -2,6 +2,25 @@
 Directory: fixtures
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: fixtures/block
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: fixtures/block/dm-0
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/block/dm-0/stat
+Lines: 1
+6447303        0 710266738  1529043   953216        0 31201176  4557464        0   796160  6088971
+Mode: 664
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: fixtures/block/sda
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: fixtures/block/sda/stat
+Lines: 1
+9652963   396792 759304206   412943  8422549  6731723 286915323 13947418        0  5658367 19174573 1 2 3 12
+Mode: 664
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: fixtures/class
 Mode: 775
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -


Test patch:
diff --git a/diskstats_test.go b/diskstats_test.go
new file mode 100644
index 0000000..f40b3fa
--- /dev/null
+++ b/diskstats_test.go
@@ -0,0 +1,42 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package procfs
+
+import (
+	"testing"
+)
+
+const (
+	failMsgFormat        = "%v, expected %v, actual %v"
+	expectedNumOfDevices = 49
+)
+
+func TestDiskstats(t *testing.T) {
+	diskstats, err := FS("fixtures").NewDiskstats()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(diskstats) != expectedNumOfDevices {
+		t.Errorf(failMsgFormat, "Incorrect number of devices", expectedNumOfDevices, len(diskstats))
+	}
+	if diskstats[0].DeviceName != "ram0" {
+		t.Errorf(failMsgFormat, "Incorrect device name", "ram0", diskstats[0].DeviceName)
+	}
+	if diskstats[24].WriteIOs != 28444756 {
+		t.Errorf(failMsgFormat, "Incorrect writes completed", 28444756, diskstats[24].WriteIOs)
+	}
+	if diskstats[48].DiscardTicks != 11130 {
+		t.Errorf(failMsgFormat, "Incorrect discard time", 11130, diskstats[48].DiscardTicks)
+	}
+}
diff --git a/sysfs/block_device_test.go b/sysfs/block_device_test.go
new file mode 100644
index 0000000..d7f2d4f
--- /dev/null
+++ b/sysfs/block_device_test.go
@@ -0,0 +1,58 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package sysfs
+
+import (
+	"testing"
+)
+
+const (
+	failMsgFormat        = "%v, expected %v, actual %v"
+	expectedNumOfDevices = 2
+)
+
+func TestBlockDevice(t *testing.T) {
+	devices, err := FS("fixtures").AllBlockDevices()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(devices) != expectedNumOfDevices {
+		t.Fatalf(failMsgFormat, "Incorrect number of devices", expectedNumOfDevices, len(devices))
+	}
+	if devices[0].DeviceName != "dm-0" {
+		t.Errorf(failMsgFormat, "Incorrect device name", "dm-0", devices[0].DeviceName)
+	}
+	device0stats, err := devices[0].NewBlockDeviceStat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if device0stats.ReadIOs != 6447303 {
+		t.Errorf(failMsgFormat, "Incorrect read I/Os", 6447303, device0stats.ReadIOs)
+	}
+	if device0stats.WeightedIOTicks != 6088971 {
+		t.Errorf(failMsgFormat, "Incorrect time in queue", 6088971, device0stats.WeightedIOTicks)
+	}
+	device1stats, err := devices[1].NewBlockDeviceStat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if device1stats.WriteSectors != 286915323 {
+		t.Errorf(failMsgFormat, "Incorrect write merges", 286915323, device1stats.WriteSectors)
+	}
+	if device1stats.DiscardTicks != 12 {
+		t.Errorf(failMsgFormat, "Incorrect discard ticks", 12, device1stats.DiscardTicks)
+	}
+}