Repository: prometheus/procfs
Pull request #558 (2023-08-08 13:54:28)

Support mountstats  add NFS RDMA protocol stats parse
fix: https://github.com/prometheus/node_exporter/issues/2749

mountstats xprt add `rdma` protocol parse

kernel version >= 4.2 , See: https://elixir.bootlin.com/linux/v6.4.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L393
kernel version <= 4.14 , See: https://elixir.bootlin.com/linux/v4.2.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L331


Patch:
diff --git a/mountstats.go b/mountstats.go
index 852c8c4..9d8af6d 100644
--- a/mountstats.go
+++ b/mountstats.go
@@ -44,6 +44,14 @@ const (
 
 	fieldTransport11TCPLen = 13
 	fieldTransport11UDPLen = 10
+
+	// kernel version >= 4.14 MaxLen
+	// See: https://elixir.bootlin.com/linux/v6.4.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L393
+	fieldTransport11RDMAMaxLen = 28
+
+	// kernel version <= 4.2 MinLen
+	// See: https://elixir.bootlin.com/linux/v4.2.8/source/net/sunrpc/xprtrdma/xprt_rdma.h#L331
+	fieldTransport11RDMAMinLen = 20
 )
 
 // A Mount is a device mount parsed from /proc/[pid]/mountstats.
@@ -233,6 +241,33 @@ type NFSTransportStats struct {
 	// A running counter, incremented on each request as the current size of the
 	// pending queue.
 	CumulativePendingQueue uint64
+
+	// Stats below only available with stat version 1.1.
+	// Transport over RDMA
+
+	// accessed when sending a call
+	ReadChunkCount   uint64
+	WriteChunkCount  uint64
+	ReplyChunkCount  uint64
+	TotalRdmaRequest uint64
+
+	// rarely accessed error counters
+	PullupCopyCount      uint64
+	HardwayRegisterCount uint64
+	FailedMarshalCount   uint64
+	BadReplyCount        uint64
+	MrsRecovered         uint64
+	MrsOrphaned          uint64
+	MrsAllocated         uint64
+	EmptySendctxQ        uint64
+
+	// accessed when receiving a reply
+	TotalRdmaReply    uint64
+	FixupCopyCount    uint64
+	ReplyWaitsForSend uint64
+	LocalInvNeeded    uint64
+	NomsgCallCount    uint64
+	BcallCount        uint64
 }
 
 // parseMountStats parses a /proc/[pid]/mountstats file and returns a slice
@@ -587,14 +622,17 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
 			expectedLength = fieldTransport11TCPLen
 		} else if protocol == "udp" {
 			expectedLength = fieldTransport11UDPLen
+		} else if protocol == "rdma" {
+			expectedLength = fieldTransport11RDMAMinLen
 		} else {
 			return nil, fmt.Errorf("%w: invalid NFS protocol \"%s\" in stats 1.1 statement: %v", ErrFileParse, protocol, ss)
 		}
-		if len(ss) != expectedLength {
-			return nil, fmt.Errorf("%w: invalid NFS transport stats 1.1 statement: %v", ErrFileParse, ss)
+		if (len(ss) != expectedLength && (protocol == "tcp" || protocol == "udp")) ||
+			(protocol == "rdma" && len(ss) < expectedLength) {
+			return nil, fmt.Errorf("%w: invalid NFS transport stats 1.1 statement: %v, protocol: %v", ErrFileParse, ss, protocol)
 		}
 	default:
-		return nil, fmt.Errorf("%s: Unrecognized NFS transport stats version: %q", ErrFileParse, statVersion)
+		return nil, fmt.Errorf("%s: Unrecognized NFS transport stats version: %q, protocol: %v", ErrFileParse, statVersion, protocol)
 	}
 
 	// Allocate enough for v1.1 stats since zero value for v1.1 stats will be okay
@@ -604,7 +642,9 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
 	// Note: slice length must be set to length of v1.1 stats to avoid a panic when
 	// only v1.0 stats are present.
 	// See: https://github.com/prometheus/node_exporter/issues/571.
-	ns := make([]uint64, fieldTransport11TCPLen)
+	//
+	// Note: NFS Over RDMA slice length is fieldTransport11RDMAMaxLen
+	ns := make([]uint64, fieldTransport11RDMAMaxLen+3)
 	for i, s := range ss {
 		n, err := strconv.ParseUint(s, 10, 64)
 		if err != nil {
@@ -622,9 +662,14 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
 	// we set them to 0 here.
 	if protocol == "udp" {
 		ns = append(ns[:2], append(make([]uint64, 3), ns[2:]...)...)
+	} else if protocol == "tcp" {
+		ns = append(ns[:fieldTransport11TCPLen], make([]uint64, fieldTransport11RDMAMaxLen-fieldTransport11TCPLen+3)...)
+	} else if protocol == "rdma" {
+		ns = append(ns[:fieldTransport10TCPLen], append(make([]uint64, 3), ns[fieldTransport10TCPLen:]...)...)
 	}
 
 	return &NFSTransportStats{
+		// NFS xprt over tcp or udp
 		Protocol:                 protocol,
 		Port:                     ns[0],
 		Bind:                     ns[1],
@@ -636,8 +681,32 @@ func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats
 		BadTransactionIDs:        ns[7],
 		CumulativeActiveRequests: ns[8],
 		CumulativeBacklog:        ns[9],
-		MaximumRPCSlotsUsed:      ns[10],
-		CumulativeSendingQueue:   ns[11],
-		CumulativePendingQueue:   ns[12],
+
+		// NFS xprt over tcp or udp
+		// And statVersion 1.1
+		MaximumRPCSlotsUsed:    ns[10],
+		CumulativeSendingQueue: ns[11],
+		CumulativePendingQueue: ns[12],
+
+		// NFS xprt over rdma
+		// And stat Version 1.1
+		ReadChunkCount:       ns[13],
+		WriteChunkCount:      ns[14],
+		ReplyChunkCount:      ns[15],
+		TotalRdmaRequest:     ns[16],
+		PullupCopyCount:      ns[17],
+		HardwayRegisterCount: ns[18],
+		FailedMarshalCount:   ns[19],
+		BadReplyCount:        ns[20],
+		MrsRecovered:         ns[21],
+		MrsOrphaned:          ns[22],
+		MrsAllocated:         ns[23],
+		EmptySendctxQ:        ns[24],
+		TotalRdmaReply:       ns[25],
+		FixupCopyCount:       ns[26],
+		ReplyWaitsForSend:    ns[27],
+		LocalInvNeeded:       ns[28],
+		NomsgCallCount:       ns[29],
+		BcallCount:           ns[30],
 	}, nil
 }


Test patch:
diff --git a/mountstats_test.go b/mountstats_test.go
index 3d23671..43903e9 100644
--- a/mountstats_test.go
+++ b/mountstats_test.go
@@ -372,6 +372,102 @@ func TestMountStats(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "NFS xprt over rdma proto",
+			s: `device <nfsserver>:<nfsmount> mounted on <mountpoint> with fstype nfs statvers=1.1
+        opts:   ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=120,acregmax=120,acdirmin=120,acdirmax=120,hard,nocto,forcerdirplus,proto=rdma,nconnect=16,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=172.16.40.20,mountvers=3,mountport=0,mountproto=tcp,local_lock=none
+        age:    1270876
+        caps:   caps=0xf,wtmult=4096,dtsize=131072,bsize=0,namlen=255
+        sec:    flavor=1,pseudoflavor=1
+        events: 512052 36601115 0 68 1498583 16514 38815015 0 41584 2654459933 0 0 0 0 1527715 0 0 1498575 0 0 0 0 0 0 0 0 0
+        bytes:  3104202770327296 0 0 0 2013200952170479 0 491504202537 0
+        RPC iostats version: 1.1  p/v: 100003/3 (nfs)
+        xprt:   rdma 0 0 5808 62 0 494490723 494490687 36 10032963746 1282789 107150285 1226637531 2673889 135120843409861 135119397156505 266368832 75716996 0 7853 0 0 0 0 119328 1336431717 0 96
+        per-op statistics
+                NULL: 16 16 0 640 384 320 11 331 0
+`,
+			mounts: []*Mount{{
+				Device: "<nfsserver>:<nfsmount>",
+				Mount:  "<mountpoint>",
+				Type:   "nfs",
+				Stats: &MountStatsNFS{
+					StatVersion: "1.1",
+					Opts: map[string]string{"acdirmax": "120", "acdirmin": "120", "acregmax": "120",
+						"acregmin": "120", "forcerdirplus": "", "hard": "", "local_lock": "none",
+						"mountaddr": "172.16.40.20", "mountport": "0", "mountproto": "tcp", "mountvers": "3",
+						"namlen": "255", "nconnect": "16", "nocto": "", "port": "20049", "proto": "rdma",
+						"retrans": "2", "ro": "", "rsize": "1048576", "sec": "sys", "timeo": "600",
+						"vers": "3", "wsize": "1048576"},
+					Age: 1270876 * time.Second,
+					Bytes: NFSBytesStats{
+						Read:      3104202770327296,
+						ReadTotal: 2013200952170479,
+						ReadPages: 491504202537,
+					},
+					Events: NFSEventsStats{
+						InodeRevalidate:     512052,
+						DnodeRevalidate:     36601115,
+						AttributeInvalidate: 68,
+						VFSOpen:             1498583,
+						VFSLookup:           16514,
+						VFSAccess:           38815015,
+						VFSReadPage:         41584,
+						VFSReadPages:        2654459933,
+						VFSFlush:            1527715,
+						VFSFileRelease:      1498575,
+					},
+					Operations: []NFSOperationStats{
+						{
+							Operation:                           "NULL",
+							Requests:                            16,
+							Transmissions:                       16,
+							MajorTimeouts:                       0,
+							BytesSent:                           640,
+							BytesReceived:                       384,
+							CumulativeQueueMilliseconds:         320,
+							CumulativeTotalResponseMilliseconds: 11,
+							CumulativeTotalRequestMilliseconds:  331,
+							AverageRTTMilliseconds:              0.6875,
+							Errors:                              0,
+						},
+					},
+					Transport: NFSTransportStats{
+						Protocol:                 "rdma",
+						Port:                     0,
+						Bind:                     0,
+						Connect:                  5808,
+						ConnectIdleTime:          62,
+						IdleTimeSeconds:          0,
+						Sends:                    494490723,
+						Receives:                 494490687,
+						BadTransactionIDs:        36,
+						CumulativeActiveRequests: 10032963746,
+						CumulativeBacklog:        1282789,
+						MaximumRPCSlotsUsed:      0,
+						CumulativeSendingQueue:   0,
+						CumulativePendingQueue:   0,
+						ReadChunkCount:           107150285,
+						WriteChunkCount:          1226637531,
+						ReplyChunkCount:          2673889,
+						TotalRdmaRequest:         135120843409861,
+						PullupCopyCount:          135119397156505,
+						HardwayRegisterCount:     266368832,
+						FailedMarshalCount:       75716996,
+						BadReplyCount:            0,
+						MrsRecovered:             7853,
+						MrsOrphaned:              0,
+						MrsAllocated:             0,
+						EmptySendctxQ:            0,
+						TotalRdmaReply:           0,
+						FixupCopyCount:           119328,
+						ReplyWaitsForSend:        1336431717,
+						LocalInvNeeded:           0,
+						NomsgCallCount:           96,
+						BcallCount:               0,
+					},
+				},
+			}},
+		},
 	}
 
 	for i, tt := range tests {
@@ -518,4 +614,61 @@ device fs.example.com:/volume4/apps/home-automation/node-red-data mounted on /va
 			LOOKUPP: 0 0 0 0 0 0 0 0 0
 		LAYOUTERROR: 0 0 0 0 0 0 0 0 0
 `
+
+	extendedRDMAExampleALLMountstats = `device <nfsserver>:<nfsmount> mounted on <mountpoint> with fstype nfs statvers=1.1
+        opts:   ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=120,acregmax=120,acdirmin=120,acdirmax=120,hard,nocto,forcerdirplus,proto=rdma,nconnect=16,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=172.16.40.20,mountvers=3,mountport=0,mountproto=tcp,local_lock=none
+        age:    1270876
+        caps:   caps=0xf,wtmult=4096,dtsize=131072,bsize=0,namlen=255
+        sec:    flavor=1,pseudoflavor=1
+        events: 512052 36601115 0 68 1498583 16514 38815015 0 41584 2654459933 0 0 0 0 1527715 0 0 1498575 0 0 0 0 0 0 0 0 0
+        bytes:  3104202770327296 0 0 0 2013200952170479 0 491504202537 0
+        RPC iostats version: 1.1  p/v: 100003/3 (nfs)
+        xprt:   rdma 0 0 5808 62 0 494490723 494490687 36 10032963746 1282789 107150285 1226637531 2673889 135120843409861 135119397156505 266368832 75716996 0 7853 0 0 0 0 119328 1336431717 0 96
+        xprt:   rdma 0 0 14094 145 0 492392334 492392307 27 7078693624 2509627 105561370 1280878332 2659446 142218924010291 142217463504063 276368040 94761838 0 7610 0 0 0 0 207977 1389069860 0 103
+        xprt:   rdma 0 0 16107 156 0 522755125 522755092 33 9119562599 1147699 109077860 1491898147 2566003 167152062826463 167149287506014 284931680 83011025 0 6229 0 0 0 0 221408 1603518232 0 82
+        xprt:   rdma 0 0 7808 82 0 441542046 441542010 36 7226132207 2519174 111096004 955223347 2676765 105741904708009 105740125663595 275613584 80373159 0 8893 0 0 0 0 149479 1068962768 0 76
+        xprt:   rdma 0 0 15018 167 0 508091827 508091764 63 19817677255 36702583 108265928 1258185459 2438516 138247436686102 138246196289594 270162080 74962306 0 13328 0 0 0 0 268433 1368837472 0 66
+        xprt:   rdma 0 0 14321 149 0 530246310 530246275 35 9723190432 2392024 111099700 1494204555 2589805 166691166581904 166689567426908 289995492 85067377 0 8010 0 0 0 0 214511 1607864447 0 100
+        xprt:   rdma 0 0 7863 84 0 459019689 459019642 47 11809253102 1716688 111825219 1032758664 2564226 114416685286438 114414936423706 290494252 73702102 0 6927 0 0 0 0 134453 1147121864 0 79
+        xprt:   rdma 0 0 7702 84 3 497598986 497598931 55 11816221496 3924722 106922130 1382063307 2506108 153967067193941 153965665472218 286222584 84094006 0 5875 0 0 0 0 127347 1491469045 0 66
+        xprt:   rdma 0 0 18341 202 0 477721151 477721073 78 15204400959 40562626 106645745 1291616653 3091375 144533696686651 144529688231163 278135800 73821525 0 6795 0 0 0 0 251097 1401327563 0 64
+        xprt:   rdma 0 0 8228 90 4 453155092 453155063 29 7884786894 1591225 112197590 1026006338 2742688 114591819605673 114590175821191 275541944 85857259 0 7487 0 0 0 0 143044 1140917892 0 76
+        xprt:   rdma 0 0 7843 83 0 446480377 446480324 53 12267986428 2958997 111971246 963162784 2693433 107176282309753 107174637802555 290269096 101100410 0 7825 0 0 0 0 141735 1077797328 0 83
+        xprt:   rdma 0 0 7582 86 0 423315608 423315567 41 10197484604 2076993 109207538 785978455 2650354 86090211449474 86088475571312 279912524 87676008 0 7491 0 0 0 0 137533 897807641 0 101
+        xprt:   rdma 0 0 7767 84 0 482538465 482538424 41 8935200479 1344778 112200583 1192341640 2644896 132860698423762 132858881459050 273354060 75337030 0 5941 0 0 0 0 127842 1307164736 0 97
+        xprt:   rdma 0 0 14526 148 2 537745063 537745007 56 20756072620 3970332320 109539564 1363647371 2503250 148793734936250 148791264145401 291888720 90344151 0 7471 0 0 0 0 211057 1475661285 0 82
+        xprt:   rdma 0 0 14300 151 0 495357347 495357316 31 8703101643 1451809 112315311 1303804607 2620502 145680743007170 145678880292235 288046696 98018259 0 7241 0 0 0 0 209396 1418712657 0 139
+        xprt:   rdma 0 0 7700 82 0 466611083 466611050 33 8540498291 4082864 114740300 1059770596 2523155 117376668239921 117375375683167 260927576 78437075 0 6691 0 0 0 0 130878 1177008175 1 76
+        per-op statistics
+                NULL: 16 16 0 640 384 320 11 331 0
+             GETATTR: 512052 512052 0 79823516 57349824 107131 612667 751847 0
+             SETATTR: 0 0 0 0 0 0 0 0 0
+              LOOKUP: 16713 16713 0 3040536 3706344 560 17488 20232 346
+              ACCESS: 211705 211705 0 33860920 25404600 37059 229754 283822 0
+            READLINK: 0 0 0 0 0 0 0 0 0
+                READ: 2654501510 2654501510 0 445911966900 2013540728551504 6347457114 31407021389 37927280438 0
+               WRITE: 0 0 0 0 0 0 0 0 0
+              CREATE: 0 0 0 0 0 0 0 0 0
+               MKDIR: 0 0 0 0 0 0 0 0 0
+             SYMLINK: 0 0 0 0 0 0 0 0 0
+               MKNOD: 0 0 0 0 0 0 0 0 0
+              REMOVE: 0 0 0 0 0 0 0 0 0
+               RMDIR: 0 0 0 0 0 0 0 0 0
+              RENAME: 0 0 0 0 0 0 0 0 0
+                LINK: 0 0 0 0 0 0 0 0 0
+             READDIR: 0 0 0 0 0 0 0 0 0
+         READDIRPLUS: 0 0 0 0 0 0 0 0 0
+              FSSTAT: 56356 56356 0 6243572 9467808 82068 74356 159001 0
+              FSINFO: 2 2 0 184 328 0 0 0 0
+            PATHCONF: 1 1 0 92 140 0 0 0 0
+              COMMIT: 0 0 0 0 0 0 0 0 0
+`
 )
+
+func TestMountStatsExtendedRDMAStats(t *testing.T) {
+	r := strings.NewReader(extendedRDMAExampleALLMountstats)
+	_, err := parseMountStats(r)
+	if err != nil {
+		t.Errorf("failed to parse mount stats with extended RDMA statistics: %v", err)
+	}
+}