Repository: elastic/go-sysinfo
Pull request #72 (2019-12-02 23:52:36)

Add Linux Networking counters
See  https://github.com/elastic/beats/issues/12210

Started this before I left for a 4-day weekend so it might be a tad all over, hopefully not.

I fear this is a case of me being a bit too 'clever' since I came up with a way to have one method that populates all of the structs. It would be possible to combine `fillStruct` and `readAndParseNetFile` , but then we would need `FieldbyName` as opposed to `Field` to get the struct, which means each call would iterate over the struct `NumField()` times as `FieldByName` just iterates until it gets a given name. I would still like to find a better way to do this.

Other than that, this is fairly straight forward. The mapping of struct field -> map is because the network counter types are known, but there's basically no documentation on the counters themselves, they're just random kernel fields. I think it would be better to have this library be more neutral, and any editorializing/filtering can be done in metricbeat.

Ah! Still need to update the readme.


Patch:
diff --git a/README.md b/README.md
index 958f11c..d80c5bf 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,7 @@ that are implemented.
 | `Memory()`       | x      | x     | x       |
 | `CPUTimer`       | x      | x     | x       |
 | `VMStat`         |        | x     |         |
+| `NetworkCounters`|        | x     |         |
 
 | `Process` Features     | Darwin | Linux | Windows |
 |------------------------|--------|-------|---------|
diff --git a/providers/linux/host_linux.go b/providers/linux/host_linux.go
index 19325b0..7727e72 100644
--- a/providers/linux/host_linux.go
+++ b/providers/linux/host_linux.go
@@ -81,6 +81,29 @@ func (h *host) VMStat() (*types.VMStatInfo, error) {
 	return parseVMStat(content)
 }
 
+// NetworkCounters reports data from /proc/net on linux
+func (h *host) NetworkCounters() (*types.NetworkCountersInfo, error) {
+	snmpRaw, err := ioutil.ReadFile(h.procFS.path("net/snmp"))
+	if err != nil {
+		return nil, err
+	}
+	snmp, err := getNetSnmpStats(snmpRaw)
+	if err != nil {
+		return nil, err
+	}
+
+	netstatRaw, err := ioutil.ReadFile(h.procFS.path("net/netstat"))
+	if err != nil {
+		return nil, err
+	}
+	netstat, err := getNetstatStats(netstatRaw)
+	if err != nil {
+		return nil, err
+	}
+
+	return &types.NetworkCountersInfo{SNMP: snmp, Netstat: netstat}, nil
+}
+
 func (h *host) CPUTime() (types.CPUTimes, error) {
 	stat, err := h.procFS.NewStat()
 	if err != nil {
diff --git a/providers/linux/procnet.go b/providers/linux/procnet.go
new file mode 100644
index 0000000..685d780
--- /dev/null
+++ b/providers/linux/procnet.go
@@ -0,0 +1,112 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package linux
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/go-sysinfo/types"
+)
+
+// fillStruct is some reflection work that can dynamically fill one of our tagged `netstat` structs with netstat data
+func fillStruct(str interface{}, data map[string]map[string]int64) {
+	val := reflect.ValueOf(str).Elem()
+	typ := reflect.TypeOf(str).Elem()
+
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		if tag := field.Tag.Get("netstat"); tag != "" {
+			if values, ok := data[tag]; ok {
+				val.Field(i).Set(reflect.ValueOf(values))
+			}
+		}
+	}
+}
+
+// parseEntry parses two lines from the net files, the first line being keys, the second being values
+func parseEntry(line1, line2 string) (map[string]int64, error) {
+	keyArr := strings.Split(strings.TrimSpace(line1), " ")
+	valueArr := strings.Split(strings.TrimSpace(line2), " ")
+
+	if len(keyArr) != len(valueArr) {
+		return nil, errors.New("key and value lines are mismatched")
+	}
+
+	counters := make(map[string]int64, len(valueArr))
+	for iter, value := range valueArr {
+		parsed, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error parsing string to int in line: %#v", valueArr)
+		}
+		counters[keyArr[iter]] = parsed
+	}
+	return counters, nil
+}
+
+// parseNetFile parses an entire file, and returns a 2D map, representing how files are sorted by protocol
+func parseNetFile(body string) (map[string]map[string]int64, error) {
+	fileMetrics := make(map[string]map[string]int64)
+	bodySplit := strings.Split(strings.TrimSpace(body), "\n")
+	// There should be an even number of lines. If not, something is wrong.
+	if len(bodySplit)%2 != 0 {
+		return nil, fmt.Errorf("badly parsed body: %s", body)
+	}
+	// in the network counters, data is divided into two-line sections: a line of keys, and a line of values
+	// With each line
+	for index := 0; index < len(bodySplit); index += 2 {
+		keysSplit := strings.Split(bodySplit[index], ":")
+		valuesSplit := strings.Split(bodySplit[index+1], ":")
+		if len(keysSplit) != 2 || len(valuesSplit) != 2 {
+			return nil, fmt.Errorf("wrong number of keys: %#v", keysSplit)
+		}
+		valMap, err := parseEntry(keysSplit[1], valuesSplit[1])
+		if err != nil {
+			return nil, errors.Wrap(err, "error parsing lines")
+		}
+		fileMetrics[valuesSplit[0]] = valMap
+	}
+	return fileMetrics, nil
+}
+
+// getNetSnmpStats pulls snmp stats from /proc/net
+func getNetSnmpStats(raw []byte) (types.SNMP, error) {
+	snmpData, err := parseNetFile(string(raw))
+	if err != nil {
+		return types.SNMP{}, errors.Wrap(err, "error parsing SNMP")
+	}
+	output := types.SNMP{}
+	fillStruct(&output, snmpData)
+
+	return output, nil
+}
+
+// getNetstatStats pulls netstat stats from /proc/net
+func getNetstatStats(raw []byte) (types.Netstat, error) {
+	netstatData, err := parseNetFile(string(raw))
+	if err != nil {
+		return types.Netstat{}, errors.Wrap(err, "error parsing netstat")
+	}
+	output := types.Netstat{}
+	fillStruct(&output, netstatData)
+	return output, nil
+}
diff --git a/types/host.go b/types/host.go
index bf5b80c..80db2c8 100644
--- a/types/host.go
+++ b/types/host.go
@@ -26,6 +26,33 @@ type Host interface {
 	Memory() (*HostMemoryInfo, error)
 }
 
+// NetworkCounters represents network stats from /proc/net
+type NetworkCounters interface {
+	NetworkCounters() (*NetworkCountersInfo, error)
+}
+
+// SNMP represents the data from /proc/net/snmp
+type SNMP struct {
+	IP      map[string]int64 `json:"ip" netstat:"Ip"`
+	ICMP    map[string]int64 `json:"icmp" netstat:"Icmp"`
+	ICMPMsg map[string]int64 `json:"icmp_msg" netstat:"IcmpMsg"`
+	TCP     map[string]int64 `json:"tcp" netstat:"Tcp"`
+	UDP     map[string]int64 `json:"udp" netstat:"Udp"`
+	UDPLite map[string]int64 `json:"udp_lite" netstat:"UdpLite"`
+}
+
+// Netstat represents the data from /proc/net/netstat
+type Netstat struct {
+	TCPExt map[string]int64 `json:"tcp_ext" netstat:"TcpExt"`
+	IPExt  map[string]int64 `json:"ip_ext" netstat:"IpExt"`
+}
+
+// NetworkCountersInfo represents available network counters from /proc/net
+type NetworkCountersInfo struct {
+	SNMP    SNMP    `json:"snmp"`
+	Netstat Netstat `json:"netstat"`
+}
+
 // VMStat is the interface wrapper for platforms that support /proc/vmstat.
 type VMStat interface {
 	VMStat() (*VMStatInfo, error)


Test patch:
diff --git a/providers/linux/host_linux_test.go b/providers/linux/host_linux_test.go
index 17bb4ad..fb36e95 100644
--- a/providers/linux/host_linux_test.go
+++ b/providers/linux/host_linux_test.go
@@ -71,3 +71,25 @@ func TestHostVMStat(t *testing.T) {
 	}
 	t.Log(string(data))
 }
+
+func TestHostNetworkCounters(t *testing.T) {
+	host, err := newLinuxSystem("testdata/fedora30").Host()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := host.(types.NetworkCounters).NetworkCounters()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotEmpty(t, s.Netstat.IPExt)
+	assert.NotEmpty(t, s.Netstat.TCPExt)
+	assert.NotEmpty(t, s.SNMP.IP)
+
+	data, err := json.MarshalIndent(s, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(string(data))
+}
diff --git a/providers/linux/procnet_test.go b/providers/linux/procnet_test.go
new file mode 100644
index 0000000..d09a189
--- /dev/null
+++ b/providers/linux/procnet_test.go
@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package linux
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/go-sysinfo/types"
+)
+
+func TestParseFile(t *testing.T) {
+	file := `Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates
+Ip: 1 64 23123056 0 19 21075 0 0 21842645 16017256 1 4 0 0 0 0 0 0 0
+Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps
+Icmp: 487 1 0 486 0 0 0 0 1 0 0 0 0 0 570 0 569 0 0 0 0 0 1 0 0 0 0
+IcmpMsg: InType3 InType8 OutType0 OutType3
+IcmpMsg: 486 1 1 569
+Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors
+Tcp: 1 200 120000 -1 4160 873 1408 334 10 21825426 34620510 10474 0 6102 0
+Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+Udp: 16755 33 0 16829 0 0 0 0
+UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+UdpLite: 0 0 0 0 0 0 0 0`
+	mapStr, err := parseNetFile(file)
+	if err != nil {
+		t.Fatalf("error: %s", err)
+	}
+	testOut := types.SNMP{}
+	fillStruct(&testOut, mapStr)
+
+	assert.NotEmpty(t, testOut.IP)
+	assert.Equal(t, int64(16755), testOut.UDP["InDatagrams"])
+}
diff --git a/providers/linux/testdata/fedora30/proc/net/netstat b/providers/linux/testdata/fedora30/proc/net/netstat
new file mode 100644
index 0000000..7b390ba
--- /dev/null
+++ b/providers/linux/testdata/fedora30/proc/net/netstat
@@ -0,0 +1,4 @@
+TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed EmbryonicRsts PruneCalled RcvPruned OfoPruned OutOfWindowIcmps LockDroppedIcmps ArpFilter TW TWRecycled TWKilled PAWSActive PAWSEstab DelayedACKs DelayedACKLocked DelayedACKLost ListenOverflows ListenDrops TCPHPHits TCPPureAcks TCPHPAcks TCPRenoRecovery TCPSackRecovery TCPSACKReneging TCPSACKReorder TCPRenoReorder TCPTSReorder TCPFullUndo TCPPartialUndo TCPDSACKUndo TCPLossUndo TCPLostRetransmit TCPRenoFailures TCPSackFailures TCPLossFailures TCPFastRetrans TCPSlowStartRetrans TCPTimeouts TCPLossProbes TCPLossProbeRecovery TCPRenoRecoveryFail TCPSackRecoveryFail TCPRcvCollapsed TCPBacklogCoalesce TCPDSACKOldSent TCPDSACKOfoSent TCPDSACKRecv TCPDSACKOfoRecv TCPAbortOnData TCPAbortOnClose TCPAbortOnMemory TCPAbortOnTimeout TCPAbortOnLinger TCPAbortFailed TCPMemoryPressures TCPMemoryPressuresChrono TCPSACKDiscard TCPDSACKIgnoredOld TCPDSACKIgnoredNoUndo TCPSpuriousRTOs TCPMD5NotFound TCPMD5Unexpected TCPMD5Failure TCPSackShifted TCPSackMerged TCPSackShiftFallback TCPBacklogDrop PFMemallocDrop TCPMinTTLDrop TCPDeferAcceptDrop IPReversePathFilter TCPTimeWaitOverflow TCPReqQFullDoCookies TCPReqQFullDrop TCPRetransFail TCPRcvCoalesce TCPOFOQueue TCPOFODrop TCPOFOMerge TCPChallengeACK TCPSYNChallenge TCPFastOpenActive TCPFastOpenActiveFail TCPFastOpenPassive TCPFastOpenPassiveFail TCPFastOpenListenOverflow TCPFastOpenCookieReqd TCPFastOpenBlackhole TCPSpuriousRtxHostQueues BusyPollRxPackets TCPAutoCorking TCPFromZeroWindowAdv TCPToZeroWindowAdv TCPWantZeroWindowAdv TCPSynRetrans TCPOrigDataSent TCPHystartTrainDetect TCPHystartTrainCwnd TCPHystartDelayDetect TCPHystartDelayCwnd TCPACKSkippedSynRecv TCPACKSkippedPAWS TCPACKSkippedSeq TCPACKSkippedFinWait2 TCPACKSkippedTimeWait TCPACKSkippedChallenge TCPWinProbe TCPKeepAlive TCPMTUPFail TCPMTUPSuccess TCPDelivered TCPDeliveredCE TCPAckCompressed TCPZeroWindowDrop TCPRcvQDrop
+TcpExt: 0 0 2 2 0 0 0 8 0 0 934 0 0 0 15 287657 86 1174 0 0 13549524 3703992 2542609 0 118 0 467 0 168 3 39 35 359 1387 0 5 1 3781 2 1955 7458 366 0 2 0 278312 1238 11 4216 2 1494 272 0 77 0 0 0 0 82 186 1869 3 0 0 0 25682 9680 660 0 0 0 0 0 0 0 0 0 3548522 127400 0 10 0 0 0 0 0 0 0 0 0 0 0 104 883 883 71010 257 29255516 148 3243 18 2382 3 1 3 0 0 0 0 204720 0 0 29260768 0 109908 0 0
+IpExt: InNoRoutes InTruncatedPkts InMcastPkts OutMcastPkts InBcastPkts OutBcastPkts InOctets OutOctets InMcastOctets OutMcastOctets InBcastOctets OutBcastOctets InCsumErrors InNoECTPkts InECT1Pkts InECT0Pkts InCEPkts ReasmOverlaps
+IpExt: 0 0 0 0 601711 0 102243038277 39839396260 0 0 143425103 0 0 76854883 0 3160941 0 0
diff --git a/providers/linux/testdata/fedora30/proc/net/snmp b/providers/linux/testdata/fedora30/proc/net/snmp
new file mode 100644
index 0000000..588937f
--- /dev/null
+++ b/providers/linux/testdata/fedora30/proc/net/snmp
@@ -0,0 +1,12 @@
+Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates
+Ip: 1 64 23685774 0 19 22122 0 0 22328773 16478755 1 4 0 0 0 0 0 0 0
+Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps
+Icmp: 487 1 0 486 0 0 0 0 1 0 0 0 0 0 571 0 570 0 0 0 0 0 1 0 0 0 0
+IcmpMsg: InType3 InType8 OutType0 OutType3
+IcmpMsg: 486 1 1 570
+Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors
+Tcp: 1 200 120000 -1 4224 885 1408 343 8 22311083 35810350 10752 0 6286 0
+Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+Udp: 17225 34 0 17301 0 0 0 0
+UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+UdpLite: 0 0 0 0 0 0 0 0
diff --git a/providers/linux/testdata/fedora30/proc/stat b/providers/linux/testdata/fedora30/proc/stat
new file mode 100644
index 0000000..afa00df
--- /dev/null
+++ b/providers/linux/testdata/fedora30/proc/stat
@@ -0,0 +1,12 @@
+cpu  31608779 223223 15924288 1134746753 223880 4348430 1995355 266903 0 0
+cpu0 7929502 52284 3994607 283776341 58002 1041339 448737 62980 0 0
+cpu1 7871778 55337 3986872 283642724 50819 1099882 580838 68839 0 0
+cpu2 7951421 68162 4002718 283618789 59074 1071996 526388 65991 0 0
+cpu3 7856076 47438 3940090 283708897 55983 1135211 439391 69093 0 0
+intr 3649757765 110 9 0 0 0 0 0 0 1 0 1493876 32 18 0 0 2918021 0 0 0 0 0 0 0 0 0 0 0 8660478 0 19957110 246 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ctxt 5607775026
+btime 1571459083
+processes 4891735
+procs_running 1
+procs_blocked 0
+softirq 1934569580 0 876016023 1299 20736488 9710454 0 130 357152191 25831 670927164