Repository: kentik/ktranslate
Pull request #63 (2021-07-16 01:17:30)

Adding CPU and MemoryUtilization (#62)
Also enforces a min polling time of 30 sec. 


Patch:
diff --git a/config/profiles/kentik_snmp/cisco/cisco-all-devices.yml b/config/profiles/kentik_snmp/cisco/cisco-all-devices.yml
index be2a809..565efa0 100644
--- a/config/profiles/kentik_snmp/cisco/cisco-all-devices.yml
+++ b/config/profiles/kentik_snmp/cisco/cisco-all-devices.yml
@@ -79,6 +79,7 @@ metrics:
         name: cpmCPUMemoryFree
       - OID: 1.3.6.1.4.1.9.9.109.1.1.1.1.7
         name: cpmCPUTotal1minRev
+        tag: CPU
     metric_tags:
       - index: 1
         tag: cpu
@@ -213,8 +214,10 @@ metrics:
     symbols:
       - OID: 1.3.6.1.4.1.9.9.48.1.1.1.5
         name: ciscoMemoryPoolUsed
+        tag: MemoryUsed
       - OID: 1.3.6.1.4.1.9.9.48.1.1.1.6
         name: ciscoMemoryPoolFree
+        tag: MemoryFree
       - OID: 1.3.6.1.4.1.9.9.48.1.1.1.7
         name: ciscoMemoryPoolLargestFree
     metric_tags:
@@ -359,7 +362,7 @@ metrics:
       - index: 1
         tag: rtt_operation_index
   - MIB: CISCO-VPC-MIB
-    table: 
+    table:
       OID: 1.3.6.1.4.1.9.9.807.1.4.2
       name: cVpcStatusHostLinkTable
     symbols:
diff --git a/config/profiles/kentik_snmp/juniper/juniper-all-devices.yml b/config/profiles/kentik_snmp/juniper/juniper-all-devices.yml
new file mode 100644
index 0000000..639d7e8
--- /dev/null
+++ b/config/profiles/kentik_snmp/juniper/juniper-all-devices.yml
@@ -0,0 +1,32 @@
+# http://oid-info.com/get/1.3.6.1.4.1.2636.3.1.13.1
+---
+extends:
+  - system-mib.yml
+  - if-mib.yml
+  - ip-mib.yml
+  - tcp-mib.yml
+  - udp-mib.yml
+  - bgp4-mib.yml
+  - ospf-mib.yml
+  - host-resources-mib.yml
+
+sysobjectid:
+  - 1.3.6.1.4.1.2636.1.1.1.*
+
+metrics:
+  - MIB: JUNIPER-MIB
+    table:
+      OID: 1.3.6.1.4.1.2636.3.1.13.1
+      name: jnxOperatingEntry
+    symbols:
+      - OID: 1.3.6.1.4.1.2636.3.1.13.1.8
+        name: jnxOperatingCPU
+        tag: CPU
+      - OID: 1.3.6.1.4.1.2636.3.1.13.1.11
+        name: jnxOperatingBuffer
+        tag: MemoryUtilization
+    metric_tags:
+      - tag: chassis
+        column:
+          OID: 1.3.6.1.4.1.2636.3.1.13.1.18
+          name: jnxOperatingChassisDescr
diff --git a/config/profiles/kentik_snmp/juniper/juniper-ex-switches.yml b/config/profiles/kentik_snmp/juniper/juniper-ex-switches.yml
index 2a87fca..b3a4829 100644
--- a/config/profiles/kentik_snmp/juniper/juniper-ex-switches.yml
+++ b/config/profiles/kentik_snmp/juniper/juniper-ex-switches.yml
@@ -4,14 +4,7 @@
 # http://oid-info.com/get/1.3.6.1.4.1.2636.3.15.10
 ---
 extends:
-  - system-mib.yml
-  - if-mib.yml
-  - ip-mib.yml
-  - tcp-mib.yml
-  - udp-mib.yml
-  - bgp4-mib.yml
-  - ospf-mib.yml
-  - host-resources-mib.yml
+  - juniper-all-devices.yml
 
 sysobjectid:
   - 1.3.6.1.4.1.2636.1.1.1.2.30
diff --git a/config/profiles/kentik_snmp/juniper/juniper-mx-router.yml b/config/profiles/kentik_snmp/juniper/juniper-mx-router.yml
index 9a0f43b..0e090d4 100644
--- a/config/profiles/kentik_snmp/juniper/juniper-mx-router.yml
+++ b/config/profiles/kentik_snmp/juniper/juniper-mx-router.yml
@@ -2,14 +2,7 @@
 # http://oid-info.com/get/1.3.6.1.4.1.2636.3.40.1.4.1
 ---
 extends:
-  - system-mib.yml
-  - if-mib.yml
-  - ip-mib.yml
-  - tcp-mib.yml
-  - udp-mib.yml
-  - bgp4-mib.yml
-  - ospf-mib.yml
-  - host-resources-mib.yml
+  - juniper-all-devices.yml
 
 sysobjectid:
   - 1.3.6.1.4.1.2636.1.1.1.2.21
diff --git a/config/profiles/kentik_snmp/juniper/juniper-srx-firewalls.yml b/config/profiles/kentik_snmp/juniper/juniper-srx-firewalls.yml
index b42bad6..605645e 100644
--- a/config/profiles/kentik_snmp/juniper/juniper-srx-firewalls.yml
+++ b/config/profiles/kentik_snmp/juniper/juniper-srx-firewalls.yml
@@ -2,14 +2,7 @@
 # http://oid-info.com/get/1.3.6.1.4.1.2636.3.16.1.1
 ---
 extends:
-  - system-mib.yml
-  - if-mib.yml
-  - ip-mib.yml
-  - tcp-mib.yml
-  - udp-mib.yml
-  - ospf-mib.yml
-  - bgp4-mib.yml
-  - host-resources-mib.yml
+  - juniper-all-devices.yml
 
 sysobjectid:
   - 1.3.6.1.4.1.2636.1.1.1.2.26
diff --git a/pkg/inputs/snmp/metrics/device_metrics.go b/pkg/inputs/snmp/metrics/device_metrics.go
index de7e501..6eaee8a 100644
--- a/pkg/inputs/snmp/metrics/device_metrics.go
+++ b/pkg/inputs/snmp/metrics/device_metrics.go
@@ -317,6 +317,20 @@ func (dm *DeviceMetrics) pollFromConfig(server *gosnmp.GoSNMP) ([]*kt.JCHF, erro
 		for k, v := range dm.conf.UserTags {
 			dst.CustomStr[k] = v
 		}
+
+		// Memory can be compound value so need to do it here if present but not already set.
+		if _, ok := dst.CustomBigInt["MemoryUtilization"]; !ok {
+			memoryUsed, oku := dst.CustomBigInt["MemoryUsed"]
+			memoryFree, okt := dst.CustomBigInt["MemoryFree"]
+			if oku && okt {
+				memoryTotal := memoryFree + memoryUsed
+				if memoryTotal > 0 {
+					dst.CustomBigInt["MemoryUtilization"] = int64(float32(memoryUsed) / float32(memoryTotal) * 100)
+					dst.CustomMetrics["MemoryUtilization"] = "computed"
+				}
+			}
+		}
+
 		flows = append(flows, dst)
 	}
 
diff --git a/pkg/inputs/snmp/metrics/poll.go b/pkg/inputs/snmp/metrics/poll.go
index 2926635..a121ac6 100644
--- a/pkg/inputs/snmp/metrics/poll.go
+++ b/pkg/inputs/snmp/metrics/poll.go
@@ -31,6 +31,11 @@ func NewPoller(server *gosnmp.GoSNMP, gconf *kt.SnmpGlobalConfig, conf *kt.SnmpD
 	} else if gconf != nil && gconf.PollTimeSec > 0 {
 		counterTimeSec = gconf.PollTimeSec
 	}
+	// Lastly, enforece a min polling interval.
+	if counterTimeSec < 30 {
+		log.Warnf("%d poll time is below min of 30. Raising to 30 seconds", counterTimeSec)
+		counterTimeSec = 30
+	}
 
 	// Default is not not drop.
 	dropIfOutside := false
diff --git a/pkg/inputs/snmp/mibs/profile.go b/pkg/inputs/snmp/mibs/profile.go
index 805f5ac..df6e010 100644
--- a/pkg/inputs/snmp/mibs/profile.go
+++ b/pkg/inputs/snmp/mibs/profile.go
@@ -322,6 +322,10 @@ func (p *Profile) GetMetrics(enabledMibs []string) (map[string]*kt.Mib, map[stri
 		}
 	}
 
+	// If we have any duplicate names, keep the longest OID.
+	prune(deviceMetrics)
+	prune(interfaceMetrics)
+
 	return deviceMetrics, interfaceMetrics
 }
 
@@ -384,6 +388,11 @@ func (p *Profile) GetMetadata(enabledMibs []string) (map[string]*kt.Mib, map[str
 			}
 		}
 	}
+
+	// If we have any duplicate names, keep the longest OID.
+	prune(deviceMetadata)
+	prune(interfaceMetadata)
+
 	return deviceMetadata, interfaceMetadata
 }
 
@@ -510,3 +519,36 @@ func newProfileFromApc(ap *apc.APC, file string, log logger.ContextL) []*Profile
 
 	return profiles
 }
+
+// Sometimes the same tag can be in multiple mibs. Run down the list and keep the longest oid if there are any collisions.
+func prune(mibs map[string]*kt.Mib) {
+	seenNames := map[string]string{}
+
+	for oid, mib := range mibs {
+		oidName := mib.Name
+		if mib.Tag != "" {
+			oidName = mib.Tag
+		}
+		if _, ok := seenNames[oidName]; !ok { // We haven't seen this name yet so mark that we have it.
+			seenNames[oidName] = oid
+		} else {
+			// There's a conflict. Keep the longest oid.
+			if len(oid) > len(seenNames[oidName]) {
+				seenNames[oidName] = oid
+			}
+		}
+	}
+
+	// Reverse this map to know who we're keeping.
+	keepNames := map[string]bool{}
+	for _, oid := range seenNames {
+		keepNames[oid] = true
+	}
+
+	// Lastly, delete if we're not keeping.
+	for oid, _ := range mibs {
+		if !keepNames[oid] {
+			delete(mibs, oid)
+		}
+	}
+}


Test patch:
diff --git a/pkg/inputs/snmp/mibs/profile_test.go b/pkg/inputs/snmp/mibs/profile_test.go
index 0f34260..26f39c7 100644
--- a/pkg/inputs/snmp/mibs/profile_test.go
+++ b/pkg/inputs/snmp/mibs/profile_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
 	lt "github.com/kentik/ktranslate/pkg/eggs/logger/testing"
+	"github.com/kentik/ktranslate/pkg/kt"
 )
 
 func TestFindProfile(t *testing.T) {
@@ -49,3 +50,20 @@ func TestFindProfile(t *testing.T) {
 		}
 	}
 }
+
+func TestPrune(t *testing.T) {
+	mibs := map[string]*kt.Mib{
+		".1.3.6.1.4.1.9.9.48.1.1.1.5": &kt.Mib{Tag: "MemoryUsed"},
+	}
+	prune(mibs)
+	assert.Equal(t, 1, len(mibs))
+
+	mibs = map[string]*kt.Mib{
+		".1.3.6.1.4.1.9.9.48.1.1.1.5":     &kt.Mib{Tag: "MemoryUsed"},
+		"1.3.6.1.4.1.9.9.305.1.1.1":       &kt.Mib{Tag: "CPU"},
+		".1.3.6.1.4.1.9.9.109.1.1.1.1.7":  &kt.Mib{Tag: "CPU"},
+		".1.3.6.1.4.1.9.9.109.1.1.1.1.13": &kt.Mib{Name: "cpmCPUMemoryFree"},
+	}
+	prune(mibs)
+	assert.Equal(t, 3, len(mibs))
+}