Repository: kentik/ktranslate
Pull request #124 (2021-09-02 22:24:43)

Dynamic instrumentation.name values
Setting `instrumentation.name` based on what profile is found for SNMP.

Also changing default snmp trap to 1620 to avoid need for root on Linux and updated windows build file. 


Patch:
diff --git a/config/snmp-base.yaml b/config/snmp-base.yaml
index c60ac31..37a6ce4 100644
--- a/config/snmp-base.yaml
+++ b/config/snmp-base.yaml
@@ -1,6 +1,6 @@
 devices:
 trap:
-  listen: 127.0.0.1:162
+  listen: 127.0.0.1:1620
   community: hello
   version: ""
   transport: ""
diff --git a/config/snmp-win.yaml b/config/snmp-win.yaml
index fe31256..a9edb7f 100644
--- a/config/snmp-win.yaml
+++ b/config/snmp-win.yaml
@@ -1,6 +1,6 @@
 devices:
 trap:
-  listen: 127.0.0.1:162
+  listen: 127.0.0.1:1620
   community: hello
   version: ""
   transport: ""
diff --git a/ktranslate.wxs b/ktranslate.wxs
index 76201e3..33f277b 100644
--- a/ktranslate.wxs
+++ b/ktranslate.wxs
@@ -56,6 +56,11 @@
                 <File Id="filCC765DB29D59E91AD84875370FB175D8" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/aruba/aruba.yml"/>
               </Component>
             </Directory>
+            <Directory Id="dir5424FE08AFD387F438440DA4CC5DCE55" Name="brocade">
+              <Component Id="cmp4463D2D7EF2E64D60B7165ED2C67921D" Guid="*">
+                <File Id="filF4A69B1D861A111861A00CBC726BEC45" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/brocade/brocade-fc-switch.yml"/>
+              </Component>
+            </Directory>
             <Directory Id="dir184326711F224C33A87911F499EC71C2" Name="apc">
               <Component Id="cmp73E19C142A042AAFF0DCE76003EFFEBB" Guid="*">
                 <File Id="fil223E3A8C3D00473AD3A5C3E7BB0A916C" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/apc/apc_ups.yml"/>
@@ -79,10 +84,18 @@
                 <File Id="fil963DD98235B75B78B64CCDB266A63EF9" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/generic/base.yml"/>
               </Component>
             </Directory>
+            <Directory Id="dirE120CB3AFB969BA1D1607DE3F0262F3D" Name="dell">
+              <Component Id="cmp93DFEBCE84E7942A476C36A274F90591" Guid="*">
+                <File Id="filEC5959567D8BA8F7AE7CAF0DF0A78AE6" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/dell/idrac.yml"/>
+              </Component>
+            </Directory>
             <Directory Id="dir339464D2982D6915DE83ABB5613157A1" Name="netgear">
               <Component Id="cmpD09A243C955CAF9F43EA7377874B6D13" Guid="*">
                 <File Id="filF01A5BC256064004B2D3CBD33FDCAC66" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/netgear/readynas.yml"/>
               </Component>
+              <Component Id="cmp6A788D7A9CFF37ACBD09B30D33572D3A" Guid="*">
+                <File Id="filEADCA998EDCA37F70BA3DA5D65AC48A5" KeyPath="yes" Source="$(var.Path)/config/profiles/kentik_snmp/netgear/netgear-generic.yml"/>
+              </Component>
             </Directory>
             <Directory Id="dirA24188538623D1A47393C1E3E3BDA301" Name="checkpoint">
               <Component Id="cmp797BDC5B5A66F93A7B18E1BE8DB9E238" Guid="*">
@@ -228,6 +241,7 @@
         <Feature Id="MainApplication" Title="Main Application" Level="1">
           <ComponentRef Id="ktranslate.exe" />
         <ComponentRef Id="cmpB46AF4C0CF9C68FA9C458C1219F7C1D1" />
+              <ComponentRef Id="cmp4463D2D7EF2E64D60B7165ED2C67921D" />
               <ComponentRef Id="cmp6052E2C43A4DDBA643B6A03B0C5D3B39" />
               <ComponentRef Id="cmp73E19C142A042AAFF0DCE76003EFFEBB" />
               <ComponentRef Id="cmpC1F0A78F692AF501FBC57AA044D2BCF1" />
@@ -274,6 +288,8 @@
         <ComponentRef Id="cmp7EA4396B8D9EC67B7910D8A242FFCF57" />
         <ComponentRef Id="cmpB7CB43E39FFEAC4402F746BD2B4A43D4" />
         <ComponentRef Id="cmp34171BB83455309CBC625CC7B8309AD4" />
+        <ComponentRef Id="cmp93DFEBCE84E7942A476C36A274F90591" />
+        <ComponentRef Id="cmp6A788D7A9CFF37ACBD09B30D33572D3A" />
         </Feature>
 
     </Product>
diff --git a/pkg/formats/nrm/nrm.go b/pkg/formats/nrm/nrm.go
index d043e60..462bdf1 100644
--- a/pkg/formats/nrm/nrm.go
+++ b/pkg/formats/nrm/nrm.go
@@ -31,7 +31,6 @@ const (
 
 	InstNameVPCMetric     = "vpc-logs-metrics"
 	InstNameNetflowMetric = "netflow-metrics"
-	InstNameSNMP          = "snmp"
 	InstNameSynthetic     = "synthetic"
 	InstNameKtranslate    = "heartbeat"
 
@@ -482,11 +481,13 @@ func (f *NRMFormat) fromSnmpInterfaceMetric(in *kt.JCHF) []NRMetric {
 	}
 
 	ms := make([]NRMetric, 0, len(metrics))
+	profileName := "snmp"
 	for m, name := range metrics {
 		if m == "" {
 			f.Errorf("Missing metric name, skipping %v", attr)
 			continue
 		}
+		profileName = name.Profile
 		if _, ok := in.CustomBigInt[m]; ok {
 			attrNew := copyAttrForSnmp(attr, m, name)
 			ms = append(ms, NRMetric{
@@ -505,7 +506,7 @@ func (f *NRMFormat) fromSnmpInterfaceMetric(in *kt.JCHF) []NRMetric {
 				if ispeed, ok := speed.(int32); ok {
 					uptimeSpeed := in.CustomBigInt["Uptime"] * (int64(ispeed) * 1000000) // Convert into bits here, from megabits.
 					if uptimeSpeed > 0 {
-						attrNew := copyAttrForSnmp(attr, "IfInUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed"})
+						attrNew := copyAttrForSnmp(attr, "IfInUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: profileName})
 						ms = append(ms, NRMetric{
 							Name:       "kentik.snmp.IfInUtilization",
 							Type:       NR_GAUGE_TYPE,
@@ -521,7 +522,7 @@ func (f *NRMFormat) fromSnmpInterfaceMetric(in *kt.JCHF) []NRMetric {
 				if ispeed, ok := speed.(int32); ok {
 					uptimeSpeed := in.CustomBigInt["Uptime"] * (int64(ispeed) * 1000000) // Convert into bits here, from megabits.
 					if uptimeSpeed > 0 {
-						attrNew := copyAttrForSnmp(attr, "IfOutUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed"})
+						attrNew := copyAttrForSnmp(attr, "IfOutUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: profileName})
 						ms = append(ms, NRMetric{
 							Name:       "kentik.snmp.IfOutUtilization",
 							Type:       NR_GAUGE_TYPE,
@@ -616,7 +617,7 @@ func copyAttrForSnmp(attr map[string]interface{}, metricName string, name kt.Met
 	attrNew := map[string]interface{}{
 		"objectIdentifier":     name.Oid,
 		"mib-name":             name.Mib,
-		"instrumentation.name": InstNameSNMP,
+		"instrumentation.name": name.Profile,
 	}
 	for k, v := range attr {
 		if metricName != "Uptime" { // Only allow Sys* attributes on uptime.
@@ -648,7 +649,7 @@ func copyAttrForSnmp(attr map[string]interface{}, metricName string, name kt.Met
 		// Force these to be back in.
 		attrNew["objectIdentifier"] = name.Oid
 		attrNew["mib-name"] = name.Mib
-		attrNew["instrumentation.name"] = InstNameSNMP
+		attrNew["instrumentation.name"] = name.Profile
 	}
 
 	if attrNew["mib-name"] == "" {
diff --git a/pkg/inputs/snmp/metrics/device_metrics.go b/pkg/inputs/snmp/metrics/device_metrics.go
index 9c6618c..df946d5 100644
--- a/pkg/inputs/snmp/metrics/device_metrics.go
+++ b/pkg/inputs/snmp/metrics/device_metrics.go
@@ -9,19 +9,21 @@ import (
 
 	"github.com/kentik/gosnmp"
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
+	"github.com/kentik/ktranslate/pkg/inputs/snmp/mibs"
 	"github.com/kentik/ktranslate/pkg/kt"
 
 	snmp_util "github.com/kentik/ktranslate/pkg/inputs/snmp/util"
 )
 
 type DeviceMetrics struct {
-	log     logger.ContextL
-	conf    *kt.SnmpDeviceConfig
-	gconf   *kt.SnmpGlobalConfig
-	metrics *kt.SnmpDeviceMetric
+	log         logger.ContextL
+	conf        *kt.SnmpDeviceConfig
+	gconf       *kt.SnmpGlobalConfig
+	metrics     *kt.SnmpDeviceMetric
+	profileName string
 }
 
-func NewDeviceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig, metrics *kt.SnmpDeviceMetric, profileMetrics map[string]*kt.Mib, log logger.ContextL) *DeviceMetrics {
+func NewDeviceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig, metrics *kt.SnmpDeviceMetric, profileMetrics map[string]*kt.Mib, profile *mibs.Profile, log logger.ContextL) *DeviceMetrics {
 	if conf.DeviceOids == nil && len(profileMetrics) > 0 {
 		conf.DeviceOids = profileMetrics
 	} else if len(profileMetrics) > 0 {
@@ -45,10 +47,11 @@ func NewDeviceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig, met
 	}
 
 	return &DeviceMetrics{
-		gconf:   gconf,
-		log:     log,
-		conf:    conf,
-		metrics: metrics,
+		gconf:       gconf,
+		log:         log,
+		conf:        conf,
+		metrics:     metrics,
+		profileName: profile.GetProfileName(),
 	}
 }
 
@@ -115,7 +118,7 @@ func (dm *DeviceMetrics) pollFromConfig(ctx context.Context, server *gosnmp.GoSN
 	}
 
 	// Map back into types we know about.
-	metricsFound := map[string]kt.MetricInfo{"Uptime": kt.MetricInfo{Oid: sysUpTime}}
+	metricsFound := map[string]kt.MetricInfo{"Uptime": kt.MetricInfo{Oid: sysUpTime, Profile: dm.profileName}}
 	for _, variable := range results {
 		if variable.Value == nil { // You can get nil w/out getting an error, though.
 			continue
@@ -153,7 +156,7 @@ func (dm *DeviceMetrics) pollFromConfig(ctx context.Context, server *gosnmp.GoSN
 		}
 
 		dmr := assureDeviceMetrics(m, idx)
-		metricsFound[oidName] = kt.MetricInfo{Oid: mib.Oid, Mib: mib.Mib}
+		metricsFound[oidName] = kt.MetricInfo{Oid: mib.Oid, Mib: mib.Mib, Profile: dm.profileName}
 		switch variable.Type {
 		case gosnmp.OctetString, gosnmp.BitString:
 			value := string(variable.Value.([]byte))
@@ -222,7 +225,7 @@ func (dm *DeviceMetrics) pollFromConfig(ctx context.Context, server *gosnmp.GoSN
 				memoryTotal := memoryFree + memoryUsed
 				if memoryTotal > 0 {
 					dst.CustomBigInt["MemoryUtilization"] = int64(float32(memoryUsed) / float32(memoryTotal) * 100)
-					dst.CustomMetrics["MemoryUtilization"] = kt.MetricInfo{Oid: "computed"}
+					dst.CustomMetrics["MemoryUtilization"] = kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: dm.profileName}
 				}
 			}
 		}
diff --git a/pkg/inputs/snmp/metrics/interface_metrics.go b/pkg/inputs/snmp/metrics/interface_metrics.go
index 7e1b546..b331679 100644
--- a/pkg/inputs/snmp/metrics/interface_metrics.go
+++ b/pkg/inputs/snmp/metrics/interface_metrics.go
@@ -10,6 +10,7 @@ import (
 	"github.com/kentik/gosnmp"
 
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
+	"github.com/kentik/ktranslate/pkg/inputs/snmp/mibs"
 	snmp_util "github.com/kentik/ktranslate/pkg/inputs/snmp/util"
 	"github.com/kentik/ktranslate/pkg/kt"
 	"github.com/kentik/ktranslate/pkg/kt/counters"
@@ -47,13 +48,14 @@ type InterfaceMetrics struct {
 	// evolve, so let's be careful.
 	mux sync.Mutex
 
-	intValues  map[string]*counters.CounterSet
-	oidMap     map[string]string
-	nameOidMap map[string]string
-	oidMibMap  map[string]*kt.Mib
+	intValues   map[string]*counters.CounterSet
+	oidMap      map[string]string
+	nameOidMap  map[string]string
+	oidMibMap   map[string]*kt.Mib
+	profileName string
 }
 
-func NewInterfaceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig, metrics *kt.SnmpDeviceMetric, profileMetrics map[string]*kt.Mib, log logger.ContextL) *InterfaceMetrics {
+func NewInterfaceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig, metrics *kt.SnmpDeviceMetric, profileMetrics map[string]*kt.Mib, profile *mibs.Profile, log logger.ContextL) *InterfaceMetrics {
 	oidMap := conf.InterfaceMetricsOidMap
 	if oidMap == nil {
 		oidMap = make(map[string]string)
@@ -84,14 +86,15 @@ func NewInterfaceMetrics(gconf *kt.SnmpGlobalConfig, conf *kt.SnmpDeviceConfig,
 	}
 
 	return &InterfaceMetrics{
-		log:        log,
-		gconf:      gconf,
-		conf:       conf,
-		metrics:    metrics,
-		oidMap:     oidMap,
-		nameOidMap: nameOidMap,
-		oidMibMap:  profileMetrics,
-		intValues:  make(map[string]*counters.CounterSet),
+		log:         log,
+		gconf:       gconf,
+		conf:        conf,
+		metrics:     metrics,
+		oidMap:      oidMap,
+		nameOidMap:  nameOidMap,
+		oidMibMap:   profileMetrics,
+		intValues:   make(map[string]*counters.CounterSet),
+		profileName: profile.GetProfileName(),
 	}
 }
 
@@ -244,9 +247,9 @@ func (im *InterfaceMetrics) convertToCHF(deltas map[string]map[string]uint64) []
 			}
 			dst.CustomBigInt[k] = int64(v)
 			if mib != nil {
-				metrics[k] = kt.MetricInfo{Oid: im.nameOidMap[k], Mib: mib.Mib}
+				metrics[k] = kt.MetricInfo{Oid: im.nameOidMap[k], Mib: mib.Mib, Profile: im.profileName}
 			} else {
-				metrics[k] = kt.MetricInfo{Oid: im.nameOidMap[k]}
+				metrics[k] = kt.MetricInfo{Oid: im.nameOidMap[k], Profile: im.profileName}
 			}
 		}
 		if uptimeDelta > 0 {
diff --git a/pkg/inputs/snmp/metrics/poll.go b/pkg/inputs/snmp/metrics/poll.go
index 4187e28..ebba01e 100644
--- a/pkg/inputs/snmp/metrics/poll.go
+++ b/pkg/inputs/snmp/metrics/poll.go
@@ -54,8 +54,8 @@ func NewPoller(server *gosnmp.GoSNMP, gconf *kt.SnmpGlobalConfig, conf *kt.SnmpD
 		log:              log,
 		metrics:          metrics,
 		server:           server,
-		interfaceMetrics: NewInterfaceMetrics(gconf, conf, metrics, interfaceMetricMibs, log),
-		deviceMetrics:    NewDeviceMetrics(gconf, conf, metrics, deviceMetricMibs, log),
+		interfaceMetrics: NewInterfaceMetrics(gconf, conf, metrics, interfaceMetricMibs, profile, log),
+		deviceMetrics:    NewDeviceMetrics(gconf, conf, metrics, deviceMetricMibs, profile, log),
 		counterTimeSec:   counterTimeSec,
 		dropIfOutside:    dropIfOutside,
 	}
diff --git a/pkg/inputs/snmp/mibs/profile.go b/pkg/inputs/snmp/mibs/profile.go
index 6b42b67..96c14b4 100644
--- a/pkg/inputs/snmp/mibs/profile.go
+++ b/pkg/inputs/snmp/mibs/profile.go
@@ -54,6 +54,17 @@ type Profile struct {
 	extended        bool
 }
 
+func (p *Profile) GetProfileName() string {
+	if p == nil {
+		return "snmp"
+	}
+	pts := strings.Split(p.From, ".")
+	if pts[0] != "" {
+		return pts[0]
+	}
+	return "snmp"
+}
+
 func (p *Profile) extend(extends map[string]*Profile) error {
 	if p.extended { // Don't extend multiple times, also halt recursion.
 		return nil
diff --git a/pkg/inputs/snmp/mibs/pymib.go b/pkg/inputs/snmp/mibs/pymib.go
index cd22a42..85e70b1 100644
--- a/pkg/inputs/snmp/mibs/pymib.go
+++ b/pkg/inputs/snmp/mibs/pymib.go
@@ -141,12 +141,20 @@ func ConvertJson2Yaml(file string, log logger.ContextL) error {
 		}
 	}
 
+	// Remove any metrics with no metrics (just tags). Needed because otherwise validation will fail.
+	gm := []MIB{}
+	for _, metric := range finals {
+		if metric.Symbol.Oid != "" || len(metric.Symbols) > 0 {
+			gm = append(gm, metric)
+		}
+	}
+
 	// Sort the mibs.
-	sort.Sort(MibList(finals))
+	sort.Sort(MibList(gm))
 
 	// Make a profile.
 	pro := Profile{
-		Metrics:     finals,
+		Metrics:     gm,
 		Sysobjectid: sysoids,
 		ContextL:    log,
 		From:        name,
diff --git a/pkg/kt/types.go b/pkg/kt/types.go
index 8720e38..190e1f2 100644
--- a/pkg/kt/types.go
+++ b/pkg/kt/types.go
@@ -171,9 +171,10 @@ type JCHF struct {
 }
 
 type MetricInfo struct {
-	Oid  string `json:"-"`
-	Mib  string `json:"-"`
-	Name string `json:"-"`
+	Oid     string `json:"-"`
+	Mib     string `json:"-"`
+	Name    string `json:"-"`
+	Profile string `json:"-"`
 }
 
 func NewJCHF() *JCHF {


Test patch:
diff --git a/pkg/inputs/snmp/mibs/profile_test.go b/pkg/inputs/snmp/mibs/profile_test.go
index 26f39c7..daf1cea 100644
--- a/pkg/inputs/snmp/mibs/profile_test.go
+++ b/pkg/inputs/snmp/mibs/profile_test.go
@@ -67,3 +67,18 @@ func TestPrune(t *testing.T) {
 	prune(mibs)
 	assert.Equal(t, 3, len(mibs))
 }
+
+func TestProfileName(t *testing.T) {
+	tests := map[string][]*Profile{
+		"foo-test": []*Profile{&Profile{From: "foo-test.yml"}},
+		"snmp":     []*Profile{nil, &Profile{}},
+		"noone":    []*Profile{&Profile{From: "noone"}},
+	}
+
+	for expected, profiles := range tests {
+		for _, profile := range profiles {
+			res := profile.GetProfileName()
+			assert.Equal(t, expected, res)
+		}
+	}
+}