Repository: kentik/ktranslate
Pull request #23 (2021-06-30 01:39:28)

Adding Default provider.
Fixing how rollups worth with only a primary dimension and a limit.
Allow string valued metrics to map with an enum set.
Fixing bug with type for device metrics.
Allow now interfaces for metadata to still get processed.
Switching to using tag names not oid names for metadata.


Patch:
diff --git a/pkg/formats/nrm/nrm.go b/pkg/formats/nrm/nrm.go
index 8b4a470..8ac0a0c 100644
--- a/pkg/formats/nrm/nrm.go
+++ b/pkg/formats/nrm/nrm.go
@@ -461,15 +461,10 @@ func (f *NRMFormat) fromSnmpDeviceMetric(in *kt.JCHF) []NRMetric {
 	for m, name := range metrics {
 		if _, ok := in.CustomBigInt[m]; ok {
 			attrNew := copyAttrForSnmp(attr, name)
-			value, ok := in.CustomBigInt[m]
-			if !ok && in.CustomStr[m] != "" {
-				attrNew[m] = in.CustomStr[m]
-				value = 0 // Hard code this one because what would we do with a string based metric?
-			}
 			ms = append(ms, NRMetric{
 				Name:       "kentik.snmp." + m,
 				Type:       NR_GAUGE_TYPE,
-				Value:      int64(value),
+				Value:      int64(in.CustomBigInt[m]),
 				Attributes: attrNew,
 			})
 		}
diff --git a/pkg/inputs/snmp/disco.go b/pkg/inputs/snmp/disco.go
index 7feeae6..b3a9615 100644
--- a/pkg/inputs/snmp/disco.go
+++ b/pkg/inputs/snmp/disco.go
@@ -171,7 +171,15 @@ func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, m
 	if mibProfile != nil {
 		log.Infof("Found profile for %s: %v", md.SysObjectID, mibProfile)
 		device.MibProfile = mibProfile.From
-		device.MibSet = mibProfile.GetMibs()
+		mibs := mibProfile.GetMibs()
+		if len(mibs) > 0 {
+			device.DiscoveredMibs = make([]string, len(mibs))
+			i := 0
+			for m, _ := range mibs {
+				device.DiscoveredMibs[i] = m
+				i++
+			}
+		}
 	}
 
 	// Now, see what mibs this sucker can use.
@@ -218,7 +226,7 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 	if conf.Disco.AddAllMibs {
 		fullMibSet := map[string]bool{}
 		for _, device := range conf.Devices {
-			for mib, _ := range device.MibSet {
+			for _, mib := range device.DiscoveredMibs {
 				fullMibSet[mib] = true
 			}
 		}
diff --git a/pkg/inputs/snmp/metadata/device_metadata.go b/pkg/inputs/snmp/metadata/device_metadata.go
index d4275e3..ff69838 100644
--- a/pkg/inputs/snmp/metadata/device_metadata.go
+++ b/pkg/inputs/snmp/metadata/device_metadata.go
@@ -100,13 +100,16 @@ func GetDeviceMetadata(log logger.ContextL, server *gosnmp.GoSNMP, deviceMetadat
 		case SNMP_sysServices:
 			md.SysServices = int(snmp_util.ToInt64(value))
 		default:
+			if oid.Tag != "" {
+				oidName = oid.Tag
+			}
 			switch vt := value.(type) {
 			case string:
-				md.Customs[oid.Name] = vt
+				md.Customs[oidName] = vt
 			case []byte:
-				md.Customs[oid.Name] = string(vt)
+				md.Customs[oidName] = string(vt)
 			default:
-				md.CustomInts[oid.Name] = snmp_util.ToInt64(value)
+				md.CustomInts[oidName] = snmp_util.ToInt64(value)
 			}
 		}
 	}
@@ -137,7 +140,12 @@ func getTable(log logger.ContextL, g *gosnmp.GoSNMP, oid string, mib *kt.Mib, md
 		return err
 	}
 
-	log.Infof("TableWalk Results: %s: %s -> %d", mib.Name, oid, len(results))
+	oidName := mib.Name
+	if mib.Tag != "" {
+		oidName = mib.Tag
+	}
+
+	log.Infof("TableWalk Results: %s: %s -> %d", oidName, oid, len(results))
 	// Save as index -> oid -> value
 	for _, variable := range results {
 		idx := variable.Name[len(oid)+1:]
@@ -147,10 +155,10 @@ func getTable(log logger.ContextL, g *gosnmp.GoSNMP, oid string, mib *kt.Mib, md
 
 		switch variable.Type {
 		case gosnmp.OctetString:
-			md.Tables[idx].Customs[mib.Name] = string(variable.Value.([]byte))
+			md.Tables[idx].Customs[oidName] = string(variable.Value.([]byte))
 		default:
 			// Try to just use as a number
-			md.Tables[idx].CustomInts[mib.Name] = gosnmp.ToBigInt(variable.Value).Int64()
+			md.Tables[idx].CustomInts[oidName] = gosnmp.ToBigInt(variable.Value).Int64()
 		}
 	}
 
diff --git a/pkg/inputs/snmp/metadata/interface_metadata.go b/pkg/inputs/snmp/metadata/interface_metadata.go
index e7d962b..78368a5 100644
--- a/pkg/inputs/snmp/metadata/interface_metadata.go
+++ b/pkg/inputs/snmp/metadata/interface_metadata.go
@@ -435,15 +435,6 @@ func (im *InterfaceMetadata) Poll(server *gosnmp.GoSNMP) (map[string]*kt.Interfa
 		}
 	}
 
-	// extra check -- if we got no interfaces in that poll, something went badly wrong.
-	// Don't delete the interfaces from previous polls from the database; report an error
-	// and bail.
-	if len(intLine) == 0 {
-		err := fmt.Errorf("zero interfaces found")
-		im.log.Errorf("SNMP: Error polling metadata: %v", err)
-		return nil, "", err
-	}
-
 	deviceManufacturer := snmp_util.GetDeviceManufacturer(server, im.log)
 	lowerManufacturer := strings.ToLower(deviceManufacturer)
 	isNokia := snmp_util.ContainsAny(lowerManufacturer, "timos", "nokia")
diff --git a/pkg/inputs/snmp/metadata/poll.go b/pkg/inputs/snmp/metadata/poll.go
index 0ae09a4..df9868d 100644
--- a/pkg/inputs/snmp/metadata/poll.go
+++ b/pkg/inputs/snmp/metadata/poll.go
@@ -2,7 +2,6 @@ package metadata
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	"github.com/kentik/gosnmp"
@@ -109,13 +108,9 @@ func (p *Poller) PollSNMPMetadata() (*kt.DeviceData, error) {
 		return nil, err
 	}
 
-	// extra check -- if we got no interfaces in that poll, something went badly wrong.
-	// Don't delete the interfaces from previous polls from the database; report an error
-	// and bail.
+	// If there's no interfaces, note this this might be an issue for some devices but keep on going.
 	if len(intLine) == 0 {
-		err := fmt.Errorf("zero interfaces found")
-		p.log.Errorf("SNMP: Error polling metadata: %v", err)
-		return nil, err
+		p.log.Warnf("SNMP: issue polling metadata: zero interfaces found")
 	}
 
 	deviceData := &kt.DeviceData{
@@ -204,6 +199,11 @@ func (p *Poller) toFlows(dd *kt.DeviceData) ([]*kt.JCHF, error) {
 		dst.CustomStr["if."+intr+".VrfName"] = id.VrfName
 		dst.CustomStr["if."+intr+".VrfDescr"] = id.VrfDescr
 		dst.CustomStr["if."+intr+".VrfRD"] = id.VrfRD
+
+		// And in anything extra which came out here.
+		for k, v := range id.ExtraInfo {
+			dst.CustomStr["if."+intr+"."+k] = v
+		}
 	}
 
 	return []*kt.JCHF{dst}, nil
diff --git a/pkg/inputs/snmp/metrics/device_metrics.go b/pkg/inputs/snmp/metrics/device_metrics.go
index a914cbd..587d232 100644
--- a/pkg/inputs/snmp/metrics/device_metrics.go
+++ b/pkg/inputs/snmp/metrics/device_metrics.go
@@ -243,36 +243,25 @@ func (dm *DeviceMetrics) pollFromConfig(server *gosnmp.GoSNMP) ([]*kt.JCHF, erro
 		}
 		dmr := assureDeviceMetrics(m, idx)
 		metricsFound[mib.Name] = mib.Oid
-		switch mib.Type {
-		case kt.String:
+		switch variable.Type {
+		case gosnmp.OctetString, gosnmp.BitString:
 			value := string(variable.Value.([]byte))
-			// Try to parse this as a number. If its not though, just store as a string.
-			if s, err := strconv.ParseInt(value, 10, 64); err == nil {
-				dmr.customBigInt[mib.Name] = s
+			if mib.Enum != nil {
+				if val, ok := mib.Enum[strings.ToLower(value)]; ok {
+					dmr.customBigInt[mib.Name] = val
+				} else {
+					dm.log.Warnf("Missing enum value for device metric %s %s", mib.Name, value)
+				}
 			} else {
-				dmr.customStr[mib.Name] = string(variable.Value.([]byte))
+				// Try to parse this as a number. If its not though, just store as a string.
+				if s, err := strconv.ParseInt(value, 10, 64); err == nil {
+					dmr.customBigInt[mib.Name] = s
+				} else {
+					dm.log.Warnf("unable to set string valued metric as numeric: %s %s", mib.Name, value)
+					dmr.customStr[mib.Name] = string(value)
+				}
 			}
-		case kt.INTEGER:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.NetAddr:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.IpAddr:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.Counter:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.Gauge:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.TimeTicks:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.Counter64:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.BitString:
-			if str, ok := snmp_util.ReadOctetString(variable, false); ok {
-				dmr.customStr[mib.Name] = str
-			}
-		case kt.Index:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.Integer32:
+		default:
 			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
 		}
 	}
diff --git a/pkg/inputs/snmp/mibs/load.go b/pkg/inputs/snmp/mibs/load.go
index 5786309..9839108 100644
--- a/pkg/inputs/snmp/mibs/load.go
+++ b/pkg/inputs/snmp/mibs/load.go
@@ -127,7 +127,7 @@ func (db *MibDB) GetForOid(oid string, profile string, description string) (map[
 	}
 	mibs := map[string]*kt.Mib{}
 	iter := db.db.NewIterator(util.BytesPrefix([]byte(oid)), nil)
-	provider := kt.ProviderRouter
+	provider := kt.ProviderDefault
 	foundProv := false
 	for iter.Next() {
 		pts := strings.SplitN(string(iter.Value()), " ", 2)
@@ -181,7 +181,7 @@ func (db *MibDB) GetForOidRecur(oid string, profile string, description string)
 		}
 	}
 
-	return nil, kt.ProviderRouter, false, nil
+	return nil, kt.ProviderDefault, false, nil
 }
 
 func (db *MibDB) checkForProvider(name string, profile string, description string) (kt.Provider, bool) {
diff --git a/pkg/inputs/snmp/mibs/profile.go b/pkg/inputs/snmp/mibs/profile.go
index b6212ac..a833fd4 100644
--- a/pkg/inputs/snmp/mibs/profile.go
+++ b/pkg/inputs/snmp/mibs/profile.go
@@ -12,8 +12,9 @@ import (
 )
 
 type OID struct {
-	Oid  string `yaml:"OID"`
-	Name string `yaml:"name"`
+	Oid  string           `yaml:"OID"`
+	Name string           `yaml:"name"`
+	Enum map[string]int64 `yaml:"enum"`
 }
 
 type Tag struct {
@@ -280,6 +281,7 @@ func (p *Profile) GetMetrics(enabledMibs []string) (map[string]*kt.Mib, map[stri
 				Oid:  metric.Symbol.Oid,
 				Name: metric.Symbol.Name,
 				Type: otype,
+				Enum: metric.Symbol.Enum,
 			}
 			if strings.HasPrefix(metric.Symbol.Name, "if") {
 				interfaceMetrics[metric.Symbol.Oid] = mib
@@ -293,6 +295,7 @@ func (p *Profile) GetMetrics(enabledMibs []string) (map[string]*kt.Mib, map[stri
 				Oid:  s.Oid,
 				Name: s.Name,
 				Type: otype,
+				Enum: s.Enum,
 			}
 			if strings.HasPrefix(s.Name, "if") {
 				interfaceMetrics[s.Oid] = mib
diff --git a/pkg/kt/snmp.go b/pkg/kt/snmp.go
index bbe6619..b75aaea 100644
--- a/pkg/kt/snmp.go
+++ b/pkg/kt/snmp.go
@@ -109,7 +109,7 @@ type SnmpDeviceConfig struct {
 	Provider               Provider          `yaml:"provider"`
 	FlowOnly               bool              `yaml:"flow_only"`
 	UserTags               map[string]string `yaml:"user_tags"`
-	MibSet                 map[string]bool   `yaml:"mib_set"`
+	DiscoveredMibs         []string          `yaml:"discovered_mibs"`
 }
 
 type SnmpTrapConfig struct {
@@ -206,6 +206,7 @@ type Mib struct {
 	Type  Oidtype
 	Extra string
 	Tag   string
+	Enum  map[string]int64
 }
 
 func (mb Mib) String() string {
diff --git a/pkg/kt/types.go b/pkg/kt/types.go
index 5140700..aa3af21 100644
--- a/pkg/kt/types.go
+++ b/pkg/kt/types.go
@@ -43,21 +43,24 @@ type Provider string
 
 const (
 	ProviderRouter             Provider = "kentik-router"
-	ProviderVPC                         = "kentik-vpc"
-	ProviderSynth                       = "kentik-network-synthetic"
-	ProviderSwitch                      = "kentik-switch"
-	ProviderFirewall                    = "kentik-firewall"
-	ProviderUPS                         = "kentik-ups"
-	ProviderPDU                         = "kentik-pdu"
-	ProviderIOT                         = "kentik-iot"
-	ProviderHost                        = "kentik-host"
-	ProviderAlert                       = "kentik-alert"
-	ProviderFlowDevice                  = "kentik-flow-device"
-	ProviderWirelessController          = "kentik-wireless-controller"
-	ProviderNas                         = "kentik-nas"
-	ProviderSan                         = "kentik-san"
-	ProviderAgent                       = "kentik-agent"
+	ProviderDefault            Provider = "kentik-default"
+	ProviderVPC                Provider = "kentik-vpc"
+	ProviderSynth              Provider = "kentik-network-synthetic"
+	ProviderSwitch             Provider = "kentik-switch"
+	ProviderFirewall           Provider = "kentik-firewall"
+	ProviderUPS                Provider = "kentik-ups"
+	ProviderPDU                Provider = "kentik-pdu"
+	ProviderIOT                Provider = "kentik-iot"
+	ProviderHost               Provider = "kentik-host"
+	ProviderAlert              Provider = "kentik-alert"
+	ProviderFlowDevice         Provider = "kentik-flow-device"
+	ProviderWirelessController Provider = "kentik-wireless-controller"
+	ProviderNas                Provider = "kentik-nas"
+	ProviderSan                Provider = "kentik-san"
+	ProviderAgent              Provider = "kentik-agent"
+)
 
+const (
 	InstProvider  = "kentik"
 	CollectorName = "ktranslate"
 	SnmpCollector = "snmp"
diff --git a/pkg/rollup/init.go b/pkg/rollup/init.go
index 50f6252..8815b33 100644
--- a/pkg/rollup/init.go
+++ b/pkg/rollup/init.go
@@ -8,7 +8,7 @@ import (
 )
 
 func init() {
-	flag.Var(&rollups, "rollups", "Any rollups to use. Format: type, metric, dimension 1, dimension 2, ..., dimension n: sum,in_bytes,dst_addr")
+	flag.Var(&rollups, "rollups", "Any rollups to use. Format: type, name, metric, dimension 1, dimension 2, ..., dimension n: sum,bytes,in_bytes,dst_addr")
 	verifyHLLConstantsOrPanic()
 }
 
diff --git a/pkg/rollup/stats.go b/pkg/rollup/stats.go
index 78d9b48..658b284 100644
--- a/pkg/rollup/stats.go
+++ b/pkg/rollup/stats.go
@@ -309,13 +309,17 @@ func (r *StatsRollup) exportSum(sum map[string]uint64, count map[string]uint64,
 func (r *StatsRollup) getTopkSum(keys []Rollup, total uint64, totalc uint64, ot time.Time, prov kt.Provider, rc chan []Rollup) {
 	top := make([]Rollup, 0, len(keys))
 	seen := map[string]int{}
+	seenPrimay := map[string]bool{}
 
 	for _, roll := range keys {
 		pts := strings.Split(roll.Dimension, r.keyJoin)
 		if seen[pts[r.primaryDim]] < r.topK { // If the primary key for this rollup has less than the topk set, add it to the list.
-			top = append(top, roll)
+			if len(seenPrimay) <= r.topK { // And, if the number of primary keys is also less than topk, add.
+				top = append(top, roll)
+			}
 		}
 		seen[pts[r.primaryDim]]++
+		seenPrimay[pts[r.primaryDim]] = true
 	}
 
 	// Fill in the total value here, unless the name has total in it.
diff --git a/profiles/kentik_snmp/netgear/readynas.yml b/profiles/kentik_snmp/netgear/readynas.yml
index 1157ff6..101d4ae 100644
--- a/profiles/kentik_snmp/netgear/readynas.yml
+++ b/profiles/kentik_snmp/netgear/readynas.yml
@@ -23,6 +23,9 @@ metrics:
         name: diskCapacity
       - OID: 1.3.6.1.4.1.4526.22.3.1.9
         name: diskState
+        enum:
+          online: 1
+          offline: 0
       - OID: 1.3.6.1.4.1.4526.22.3.1.10
         name: diskTemperature
     metric_tags:


Test patch:
diff --git a/pkg/inputs/snmp/mibs/load_test.go b/pkg/inputs/snmp/mibs/load_test.go
index 5aed0a1..40caa0b 100644
--- a/pkg/inputs/snmp/mibs/load_test.go
+++ b/pkg/inputs/snmp/mibs/load_test.go
@@ -54,9 +54,11 @@ func TestFullProvider(t *testing.T) {
 		".1.3.6.1.4.1.318.1.3.5.4": []string{"apc_ups.yaml", "APC SNMP Agent"},
 		".1.3.6.1.4.1.9.1.275":     []string{"cisco-catalyst.yml", "Cisco IOS Software, s72033_rp Software (s72033_rp-ADVIPSERVICESK9_WAN-M), Version 12.2(33)SXI14, RELEASE SOFTWARE (fc2)/Technica"},
 		".1.3.6.1.4.1.9.1.1287":    []string{"cisco-catalyst.yml", "Cisco ios xr"},
+		".1.3.6.1.4.1.8072.3.2.10": []string{"base.yml", "Linux"},
 	}
 
 	inputs := map[string]kt.Provider{
+		".1.3.6.1.4.1.8072.3.2.10": kt.ProviderDefault,
 		".1.3.6.1.4.1.9.1.2494":    kt.ProviderSwitch,
 		".1.3.6.1.4.1.9.1.1639":    kt.ProviderRouter,
 		".1.3.6.1.4.1.9.1.449":     kt.ProviderSwitch,