Repository: kentik/ktranslate
Pull request #24 (2021-06-30 01:40:29)

Staging to main


Patch:
diff --git a/pkg/formats/nrm/nrm.go b/pkg/formats/nrm/nrm.go
index 8b4a470..8ac0a0c 100644
--- a/pkg/formats/nrm/nrm.go
+++ b/pkg/formats/nrm/nrm.go
@@ -461,15 +461,10 @@ func (f *NRMFormat) fromSnmpDeviceMetric(in *kt.JCHF) []NRMetric {
 	for m, name := range metrics {
 		if _, ok := in.CustomBigInt[m]; ok {
 			attrNew := copyAttrForSnmp(attr, name)
-			value, ok := in.CustomBigInt[m]
-			if !ok && in.CustomStr[m] != "" {
-				attrNew[m] = in.CustomStr[m]
-				value = 0 // Hard code this one because what would we do with a string based metric?
-			}
 			ms = append(ms, NRMetric{
 				Name:       "kentik.snmp." + m,
 				Type:       NR_GAUGE_TYPE,
-				Value:      int64(value),
+				Value:      int64(in.CustomBigInt[m]),
 				Attributes: attrNew,
 			})
 		}
diff --git a/pkg/inputs/snmp/disco.go b/pkg/inputs/snmp/disco.go
index 7feeae6..b3a9615 100644
--- a/pkg/inputs/snmp/disco.go
+++ b/pkg/inputs/snmp/disco.go
@@ -171,7 +171,15 @@ func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, m
 	if mibProfile != nil {
 		log.Infof("Found profile for %s: %v", md.SysObjectID, mibProfile)
 		device.MibProfile = mibProfile.From
-		device.MibSet = mibProfile.GetMibs()
+		mibs := mibProfile.GetMibs()
+		if len(mibs) > 0 {
+			device.DiscoveredMibs = make([]string, len(mibs))
+			i := 0
+			for m, _ := range mibs {
+				device.DiscoveredMibs[i] = m
+				i++
+			}
+		}
 	}
 
 	// Now, see what mibs this sucker can use.
@@ -218,7 +226,7 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 	if conf.Disco.AddAllMibs {
 		fullMibSet := map[string]bool{}
 		for _, device := range conf.Devices {
-			for mib, _ := range device.MibSet {
+			for _, mib := range device.DiscoveredMibs {
 				fullMibSet[mib] = true
 			}
 		}
diff --git a/pkg/inputs/snmp/metadata/device_metadata.go b/pkg/inputs/snmp/metadata/device_metadata.go
index d4275e3..ff69838 100644
--- a/pkg/inputs/snmp/metadata/device_metadata.go
+++ b/pkg/inputs/snmp/metadata/device_metadata.go
@@ -100,13 +100,16 @@ func GetDeviceMetadata(log logger.ContextL, server *gosnmp.GoSNMP, deviceMetadat
 		case SNMP_sysServices:
 			md.SysServices = int(snmp_util.ToInt64(value))
 		default:
+			if oid.Tag != "" {
+				oidName = oid.Tag
+			}
 			switch vt := value.(type) {
 			case string:
-				md.Customs[oid.Name] = vt
+				md.Customs[oidName] = vt
 			case []byte:
-				md.Customs[oid.Name] = string(vt)
+				md.Customs[oidName] = string(vt)
 			default:
-				md.CustomInts[oid.Name] = snmp_util.ToInt64(value)
+				md.CustomInts[oidName] = snmp_util.ToInt64(value)
 			}
 		}
 	}
@@ -137,7 +140,12 @@ func getTable(log logger.ContextL, g *gosnmp.GoSNMP, oid string, mib *kt.Mib, md
 		return err
 	}
 
-	log.Infof("TableWalk Results: %s: %s -> %d", mib.Name, oid, len(results))
+	oidName := mib.Name
+	if mib.Tag != "" {
+		oidName = mib.Tag
+	}
+
+	log.Infof("TableWalk Results: %s: %s -> %d", oidName, oid, len(results))
 	// Save as index -> oid -> value
 	for _, variable := range results {
 		idx := variable.Name[len(oid)+1:]
@@ -147,10 +155,10 @@ func getTable(log logger.ContextL, g *gosnmp.GoSNMP, oid string, mib *kt.Mib, md
 
 		switch variable.Type {
 		case gosnmp.OctetString:
-			md.Tables[idx].Customs[mib.Name] = string(variable.Value.([]byte))
+			md.Tables[idx].Customs[oidName] = string(variable.Value.([]byte))
 		default:
 			// Try to just use as a number
-			md.Tables[idx].CustomInts[mib.Name] = gosnmp.ToBigInt(variable.Value).Int64()
+			md.Tables[idx].CustomInts[oidName] = gosnmp.ToBigInt(variable.Value).Int64()
 		}
 	}
 
diff --git a/pkg/inputs/snmp/metadata/interface_metadata.go b/pkg/inputs/snmp/metadata/interface_metadata.go
index e7d962b..78368a5 100644
--- a/pkg/inputs/snmp/metadata/interface_metadata.go
+++ b/pkg/inputs/snmp/metadata/interface_metadata.go
@@ -435,15 +435,6 @@ func (im *InterfaceMetadata) Poll(server *gosnmp.GoSNMP) (map[string]*kt.Interfa
 		}
 	}
 
-	// extra check -- if we got no interfaces in that poll, something went badly wrong.
-	// Don't delete the interfaces from previous polls from the database; report an error
-	// and bail.
-	if len(intLine) == 0 {
-		err := fmt.Errorf("zero interfaces found")
-		im.log.Errorf("SNMP: Error polling metadata: %v", err)
-		return nil, "", err
-	}
-
 	deviceManufacturer := snmp_util.GetDeviceManufacturer(server, im.log)
 	lowerManufacturer := strings.ToLower(deviceManufacturer)
 	isNokia := snmp_util.ContainsAny(lowerManufacturer, "timos", "nokia")
diff --git a/pkg/inputs/snmp/metadata/poll.go b/pkg/inputs/snmp/metadata/poll.go
index 0ae09a4..df9868d 100644
--- a/pkg/inputs/snmp/metadata/poll.go
+++ b/pkg/inputs/snmp/metadata/poll.go
@@ -2,7 +2,6 @@ package metadata
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	"github.com/kentik/gosnmp"
@@ -109,13 +108,9 @@ func (p *Poller) PollSNMPMetadata() (*kt.DeviceData, error) {
 		return nil, err
 	}
 
-	// extra check -- if we got no interfaces in that poll, something went badly wrong.
-	// Don't delete the interfaces from previous polls from the database; report an error
-	// and bail.
+	// If there's no interfaces, note this this might be an issue for some devices but keep on going.
 	if len(intLine) == 0 {
-		err := fmt.Errorf("zero interfaces found")
-		p.log.Errorf("SNMP: Error polling metadata: %v", err)
-		return nil, err
+		p.log.Warnf("SNMP: issue polling metadata: zero interfaces found")
 	}
 
 	deviceData := &kt.DeviceData{
@@ -204,6 +199,11 @@ func (p *Poller) toFlows(dd *kt.DeviceData) ([]*kt.JCHF, error) {
 		dst.CustomStr["if."+intr+".VrfName"] = id.VrfName
 		dst.CustomStr["if."+intr+".VrfDescr"] = id.VrfDescr
 		dst.CustomStr["if."+intr+".VrfRD"] = id.VrfRD
+
+		// And in anything extra which came out here.
+		for k, v := range id.ExtraInfo {
+			dst.CustomStr["if."+intr+"."+k] = v
+		}
 	}
 
 	return []*kt.JCHF{dst}, nil
diff --git a/pkg/inputs/snmp/metrics/device_metrics.go b/pkg/inputs/snmp/metrics/device_metrics.go
index a914cbd..587d232 100644
--- a/pkg/inputs/snmp/metrics/device_metrics.go
+++ b/pkg/inputs/snmp/metrics/device_metrics.go
@@ -243,36 +243,25 @@ func (dm *DeviceMetrics) pollFromConfig(server *gosnmp.GoSNMP) ([]*kt.JCHF, erro
 		}
 		dmr := assureDeviceMetrics(m, idx)
 		metricsFound[mib.Name] = mib.Oid
-		switch mib.Type {
-		case kt.String:
+		switch variable.Type {
+		case gosnmp.OctetString, gosnmp.BitString:
 			value := string(variable.Value.([]byte))
-			// Try to parse this as a number. If its not though, just store as a string.
-			if s, err := strconv.ParseInt(value, 10, 64); err == nil {
-				dmr.customBigInt[mib.Name] = s
+			if mib.Enum != nil {
+				if val, ok := mib.Enum[strings.ToLower(value)]; ok {
+					dmr.customBigInt[mib.Name] = val
+				} else {
+					dm.log.Warnf("Missing enum value for device metric %s %s", mib.Name, value)
+				}
 			} else {
-				dmr.customStr[mib.Name] = string(variable.Value.([]byte))
+				// Try to parse this as a number. If its not though, just store as a string.
+				if s, err := strconv.ParseInt(value, 10, 64); err == nil {
+					dmr.customBigInt[mib.Name] = s
+				} else {
+					dm.log.Warnf("unable to set string valued metric as numeric: %s %s", mib.Name, value)
+					dmr.customStr[mib.Name] = string(value)
+				}
 			}
-		case kt.INTEGER:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.NetAddr:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.IpAddr:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.Counter:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.Gauge:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.TimeTicks:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.Counter64:
-			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
-		case kt.BitString:
-			if str, ok := snmp_util.ReadOctetString(variable, false); ok {
-				dmr.customStr[mib.Name] = str
-			}
-		case kt.Index:
-			dm.log.Debugf("No handler for %+v, Value: %T %+v", variable, variable.Value, variable.Value)
-		case kt.Integer32:
+		default:
 			dmr.customBigInt[mib.Name] = snmp_util.ToInt64(variable.Value)
 		}
 	}
diff --git a/pkg/inputs/snmp/mibs/load.go b/pkg/inputs/snmp/mibs/load.go
index 5786309..9839108 100644
--- a/pkg/inputs/snmp/mibs/load.go
+++ b/pkg/inputs/snmp/mibs/load.go
@@ -127,7 +127,7 @@ func (db *MibDB) GetForOid(oid string, profile string, description string) (map[
 	}
 	mibs := map[string]*kt.Mib{}
 	iter := db.db.NewIterator(util.BytesPrefix([]byte(oid)), nil)
-	provider := kt.ProviderRouter
+	provider := kt.ProviderDefault
 	foundProv := false
 	for iter.Next() {
 		pts := strings.SplitN(string(iter.Value()), " ", 2)
@@ -181,7 +181,7 @@ func (db *MibDB) GetForOidRecur(oid string, profile string, description string)
 		}
 	}
 
-	return nil, kt.ProviderRouter, false, nil
+	return nil, kt.ProviderDefault, false, nil
 }
 
 func (db *MibDB) checkForProvider(name string, profile string, description string) (kt.Provider, bool) {
diff --git a/pkg/inputs/snmp/mibs/profile.go b/pkg/inputs/snmp/mibs/profile.go
index b6212ac..a833fd4 100644
--- a/pkg/inputs/snmp/mibs/profile.go
+++ b/pkg/inputs/snmp/mibs/profile.go
@@ -12,8 +12,9 @@ import (
 )
 
 type OID struct {
-	Oid  string `yaml:"OID"`
-	Name string `yaml:"name"`
+	Oid  string           `yaml:"OID"`
+	Name string           `yaml:"name"`
+	Enum map[string]int64 `yaml:"enum"`
 }
 
 type Tag struct {
@@ -280,6 +281,7 @@ func (p *Profile) GetMetrics(enabledMibs []string) (map[string]*kt.Mib, map[stri
 				Oid:  metric.Symbol.Oid,
 				Name: metric.Symbol.Name,
 				Type: otype,
+				Enum: metric.Symbol.Enum,
 			}
 			if strings.HasPrefix(metric.Symbol.Name, "if") {
 				interfaceMetrics[metric.Symbol.Oid] = mib
@@ -293,6 +295,7 @@ func (p *Profile) GetMetrics(enabledMibs []string) (map[string]*kt.Mib, map[stri
 				Oid:  s.Oid,
 				Name: s.Name,
 				Type: otype,
+				Enum: s.Enum,
 			}
 			if strings.HasPrefix(s.Name, "if") {
 				interfaceMetrics[s.Oid] = mib
diff --git a/pkg/kt/snmp.go b/pkg/kt/snmp.go
index bbe6619..b75aaea 100644
--- a/pkg/kt/snmp.go
+++ b/pkg/kt/snmp.go
@@ -109,7 +109,7 @@ type SnmpDeviceConfig struct {
 	Provider               Provider          `yaml:"provider"`
 	FlowOnly               bool              `yaml:"flow_only"`
 	UserTags               map[string]string `yaml:"user_tags"`
-	MibSet                 map[string]bool   `yaml:"mib_set"`
+	DiscoveredMibs         []string          `yaml:"discovered_mibs"`
 }
 
 type SnmpTrapConfig struct {
@@ -206,6 +206,7 @@ type Mib struct {
 	Type  Oidtype
 	Extra string
 	Tag   string
+	Enum  map[string]int64
 }
 
 func (mb Mib) String() string {
diff --git a/pkg/kt/types.go b/pkg/kt/types.go
index 5140700..aa3af21 100644
--- a/pkg/kt/types.go
+++ b/pkg/kt/types.go
@@ -43,21 +43,24 @@ type Provider string
 
 const (
 	ProviderRouter             Provider = "kentik-router"
-	ProviderVPC                         = "kentik-vpc"
-	ProviderSynth                       = "kentik-network-synthetic"
-	ProviderSwitch                      = "kentik-switch"
-	ProviderFirewall                    = "kentik-firewall"
-	ProviderUPS                         = "kentik-ups"
-	ProviderPDU                         = "kentik-pdu"
-	ProviderIOT                         = "kentik-iot"
-	ProviderHost                        = "kentik-host"
-	ProviderAlert                       = "kentik-alert"
-	ProviderFlowDevice                  = "kentik-flow-device"
-	ProviderWirelessController          = "kentik-wireless-controller"
-	ProviderNas                         = "kentik-nas"
-	ProviderSan                         = "kentik-san"
-	ProviderAgent                       = "kentik-agent"
+	ProviderDefault            Provider = "kentik-default"
+	ProviderVPC                Provider = "kentik-vpc"
+	ProviderSynth              Provider = "kentik-network-synthetic"
+	ProviderSwitch             Provider = "kentik-switch"
+	ProviderFirewall           Provider = "kentik-firewall"
+	ProviderUPS                Provider = "kentik-ups"
+	ProviderPDU                Provider = "kentik-pdu"
+	ProviderIOT                Provider = "kentik-iot"
+	ProviderHost               Provider = "kentik-host"
+	ProviderAlert              Provider = "kentik-alert"
+	ProviderFlowDevice         Provider = "kentik-flow-device"
+	ProviderWirelessController Provider = "kentik-wireless-controller"
+	ProviderNas                Provider = "kentik-nas"
+	ProviderSan                Provider = "kentik-san"
+	ProviderAgent              Provider = "kentik-agent"
+)
 
+const (
 	InstProvider  = "kentik"
 	CollectorName = "ktranslate"
 	SnmpCollector = "snmp"
diff --git a/pkg/rollup/init.go b/pkg/rollup/init.go
index 50f6252..8815b33 100644
--- a/pkg/rollup/init.go
+++ b/pkg/rollup/init.go
@@ -8,7 +8,7 @@ import (
 )
 
 func init() {
-	flag.Var(&rollups, "rollups", "Any rollups to use. Format: type, metric, dimension 1, dimension 2, ..., dimension n: sum,in_bytes,dst_addr")
+	flag.Var(&rollups, "rollups", "Any rollups to use. Format: type, name, metric, dimension 1, dimension 2, ..., dimension n: sum,bytes,in_bytes,dst_addr")
 	verifyHLLConstantsOrPanic()
 }
 
diff --git a/pkg/rollup/stats.go b/pkg/rollup/stats.go
index 78d9b48..658b284 100644
--- a/pkg/rollup/stats.go
+++ b/pkg/rollup/stats.go
@@ -309,13 +309,17 @@ func (r *StatsRollup) exportSum(sum map[string]uint64, count map[string]uint64,
 func (r *StatsRollup) getTopkSum(keys []Rollup, total uint64, totalc uint64, ot time.Time, prov kt.Provider, rc chan []Rollup) {
 	top := make([]Rollup, 0, len(keys))
 	seen := map[string]int{}
+	seenPrimay := map[string]bool{}
 
 	for _, roll := range keys {
 		pts := strings.Split(roll.Dimension, r.keyJoin)
 		if seen[pts[r.primaryDim]] < r.topK { // If the primary key for this rollup has less than the topk set, add it to the list.
-			top = append(top, roll)
+			if len(seenPrimay) <= r.topK { // And, if the number of primary keys is also less than topk, add.
+				top = append(top, roll)
+			}
 		}
 		seen[pts[r.primaryDim]]++
+		seenPrimay[pts[r.primaryDim]] = true
 	}
 
 	// Fill in the total value here, unless the name has total in it.
diff --git a/profiles/kentik_snmp/meraki/meraki-cloud-controller.yml b/profiles/kentik_snmp/meraki/meraki-cloud-controller.yml
index a60ca9d..d7a4352 100644
--- a/profiles/kentik_snmp/meraki/meraki-cloud-controller.yml
+++ b/profiles/kentik_snmp/meraki/meraki-cloud-controller.yml
@@ -1,4 +1,5 @@
-# http://oid-info.com/get/1.3.6.1.4.1.29671.1.1
+# http://oid-info.com/get/1.3.6.1.4.1.29671.1.1.4.1
+# http://oid-info.com/get/1.3.6.1.4.1.29671.1.1.5.1
 ---
 extends:
   - system-mib.yml
@@ -23,15 +24,23 @@ metrics:
       - column:
           OID: 1.3.6.1.4.1.29671.1.1.4.1.2
           name: devName
-        tag: device_name
+        tag: ap_name
+      - column:
+          OID: 1.3.6.1.4.1.29671.1.1.4.1.7
+          name: devPublicIp
+        tag: ap_public_ip
+      - column:
+          OID: 1.3.6.1.4.1.29671.1.1.4.1.8
+          name: devSerial
+        tag: ap_serial
       - column:
           OID: 1.3.6.1.4.1.29671.1.1.4.1.9
           name: devProductCode
-        tag: product_code
+        tag: ap_model
       - column:
           OID: 1.3.6.1.4.1.29671.1.1.4.1.11
           name: devNetworkName
-        tag: network_name
+        tag: ap_network_name
   - MIB: MERAKI-CLOUD-CONTROLLER-MIB
     table:
       OID: 1.3.6.1.4.1.29671.1.1.5
@@ -46,11 +55,11 @@ metrics:
       - OID: 1.3.6.1.4.1.29671.1.1.5.1.7
         name: devInterfaceRecvBytes
     metric_tags:
-      - column:
-          OID: 1.3.6.1.4.1.29671.1.1.5.1.3
-          name: devInterfaceName
-        tag: interface_name
       - column:
           OID: 1.3.6.1.4.1.29671.1.1.5.1.2
           name: devInterfaceIndex
         tag: interface_index
+      - column:
+          OID: 1.3.6.1.4.1.29671.1.1.5.1.3
+          name: devInterfaceName
+        tag: interface_name
diff --git a/profiles/kentik_snmp/netgear/readynas.yml b/profiles/kentik_snmp/netgear/readynas.yml
index 1157ff6..101d4ae 100644
--- a/profiles/kentik_snmp/netgear/readynas.yml
+++ b/profiles/kentik_snmp/netgear/readynas.yml
@@ -23,6 +23,9 @@ metrics:
         name: diskCapacity
       - OID: 1.3.6.1.4.1.4526.22.3.1.9
         name: diskState
+        enum:
+          online: 1
+          offline: 0
       - OID: 1.3.6.1.4.1.4526.22.3.1.10
         name: diskTemperature
     metric_tags:


Test patch:
diff --git a/pkg/inputs/snmp/mibs/load_test.go b/pkg/inputs/snmp/mibs/load_test.go
index 5aed0a1..40caa0b 100644
--- a/pkg/inputs/snmp/mibs/load_test.go
+++ b/pkg/inputs/snmp/mibs/load_test.go
@@ -54,9 +54,11 @@ func TestFullProvider(t *testing.T) {
 		".1.3.6.1.4.1.318.1.3.5.4": []string{"apc_ups.yaml", "APC SNMP Agent"},
 		".1.3.6.1.4.1.9.1.275":     []string{"cisco-catalyst.yml", "Cisco IOS Software, s72033_rp Software (s72033_rp-ADVIPSERVICESK9_WAN-M), Version 12.2(33)SXI14, RELEASE SOFTWARE (fc2)/Technica"},
 		".1.3.6.1.4.1.9.1.1287":    []string{"cisco-catalyst.yml", "Cisco ios xr"},
+		".1.3.6.1.4.1.8072.3.2.10": []string{"base.yml", "Linux"},
 	}
 
 	inputs := map[string]kt.Provider{
+		".1.3.6.1.4.1.8072.3.2.10": kt.ProviderDefault,
 		".1.3.6.1.4.1.9.1.2494":    kt.ProviderSwitch,
 		".1.3.6.1.4.1.9.1.1639":    kt.ProviderRouter,
 		".1.3.6.1.4.1.9.1.449":     kt.ProviderSwitch,