Repository: kentik/ktranslate
Pull request #411 (2022-08-30 00:01:43)

Adding asa,nbar,pan ent field support as native flow
This is a way to get extended IPFIX fields from a few common senders: ASA, NBAR and Palo Alto.

The flag `-nf.source` is extended with 3 new values: `asa,nbar,pan`. 

These will accept specific flow from a Cisco ASA, Nbar and Palo Alto firewall. See the special.go file for how these fields are mapped in. Eliminates the need for a config json file to get passed in for every one of these boxes. 

i3149: Closes #407
i3149: @thezackm - its not super possible to look at all fields sent because of how the goflow2 library works. The model it has is explicit opt in and anything not opted in is dropped before the top layer application can see anything. This makes it work well with the static protobuf it uses for passing data around internally but also limits things like you call out. 

I made one change though with nbar, mapping `application` now to the current application field and not overriding it with port based app if set. This should allow existing dashboards to pick up and use `application` without modification for nbar speaking devices. 

Patch:
diff --git a/cmd/ktranslate/main.go b/cmd/ktranslate/main.go
index 47d66c9..32c08f0 100644
--- a/cmd/ktranslate/main.go
+++ b/cmd/ktranslate/main.go
@@ -90,7 +90,7 @@ func init() {
 	flag.StringVar(&sslKeyFile, "ssl_key_file", "", "SSL Key file to use for serving HTTPS traffic")
 	flag.StringVar(&tagMapType, "tag_map_type", "", "type of mapping to use for tag values. file|null")
 	flag.StringVar(&vpcSource, "vpc", kt.LookupEnvString("KENTIK_VPC", ""), "Run VPC Flow Ingest")
-	flag.StringVar(&flowSource, "nf.source", "", "Run NetFlow Ingest Directly. Valid values here are netflow5|netflow9|ipfix|sflow")
+	flag.StringVar(&flowSource, "nf.source", "", "Run NetFlow Ingest Directly. Valid values here are netflow5|netflow9|ipfix|sflow|nbar|asa|pan")
 	flag.BoolVar(&teeLog, "tee_logs", false, "Tee log messages to sink")
 	flag.StringVar(&appMap, "application_map", "", "File containing custom application mappings")
 	flag.StringVar(&syslog, "syslog.source", "", "Run Syslog Server at this IP:Port or unix socket.")
@@ -420,6 +420,9 @@ func applyFlags(cfg *ktranslate.Config) error {
 					return
 				}
 				cfg.PrometheusFormat.FlowsNeeded = v
+			// pkg/formats/influxdb
+			case "influxdb_measurement_prefix":
+				cfg.InfluxDBFormat.MeasurementPrefix = val
 			// pkg/sinks/prom
 			case "prom_listen":
 				cfg.PrometheusSink.ListenAddr = val
diff --git a/config.go b/config.go
index 671b345..185c0aa 100644
--- a/config.go
+++ b/config.go
@@ -20,6 +20,11 @@ type NetflowFormatConfig struct {
 	Version string
 }
 
+// InfluxDBFormatConfig is the config format for influxdb
+type InfluxDBFormatConfig struct {
+	MeasurementPrefix string
+}
+
 // PrometheusFormatConfig is the config for the prometheus format
 type PrometheusFormatConfig struct {
 	EnableCollectorStats bool
@@ -220,6 +225,8 @@ type Config struct {
 	NetflowFormat *NetflowFormatConfig
 	// pkg/formats/prom
 	PrometheusFormat *PrometheusFormatConfig
+	// pkg/formats/influxdb
+	InfluxDBFormat *InfluxDBFormatConfig
 
 	// pkg/sinks/prom
 	PrometheusSink *PrometheusSinkConfig
@@ -306,6 +313,9 @@ func DefaultConfig() *Config {
 			EnableCollectorStats: false,
 			FlowsNeeded:          10,
 		},
+		InfluxDBFormat: &InfluxDBFormatConfig{
+			MeasurementPrefix: "",
+		},
 		PrometheusSink: &PrometheusSinkConfig{
 			ListenAddr: ":8082",
 		},
diff --git a/hack/influxdb/docker-compose.yml b/hack/influxdb/docker-compose.yml
index e794d0b..3739f3f 100644
--- a/hack/influxdb/docker-compose.yml
+++ b/hack/influxdb/docker-compose.yml
@@ -35,7 +35,7 @@ services:
         maxthreads: 1
         format: "influx"
         compression: "none"
-        sinks: "http"
+        sinks: "stdout"
         maxflowspermessage: 10000
         rollupinterval: 0
         rollupandalpha: false
diff --git a/pkg/api/api.go b/pkg/api/api.go
index dc19f79..52381d0 100644
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@@ -236,7 +236,7 @@ func (api *KentikApi) GetDevice(cid kt.Cid, did kt.DeviceID) *kt.Device {
 
 func (api *KentikApi) getDevices(ctx context.Context) error {
 	stime := time.Now()
-	res, err := api.getDeviceInfo(ctx, api.conf.ApiRoot+"/api/v5/devices")
+	res, err := api.getDeviceInfo(ctx, api.conf.ApiRoot+"/api/internal/devices")
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cat/jchf.go b/pkg/cat/jchf.go
index 717043d..1d29515 100644
--- a/pkg/cat/jchf.go
+++ b/pkg/cat/jchf.go
@@ -579,13 +579,15 @@ func (kc *KTranslate) doEnrichments(ctx context.Context, citycache map[uint32]st
 		}
 
 		// See if we know what service this is based on proto and port.
-		if msg.L4SrcPort < msg.L4DstPort {
-			if app, ok := kc.rule.GetService(sip, msg.L4SrcPort, ic.PROTO_NUMS[msg.Protocol]); ok {
-				msg.CustomStr["application"] = app
-			}
-		} else {
-			if app, ok := kc.rule.GetService(dip, msg.L4DstPort, ic.PROTO_NUMS[msg.Protocol]); ok {
-				msg.CustomStr["application"] = app
+		if msg.CustomStr["application"] == "" {
+			if msg.L4SrcPort > 0 && msg.L4SrcPort < msg.L4DstPort {
+				if app, ok := kc.rule.GetService(sip, msg.L4SrcPort, ic.PROTO_NUMS[msg.Protocol]); ok {
+					msg.CustomStr["application"] = app
+				}
+			} else {
+				if app, ok := kc.rule.GetService(dip, msg.L4DstPort, ic.PROTO_NUMS[msg.Protocol]); ok {
+					msg.CustomStr["application"] = app
+				}
 			}
 		}
 
diff --git a/pkg/cat/kkc.go b/pkg/cat/kkc.go
index 020f85a..649937c 100644
--- a/pkg/cat/kkc.go
+++ b/pkg/cat/kkc.go
@@ -658,7 +658,7 @@ func (kc *KTranslate) Run(ctx context.Context) error {
 		}
 		assureInput()
 		kc.metrics.SnmpDeviceData = kt.NewSnmpMetricSet(kc.registry)
-		err := snmp.StartSNMPPolls(ctx, kc.inputChan, kc.metrics.SnmpDeviceData, kc.registry, kc.apic, kc.log, kc.config.SNMPInput)
+		err := snmp.StartSNMPPolls(ctx, kc.inputChan, kc.metrics.SnmpDeviceData, kc.registry, kc.apic, kc.log, kc.config.SNMPInput, kc.resolver)
 		if err != nil {
 			return err
 		}
diff --git a/pkg/formats/carbon/carbon.go b/pkg/formats/carbon/carbon.go
index 66702ca..cb5fc38 100644
--- a/pkg/formats/carbon/carbon.go
+++ b/pkg/formats/carbon/carbon.go
@@ -79,6 +79,8 @@ func (f *CarbonFormat) To(msgs []*kt.JCHF, serBuf []byte) (*kt.Output, error) {
 
 func (f *CarbonFormat) toCarbonMetric(in *kt.JCHF) []*CarbonData {
 	switch in.EventType {
+	case kt.KENTIK_EVENT_SYNTH, kt.KENTIK_EVENT_TRACE:
+		return f.fromKsynth(in)
 	case kt.KENTIK_EVENT_TYPE:
 		return f.fromKflow(in)
 	default:
@@ -197,6 +199,80 @@ func (f *CarbonFormat) fromKflow(in *kt.JCHF) []*CarbonData {
 	return metricData
 }
 
+func (f *CarbonFormat) fromKsynth(in *kt.JCHF) []*CarbonData {
+	metrics := []string{
+		"ping_avg_rtt",
+		"ping_jit_rtt",
+		"ping_max_rtt",
+		"ping_std_rtt",
+	}
+	metricData := []*CarbonData{}
+	for _, m := range metrics {
+		val := int64(0)
+		mtype := "gauge"
+		unit := "μ"
+		if v, ok := in.CustomInt[m]; ok {
+			val = int64(v)
+		}
+
+		// intrinsic tags are the metric dimensions
+		intrinsicTags := map[string]interface{}{}
+		if v, ok := in.CustomStr["application_type"]; ok && v != "" {
+			intrinsicTags["application_type"] = v
+		}
+		if v := in.DstAddr; v != "" {
+			intrinsicTags["dst_addr"] = v
+		}
+		if v := in.SrcAddr; v != "" {
+			intrinsicTags["src_addr"] = v
+		}
+
+		// meta tags are used to decorate and add context
+		metaTags := map[string]interface{}{
+			"src_as": in.SrcAs,
+			"dst_as": in.DstAs,
+		}
+		if v := in.DstGeo; v != "" {
+			metaTags["dst_geo"] = v
+		}
+		if v := in.SrcGeo; v != "" {
+			metaTags["src_geo"] = v
+		}
+		if v := in.SrcGeo; v != "" {
+			metaTags["src_geo"] = v
+		}
+		if v := in.SrcGeoRegion; v != "" {
+			metaTags["src_geo_region"] = v
+		}
+		if v := in.DstGeoRegion; v != "" {
+			metaTags["dst_geo_region"] = v
+		}
+		if v := in.SrcGeoCity; v != "" {
+			metaTags["src_geo_city"] = v
+		}
+		if v := in.DstGeoCity; v != "" {
+			metaTags["dst_geo_city"] = v
+		}
+		for k, v := range in.CustomStr {
+			if v == "" {
+				continue
+			}
+			metaTags[k] = v
+		}
+		metricData = append(metricData, &CarbonData{
+			Type:          mtype,
+			Name:          m,
+			Value:         val,
+			Unit:          unit,
+			Timestamp:     in.Timestamp,
+			IntrinsicTags: intrinsicTags,
+			MetaTags:      metaTags,
+		})
+	}
+
+	return metricData
+}
+
 func parseTags(tags map[string]interface{}) []string {
 	parsedTags := make([]string, len(tags))
 	x := 0
diff --git a/pkg/formats/format.go b/pkg/formats/format.go
index 87cbe5c..4a273d3 100644
--- a/pkg/formats/format.go
+++ b/pkg/formats/format.go
@@ -56,7 +56,7 @@ func NewFormat(format Format, log logger.Underlying, registry go_metrics.Registr
 	case FORMAT_NETFLOW:
 		return netflow.NewFormat(log, compression, cfg.NetflowFormat)
 	case FORMAT_INFLUX:
-		return influx.NewFormat(log, registry, compression)
+		return influx.NewFormat(log, registry, compression, cfg.InfluxDBFormat)
 	case FORMAT_CARBON:
 		return carbon.NewFormat(log, compression)
 	case FORMAT_PROM:
diff --git a/pkg/formats/influx/influx.go b/pkg/formats/influx/influx.go
index ee1d768..3b84895 100644
--- a/pkg/formats/influx/influx.go
+++ b/pkg/formats/influx/influx.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/influxdata/line-protocol/v2/lineprotocol"
 	go_metrics "github.com/kentik/go-metrics"
+	"github.com/kentik/ktranslate"
 	"github.com/kentik/ktranslate/pkg/formats/util"
 	"github.com/kentik/ktranslate/pkg/kt"
 	"github.com/kentik/ktranslate/pkg/rollup"
@@ -18,6 +19,15 @@ import (
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
 )
 
+var (
+	measurementPrefix string
+)
+
+func init() {
+	flag.StringVar(&measurementPrefix, "influxdb_measurement_prefix", "", "Prefix metric names with this")
+
+}
+
 type InfluxFormat struct {
 	logger.ContextL
 	invalids     map[string]bool
@@ -25,6 +35,7 @@ type InfluxFormat struct {
 	lastMetadata map[string]*kt.LastMetadata
 	mux          sync.RWMutex
 	metrics      *InfluxMetrics
+	config       *ktranslate.InfluxDBFormatConfig
 }
 
 type InfluxMetrics struct {
@@ -40,10 +51,6 @@ type InfluxData struct {
 	Timestamp    int64
 }
 
-var (
-	Prefix = flag.String("influxdb_measurement_prefix", "", "Prefix metric names with this")
-)
-
 func (d *InfluxData) GetTags() string {
 	var tags []string
 	for key, v := range d.Tags {
@@ -241,7 +248,10 @@ func prepareTagValueMap(s string) string {
 	return s
 }
 
-func NewFormat(log logger.Underlying, registry go_metrics.Registry, compression kt.Compression) (*InfluxFormat, error) {
+func NewFormat(log logger.Underlying, registry go_metrics.Registry, compression kt.Compression, cfg *ktranslate.InfluxDBFormatConfig) (*InfluxFormat, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("influxdb format cannot be nil")
+	}
 	jf := &InfluxFormat{
 		ContextL:     logger.NewContextLFromUnderlying(logger.SContext{S: "influxFormat"}, log),
 		invalids:     map[string]bool{},
@@ -249,6 +259,7 @@ func NewFormat(log logger.Underlying, registry go_metrics.Registry, compression
 		metrics: &InfluxMetrics{
 			EncoderErrors: go_metrics.GetOrRegisterCounter("influx_encoder_errors", registry),
 		},
+		config: cfg,
 	}
 
 	return jf, nil
@@ -383,7 +394,7 @@ func (f *InfluxFormat) fromKtranslate(in *kt.JCHF) []InfluxData {
 	case "counter":
 		if in.CustomStr["force"] == "true" || in.CustomBigInt["count"] > 0 {
 			ms = append(ms, InfluxData{
-				Name:        *Prefix + "ktranslate",
+				Name:        f.config.MeasurementPrefix + "ktranslate",
 				FieldsFloat: map[string]float64{name: float64(in.CustomBigInt["count"]) / 100},
 				Timestamp:   in.Timestamp,
 				Tags:        attr,
@@ -392,7 +403,7 @@ func (f *InfluxFormat) fromKtranslate(in *kt.JCHF) []InfluxData {
 	case "gauge":
 		if in.CustomStr["force"] == "true" || in.CustomBigInt["value"] > 0 {
 			ms = append(ms, InfluxData{
-				Name:        *Prefix + "ktranslate",
+				Name:        f.config.MeasurementPrefix + "ktranslate",
 				FieldsFloat: map[string]float64{name: float64(in.CustomBigInt["value"]) / 100},
 				Timestamp:   in.Timestamp,
 				Tags:        attr,
@@ -401,7 +412,7 @@ func (f *InfluxFormat) fromKtranslate(in *kt.JCHF) []InfluxData {
 	case "histogram":
 		if in.CustomStr["force"] == "true" || in.CustomBigInt["95-percentile"] > 0 {
 			ms = append(ms, InfluxData{
-				Name:        *Prefix + "ktranslate",
+				Name:        f.config.MeasurementPrefix + "ktranslate",
 				FieldsFloat: map[string]float64{name: float64(in.CustomBigInt["95-percentile"]) / 100},
 				Timestamp:   in.Timestamp,
 				Tags:        attr,
@@ -410,7 +421,7 @@ func (f *InfluxFormat) fromKtranslate(in *kt.JCHF) []InfluxData {
 	case "meter":
 		if in.CustomStr["force"] == "true" || in.CustomBigInt["one-minute"] > 0 {
 			ms = append(ms, InfluxData{
-				Name:        *Prefix + "ktranslate",
+				Name:        f.config.MeasurementPrefix + "ktranslate",
 				FieldsFloat: map[string]float64{name: float64(in.CustomBigInt["one-minute"]) / 100},
 				Timestamp:   in.Timestamp,
 				Tags:        attr,
@@ -419,7 +430,7 @@ func (f *InfluxFormat) fromKtranslate(in *kt.JCHF) []InfluxData {
 	case "timer":
 		if in.CustomStr["force"] == "true" || in.CustomBigInt["95-percentile"] > 0 {
 			ms = append(ms, InfluxData{
-				Name:        *Prefix + "ktranslate",
+				Name:        f.config.MeasurementPrefix + "ktranslate",
 				FieldsFloat: map[string]float64{name: float64(in.CustomBigInt["95-percentile"]) / 100},
 				Timestamp:   in.Timestamp,
 				Tags:        attr,
@@ -449,7 +460,7 @@ func (f *InfluxFormat) fromKSynth(in *kt.JCHF) []InfluxData {
 	}
 
 	return []InfluxData{{
-		Name:      *Prefix,
+		Name:      f.config.MeasurementPrefix + "ksynth",
 		Fields:    ms,
 		Timestamp: in.Timestamp * 1000000000,
 		Tags:      attr,
@@ -480,7 +491,7 @@ func (f *InfluxFormat) fromKflow(in *kt.JCHF) []InfluxData {
 	}
 
 	return []InfluxData{{
-		Name:      *Prefix,
+		Name:      f.config.MeasurementPrefix + "flow",
 		Fields:    ms,
 		Timestamp: in.Timestamp * 1000000000,
 		Tags:      attr,
@@ -510,14 +521,14 @@ func (f *InfluxFormat) fromSnmpDeviceMetric(in *kt.JCHF) []InfluxData {
 			mib := getMib(attrNew, ip)
 			if name.Format == kt.FloatMS {
 				results = append(results, InfluxData{
-					Name:        *Prefix + mib,
+					Name:        f.config.MeasurementPrefix + mib,
 					FieldsFloat: map[string]float64{m: float64(float64(in.CustomBigInt[m]) / 1000)},
 					Timestamp:   in.Timestamp * 1000000000,
 					Tags:        attrNew,
 				})
 			} else {
 				results = append(results, InfluxData{
-					Name:      *Prefix + mib,
+					Name:      f.config.MeasurementPrefix + mib,
 					Fields:    map[string]int64{m: int64(in.CustomBigInt[m])},
 					Timestamp: in.Timestamp * 1000000000,
 					Tags:      attrNew,
@@ -554,14 +565,14 @@ func (f *InfluxFormat) fromSnmpInterfaceMetric(in *kt.JCHF) []InfluxData {
 			mib := getMib(attrNew, ip)
 			if name.Format == kt.FloatMS {
 				results = append(results, InfluxData{
-					Name:        *Prefix + mib,
+					Name:        f.config.MeasurementPrefix + mib,
 					FieldsFloat: map[string]float64{m: float64(float64(in.CustomBigInt[m]) / 1000)},
 					Timestamp:   in.Timestamp * 1000000000,
 					Tags:        attrNew,
 				})
 			} else {
 				results = append(results, InfluxData{
-					Name:      *Prefix + mib,
+					Name:      f.config.MeasurementPrefix + mib,
 					Fields:    map[string]int64{m: int64(in.CustomBigInt[m])},
 					Timestamp: in.Timestamp * 1000000000,
 					Tags:      attrNew,
@@ -573,45 +584,58 @@ func (f *InfluxFormat) fromSnmpInterfaceMetric(in *kt.JCHF) []InfluxData {
 		}
 	}
 
-	// Grap capacity utilization if possible.
+	// Grap varioius rates computed over time here if possible.
 	if f.lastMetadata[in.DeviceName] != nil {
-		if ii, ok := f.lastMetadata[in.DeviceName].InterfaceInfo[in.InputPort]; ok {
-			if speed, ok := ii["Speed"]; ok {
-				if ispeed, ok := speed.(int32); ok {
-					uptimeSpeed := in.CustomBigInt["Uptime"] * (int64(ispeed) * 10000) // Convert into bits here, from megabits. Also divide by 100 to convert uptime into seconds, from centi-seconds.
-					if uptimeSpeed > 0 {
-						attrNew := util.CopyAttrForSnmp(attr, "IfInUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: profileName, Table: "if"}, f.lastMetadata[in.DeviceName], false)
-						if inBytes, ok := in.CustomBigInt["ifHCInOctets"]; ok {
-							if !util.DropOnFilter(attrNew, f.lastMetadata[in.DeviceName], true) {
-								getMib(attrNew, ip)
-								results = append(results, InfluxData{
-									Name:        *Prefix + "IF-MIB::if",
-									FieldsFloat: map[string]float64{"IfInUtilization": float64(inBytes*8*100) / float64(uptimeSpeed)},
-									Timestamp:   in.Timestamp * 1000000000,
-									Tags:        attrNew,
-								})
-							}
+		results = f.setRates("In", in, results, attr, profileName, ip)
+		results = f.setRates("Out", in, results, attr, profileName, ip)
+	}
+
+	return results
+}
+
+func (f *InfluxFormat) setRates(direction string, in *kt.JCHF, results []InfluxData, attr map[string]interface{}, profileName string, ip interface{}) []InfluxData {
+	var port kt.IfaceID
+	if direction == "In" {
+		port = in.InputPort
+	} else {
+		port = in.OutputPort
+	}
+	utilName := fmt.Sprintf("If%sUtilization", direction)
+	bitRate := fmt.Sprintf("If%sBitRate", direction)
+	pktRate := fmt.Sprintf("If%sPktRate", direction)
+	totalBytes := in.CustomBigInt[fmt.Sprintf("ifHC%sOctets", direction)]
+	totalPkts := in.CustomBigInt[fmt.Sprintf("ifHC%sUcastPkts", direction)] + in.CustomBigInt[fmt.Sprintf("ifHC%sMulticastPkts", direction)] + in.CustomBigInt[fmt.Sprintf("ifHC%sBroadcastPkts", direction)]
+
+	if ii, ok := f.lastMetadata[in.DeviceName].InterfaceInfo[port]; ok {
+		if speed, ok := ii["Speed"]; ok {
+			if ispeed, ok := speed.(int32); ok {
+				uptime := in.CustomBigInt["Uptime"]
+				uptimeSpeed := uptime * (int64(ispeed) * 10000) // Convert into bits here, from megabits. Also divide by 100 to convert uptime into seconds, from centi-seconds.
+				if uptimeSpeed > 0 {
+					attrNew := util.CopyAttrForSnmp(attr, utilName, kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: profileName, Table: "if"}, f.lastMetadata[in.DeviceName], false)
+					if !util.DropOnFilter(attrNew, f.lastMetadata[in.DeviceName], true) {
+						getMib(attrNew, ip)
+						if totalBytes > 0 {
+							results = append(results, InfluxData{
+								Name:        f.config.MeasurementPrefix + "IF-MIB::if",
+								FieldsFloat: map[string]float64{utilName: float64(totalBytes*8*100) / float64(uptimeSpeed)},
+								Timestamp:   in.Timestamp * 1000000000,
+								Tags:        attrNew,
+							})
+							results = append(results, InfluxData{
+								Name:        f.config.MeasurementPrefix + "IF-MIB::if",
+								FieldsFloat: map[string]float64{bitRate: float64(totalBytes*8*100) / float64(uptime)},
+								Timestamp:   in.Timestamp * 1000000000,
+								Tags:        attrNew,
+							})
 						}
-					}
-				}
-			}
-		}
-		if oi, ok := f.lastMetadata[in.DeviceName].InterfaceInfo[in.OutputPort]; ok {
-			if speed, ok := oi["Speed"]; ok {
-				if ispeed, ok := speed.(int32); ok {
-					uptimeSpeed := in.CustomBigInt["Uptime"] * (int64(ispeed) * 10000) // Convert into bits here, from megabits. Also divide by 100 to convert uptime into seconds, from centi-seconds.
-					if uptimeSpeed > 0 {
-						attrNew := util.CopyAttrForSnmp(attr, "IfOutUtilization", kt.MetricInfo{Oid: "computed", Mib: "computed", Profile: profileName, Table: "if"}, f.lastMetadata[in.DeviceName], false)
-						if outBytes, ok := in.CustomBigInt["ifHCOutOctets"]; ok {
-							if !util.DropOnFilter(attrNew, f.lastMetadata[in.DeviceName], true) {
-								getMib(attrNew, ip)
-								results = append(results, InfluxData{
-									Name:        *Prefix + "IF-MIB::if",
-									FieldsFloat: map[string]float64{"IfOutUtilization": float64(outBytes*8*100) / float64(uptimeSpeed)},
-									Timestamp:   in.Timestamp * 1000000000,
-									Tags:        attrNew,
-								})
-							}
+						if totalPkts > 0 {
+							results = append(results, InfluxData{
+								Name:        f.config.MeasurementPrefix + "IF-MIB::if",
+								FieldsFloat: map[string]float64{pktRate: float64(totalPkts*100) / float64(uptime)},
+								Timestamp:   in.Timestamp * 1000000000,
+								Tags:        attrNew,
+							})
 						}
 					}
 				}
diff --git a/pkg/inputs/flow/flow.go b/pkg/inputs/flow/flow.go
index ef3d9a0..984e08d 100644
--- a/pkg/inputs/flow/flow.go
+++ b/pkg/inputs/flow/flow.go
@@ -29,6 +29,9 @@ const (
 	Sflow               = "sflow"
 	Netflow5            = "netflow5"
 	Netflow9            = "netflow9"
+	NBar                = "nbar"
+	ASA                 = "asa"
+	PAN                 = "pan"
 )
 
 var (
@@ -52,9 +55,6 @@ func init() {
 }
 
 func NewFlowSource(ctx context.Context, proto FlowSource, maxBatchSize int, log logger.Underlying, registry go_metrics.Registry, jchfChan chan []*kt.JCHF, apic *api.KentikApi, resolv *resolv.Resolver, cfg *ktranslate.FlowInputConfig) (*KentikDriver, error) {
-	kt := NewKentikDriver(ctx, proto, maxBatchSize, log, registry, jchfChan, apic, cfg.MessageFields, resolv, cfg)
-	kt.Infof("Netflow listener running on %s:%d for format %s and a batch size of %d", cfg.ListenIP, cfg.ListenPort, proto, maxBatchSize)
-	kt.Infof("Netflow listener sending fields %s", cfg.MessageFields)
 
 	defer func() {
 		if v := cfg.PrometheusListenAddr; v != "" {
@@ -65,6 +65,20 @@ func NewFlowSource(ctx context.Context, proto FlowSource, maxBatchSize int, log
 
 	// Allow processing of custom ipfix templates here.
 	var config EntConfig
+
+	// Load a special config if there's a known enriched flow.
+	switch proto {
+	case ASA:
+		config = loadASA(cfg)
+	case NBar:
+		config = loadNBar(cfg)
+	case PAN:
+		config = loadPAN(cfg)
+	}
+
+	kt := NewKentikDriver(ctx, proto, maxBatchSize, log, registry, jchfChan, apic, cfg.MessageFields, resolv, cfg)
+
+	// Or pull up a special file if needed.
 	if v := cfg.MappingFile; v != "" {
 		f, err := os.Open(v)
 		if err != nil {
@@ -77,21 +91,25 @@ func NewFlowSource(ctx context.Context, proto FlowSource, maxBatchSize int, log
 			kt.Errorf("Invalid yaml for netflow mapping file: %v", err)
 			return nil, err
 		}
-		kt.SetConfig(config)
 	}
 
+	kt.Infof("Netflow listener running on %s:%d for format %s and a batch size of %d", cfg.ListenIP, cfg.ListenPort, proto, maxBatchSize)
+	kt.Infof("Netflow listener sending fields %s", cfg.MessageFields)
+	kt.SetConfig(config)
+
 	switch proto {
-	case Ipfix, Netflow9:
+	case Ipfix, Netflow9, ASA, NBar, PAN:
 		sNF := &utils.StateNetFlow{
 			Format: kt,
 			Logger: &KentikLog{l: kt},
 			Config: &config.FlowConfig,
 		}
-		if proto == Ipfix {
+		switch proto {
+		case Ipfix, ASA, NBar, PAN:
 			for _, v := range config.FlowConfig.IPFIX.Mapping {
 				kt.Infof("Custom IPFIX Field Mapping: Field=%v, Pen=%v -> %v", v.Type, v.Pen, config.NameMap[v.Destination])
 			}
-		} else {
+		case Netflow9:
 			for _, v := range config.FlowConfig.NetFlowV9.Mapping {
 				kt.Infof("Custom Netflow9 Field Mapping: Field=%v -> %v", v.Type, config.NameMap[v.Destination])
 			}
@@ -150,7 +168,7 @@ func loadMapping(f io.Reader, proto FlowSource) (EntConfig, error) {
 		config.NameMap = map[string]string{}
 	}
 	switch proto {
-	case Ipfix:
+	case Ipfix, ASA, NBar, PAN:
 		for _, v := range config.FlowConfig.IPFIX.Mapping {
 			if _, ok := config.NameMap[v.Destination]; !ok {
 				config.NameMap[v.Destination] = v.Destination
diff --git a/pkg/inputs/flow/specials.go b/pkg/inputs/flow/specials.go
new file mode 100644
index 0000000..dc16567
--- /dev/null
+++ b/pkg/inputs/flow/specials.go
@@ -0,0 +1,170 @@
+package flow
+
+import (
+	"strings"
+
+	"github.com/kentik/ktranslate"
+
+	"github.com/netsampler/goflow2/producer"
+)
+
+func loadASA(cfg *ktranslate.FlowInputConfig) EntConfig {
+	config := EntConfig{
+		FlowConfig: producer.ProducerConfig{
+			IPFIX: producer.IPFIXProducerConfig{
+				Mapping: []producer.NetFlowMapField{
+					producer.NetFlowMapField{
+						Type:        231,
+						Destination: "CustomInteger1",
+					},
+					producer.NetFlowMapField{
+						Type:        232,
+						Destination: "CustomInteger2",
+					},
+					producer.NetFlowMapField{
+						Type:        298,
+						Destination: "CustomInteger3",
+					},
+					producer.NetFlowMapField{
+						Type:        299,
+						Destination: "CustomInteger4",
+					},
+					producer.NetFlowMapField{
+						Type:        233,
+						Destination: "CustomInteger5",
+					},
+					producer.NetFlowMapField{
+						Type:        33000,
+						Destination: "CustomBytes1",
+					},
+					producer.NetFlowMapField{
+						Type:        33001,
+						Destination: "CustomBytes2",
+					},
+				},
+			},
+		},
+		NameMap: map[string]string{
+			"CustomInteger1": "in_bytes",
+			"CustomInteger2": "out_bytes",
+			"CustomInteger3": "in_pkts",
+			"CustomInteger4": "out_pkts",
+			"CustomInteger5": "Firewall Event",
+			"CustomBytes1":   "Ingress ACL",
+			"CustomBytes2":   "Egress ACL",
+		},
+	}
+
+	for field, _ := range config.NameMap {
+		if !strings.Contains(cfg.MessageFields, field) {
+			cfg.MessageFields = cfg.MessageFields + "," + field
+		}
+	}
+
+	return config
+}
+
+func loadNBar(cfg *ktranslate.FlowInputConfig) EntConfig {
+	config := EntConfig{
+		FlowConfig: producer.ProducerConfig{
+			IPFIX: producer.IPFIXProducerConfig{
+				Mapping: []producer.NetFlowMapField{
+					producer.NetFlowMapField{
+						Type:        96,
+						Destination: "CustomBytes1",
+					},
+					producer.NetFlowMapField{
+						Type:        12232,
+						Destination: "CustomBytes2",
+						PenProvided: true,
+						Pen:         9,
+					},
+					producer.NetFlowMapField{
+						Type:        12233,
+						Destination: "CustomBytes3",
+						PenProvided: true,
+						Pen:         9,
+					},
+					producer.NetFlowMapField{
+						Type:        12234,
+						Destination: "CustomBytes4",
+						PenProvided: true,
+						Pen:         9,
+					},
+					producer.NetFlowMapField{
+						Type:        12243,
+						Destination: "CustomBytes5",
+						PenProvided: true,
+						Pen:         9,
+					},
+				},
+			},
+		},
+		NameMap: map[string]string{
+			"CustomBytes1": "application",
+			"CustomBytes2": "Application Category",
+			"CustomBytes3": "Application Subcategory",
+			"CustomBytes4": "Application Group",
+			"CustomBytes5": "Application Traffic Class",
+		},
+	}
+
+	for field, _ := range config.NameMap {
+		if !strings.Contains(cfg.MessageFields, field) {
+			cfg.MessageFields = cfg.MessageFields + "," + field
+		}
+	}
+
+	return config
+}
+
+func loadPAN(cfg *ktranslate.FlowInputConfig) EntConfig {
+	config := EntConfig{
+		FlowConfig: producer.ProducerConfig{
+			IPFIX: producer.IPFIXProducerConfig{
+				Mapping: []producer.NetFlowMapField{
+					producer.NetFlowMapField{
+						Type:        32,
+						Destination: "CustomInteger1",
+					},
+					producer.NetFlowMapField{
+						Type:        148,
+						Destination: "CustomInteger2",
+					},
+					producer.NetFlowMapField{
+						Type:        233,
+						Destination: "CustomInteger3",
+					},
+					producer.NetFlowMapField{
+						Type:        61,
+						Destination: "CustomInteger4",
+					},
+					producer.NetFlowMapField{
+						Type:        56701,
+						Destination: "CustomBytes1",
+					},
+					producer.NetFlowMapField{
+						Type:        56702,
+						Destination: "CustomBytes2",
+					},
+				},
+			},
+		},
+		NameMap: map[string]string{
+			"CustomInteger1": "ICMP Type",
+			"CustomInteger2": "Flow ID",
+			"CustomInteger3": "Firewall Event",
+			"CustomInteger4": "Direction",
+			"CustomBytes1":   "Application ID",
+			"CustomBytes2":   "User ID",
+		},
+	}
+
+	for field, _ := range config.NameMap {
+		if !strings.Contains(cfg.MessageFields, field) {
+			cfg.MessageFields = cfg.MessageFields + "," + field
+		}
+	}
+
+	return config
+}
diff --git a/pkg/inputs/snmp/metadata/device_metadata.go b/pkg/inputs/snmp/metadata/device_metadata.go
index f8621a1..3d51b51 100644
--- a/pkg/inputs/snmp/metadata/device_metadata.go
+++ b/pkg/inputs/snmp/metadata/device_metadata.go
@@ -158,7 +158,7 @@ func (dm *DeviceMetadata) poll(ctx context.Context, server *gosnmp.GoSNMP) (*kt.
 		case SNMP_sysServices:
 			md.SysServices = int(snmp_util.ToInt64(value))
 		case SNMP_engineID:
-			_, md.EngineID = snmp_util.GetFromConv(wrapper.variable, snmp_util.CONV_ENGINE_ID, dm.log)
+			_, md.EngineID, _ = snmp_util.GetFromConv(wrapper.variable, snmp_util.CONV_ENGINE_ID, dm.log)
 		default:
 			// Now we're actually in the range of custom fields.
 			if wrapper.mib == nil { // This should never happen here.
@@ -174,12 +174,18 @@ func (dm *DeviceMetadata) poll(ctx context.Context, server *gosnmp.GoSNMP) (*kt.
 					md.Customs[oidName] = vt
 				case []byte:
 					if wrapper.mib.Conversion != "" { // Adjust for any hard coded values here.
-						ival, sval := snmp_util.GetFromConv(wrapper.variable, wrapper.mib.Conversion, dm.log)
+						ival, sval, mval := snmp_util.GetFromConv(wrapper.variable, wrapper.mib.Conversion, dm.log)
 						if ival > 0 {
 							md.CustomInts[oidName] = ival
 							md.Customs[kt.StringPrefix+oidName] = sval
 						} else {
-							md.Customs[oidName] = sval
+							if len(mval) > 0 {
+								for k, v := range mval {
+									md.Customs[k] = v
+								}
+							} else {
+								md.Customs[oidName] = sval
+							}
 						}
 					} else {
 						md.Customs[oidName] = string(vt)
@@ -226,7 +232,7 @@ func (dm *DeviceMetadata) handleTable(idx string, value wrapper, oidName string,
 	case gosnmp.OctetString:
 		val := string(value.variable.Value.([]byte))
 		if value.mib.Conversion != "" { // Adjust for any hard coded values here.
-			_, val = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
+			_, val, _ = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
 		}
 		if utf8.ValidString(val) {
 			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val, 0)
@@ -243,7 +249,7 @@ func (dm *DeviceMetadata) handleTable(idx string, value wrapper, oidName string,
 	case gosnmp.ObjectIdentifier:
 		val := string(value.variable.Value.(string))
 		if value.mib.Conversion != "" { // Adjust for any hard coded values here.
-			_, val = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
+			_, val, _ = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
 		}
 		if utf8.ValidString(val) {
 			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val, 0)
@@ -303,7 +309,7 @@ func GetBasicDeviceMetadata(log logger.ContextL, server *gosnmp.GoSNMP) (*kt.Dev
 		case SNMP_sysServices:
 			md.SysServices = int(snmp_util.ToInt64(value))
 		case SNMP_engineID:
-			_, md.EngineID = snmp_util.GetFromConv(pdu, snmp_util.CONV_ENGINE_ID, log)
+			_, md.EngineID, _ = snmp_util.GetFromConv(pdu, snmp_util.CONV_ENGINE_ID, log)
 		}
 	}
 
diff --git a/pkg/inputs/snmp/metadata/interface_metadata.go b/pkg/inputs/snmp/metadata/interface_metadata.go
index 358cd1f..35446b3 100644
--- a/pkg/inputs/snmp/metadata/interface_metadata.go
+++ b/pkg/inputs/snmp/metadata/interface_metadata.go
@@ -247,7 +247,7 @@ func (im *InterfaceMetadata) Poll(ctx context.Context, conf *kt.SnmpDeviceConfig
 				case SNMP_ifPhysAddress:
 					switch variable.Type {
 					case gosnmp.OctetString:
-						_, sval := snmp_util.GetFromConv(variable, "hwaddr", im.log)
+						_, sval, _ := snmp_util.GetFromConv(variable, "hwaddr", im.log)
 						data.ExtraInfo[SNMP_ifPhysAddress] = sval
 					}
 				case SNMP_ifLastChange:
diff --git a/pkg/inputs/snmp/metrics/device_metrics.go b/pkg/inputs/snmp/metrics/device_metrics.go
index 764526b..fbb17c1 100644
--- a/pkg/inputs/snmp/metrics/device_metrics.go
+++ b/pkg/inputs/snmp/metrics/device_metrics.go
@@ -165,7 +165,7 @@ func (dm *DeviceMetrics) pollFromConfig(ctx context.Context, server *gosnmp.GoSN
 		case gosnmp.OctetString, gosnmp.BitString:
 			value := string(wrapper.variable.Value.([]byte))
 			if wrapper.mib.Conversion != "" { // Adjust for any hard coded values here.
-				ival, sval := snmp_util.GetFromConv(wrapper.variable, wrapper.mib.Conversion, dm.log)
+				ival, sval, _ := snmp_util.GetFromConv(wrapper.variable, wrapper.mib.Conversion, dm.log)
 				if ival > 0 {
 					dmr.customBigInt[oidName] = ival
 					dmr.customStr[kt.StringPrefix+oidName] = sval
diff --git a/pkg/inputs/snmp/mibs/load.go b/pkg/inputs/snmp/mibs/load.go
index 883bdea..260d547 100644
--- a/pkg/inputs/snmp/mibs/load.go
+++ b/pkg/inputs/snmp/mibs/load.go
@@ -98,6 +98,15 @@ func (db *MibDB) GetForKey(oid string) (*kt.Mib, error) {
 		return res, nil
 	}
 
+	// Now walk resursivly up the tree, seeing what profiles are found via a wildcard
+	pts := strings.Split(oid, ".")
+	for i := len(pts); i > 0; i-- {
+		check := strings.Join(pts[0:i], ".") + ".*"
+		if t, ok := db.trapMibs[check]; ok {
+			return t, nil
+		}
+	}
+
 	if db.db == nil { // We might not have set up a db here.
 		return nil, nil
 	}
@@ -109,7 +118,7 @@ func (db *MibDB) GetForKey(oid string) (*kt.Mib, error) {
 		return nil, err
 	}
 
-	pts := strings.SplitN(string(data), " ", 2)
+	pts = strings.SplitN(string(data), " ", 2)
 	if len(pts) >= 2 {
 		res := reType.FindAllStringSubmatch(pts[1], -1)
 		if len(res) > 0 {
diff --git a/pkg/inputs/snmp/mibs/profile.go b/pkg/inputs/snmp/mibs/profile.go
index 46a426f..4dc79a1 100644
--- a/pkg/inputs/snmp/mibs/profile.go
+++ b/pkg/inputs/snmp/mibs/profile.go
@@ -389,6 +389,7 @@ func (p *Profile) GetMetrics(enabledMibs []string, counterTimeSec int) (map[stri
 
 	enabled := map[string]bool{}
 	enabledTables := map[string]map[string]bool{}
+	allMibsEnabled := false
 	for _, mib := range enabledMibs {
 		pts := strings.Split(mib, ".")
 		if len(pts) == 1 {
@@ -400,10 +401,14 @@ func (p *Profile) GetMetrics(enabledMibs []string, counterTimeSec int) (map[stri
 			}
 			enabledTables[pts[0]][pts[1]] = true
 		}
+
+		if mib == kt.EnableAllMibs {
+			allMibsEnabled = true
+		}
 	}
 
 	for _, metric := range p.Metrics {
-		if !enabled[metric.Mib] { // Only look at mibs we have opted into caring about.
+		if !allMibsEnabled && !enabled[metric.Mib] { // Only look at mibs we have opted into caring about.
 			continue
 		}
 		if enabledTables[metric.Mib] != nil {
@@ -531,6 +536,7 @@ func (p *Profile) GetMetadata(enabledMibs []string) (map[string]*kt.Mib, map[str
 
 	enabled := map[string]bool{}
 	enabledTables := map[string]map[string]bool{}
+	allMibsEnabled := false
 	for _, mib := range enabledMibs {
 		pts := strings.Split(mib, ".")
 		if len(pts) == 1 {
@@ -542,6 +548,10 @@ func (p *Profile) GetMetadata(enabledMibs []string) (map[string]*kt.Mib, map[str
 			}
 			enabledTables[pts[0]][pts[1]] = true
 		}
+
+		if mib == kt.EnableAllMibs {
+			allMibsEnabled = true
+		}
 	}
 
 	// Top level tags here.
@@ -580,7 +590,7 @@ func (p *Profile) GetMetadata(enabledMibs []string) (map[string]*kt.Mib, map[str
 
 	// per metric tags here.
 	for _, metric := range p.Metrics {
-		if !enabled[metric.Mib] { // Only look at mibs we have opted into caring about.
+		if !allMibsEnabled && !enabled[metric.Mib] { // Only look at mibs we have opted into caring about.
 			continue
 		}
 		if enabledTables[metric.Mib] != nil {
diff --git a/pkg/inputs/snmp/snmp.go b/pkg/inputs/snmp/snmp.go
index d10a3b4..51f349d 100644
--- a/pkg/inputs/snmp/snmp.go
+++ b/pkg/inputs/snmp/snmp.go
@@ -20,6 +20,7 @@ import (
 	"github.com/kentik/ktranslate/pkg/inputs/snmp/traps"
 	snmp_util "github.com/kentik/ktranslate/pkg/inputs/snmp/util"
 	"github.com/kentik/ktranslate/pkg/kt"
+	"github.com/kentik/ktranslate/pkg/util/resolv"
 
 	"gopkg.in/yaml.v2"
 )
@@ -55,7 +56,7 @@ func init() {
 	flag.BoolVar(&validateMib, "snmp_validate", false, "If true, validate mib profiles and exit.")
 }
 
-func StartSNMPPolls(ctx context.Context, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL, cfg *ktranslate.SNMPInputConfig) error {
+func StartSNMPPolls(ctx context.Context, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL, cfg *ktranslate.SNMPInputConfig, resolv *resolv.Resolver) error {
 	snmpFile := cfg.SNMPFile
 	// Do this once here just to see if we need to exit right away.
 	conf, connectTimeout, retries, err := initSnmp(ctx, snmpFile, log)
@@ -98,7 +99,7 @@ func StartSNMPPolls(ctx context.Context, jchfChan chan []*kt.JCHF, metrics *kt.S
 
 	// Run a trap listener?
 	if conf.Trap != nil && !cfg.FlowOnly {
-		err := launchSnmpTrap(conf, jchfChan, metrics, log)
+		err := launchSnmpTrap(ctx, conf, jchfChan, metrics, log, resolv)
 		if err != nil {
 			return err
 		}
@@ -243,9 +244,9 @@ func runSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JC
 	return nil
 }
 
-func launchSnmpTrap(conf *kt.SnmpConfig, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, log logger.ContextL) error {
+func launchSnmpTrap(ctx context.Context, conf *kt.SnmpConfig, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, log logger.ContextL, resolv *resolv.Resolver) error {
 	log.Infof("Client SNMP: Running SNMP Trap listener on %s", conf.Trap.Listen)
-	tl, err := traps.NewSnmpTrapListener(conf, jchfChan, metrics, mibdb, log)
+	tl, err := traps.NewSnmpTrapListener(ctx, conf, jchfChan, metrics, mibdb, log, resolv)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/inputs/snmp/traps/traps.go b/pkg/inputs/snmp/traps/traps.go
index 6df9848..5f7cec6 100644
--- a/pkg/inputs/snmp/traps/traps.go
+++ b/pkg/inputs/snmp/traps/traps.go
@@ -1,6 +1,7 @@
 package traps
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"sync"
@@ -12,6 +13,7 @@ import (
 	"github.com/kentik/ktranslate/pkg/inputs/snmp/mibs"
 	snmp_util "github.com/kentik/ktranslate/pkg/inputs/snmp/util"
 	"github.com/kentik/ktranslate/pkg/kt"
+	"github.com/kentik/ktranslate/pkg/util/resolv"
 )
 
 const (
@@ -28,6 +30,8 @@ type SnmpTrap struct {
 	conf      *kt.SnmpConfig
 	mux       sync.RWMutex
 	mibdb     *mibs.MibDB
+	resolv    *resolv.Resolver
+	ctx       context.Context
 	deviceMap map[string]*kt.SnmpDeviceConfig
 }
 
@@ -45,7 +49,7 @@ func (l logWrapper) Printf(format string, v ...interface{}) {
 	l.printf(format, v...)
 }
 
-func NewSnmpTrapListener(conf *kt.SnmpConfig, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, mibdb *mibs.MibDB, log logger.ContextL) (*SnmpTrap, error) {
+func NewSnmpTrapListener(ctx context.Context, conf *kt.SnmpConfig, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, mibdb *mibs.MibDB, log logger.ContextL, resolv *resolv.Resolver) (*SnmpTrap, error) {
 	st := &SnmpTrap{
 		jchfChan:  jchfChan,
 		log:       log,
@@ -53,6 +57,8 @@ func NewSnmpTrapListener(conf *kt.SnmpConfig, jchfChan chan []*kt.JCHF, metrics
 		listen:    conf.Trap.Listen,
 		metrics:   metrics,
 		deviceMap: map[string]*kt.SnmpDeviceConfig{},
+		resolv:    resolv,
+		ctx:       ctx,
 	}
 
 	// Some quick defaults.
@@ -141,6 +147,12 @@ func (s *SnmpTrap) handle(packet *gosnmp.SnmpPacket, addr *net.UDPAddr) {
 		dev.SetUserTags(dst.CustomStr)
 	} else {
 		dst.DeviceName = addr.IP.String()
+		if s.resolv != nil { // Try pulling device name from reverse IP if possible.
+			dm := s.resolv.Resolve(s.ctx, dst.DeviceName, true)
+			if dm != "" {
+				dst.DeviceName = dm
+			}
+		}
 		dst.Provider = kt.ProviderTrapUnknown
 	}
 
diff --git a/pkg/inputs/snmp/util/util.go b/pkg/inputs/snmp/util/util.go
index 717005a..f06301d 100644
--- a/pkg/inputs/snmp/util/util.go
+++ b/pkg/inputs/snmp/util/util.go
@@ -198,7 +198,7 @@ func GetDeviceManufacturer(server snmpWalker, log logger.ContextL) string {
 func PrettyPrint(pdu gosnmp.SnmpPDU, format string, log logger.ContextL) string {
 	switch pdu.Type {
 	case gosnmp.OctetString:
-		_, s := GetFromConv(pdu, format, log)
+		_, s, _ := GetFromConv(pdu, format, log)
 		return s
 	case gosnmp.IPAddress:
 		return pdu.Value.(string)
@@ -236,7 +236,7 @@ func DoWalk(device string, baseOid string, format string, conf *kt.SnmpConfig, c
 }
 
 // Handle the case of wierd ints encoded as byte arrays.
-func GetFromConv(pdu gosnmp.SnmpPDU, conv string, log logger.ContextL) (int64, string) {
+func GetFromConv(pdu gosnmp.SnmpPDU, conv string, log logger.ContextL) (int64, string, map[string]string) {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Warnf("Invalid Conversion: %s %v %v", pdu.Name, pdu.Value, r)
@@ -245,12 +245,12 @@ func GetFromConv(pdu gosnmp.SnmpPDU, conv string, log logger.ContextL) (int64, s
 
 	bv, ok := pdu.Value.([]byte)
 	if !ok || len(bv) == 0 {
-		return 0, ""
+		return 0, "", nil
 	}
 
 	switch conv {
 	case CONV_HWADDR: // If there's an encoded mac addr here.
-		return 0, net.HardwareAddr(bv).String()
+		return 0, net.HardwareAddr(bv).String(), nil
 	case CONV_POWERSET:
 		return vendor.HandlePowersetStatus(bv)
 	case CONV_HEXTOIP:
@@ -269,37 +269,37 @@ func GetFromConv(pdu gosnmp.SnmpPDU, conv string, log logger.ContextL) (int64, s
 			if endian == "LittleEndian" {
 				switch bit {
 				case "uint64":
-					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint64(bv))
+					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint64(bv)), nil
 				case "uint32":
-					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint32(bv))
+					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint32(bv)), nil
 				case "uint16":
-					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint16(bv))
+					return 0, fmt.Sprintf("%d", binary.LittleEndian.Uint16(bv)), nil
 				default:
 					log.Errorf("invalid bit value (%s) for hex to int conversion", bit)
-					return 0, ""
+					return 0, "", nil
 				}
 			} else if endian == "BigEndian" {
 				switch bit {
 				case "uint64":
-					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint64(bv))
+					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint64(bv)), nil
 				case "uint32":
-					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint32(bv))
+					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint32(bv)), nil
 				case "uint16":
-					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint16(bv))
+					return 0, fmt.Sprintf("%d", binary.BigEndian.Uint16(bv)), nil
 				default:
 					log.Errorf("invalid bit value (%s) for hex to int conversion", bit)
-					return 0, ""
+					return 0, "", nil
 				}
 			} else {
 				log.Errorf("invalid Endian value (%s) for hex to int conversion", endian)
-				return 0, ""
+				return 0, "", nil
 			}
 		} else if split[0] == CONV_REGEXP && len(split) >= 2 {
 			return fromRegexp(bv, strings.Join(split[1:], ":")) // Put back together just in case RE has a : in it.
 		}
 	}
 
-	return 0, string(bv) // Default down to here.
+	return 0, string(bv), nil // Default down to here.
 }
 
 /**
@@ -308,7 +308,7 @@ I need to take this:
 .1.3.6.1.4.1.9.9.42.1.2.2.1.2.1 = Hex-String: 0A00640A
 and display it as a string 10.0.100.10
 */
-func hexToIP(bv []byte) (int64, string) {
+func hexToIP(bv []byte) (int64, string, map[string]string) {
 	switch len(bv) {
 	case 8:
 		return 0, fmt.Sprintf("%d.%d.%d.%d",
@@ -316,7 +316,7 @@ func hexToIP(bv []byte) (int64, string) {
 			binary.BigEndian.Uint16(bv[2:4]),
 			binary.BigEndian.Uint16(bv[4:6]),
 			binary.BigEndian.Uint16(bv[6:8]),
-		)
+		), nil
 	case 4:
 		nv := []byte{0x00, bv[0], 0x00, bv[1], 0x00, bv[2], 0x00, bv[3]}
 		return 0, fmt.Sprintf("%d.%d.%d.%d",
@@ -324,20 +324,20 @@ func hexToIP(bv []byte) (int64, string) {
 			binary.BigEndian.Uint16(nv[2:4]),
 			binary.BigEndian.Uint16(nv[4:6]),
 			binary.BigEndian.Uint16(nv[6:8]),
-		)
+		), nil
 	default:
-		return 0, ""
+		return 0, "", nil
 	}
 }
 
-func engineID(bv []byte) (int64, string) {
+func engineID(bv []byte) (int64, string, map[string]string) {
 	buf := make([]byte, 0, 3*len(bv))
 	x := buf[1*len(bv) : 3*len(bv)]
 	hex.Encode(x, bv)
 	for i := 0; i < len(x); i += 2 {
 		buf = append(buf, x[i], x[i+1], ':')
 	}
-	return 0, string(buf[:len(buf)-1])
+	return 0, string(buf[:len(buf)-1]), nil
 }
 
 /**
@@ -347,30 +347,46 @@ This lets people parse it out.
 	agentSwitchCpuProcessTotalUtilization
 	1.3.6.1.4.1.4413.1.1.1.1.4.9.0 = STRING: "    5 Secs ( 96.3762%)   60 Secs ( 62.8549%)  300 Secs ( 25.2877%)"
 */
-func fromRegexp(bv []byte, reg string) (int64, string) {
+func fromRegexp(bv []byte, reg string) (int64, string, map[string]string) {
 	r := reCache[reg]
 	if r == nil {
 		rn, err := regexp.Compile(reg)
 		if err != nil {
-			return 0, ""
+			return 0, "", nil
 		}
 		reCache[reg] = rn
 		r = rn
 	}
 
 	// Now, lets run some regexps.
+	named := r.SubexpNames()
 	res := r.FindSubmatch(bv)
+	matches := map[string]string{}
 	if len(res) < 2 {
-		return 0, ""
+		return 0, "", nil
 	}
-	ival, err := strconv.Atoi(string(res[1]))
-	if err != nil { // If we can't parse as int, return as string.
-		return 0, string(res[1])
+
+	for i, match := range res {
+		if i == 0 {
+			continue // This is always the whole matched expression.
+		}
+
+		if named[i] != "" { // If we have a list of named matches to deal with, don't return here but keep going.
+			matches[named[i]] = string(match) // Store this as a string.
+			continue
+		}
+
+		ival, err := strconv.Atoi(string(match))
+		if err != nil { // If we can't parse as int, return as string.
+			return 0, string(match), nil
+		}
+		return int64(ival), string(match), nil // Parsed as int but return both just in case.
 	}
-	return int64(ival), string(res[1]) // Parsed as int but return both just in case.
+
+	return 0, "", matches
 }
 
 // This one is used for certain string valued oids which we want to poll as metrics. Just converts to 1.
-func toOne(bv []byte) (int64, string) {
-	return 1, string(bv)
+func toOne(bv []byte) (int64, string, map[string]string) {
+	return 1, string(bv), nil
 }
diff --git a/pkg/inputs/snmp/util/vendor/powerset.go b/pkg/inputs/snmp/util/vendor/powerset.go
index bce948e..462d409 100644
--- a/pkg/inputs/snmp/util/vendor/powerset.go
+++ b/pkg/inputs/snmp/util/vendor/powerset.go
@@ -4,7 +4,7 @@ import (
 	"strings"
 )
 
-func HandlePowersetStatus(bv []byte) (int64, string) {
+func HandlePowersetStatus(bv []byte) (int64, string, map[string]string) {
 	actives := make([]*UpsCode, len(upsCodes))
 
 	for i, v := range bv {
@@ -43,12 +43,12 @@ func HandlePowersetStatus(bv []byte) (int64, string) {
 
 	for i := len(severities); i > 0; i-- {
 		if len(severity[i]) > 0 { // Return the highest severity present which has flags set.
-			return int64(i), severities[i-1] + ": " + strings.Join(severity[i], ",")
+			return int64(i), severities[i-1] + ": " + strings.Join(severity[i], ","), nil
 		}
 	}
 
 	// Generally assume you don't get here.
-	return 0, ""
+	return 0, "", nil
 }
 
 type UpsCode struct {
diff --git a/pkg/kt/device_types.go b/pkg/kt/device_types.go
index 4182fba..1fbc558 100644
--- a/pkg/kt/device_types.go
+++ b/pkg/kt/device_types.go
@@ -29,6 +29,9 @@ type Device struct {
 	MaxFlowRate   int                   `json:"max_flow_rate"`
 	Customs       []Column              `json:"custom_column_data,omitempty"`
 	CustomStr     string                `json:"custom_columns"`
+	SnmpCommunity string                `json:"device_snmp_community"`
+	SnmpIp        string                `json:"device_snmp_ip"`
+	SnmpV3        *V3SNMPConfig         `json:"device_snmp_v3_conf"`
 	allUserTags   map[string]string
 }
 
diff --git a/pkg/kt/snmp.go b/pkg/kt/snmp.go
index c2539b5..7fe765e 100644
--- a/pkg/kt/snmp.go
+++ b/pkg/kt/snmp.go
@@ -21,6 +21,7 @@ const (
 	ProviderPrefix = "provider" + ProviderToken
 	GlobalProvider = "global"
 	DeviceProvider = "device"
+	EnableAllMibs  = "USE_ALL_MIBS"
 )
 
 // DeviceData holds information about a device, sent via ST, and sent
@@ -127,13 +128,13 @@ type TopValue struct {
 }
 
 type V3SNMPConfig struct {
-	UserName                 string `yaml:"user_name"`
-	AuthenticationProtocol   string `yaml:"authentication_protocol"`
-	AuthenticationPassphrase string `yaml:"authentication_passphrase"`
-	PrivacyProtocol          string `yaml:"privacy_protocol"`
-	PrivacyPassphrase        string `yaml:"privacy_passphrase"`
-	ContextEngineID          string `yaml:"context_engine_id"`
-	ContextName              string `yaml:"context_name"`
+	UserName                 string `yaml:"user_name" json:"UserName"`
+	AuthenticationProtocol   string `yaml:"authentication_protocol" json:"AuthenticationProtocol"`
+	AuthenticationPassphrase string `yaml:"authentication_passphrase" json:"AuthenticationPassphrase"`
+	PrivacyProtocol          string `yaml:"privacy_protocol" json:"PrivacyProtocol"`
+	PrivacyPassphrase        string `yaml:"privacy_passphrase" json:"PrivacyPassphrase"`
+	ContextEngineID          string `yaml:"context_engine_id" json:"ContextEngineID"`
+	ContextName              string `yaml:"context_name" json:"ContextName"`
 	useGlobal                bool
 	origStr                  string
 }


Test patch:
diff --git a/pkg/formats/influx/influx_test.go b/pkg/formats/influx/influx_test.go
index e039d2c..f9f59bc 100644
--- a/pkg/formats/influx/influx_test.go
+++ b/pkg/formats/influx/influx_test.go
@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	go_metrics "github.com/kentik/go-metrics"
+	"github.com/kentik/ktranslate"
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
 	lt "github.com/kentik/ktranslate/pkg/eggs/logger/testing"
 	"github.com/kentik/ktranslate/pkg/kt"
@@ -17,11 +18,10 @@ func TestSeriToInflux(t *testing.T) {
 	assert := assert.New(t)
 	l := lt.NewTestContextL(logger.NilContext, t).GetLogger().GetUnderlyingLogger()
 
-	f, err := NewFormat(l, go_metrics.DefaultRegistry, kt.CompressionNone)
+	cfg := ktranslate.InfluxDBFormatConfig{MeasurementPrefix: "notempty"}
+	f, err := NewFormat(l, go_metrics.DefaultRegistry, kt.CompressionNone, &cfg)
 	assert.NoError(err)
 
-	prefix := "notempty"
-	Prefix = &prefix
 	res, err := f.To(kt.InputTesting, serBuf)
 	assert.NoError(err)
 	assert.NotNil(res)
@@ -36,7 +36,8 @@ func TestNewline(t *testing.T) {
 	assert := assert.New(t)
 	l := lt.NewTestContextL(logger.NilContext, t).GetLogger().GetUnderlyingLogger()
 
-	f, err := NewFormat(l, go_metrics.DefaultRegistry, kt.CompressionNone)
+	cfg := ktranslate.InfluxDBFormatConfig{MeasurementPrefix: "notempty"}
+	f, err := NewFormat(l, go_metrics.DefaultRegistry, kt.CompressionNone, &cfg)
 	assert.NoError(err)
 
 	input := InfluxDataSet{
diff --git a/pkg/inputs/snmp/util/util_test.go b/pkg/inputs/snmp/util/util_test.go
index 88fc7eb..050627d 100644
--- a/pkg/inputs/snmp/util/util_test.go
+++ b/pkg/inputs/snmp/util/util_test.go
@@ -106,7 +106,7 @@ func TestHandlePowerset(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		val, str := GetFromConv(pdu, "powerset_status", l)
+		val, str, _ := GetFromConv(pdu, "powerset_status", l)
 		assert.Equal(expt, str, "%s", in)
 		pts := strings.SplitN(str, ":", 2)
 		assert.Equal(levels[pts[0]], val, "%s", in)
@@ -123,7 +123,7 @@ func TestHWAddr(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		_, str := GetFromConv(pdu, "hwaddr", l)
+		_, str, _ := GetFromConv(pdu, "hwaddr", l)
 		assert.Equal(expt, str, "%s", in)
 	}
 }
@@ -138,7 +138,7 @@ func TestHexToInt(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		_, str := GetFromConv(pdu, "hextoint:LittleEndian:uint64", l)
+		_, str, _ := GetFromConv(pdu, "hextoint:LittleEndian:uint64", l)
 		assert.Equal(expt, str, "%s", in)
 	}
 }
@@ -154,7 +154,7 @@ func TestHexToIP(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		_, str := GetFromConv(pdu, CONV_HEXTOIP, l)
+		_, str, _ := GetFromConv(pdu, CONV_HEXTOIP, l)
 		assert.Equal(expt, str, "%s", in)
 	}
 }
@@ -171,11 +171,17 @@ func TestEngineID(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		_, str := GetFromConv(pdu, CONV_ENGINE_ID, l)
+		_, str, _ := GetFromConv(pdu, CONV_ENGINE_ID, l)
 		assert.Equal(expt, str, "%s", in)
 	}
 }
 
+type multiRe struct {
+	input   []byte
+	re      string
+	outputs map[string]string
+}
+
 func TestRegex(t *testing.T) {
 	assert := assert.New(t)
 	l := lt.NewTestContextL(logger.NilContext, t)
@@ -187,8 +193,27 @@ func TestRegex(t *testing.T) {
 
 	for expt, in := range tests {
 		pdu := gosnmp.SnmpPDU{Value: in}
-		ival, _ := GetFromConv(pdu, CONV_REGEXP+`:60 Secs.*?(\d+)`, l)
-		assert.Equal(expt, ival, "%s", in)
+		ival, _, named := GetFromConv(pdu, CONV_REGEXP+`:60 Secs.*?(\d+)`, l)
+		assert.Equal(expt, ival, "%s %v", in, named)
+	}
+
+	testStr := []multiRe{
+		multiRe{
+			input: []byte("Juniper Networks, Inc. mx80-48t , version 13.3R9.13 Build date: 2016-03-01 08:36:50 UTC "),
+			re:    `:Juniper Networks, Inc. (?P<model>\S+?) , version (?P<version>\S+)`,
+			outputs: map[string]string{
+				"model":   "mx80-48t",
+				"version": "13.3R9.13",
+			},
+		},
+	}
+
+	for _, in := range testStr {
+		pdu := gosnmp.SnmpPDU{Value: in.input}
+		_, str, mult := GetFromConv(pdu, CONV_REGEXP+in.re, l)
+		for k, v := range in.outputs {
+			assert.Equal(v, mult[k], "%s %s %v %s", k, string(in.input), mult, str)
+		}
 	}
 }
 
@@ -203,7 +228,7 @@ func TestToOne(t *testing.T) {
 
 	for in, expt := range tests {
 		pdu := gosnmp.SnmpPDU{Value: []byte(in)}
-		ival, _ := GetFromConv(pdu, CONV_ONE, l)
+		ival, _, _ := GetFromConv(pdu, CONV_ONE, l)
 		assert.Equal(expt, ival, "%s", in)
 	}
 }