Repository: kentik/ktranslate
Pull request #283 (2022-03-08 02:11:56)

S3, Http config; looping polling for SNMP
Allow snmp config files to be prefixed s3:// or http(s):// in addition to a local path. If a remote path, fetch the resource from the remote place. 

Also, adds a flag `--snmp_discovery_min` which will run discovery every X minutes and restart the snmp polling system if a new device is found. 


Patch:
diff --git a/deployment/kubernetes/ktranslate-pv.yaml b/deployment/kubernetes/ktranslate-pv.yaml
new file mode 100644
index 0000000..e0ad902
--- /dev/null
+++ b/deployment/kubernetes/ktranslate-pv.yaml
@@ -0,0 +1,194 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ktranslate-discovery
+spec:
+  selector:
+    matchLabels:
+      app: ktranslate-discovery
+  template:
+    metadata:
+      labels:
+        app: ktranslate-discovery
+    spec:
+      containers:
+      - name: discovery
+        image: docker.io/kentik/ktranslate:v2
+        imagePullPolicy: Always
+        args:
+          - -snmp
+          - /data/snmp.yaml
+          - -log_level=info
+          - -snmp_discovery=true
+        resources:
+          limits:
+            cpu: 1000m
+            memory: 256Mi
+          requests:
+            cpu: 500m
+            memory: 128Mi
+        volumeMounts:
+          - name: ktranslate-discovery-claim
+            mountPath: /data
+      volumes:
+        - name: ktranslate-discovery-claim
+          hostPath:
+            path: /mnt/data/
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ktranslate
+spec:
+  selector:
+    matchLabels:
+      app: ktranslate
+  template:
+    metadata:
+      labels:
+        app: ktranslate
+    spec:
+      containers:
+      - name: ktranslate
+        image: docker.io/kentik/ktranslate:v2
+        imagePullPolicy: Always
+        args:
+          - --listen=0.0.0.0:8082
+          - --metalisten=0.0.0.0:8083
+          - --snmp=/data/snmp.yaml
+          - --log_level=info
+          - --nf.source=sflow
+          - --nf.addr=0.0.0.0
+          - --nf.port=6343
+          - --sinks=prometheus
+          - --format=prometheus
+          - --prom_listen=:8084
+        resources:
+          limits:
+            cpu: 1000m
+            memory: 256Mi
+          requests:
+            cpu: 500m
+            memory: 128Mi
+        ports:
+        - name: firehose
+          containerPort: 8082
+        - name: metadata
+          containerPort: 8083
+        - name: prometheus
+          containerPort: 8084
+        - name: sflow
+          containerPort: 6343
+          protocol: UDP
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: metadata
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: metadata
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+          - name: ktranslate-discovery-claim
+            mountPath: /data
+      volumes:
+        - name: ktranslate-discovery-claim
+          hostPath:
+            path: /mnt/data/
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-firehose
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8082
+    name: firehose
+    targetPort: 8082
+    protocol: TCP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-metadata
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8083
+    name: metadata
+    targetPort: 8083
+    protocol: TCP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-prom
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8084
+    name: prom
+    targetPort: 8084
+    protocol: TCP
+  selector:
+    app: ktranslate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-sflow
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 6343
+    name: sflow
+    targetPort: 6343
+    protocol: UDP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ktranslate-discovery-claim
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ktranslate-pv-volume
+  labels:
+    type: local
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: "/mnt/data"
diff --git a/deployment/kubernetes/ktranslate.yaml b/deployment/kubernetes/ktranslate.yaml
new file mode 100644
index 0000000..c4c3853
--- /dev/null
+++ b/deployment/kubernetes/ktranslate.yaml
@@ -0,0 +1,151 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ktranslate
+spec:
+  selector:
+    matchLabels:
+      app: ktranslate
+  template:
+    metadata:
+      labels:
+        app: ktranslate
+    spec:
+      containers:
+      - name: ktranslate
+        image: docker.io/kentik/ktranslate:v2
+        imagePullPolicy: Always
+        args:
+          - --listen=0.0.0.0:8082
+          - --metalisten=0.0.0.0:8083
+          - --snmp=/etc/ktranslate/snmp.yml
+          - --log_level=info
+          - --nf.source=sflow
+          - --nf.addr=0.0.0.0
+          - --nf.port=6343
+          - --sinks=prometheus
+          - --format=prometheus
+          - --prom_listen=:8084
+        resources:
+          limits:
+            cpu: 1000m
+            memory: 256Mi
+          requests:
+            cpu: 500m
+            memory: 128Mi
+        ports:
+        - name: firehose
+          containerPort: 8082
+        - name: metadata
+          containerPort: 8083
+        - name: prometheus
+          containerPort: 8084
+        - name: sflow
+          containerPort: 6343
+          protocol: UDP
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: metadata
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: metadata
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+          - name: ktranslate-config
+            mountPath: /etc/ktranslate/snmp.yml
+            subPath: snmp.yml
+      volumes:
+        - name: ktranslate-config
+          configMap:
+            name: ktranslate-config
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-firehose
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8082
+    name: firehose
+    targetPort: 8082
+    protocol: TCP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-metadata
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8083
+    name: metadata
+    targetPort: 8083
+    protocol: TCP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-prom
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 8084
+    name: prom
+    targetPort: 8084
+    protocol: TCP
+  selector:
+    app: ktranslate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ktranslate-sflow
+  labels:
+    app: ktranslate
+spec:
+  ports:
+  - port: 6343
+    name: sflow
+    targetPort: 6343
+    protocol: UDP
+  selector:
+    app: ktranslate
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ktranslate-config
+data:
+  snmp.yml: |
+    devices:
+      switch:
+        device_name: switch
+        device_ip: 10.10.0.10
+        flow_only: true
+        user_tags: {}
+    global:
+      poll_time_sec: 30
+      drop_if_outside_poll: false
+      mib_profile_dir: /etc/ktranslate/profiles
+      mibs_db: /etc/ktranslate/mibs.db
+      mibs_enabled:
+      - IF-MIB
+      timeout_ms: 3000
+      retries: 0
diff --git a/pkg/cat/kkc.go b/pkg/cat/kkc.go
index 65806f0..3d9397a 100644
--- a/pkg/cat/kkc.go
+++ b/pkg/cat/kkc.go
@@ -28,6 +28,7 @@ import (
 	"github.com/kentik/ktranslate/pkg/sinks/kentik"
 	"github.com/kentik/ktranslate/pkg/util/enrich"
 	"github.com/kentik/ktranslate/pkg/util/gopatricia/patricia"
+	"github.com/kentik/ktranslate/pkg/util/resolv"
 	"github.com/kentik/ktranslate/pkg/util/rule"
 
 	"github.com/judwhite/go-svc"
@@ -557,7 +558,7 @@ func (kc *KTranslate) Run(ctx context.Context) error {
 
 	// DNS mapper if set.
 	if kc.config.DnsResolver != "" {
-		res, err := NewResolver(ctx, kc.log.GetLogger().GetUnderlyingLogger(), kc.config.DnsResolver)
+		res, err := resolv.NewResolver(ctx, kc.log.GetLogger().GetUnderlyingLogger(), kc.config.DnsResolver)
 		if err != nil {
 			return err
 		}
@@ -625,7 +626,8 @@ func (kc *KTranslate) Run(ctx context.Context) error {
 	// If SNMP is configured, start this system too. Poll for metrics and metadata, also handle traps.
 	if kc.config.SNMPFile != "" {
 		if kc.config.SNMPDisco { // Here, we're just returning the list of devices on the network which might speak snmp.
-			return snmp.Discover(ctx, kc.config.SNMPFile, kc.log)
+			_, err := snmp.Discover(ctx, kc.config.SNMPFile, kc.log)
+			return err
 		}
 		assureInput()
 		kc.metrics.SnmpDeviceData = kt.NewSnmpMetricSet(kc.registry)
@@ -664,7 +666,7 @@ func (kc *KTranslate) Run(ctx context.Context) error {
 	// If we're looking for syslog flows coming in
 	if kc.config.SyslogSource != "" {
 		assureInput()
-		ss, err := syslog.NewSyslogSource(ctx, kc.config.SyslogSource, kc.log.GetLogger().GetUnderlyingLogger(), kc.config.LogTee, kc.registry, kc.apic)
+		ss, err := syslog.NewSyslogSource(ctx, kc.config.SyslogSource, kc.log.GetLogger().GetUnderlyingLogger(), kc.config.LogTee, kc.registry, kc.apic, kc.resolver)
 		if err != nil {
 			return err
 		}
diff --git a/pkg/cat/types.go b/pkg/cat/types.go
index 9bc92ad..7ab1314 100644
--- a/pkg/cat/types.go
+++ b/pkg/cat/types.go
@@ -21,6 +21,7 @@ import (
 	"github.com/kentik/ktranslate/pkg/sinks/kentik"
 	"github.com/kentik/ktranslate/pkg/util/enrich"
 	"github.com/kentik/ktranslate/pkg/util/gopatricia/patricia"
+	"github.com/kentik/ktranslate/pkg/util/resolv"
 	"github.com/kentik/ktranslate/pkg/util/rule"
 
 	model "github.com/kentik/ktranslate/pkg/util/kflow2"
@@ -98,7 +99,7 @@ type KTranslate struct {
 	filters      []filter.Filter
 	geo          *patricia.MMMap
 	asn          *patricia.MMMap
-	resolver     *Resolver
+	resolver     *resolv.Resolver
 	auth         *auth.Server
 	apic         *api.KentikApi
 	tooBig       chan int
diff --git a/pkg/inputs/http/http.go b/pkg/inputs/http/http.go
index 5410ee3..cc60d64 100644
--- a/pkg/inputs/http/http.go
+++ b/pkg/inputs/http/http.go
@@ -54,6 +54,7 @@ func NewHttpListener(ctx context.Context, host string, log logger.Underlying, re
 func (ks *KentikHttpListener) RegisterRoutes(r *kmux.Router) {
 	r.HandleFunc(Listen+"/telegraf/standard", ks.wrap(ks.readStandard))
 	r.HandleFunc(Listen+"/telegraf/batch", ks.wrap(ks.readBatch))
+	r.HandleFunc(Listen+"/ktranslate/jchf", ks.wrap(ks.readJCHF))
 }
 
 type basic struct {
@@ -153,6 +154,34 @@ func (ks *KentikHttpListener) getJCHF(wrapper *basic, remoteIP string) *kt.JCHF
 	return in
 }
 
+// Get the JCHF content directly.
+func (ks *KentikHttpListener) readJCHF(w http.ResponseWriter, r *http.Request) {
+	wrapper := []*kt.JCHF{}
+
+	// Decode body in gzip format if the request header is set this way.
+	body := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		z, err := gzip.NewReader(r.Body)
+		if err != nil {
+			panic(http.StatusInternalServerError)
+		}
+		body = z
+	}
+	defer body.Close()
+
+	if err := json.NewDecoder(body).Decode(&wrapper); err != nil {
+		panic(http.StatusInternalServerError)
+	}
+	w.WriteHeader(http.StatusOK)
+
+	ks.metrics.Messages.Mark(int64(len(wrapper)))
+	for _, chf := range wrapper {
+		chf.SetMap()
+	}
+
+	ks.jchfChan <- wrapper
+}
+
 func (ks *KentikHttpListener) Close() {}
 
 func (ks *KentikHttpListener) HttpInfo() map[string]float64 {
diff --git a/pkg/inputs/snmp/disco.go b/pkg/inputs/snmp/disco.go
index 6ce557c..f60f164 100644
--- a/pkg/inputs/snmp/disco.go
+++ b/pkg/inputs/snmp/disco.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
 	"sort"
 	"strings"
 	"sync"
@@ -25,20 +26,26 @@ const (
 	permissions = 0644
 )
 
-func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
+type SnmpDiscoDeviceStat struct {
+	added    int
+	replaced int
+	delta    int
+}
+
+func Discover(ctx context.Context, snmpFile string, log logger.ContextL) (*SnmpDiscoDeviceStat, error) {
 	// First, parse the config file and see what we're doing.
 	log.Infof("SNMP Discovery, loading config from %s", snmpFile)
-	conf, err := parseConfig(snmpFile, log)
+	conf, err := parseConfig(ctx, snmpFile, log)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if conf.Disco == nil {
-		return fmt.Errorf("The discovery configuration is not set: %+v.", conf)
+		return nil, fmt.Errorf("The discovery configuration is not set: %+v.", conf)
 	}
 
 	if conf.Global == nil || conf.Global.MibProfileDir == "" {
-		return fmt.Errorf("You need to specify a global section and mib profile directory: %v.", conf)
+		return nil, fmt.Errorf("You need to specify a global section and mib profile directory: %v.", conf)
 	}
 
 	if *snmpOutFile != "" { // If we want to write somewhere else, swap the output file in here.
@@ -47,8 +54,8 @@ func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
 	}
 
 	if conf.Disco.AddDevices { // Verify that the output is writeable before diving into discoing.
-		if err := addDevices(nil, snmpFile, conf, true, log); err != nil {
-			return fmt.Errorf("There was an error when writing the %s SNMP configuration file: %v.", snmpFile, err)
+		if _, err := addDevices(ctx, nil, snmpFile, conf, true, log); err != nil {
+			return nil, fmt.Errorf("There was an error when writing the %s SNMP configuration file: %v.", snmpFile, err)
 		}
 	}
 
@@ -65,10 +72,15 @@ func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
 	// Use this for auto-discovering metrics to pull.
 	mdb, err := mibs.NewMibDB(conf.Global.MibDB, conf.Global.MibProfileDir, log)
 	if err != nil {
-		return fmt.Errorf("There was an error when setting up the %s mibDB database and the %s profiles: %v.", conf.Global.MibDB, conf.Global.MibProfileDir, err)
+		return nil, fmt.Errorf("There was an error when setting up the %s mibDB database and the %s profiles: %v.", conf.Global.MibDB, conf.Global.MibProfileDir, err)
 	}
 	defer mdb.Close()
 
+	ignoreMap := map[string]bool{}
+	for _, ip := range conf.Disco.IgnoreList {
+		ignoreMap[ip] = true
+	}
+
 	foundDevices := map[string]*kt.SnmpDeviceConfig{}
 	for _, ipr := range conf.Disco.Cidrs {
 		_, _, err := net.ParseCIDR(ipr)
@@ -90,11 +102,11 @@ func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
 		timeout := time.Millisecond * time.Duration(conf.Global.TimeoutMS)
 		scanner := scan.NewDeviceScanner(targetIterator, timeout)
 		if err := scanner.Start(); err != nil {
-			return err
+			return nil, err
 		}
 		results, err := scanner.Scan(ctx, conf.Disco.Ports)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		var wg sync.WaitGroup
 		var mux sync.RWMutex
@@ -102,6 +114,9 @@ func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
 		log.Infof("Starting to check %d ips in %s", len(results), ipr)
 		for i, result := range results {
 			if strings.HasSuffix(ipr, "/32") || result.IsHostUp() {
+				if ignoreMap[result.Host.String()] { // If we have marked this ip as to be ignored, don't do anything more with it.
+					continue
+				}
 				wg.Add(1)
 				posit := fmt.Sprintf("%d/%d)", i+1, len(results))
 				go doubleCheckHost(result, timeout, ctl, &mux, &wg, foundDevices, mdb, conf, posit, log)
@@ -111,16 +126,43 @@ func Discover(ctx context.Context, snmpFile string, log logger.ContextL) error {
 		log.Infof("Checked %d ips in %v (from start: %v)", len(results), time.Now().Sub(st), time.Now().Sub(stb))
 	}
 
+	var stats *SnmpDiscoDeviceStat
 	if conf.Disco.AddDevices {
-		err := addDevices(foundDevices, snmpFile, conf, false, log)
+		stats, err = addDevices(ctx, foundDevices, snmpFile, conf, false, log)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	time.Sleep(2 * time.Second) // Give logs time to get sent back.
 
-	return nil
+	return stats, nil
+}
+
+func RunDiscoOnTimer(ctx context.Context, c chan os.Signal, snmpFile string, log logger.ContextL, pollTimeMin int) {
+	pt := time.Duration(pollTimeMin) * time.Minute
+	log.Infof("Running SNMP Discovery Loop every %v", pt)
+	discoCheck := time.NewTicker(pt)
+	defer discoCheck.Stop()
+	for {
+		select {
+		case _ = <-discoCheck.C:
+			stats, err := Discover(ctx, snmpFile, log)
+			if err != nil {
+				log.Errorf("Discovery SNMP Error: %v", err)
+			} else {
+				if stats.delta != 0 || stats.added > 0 { // Only restart if there's a different configuration.
+					log.Infof("Discovery SNMP reloading: added: %d replaced: %d delta: %d", stats.added, stats.replaced, stats.delta)
+					c <- kt.SIGUSR2 // Restart the main loop with a new config.
+				} else {
+					log.Infof("Discovery SNMP no change so not reloading: added: %d replaced: %d delta: %d", stats.added, stats.replaced, stats.delta)
+				}
+			}
+		case <-ctx.Done():
+			log.Infof("Discovery Loop Done")
+			return
+		}
+	}
 }
 
 func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, mux *sync.RWMutex, wg *sync.WaitGroup,
@@ -163,7 +205,7 @@ func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, m
 				log.Warnf("There was an error when starting SNMP interface component -- %v.", err)
 				return
 			}
-			md, err = metadata.GetDeviceMetadata(log, serv, nil)
+			md, err = metadata.GetBasicDeviceMetadata(log, serv)
 			if err != nil {
 				log.Warnf("Cannot get device metadata on %s: %v. Check for correct snmp credentials.", result.Host.String(), err)
 				continue
@@ -185,7 +227,7 @@ func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, m
 				log.Warnf("There was an error when starting SNMP interface component -- %v.", err)
 				return
 			}
-			md, err = metadata.GetDeviceMetadata(log, serv, nil)
+			md, err = metadata.GetBasicDeviceMetadata(log, serv)
 			if err != nil {
 				log.Warnf("Cannot get device metadata on %s: %v. Check for correct snmp credentials.", result.Host.String(), err)
 				continue
@@ -248,10 +290,9 @@ func doubleCheckHost(result scan.Result, timeout time.Duration, ctl chan bool, m
 	foundDevices[result.Host.String()] = &device
 }
 
-func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, conf *kt.SnmpConfig, isTest bool, log logger.ContextL) error {
+func addDevices(ctx context.Context, foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, conf *kt.SnmpConfig, isTest bool, log logger.ContextL) (*SnmpDiscoDeviceStat, error) {
 	// Now add the new.
-	added := 0
-	replaced := 0
+	stats := SnmpDiscoDeviceStat{}
 	if conf.Devices == nil {
 		conf.Devices = map[string]*kt.SnmpDeviceConfig{}
 	}
@@ -260,6 +301,7 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 	for _, d := range conf.Devices {
 		byIP[d.DeviceIP] = d
 	}
+	origCount := len(conf.Devices)
 
 	for dip, d := range foundDevices {
 		key := d.DeviceName
@@ -272,28 +314,25 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 		if conf.Devices[key] == nil && conf.Devices[keyAlt] == nil {
 			// Start adding new devices based on deviceName__ip
 			conf.Devices[keyAlt] = d
-			added++
+			stats.added++
 		} else if conf.Devices[key] != nil {
 			if conf.Disco.ReplaceDevices { // But keep backwards compatible with existing devices and don't change their entries.
-				d.UpdateFrom(conf.Devices[key])
+				d.UpdateFrom(conf.Devices[key], conf)
 				conf.Devices[key] = d
-				replaced++
+				stats.replaced++
 			} else {
 				conf.Devices[key].Checked = time.Now()
 			}
 		} else {
 			if conf.Disco.ReplaceDevices { // Else, new style keys all use keyAlt.
-				d.UpdateFrom(conf.Devices[keyAlt])
+				d.UpdateFrom(conf.Devices[keyAlt], conf)
 				conf.Devices[keyAlt] = d
-				replaced++
+				stats.replaced++
 			} else {
 				conf.Devices[keyAlt].Checked = time.Now()
 			}
 		}
 	}
-	if !isTest {
-		log.Infof("Adding %d new SNMP devices to the configuration. %d replaced from %d.", added, replaced, len(foundDevices))
-	}
 
 	// Remove any duplicate devices based on Engine ID here.
 	for dip, d := range conf.Devices {
@@ -302,12 +341,19 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 				// Someone else has this engine ID. Delete this device.
 				log.Warnf("Removing device %s because of duplicate EngineID %s.", d.DeviceName, d.EngineID)
 				delete(conf.Devices, dip)
+				stats.added--
 			} else {
 				byEngineID[d.EngineID] = d
 			}
 		}
 	}
 
+	// Calculate total number of new devices.
+	stats.delta = origCount - len(conf.Devices)
+	if !isTest {
+		log.Infof("Adding %d new SNMP devices to the configuration. %d replaced from %d. Delta: %d", stats.added, stats.replaced, len(foundDevices), stats.delta)
+	}
+
 	// Fill up list of mibs to run on here.
 	if conf.Disco.AddAllMibs {
 		fullMibSet := map[string]bool{}
@@ -333,40 +379,61 @@ func addDevices(foundDevices map[string]*kt.SnmpDeviceConfig, snmpFile string, c
 	if conf.Disco.CidrOrig != "" {
 		t, err := yaml.Marshal(conf.Disco.Cidrs)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		err = ioutil.WriteFile(conf.Disco.CidrOrig, t, permissions)
+		if err != nil {
+			return nil, err
+		}
+		if !isTest {
+			conf.Disco.Cidrs = nil
+		}
+	}
+
+	if conf.Disco.IgnoreOrig != "" {
+		t, err := yaml.Marshal(conf.Disco.IgnoreList)
 		if err != nil {
 			return err
 		}
-		conf.Disco.Cidrs = nil
+		err = ioutil.WriteFile(conf.Disco.IgnoreOrig, t, permissions)
+		if err != nil {
+			return err
+		}
+		if !isTest {
+			conf.Disco.IgnoreList = nil
+		}
 	}
 
 	if conf.DeviceOrig != "" {
 		t, err := yaml.Marshal(conf.Devices)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		err = ioutil.WriteFile(conf.DeviceOrig, t, permissions)
 		if err != nil {
-			return err
+			return nil, err
+		}
+		if !isTest {
+			conf.Devices = nil
 		}
-		conf.Devices = nil
 	}
 
 	// Save out the config file.
 	t, err := yaml.Marshal(conf)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// Swap for our external sections.
 	if conf.Disco.CidrOrig != "" {
 		t = bytes.Replace(t, []byte("cidrs: []"), []byte(`cidrs: "@`+conf.Disco.CidrOrig+`"`), 1)
 	}
+	if conf.Disco.IgnoreOrig != "" {
+		t = bytes.Replace(t, []byte("ignore_list: []"), []byte(`ignore_list: "@`+conf.Disco.IgnoreOrig+`"`), 1)
+	}
 	if conf.DeviceOrig != "" {
 		t = bytes.Replace(t, []byte("devices: {}"), []byte(`devices: "@`+conf.DeviceOrig+`"`), 1)
 	}
 
-	return ioutil.WriteFile(snmpFile, t, permissions)
+	return &stats, snmp_util.WriteFile(ctx, snmpFile, t, permissions)
 }
diff --git a/pkg/inputs/snmp/metadata/device_metadata.go b/pkg/inputs/snmp/metadata/device_metadata.go
index fe7d8c9..a27bdbc 100644
--- a/pkg/inputs/snmp/metadata/device_metadata.go
+++ b/pkg/inputs/snmp/metadata/device_metadata.go
@@ -1,6 +1,7 @@
 package metadata
 
 import (
+	"context"
 	"net"
 	"sort"
 	"unicode/utf8"
@@ -35,92 +36,114 @@ var (
 	}()
 )
 
-// Poll device-level metadata.
-func GetDeviceMetadata(log logger.ContextL, server *gosnmp.GoSNMP, deviceMetadataMibs map[string]*kt.Mib) (*kt.DeviceMetricsMetadata, error) {
-	md := kt.DeviceMetricsMetadata{
-		Customs:    map[string]string{},
-		CustomInts: map[string]int64{},
-		Tables:     map[string]kt.DeviceTableMetadata{},
-	}
+type DeviceMetadata struct {
+	log     logger.ContextL
+	mibs    map[string]*kt.Mib
+	conf    *kt.SnmpDeviceConfig
+	metrics *kt.SnmpDeviceMetric
+	missing map[string]bool
+	basic   map[string]string
+}
 
-	var oids []string
-	if len(deviceMetadataMibs) == 0 {
-		for el := SNMP_device_metadata_oids.Front(); el != nil; el = el.Next() {
-			oids = append(oids, el.Key.(string))
-		}
-	} else {
-		log.Infof("Getting device metadata from custom map: %v", deviceMetadataMibs)
-		oids = getFromCustomMap(deviceMetadataMibs)
+func NewDeviceMetadata(deviceMetadataMibs map[string]*kt.Mib, conf *kt.SnmpDeviceConfig, metrics *kt.SnmpDeviceMetric, log logger.ContextL) *DeviceMetadata {
+	mibs := map[string]*kt.Mib{}
+	basic := map[string]string{}
+	for el := SNMP_device_metadata_oids.Front(); el != nil; el = el.Next() {
+		basic[el.Key.(string)] = el.Value.(string)
 	}
 
-	hasDataFull := false
-	max := gosnmp.MaxOids // Some profiles can get pretty big. @TODO, make max user selectable?
-	for i := 0; i < len(oids); i += max {
-		limit := i + max
-		if limit > len(oids) {
-			limit = len(oids)
-		}
-		hasData, err := pollDevice(oids[i:limit], log, server, deviceMetadataMibs, &md)
-		if err != nil {
-			return nil, err
+	if len(deviceMetadataMibs) > 0 {
+		oids := getFromCustomMap(deviceMetadataMibs)
+		for _, oid := range oids {
+			mib := deviceMetadataMibs[oid]
+			mibs[oid] = mib
+			log.Infof("Adding custom device metadata oid: %s -> %s %v", oid, mib.GetName(), mib.OtherTables)
 		}
-		if hasData {
-			hasDataFull = hasData
+	} else {
+		// Copy the global values into this map which is per device.
+		for el := SNMP_device_metadata_oids.Front(); el != nil; el = el.Next() {
+			mibs[el.Key.(string)] = nil
 		}
 	}
 
-	// If no fields in md were set, return nil.  (Trust me on the (). :)
-	if !hasDataFull {
-		log.Infof("SNMP Device Metadata: No data received")
-		return nil, nil
+	return &DeviceMetadata{
+		log:     log,
+		mibs:    mibs,
+		conf:    conf,
+		basic:   basic,
+		metrics: metrics,
+		missing: map[string]bool{},
 	}
-	log.Infof("SNMP Device Metadata: Data received: %+v", md)
+}
 
-	return &md, nil
+// Poll device-level metadata.
+func (dm *DeviceMetadata) Poll(ctx context.Context, server *gosnmp.GoSNMP) (*kt.DeviceMetricsMetadata, error) {
+	return dm.poll(ctx, server)
 }
 
-func pollDevice(oids []string, log logger.ContextL, server *gosnmp.GoSNMP, deviceMetadataMibs map[string]*kt.Mib, md *kt.DeviceMetricsMetadata) (bool, error) {
-	result, err := server.Get(oids)
-	if err != nil {
-		return false, err
-	}
+type wrapper struct {
+	variable gosnmp.SnmpPDU
+	mib      *kt.Mib
+	oid      string
+}
 
-	hasData := false
-	for _, pdu := range result.Variables {
-		log.Debugf("pdu: %+v", pdu)
+func (dm *DeviceMetadata) poll(ctx context.Context, server *gosnmp.GoSNMP) (*kt.DeviceMetricsMetadata, error) {
+	var results []wrapper
+	md := kt.DeviceMetricsMetadata{
+		Customs:    map[string]string{},
+		CustomInts: map[string]int64{},
+		Tables:     map[string]kt.DeviceTableMetadata{},
+	}
 
-		oidVal, value := pdu.Name, pdu.Value
+	missing := int64(0)
+	for oid, mib := range dm.mibs {
+		if !mib.IsPollReady() { // Skip this mib because its time to poll hasn't elapsed yet.
+			continue
+		}
+		oidResults, err := snmp_util.WalkOID(ctx, dm.conf, oid, server, dm.log, "CustomDeviceMetadata")
+		if err != nil {
+			dm.metrics.Errors.Mark(1)
+			continue
+		}
 
-		// You can get a nil value w/out getting an error.
-		if value == nil || pdu.Type == gosnmp.NoSuchObject {
-			if oidInfo, ok := deviceMetadataMibs[oidVal[1:]]; ok {
-				log.Infof("Trying to walk %s -> %s as a table", oidInfo.Name, oidVal)
-				err := getTable(log, server, oidVal, oidInfo, md)
-				if err != nil {
-					log.Warnf("Dropping %s because of nil value or missing object: %+v %v", oidVal, pdu, err)
-				}
+		if len(oidResults) == 0 {
+			missing++
+			if _, ok := dm.missing[oid]; ok {
+				dm.log.Debugf("OID %s failed to return results, Metric Name: %s", oid, mib.Name)
+			} else {
+				dm.missing[oid] = true
+				dm.log.Warnf("OID %s failed to return results, Metric Name: %s", oid, mib.Name)
 			}
+		}
+		for _, result := range oidResults {
+			results = append(results, wrapper{variable: result, mib: mib, oid: oid})
+		}
+	}
+
+	// Update the number of missing metrics metric here.
+	dm.metrics.MissingMeta.Update(missing)
+
+	// Map back into types we know about.
+	for _, wrapper := range results {
+		if wrapper.variable.Value == nil { // You can get nil w/out getting an error, though.
 			continue
 		}
 
-		var oidName string
-		oid, ok := deviceMetadataMibs[oidVal[1:]]
-		if !ok {
-			thing, ok := SNMP_device_metadata_oids.Get(oidVal)
-			if !ok {
-				if oidVal == ".1.3.6.1.6.3.15.1.1.3.0" { // This is a bad v3 config.
-					log.Errorf("User found who is not known to the SNMP engine. Likely this is an invalid v3 config.")
-				} else {
-					log.Errorf("SNMP Device Metadata: Unknown oid retrieved: %v %v", oidVal, value)
-				}
-				continue
-			}
-			oidName = thing.(string)
-		} else {
-			oidName = oid.Name
+		// Calculate the index first out here.
+		idx := snmp_util.GetIndex(wrapper.variable.Name[1:], wrapper.oid)
+
+		oidName := ""
+		if name, ok := dm.basic[wrapper.variable.Name]; ok {
+			oidName = name
+		} else if wrapper.mib != nil {
+			oidName = wrapper.mib.GetName()
+		}
+		if oidName == "" {
+			dm.log.Warnf("Missing metadata name: %s", wrapper.oid)
+			continue
 		}
 
-		hasData = true
+		value := wrapper.variable.Value
 		switch oidName {
 		case SNMP_sysDescr:
 			md.SysDescr = string(value.([]byte))
@@ -135,35 +158,42 @@ func pollDevice(oids []string, log logger.ContextL, server *gosnmp.GoSNMP, devic
 		case SNMP_sysServices:
 			md.SysServices = int(snmp_util.ToInt64(value))
 		case SNMP_engineID:
-			_, md.EngineID = snmp_util.GetFromConv(pdu, snmp_util.CONV_ENGINE_ID, log)
+			_, md.EngineID = snmp_util.GetFromConv(wrapper.variable, snmp_util.CONV_ENGINE_ID, dm.log)
 		default:
-			if oid.Tag != "" {
-				oidName = oid.Tag
+			// Now we're actually in the range of custom fields.
+			if wrapper.mib == nil { // This should never happen here.
+				dm.log.Warnf("Missing Custom metadata oid: %+v, Value: %T %+v", wrapper.variable, wrapper.variable.Value, wrapper.variable.Value)
+				continue
 			}
-			switch vt := value.(type) {
-			case string:
-				md.Customs[oidName] = vt
-			case []byte:
-				if oid.Conversion != "" { // Adjust for any hard coded values here.
-					ival, sval := snmp_util.GetFromConv(pdu, oid.Conversion, log)
-					if ival > 0 {
-						md.CustomInts[oidName] = ival
-						md.Customs[kt.StringPrefix+oidName] = sval
+
+			if idx != "" {
+				dm.handleTable(idx, wrapper, oidName, &md)
+			} else {
+				switch vt := value.(type) {
+				case string:
+					md.Customs[oidName] = vt
+				case []byte:
+					if wrapper.mib.Conversion != "" { // Adjust for any hard coded values here.
+						ival, sval := snmp_util.GetFromConv(wrapper.variable, wrapper.mib.Conversion, dm.log)
+						if ival > 0 {
+							md.CustomInts[oidName] = ival
+							md.Customs[kt.StringPrefix+oidName] = sval
+						} else {
+							md.Customs[oidName] = sval
+						}
 					} else {
-						md.Customs[oidName] = sval
+						md.Customs[oidName] = string(vt)
 					}
-				} else {
-					md.Customs[oidName] = string(vt)
+				case net.IP:
+					md.Customs[oidName] = vt.String()
+				default:
+					md.CustomInts[oidName] = snmp_util.ToInt64(value)
 				}
-			case net.IP:
-				md.Customs[oidName] = vt.String()
-			default:
-				md.CustomInts[oidName] = snmp_util.ToInt64(value)
 			}
 		}
 	}
 
-	return hasData, nil
+	return &md, nil
 }
 
 func getFromCustomMap(mibs map[string]*kt.Mib) []string {
@@ -176,57 +206,97 @@ func getFromCustomMap(mibs map[string]*kt.Mib) []string {
 	return keys
 }
 
-func getTable(log logger.ContextL, g *gosnmp.GoSNMP, oid string, mib *kt.Mib, md *kt.DeviceMetricsMetadata) error {
-	results, err := g.WalkAll(oid)
+func (dm *DeviceMetadata) handleTable(idx string, value wrapper, oidName string, md *kt.DeviceMetricsMetadata) {
+	if idx[0:1] == "." {
+		idx = idx[1:]
+	}
+	if _, ok := md.Tables[idx]; !ok {
+		md.Tables[idx] = kt.NewDeviceTableMetadata()
+	}
+	switch value.variable.Type {
+	case gosnmp.OctetString:
+		val := string(value.variable.Value.([]byte))
+		if value.mib.Conversion != "" { // Adjust for any hard coded values here.
+			_, val = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
+		}
+		if utf8.ValidString(val) {
+			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val, 0)
+		}
+	case gosnmp.IPAddress: // Does this work?
+		switch val := value.variable.Value.(type) {
+		case string:
+			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val, 0)
+		case []byte:
+			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, string(val), 0)
+		case net.IP:
+			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val.String(), 0)
+		}
+	case gosnmp.ObjectIdentifier:
+		val := string(value.variable.Value.(string))
+		if value.mib.Conversion != "" { // Adjust for any hard coded values here.
+			_, val = snmp_util.GetFromConv(value.variable, value.mib.Conversion, dm.log)
+		}
+		if utf8.ValidString(val) {
+			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, val, 0)
+		}
+	default:
+		// Try to just use as a number
+		md.Tables[idx].Customs[oidName] = kt.NewMetaValue(value.mib, "", gosnmp.ToBigInt(value.variable.Value).Int64())
+	}
+}
+
+// Super basic loop to get info for discovery.
+func GetBasicDeviceMetadata(log logger.ContextL, server *gosnmp.GoSNMP) (*kt.DeviceMetricsMetadata, error) {
+	md := kt.DeviceMetricsMetadata{}
+
+	var oids []string
+	for el := SNMP_device_metadata_oids.Front(); el != nil; el = el.Next() {
+		oids = append(oids, el.Key.(string))
+	}
+
+	result, err := server.Get(oids)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	oidName := mib.GetName()
+	for _, pdu := range result.Variables {
+		log.Debugf("pdu: %+v", pdu)
+		oidVal, value := pdu.Name, pdu.Value
 
-	log.Infof("TableWalk Results: %s: %s -> %d", oidName, oid, len(results))
-	// Save as index -> oid -> value
-	for _, variable := range results {
-		if len(variable.Name) <= len(oid)+1 {
-			log.Warnf("Skipping invalid table, could not get index: %s -> %s", oid, variable.Name)
+		// You can get a nil value w/out getting an error.
+		if value == nil || pdu.Type == gosnmp.NoSuchObject {
 			continue
 		}
-		idx := variable.Name[len(oid)+1:]
-		if _, ok := md.Tables[idx]; !ok {
-			md.Tables[idx] = kt.NewDeviceTableMetadata()
-		}
 
-		switch variable.Type {
-		case gosnmp.OctetString:
-			value := string(variable.Value.([]byte))
-			if mib.Conversion != "" { // Adjust for any hard coded values here.
-				_, value = snmp_util.GetFromConv(variable, mib.Conversion, log)
-			}
-			if utf8.ValidString(value) {
-				md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, value, 0)
-			}
-		case gosnmp.IPAddress: // Does this work?
-			switch val := variable.Value.(type) {
-			case string:
-				md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, val, 0)
-			case []byte:
-				md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, string(val), 0)
-			case net.IP:
-				md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, val.String(), 0)
-			}
-		case gosnmp.ObjectIdentifier:
-			value := string(variable.Value.(string))
-			if mib.Conversion != "" { // Adjust for any hard coded values here.
-				_, value = snmp_util.GetFromConv(variable, mib.Conversion, log)
-			}
-			if utf8.ValidString(value) {
-				md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, value, 0)
+		var oidName string
+		thing, ok := SNMP_device_metadata_oids.Get(oidVal)
+		if !ok {
+			if oidVal == ".1.3.6.1.6.3.15.1.1.3.0" { // This is a bad v3 config.
+				log.Errorf("User found who is not known to the SNMP engine. Likely this is an invalid v3 config.")
+			} else {
+				log.Errorf("SNMP Device Metadata: Unknown oid retrieved: %v %v", oidVal, value)
 			}
-		default:
-			// Try to just use as a number
-			md.Tables[idx].Customs[oidName] = kt.NewMetaValue(mib, "", gosnmp.ToBigInt(variable.Value).Int64())
+			continue
+		}
+		oidName = thing.(string)
+
+		switch oidName {
+		case SNMP_sysDescr:
+			md.SysDescr = string(value.([]byte))
+		case SNMP_sysObjectID:
+			md.SysObjectID = value.(string)
+		case SNMP_sysContact:
+			md.SysContact = string(value.([]byte))
+		case SNMP_sysName:
+			md.SysName = string(value.([]byte))
+		case SNMP_sysLocation:
+			md.SysLocation = string(value.([]byte))
+		case SNMP_sysServices:
+			md.SysServices = int(snmp_util.ToInt64(value))
+		case SNMP_engineID:
+			_, md.EngineID = snmp_util.GetFromConv(pdu, snmp_util.CONV_ENGINE_ID, log)
 		}
 	}
 
-	return nil
+	return &md, nil
 }
diff --git a/pkg/inputs/snmp/metadata/poll.go b/pkg/inputs/snmp/metadata/poll.go
index eeb0d72..2fdfb1c 100644
--- a/pkg/inputs/snmp/metadata/poll.go
+++ b/pkg/inputs/snmp/metadata/poll.go
@@ -21,6 +21,7 @@ type Poller struct {
 	server                *gosnmp.GoSNMP
 	interval              time.Duration
 	interfaceMetadata     *InterfaceMetadata
+	deviceMetadata        *DeviceMetadata
 	jchfChan              chan []*kt.JCHF
 	conf                  *kt.SnmpDeviceConfig
 	metrics               *kt.SnmpDeviceMetric
@@ -92,6 +93,7 @@ func NewPoller(server *gosnmp.GoSNMP, gconf *kt.SnmpGlobalConfig, conf *kt.SnmpD
 		server:                server,
 		interval:              DEFAULT_INTERVAL,
 		interfaceMetadata:     NewInterfaceMetadata(interfaceMetadataMibs, log),
+		deviceMetadata:        NewDeviceMetadata(deviceMetadataMibs, conf, metrics, log),
 		jchfChan:              jchfChan,
 		metrics:               metrics,
 		deviceMetadataMibs:    deviceMetadataMibs,
@@ -168,7 +170,7 @@ func (p *Poller) PollSNMPMetadata(ctx context.Context) (*kt.DeviceData, error) {
 		}
 	}
 
-	deviceMetadata, err := GetDeviceMetadata(p.log, p.server, p.deviceMetadataMibs)
+	deviceMetadata, err := p.deviceMetadata.Poll(ctx, p.server)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/inputs/snmp/snmp.go b/pkg/inputs/snmp/snmp.go
index 679d5a6..960eb6c 100644
--- a/pkg/inputs/snmp/snmp.go
+++ b/pkg/inputs/snmp/snmp.go
@@ -33,6 +33,7 @@ var (
 	snmpWalkFormat = flag.String("snmp_walk_format", "", "use this format for walked values if -snmp_do_walk is set.")
 	snmpOutFile    = flag.String("snmp_out_file", "", "If set, write updated snmp file here.")
 	snmpPollNow    = flag.String("snmp_poll_now", "", "If set, run one snmp poll for the specified device and then exit.")
+	snmpDiscoDur   = flag.Int("snmp_discovery_min", 0, "If set, run snmp discovery on this interval (in minutes).")
 	ServiceName    = ""
 )
 
@@ -68,7 +69,7 @@ func StartSNMPPolls(ctx context.Context, snmpFile string, jchfChan chan []*kt.JC
 	}
 
 	// Now, launch a metadata and metrics server for each configured or discovered device.
-	go wrapSnmpPolling(ctx, snmpFile, jchfChan, metrics, registry, apic, log)
+	go wrapSnmpPolling(ctx, snmpFile, jchfChan, metrics, registry, apic, log, 0)
 
 	// Run a trap listener?
 	if conf.Trap != nil && !*flowOnly {
@@ -90,7 +91,7 @@ func Close() {
 func initSnmp(ctx context.Context, snmpFile string, log logger.ContextL) (*kt.SnmpConfig, time.Duration, int, error) {
 	// First, parse the config file and see what we're doing.
 	log.Infof("Client SNMP: Running SNMP interface polling, loading config from %s", snmpFile)
-	conf, err := parseConfig(snmpFile, log)
+	conf, err := parseConfig(ctx, snmpFile, log)
 	if err != nil {
 		return nil, 0, 0, err
 	}
@@ -117,16 +118,19 @@ func initSnmp(ctx context.Context, snmpFile string, log logger.ContextL) (*kt.Sn
 	return conf, connectTimeout, retries, nil
 }
 
-func wrapSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL) {
+func wrapSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL, restartCount int) {
 	ctxSnmp, cancel := context.WithCancel(ctx)
-	err := runSnmpPolling(ctxSnmp, snmpFile, jchfChan, metrics, registry, apic, log)
+	err := runSnmpPolling(ctxSnmp, snmpFile, jchfChan, metrics, registry, apic, log, restartCount)
 	if err != nil {
 		log.Errorf("There was an error when polling for SNMP devices: %v.", err)
 	}
 
-	// Now, wait for sigusr2 to re-do.
+	// Now, wait for sigusr2 to re-do or if there's a discovery with new devices.
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, kt.SIGUSR2)
+	if *snmpDiscoDur > 0 { // If we are re-running snmp discovery every interval, start the ticker here.
+		go RunDiscoOnTimer(ctxSnmp, c, snmpFile, log, *snmpDiscoDur)
+	}
 
 	// Block here
 	_ = <-c
@@ -134,10 +138,10 @@ func wrapSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.J
 	// If we got this signal, redo the snmp system.
 	cancel()
 
-	go wrapSnmpPolling(ctx, snmpFile, jchfChan, metrics, registry, apic, log)
+	go wrapSnmpPolling(ctx, snmpFile, jchfChan, metrics, registry, apic, log, restartCount+1) // Track how many times through here we've been.
 }
 
-func runSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL) error {
+func runSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JCHF, metrics *kt.SnmpMetricSet, registry go_metrics.Registry, apic *api.KentikApi, log logger.ContextL, restartCount int) error {
 	// Parse again to make sure nothing's changed.
 	conf, connectTimeout, retries, err := initSnmp(ctx, snmpFile, log)
 	if err != nil || conf == nil || conf.Global == nil {
@@ -155,16 +159,21 @@ func runSnmpPolling(ctx context.Context, snmpFile string, jchfChan chan []*kt.JC
 		}
 
 		log.Infof("Client SNMP: Running SNMP for %s on %s (type=%s)", device.DeviceName, device.DeviceIP, device.Provider)
-		metrics.Mux.Lock()
-		nm := kt.NewSnmpDeviceMetric(registry, device.DeviceName)
 
 		// Check for duplicate device names here.
+		metrics.Mux.Lock()
+		var nm *kt.SnmpDeviceMetric
 		if _, ok := metrics.Devices[device.DeviceName]; ok {
-			log.Errorf("Duplicate device name detected (%s). Is this a misconfiguration?", device.DeviceName)
+			nm = metrics.Devices[device.DeviceName]
+			if restartCount == 0 {
+				log.Errorf("Duplicate device name detected (%s). Is this a misconfiguration?", device.DeviceName)
+			}
+		} else {
+			nm = kt.NewSnmpDeviceMetric(registry, device.DeviceName)
+			metrics.Devices[device.DeviceName] = nm
 		}
-
-		metrics.Devices[device.DeviceName] = nm
 		metrics.Mux.Unlock()
+
 		cl := logger.NewSubContextL(logger.SContext{S: device.DeviceName}, log)
 		var profile *mibs.Profile
 		if mibdb != nil {
@@ -266,9 +275,9 @@ func launchSnmp(ctx context.Context, conf *kt.SnmpGlobalConfig, device *kt.SnmpD
 	return nil
 }
 
-func parseConfig(file string, log logger.ContextL) (*kt.SnmpConfig, error) {
+func parseConfig(ctx context.Context, file string, log logger.ContextL) (*kt.SnmpConfig, error) {
 	ms := kt.SnmpConfig{}
-	by, err := ioutil.ReadFile(file)
+	by, err := snmp_util.LoadFile(ctx, file)
 	if err != nil {
 		return nil, err
 	}
@@ -292,6 +301,20 @@ func parseConfig(file string, log logger.ContextL) (*kt.SnmpConfig, error) {
 		ms.Disco.Cidrs = cidrList
 	}
 
+	if ms.Disco != nil && len(ms.Disco.IgnoreList) > 0 && strings.HasPrefix(ms.Disco.IgnoreList[0], "@") {
+		ignoreList := []string{}
+		byc, err := ioutil.ReadFile(ms.Disco.IgnoreList[0][1:])
+		if err != nil {
+			return nil, err
+		}
+		err = yaml.Unmarshal(byc, &ignoreList)
+		if err != nil {
+			return nil, err
+		}
+		ms.Disco.IgnoreOrig = ms.Disco.IgnoreList[0][1:]
+		ms.Disco.IgnoreList = ignoreList
+	}
+
 	fullDevices := map[string]*kt.SnmpDeviceConfig{}
 	for name, devFile := range ms.Devices {
 		if strings.HasPrefix(name, "file_") && strings.HasPrefix(devFile.DeviceName, "@") {
@@ -379,5 +402,5 @@ func launchPingOnly(ctx context.Context, conf *kt.SnmpGlobalConfig, device *kt.S
 
 // Public wrapper for calling this other places.
 func ParseConfig(file string, log logger.ContextL) (*kt.SnmpConfig, error) {
-	return parseConfig(file, log)
+	return parseConfig(context.Background(), file, log)
 }
diff --git a/pkg/inputs/snmp/util/file.go b/pkg/inputs/snmp/util/file.go
new file mode 100644
index 0000000..ef63637
--- /dev/null
+++ b/pkg/inputs/snmp/util/file.go
@@ -0,0 +1,166 @@
+package util
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// Utility to load a config file from various places
+func LoadFile(ctx context.Context, file string) ([]byte, error) {
+	u, err := url.Parse(file)
+	if err != nil {
+		return nil, err
+	}
+
+	switch u.Scheme {
+	case "http", "https":
+		return loadFromHttp(ctx, file)
+	case "s3":
+		return loadFromS3(ctx, u, getS3Downloader())
+	case "s3m":
+		return loadFromS3(ctx, u, getMockS3Client())
+	default:
+		return ioutil.ReadFile(file)
+	}
+}
+
+// Utility to write a config file to various places
+func WriteFile(ctx context.Context, file string, payload []byte, perms fs.FileMode) error {
+	u, err := url.Parse(file)
+	if err != nil {
+		return err
+	}
+
+	switch u.Scheme {
+	case "http", "https":
+		return writeToHttp(ctx, file, payload)
+	case "s3":
+		return writeToS3(ctx, u, payload, getS3Uploader())
+	case "s3m":
+		return writeToS3(ctx, u, payload, getMockS3Client())
+	default:
+		return ioutil.WriteFile(file, payload, perms)
+	}
+}
+
+func loadFromHttp(ctx context.Context, file string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, file, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}
+
+func writeToHttp(ctx context.Context, file string, payload []byte) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, file, bytes.NewBuffer(payload))
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	_, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("Cannot send to %s: %d", file, resp.StatusCode)
+	}
+	return nil
+}
+
+func loadFromS3(ctx context.Context, url *url.URL, client s3ClientDown) ([]byte, error) {
+	buf := aws.NewWriteAtBuffer([]byte{})
+	size, err := client.DownloadWithContext(ctx, buf, &s3.GetObjectInput{
+		Bucket: aws.String(url.Host),
+		Key:    aws.String(url.Path),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	bufr := buf.Bytes()
+	return bufr[0:size], nil
+}
+
+func writeToS3(ctx context.Context, url *url.URL, payload []byte, client s3ClientUp) error {
+	_, err := client.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket: aws.String(url.Host),
+		Body:   bytes.NewBuffer(payload),
+		Key:    aws.String(url.Path),
+	})
+	return err
+}
+
+type s3ClientDown interface {
+	DownloadWithContext(aws.Context, io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader)) (int64, error)
+}
+
+type s3ClientUp interface {
+	UploadWithContext(aws.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
+}
+
+type s3Full interface {
+	s3ClientUp
+	s3ClientDown
+}
+
+func getS3Downloader() s3ClientDown {
+	sess := session.Must(session.NewSession())
+	client := s3manager.NewDownloader(sess)
+	return client
+}
+
+func getS3Uploader() s3ClientUp {
+	sess := session.Must(session.NewSession())
+	client := s3manager.NewUploader(sess)
+	return client
+}
+
+type mockS3 struct {
+	lastContent []byte
+}
+
+var (
+	sMock *mockS3
+)
+
+func (m *mockS3) UploadWithContext(ctx aws.Context, in *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+	m.lastContent = in.Body.(*bytes.Buffer).Bytes()
+	return nil, nil
+}
+
+func (m *mockS3) DownloadWithContext(ctx aws.Context, w io.WriterAt, in *s3.GetObjectInput, options ...func(*s3manager.Downloader)) (int64, error) {
+	w.WriteAt(m.lastContent, 0)
+	return int64(len(m.lastContent)), nil
+}
+
+func getMockS3Client() s3Full {
+	return sMock
+}
diff --git a/pkg/inputs/syslog/syslog.go b/pkg/inputs/syslog/syslog.go
index 45f4aa8..5d2a0cb 100644
--- a/pkg/inputs/syslog/syslog.go
+++ b/pkg/inputs/syslog/syslog.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net"
 	"strings"
 	"time"
 
@@ -14,17 +15,19 @@ import (
 	"github.com/kentik/ktranslate/pkg/api"
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
 	"github.com/kentik/ktranslate/pkg/kt"
+	"github.com/kentik/ktranslate/pkg/util/resolv"
 )
 
 type KentikSyslog struct {
 	logger.ContextL
-	server  *syslog.Server
-	handler *syslog.ChannelHandler
-	channel syslog.LogPartsChannel
-	logchan chan string
-	metrics SyslogMetric
-	apic    *api.KentikApi
-	devices map[string]*kt.Device
+	server   *syslog.Server
+	handler  *syslog.ChannelHandler
+	channel  syslog.LogPartsChannel
+	logchan  chan string
+	metrics  SyslogMetric
+	apic     *api.KentikApi
+	devices  map[string]*kt.Device
+	resolver *resolv.Resolver
 }
 
 type SyslogMetric struct {
@@ -47,7 +50,7 @@ const (
 	InstNameSyslog       = "ktranslate-syslog"
 )
 
-func NewSyslogSource(ctx context.Context, host string, log logger.Underlying, logchan chan string, registry go_metrics.Registry, apic *api.KentikApi) (*KentikSyslog, error) {
+func NewSyslogSource(ctx context.Context, host string, log logger.Underlying, logchan chan string, registry go_metrics.Registry, apic *api.KentikApi, resolver *resolv.Resolver) (*KentikSyslog, error) {
 	ks := KentikSyslog{
 		ContextL: logger.NewContextLFromUnderlying(logger.SContext{S: "Syslog"}, log),
 		logchan:  logchan,
@@ -56,8 +59,9 @@ func NewSyslogSource(ctx context.Context, host string, log logger.Underlying, lo
 			Errors:   go_metrics.GetOrRegisterMeter(fmt.Sprintf("syslog_errors^force=true"), registry),
 			Queue:    go_metrics.GetOrRegisterGauge(fmt.Sprintf("syslog_queue^force=true"), registry),
 		},
-		apic:    apic,
-		devices: apic.GetDevicesAsMap(0),
+		apic:     apic,
+		devices:  apic.GetDevicesAsMap(0),
+		resolver: resolver,
 	}
 
 	if logchan == nil {
@@ -138,7 +142,7 @@ func (ks *KentikSyslog) process(ctx context.Context, id int, channel syslog.LogP
 		select {
 		case logParts := <-channel:
 			ks.metrics.Messages.Mark(1)
-			msg, err := ks.formatMessage(logParts)
+			msg, err := ks.formatMessage(ctx, logParts)
 			if err != nil {
 				ks.Errorf("Cannot format syslog: %v", err)
 			}
@@ -176,22 +180,40 @@ func (ks *KentikSyslog) run(ctx context.Context, host string) {
 	ks.server.Wait()
 }
 
-func (ks *KentikSyslog) formatMessage(msg sfmt.LogParts) ([]byte, error) {
+func (ks *KentikSyslog) formatMessage(ctx context.Context, msg sfmt.LogParts) ([]byte, error) {
 	if client, ok := msg["client"].(string); ok { // Look up device_name here.
 		pts := strings.Split(client, ":")
 		if dev, ok := ks.devices[pts[0]]; ok {
 			msg["device_name"] = dev.Name // Copy in any of these info we get
 			dev.SetMsgUserTags(msg)
 		}
+		if ks.resolver != nil {
+			msg["client_name"] = ks.resolver.Resolve(ctx, pts[0])
+		}
 	}
 
 	// Fall back to hostname if this is set.
 	if _, ok := msg["device_name"]; !ok {
-		if _, ok := msg["hostname"]; ok {
-			msg["device_name"] = msg["hostname"]
+		if hostname, ok := msg["hostname"]; ok {
+			if hs, ok := hostname.(string); ok {
+				if ipr := net.ParseIP(hs); ipr != nil {
+					if ks.resolver != nil {
+						msg["device_name"] = ks.resolver.Resolve(ctx, hs)
+					} else {
+						msg["device_name"] = hs
+					}
+				} else {
+					msg["device_name"] = hs
+				}
+			}
 		}
 	}
 
+	// One more time for sure.
+	if _, ok := msg["device_name"]; !ok {
+		msg["device_name"] = msg["client_name"]
+	}
+
 	msg["message"] = msg["content"] // Swap these around for NR.
 	delete(msg, "content")
 	msg["plugin.type"] = kt.PluginSyslog         // NR Processing.
diff --git a/pkg/kt/snmp.go b/pkg/kt/snmp.go
index 5c6a146..827a44d 100644
--- a/pkg/kt/snmp.go
+++ b/pkg/kt/snmp.go
@@ -173,6 +173,7 @@ type SnmpTrapConfig struct {
 
 type SnmpDiscoConfig struct {
 	Cidrs              StringArray     `yaml:"cidrs"`
+	IgnoreList         StringArray     `yaml:"ignore_list"`
 	Debug              bool            `yaml:"debug"`
 	Ports              []int           `yaml:"ports"`
 	DefaultCommunities []string        `yaml:"default_communities"`
@@ -185,6 +186,7 @@ type SnmpDiscoConfig struct {
 	ReplaceDevices     bool            `yaml:"replace_devices"`
 	NoDedup            bool            `yaml:"no_dedup_engine_id,omitempty"`
 	CidrOrig           string          `yaml:"-"`
+	IgnoreOrig         string          `yaml:"-"`
 }
 
 type SnmpGlobalConfig struct {
@@ -229,6 +231,7 @@ type SnmpDeviceMetric struct {
 	Errors           go_metrics.Meter
 	Fail             go_metrics.Gauge
 	Missing          go_metrics.Gauge
+	MissingMeta      go_metrics.Gauge
 }
 
 const (
@@ -251,6 +254,7 @@ func NewSnmpDeviceMetric(registry go_metrics.Registry, deviceName string) *SnmpD
 		Errors:           go_metrics.GetOrRegisterMeter("snmp_errors^device_name="+deviceName, registry),
 		Fail:             go_metrics.GetOrRegisterGauge("snmp_fail^device_name="+deviceName, registry),
 		Missing:          go_metrics.GetOrRegisterGauge("snmp_missing^force=true^device_name="+deviceName, registry),
+		MissingMeta:      go_metrics.GetOrRegisterGauge("snmp_missing_meta^force=true^device_name="+deviceName, registry),
 	}
 	sm.Fail.Update(SNMP_GOOD) // 1 means that this device is not failing.
 	return &sm
@@ -478,7 +482,7 @@ func (a *V3SNMPConfig) InheritGlobal() bool {
 }
 
 // Save any hard coded parts of this profile which might get overriten by an automatic process.
-func (d *SnmpDeviceConfig) UpdateFrom(old *SnmpDeviceConfig) {
+func (d *SnmpDeviceConfig) UpdateFrom(old *SnmpDeviceConfig, conf *SnmpConfig) {
 	if old == nil {
 		return
 	}
@@ -486,6 +490,53 @@ func (d *SnmpDeviceConfig) UpdateFrom(old *SnmpDeviceConfig) {
 	if strings.HasPrefix(old.MibProfile, "!") {
 		d.MibProfile = old.MibProfile
 	}
+	if old.SampleRate > 0 {
+		d.SampleRate = old.SampleRate
+	}
+	if old.FlowOnly {
+		d.FlowOnly = old.FlowOnly
+	}
+	if old.PingOnly {
+		d.PingOnly = old.PingOnly
+	}
+	if old.PollTimeSec > 0 {
+		d.PollTimeSec = old.PollTimeSec
+	}
+	if old.TimeoutMS > 0 {
+		d.TimeoutMS = old.TimeoutMS
+	}
+	if old.Retries > 0 {
+		d.Retries = old.Retries
+	}
+	if old.MonitorAdminShut {
+		d.MonitorAdminShut = old.MonitorAdminShut
+	}
+	if old.NoUseBulkWalkAll {
+		d.NoUseBulkWalkAll = old.NoUseBulkWalkAll
+	}
+	if old.RunPing {
+		d.RunPing = old.RunPing
+	}
+	if d.UserTags == nil && old.UserTags != nil {
+		d.UserTags = map[string]string{}
+	}
+	for k, v := range old.UserTags {
+		if _, ok := d.UserTags[k]; !ok {
+			if _, ok := conf.Global.UserTags[k]; !ok {
+				d.UserTags[k] = v
+			}
+		}
+	}
+	if d.MatchAttr == nil && old.MatchAttr != nil {
+		d.MatchAttr = map[string]string{}
+	}
+	for k, v := range old.MatchAttr {
+		if _, ok := d.MatchAttr[k]; !ok {
+			if _, ok := conf.Global.MatchAttr[k]; !ok {
+				d.MatchAttr[k] = v
+			}
+		}
+	}
 }
 
 func (d *SnmpDeviceConfig) AddUserTag(k string, v string) {
diff --git a/pkg/kt/types.go b/pkg/kt/types.go
index 13f9222..0ec20c7 100644
--- a/pkg/kt/types.go
+++ b/pkg/kt/types.go
@@ -208,7 +208,6 @@ func NewJCHF() *JCHF {
 
 func (j *JCHF) Reset() {
 	j.hasSetAvro = false
-	//j.SetMap() @TODO -- don't think this is needed?
 }
 
 func (j *JCHF) Flatten() map[string]interface{} {
@@ -314,6 +313,12 @@ func (j *JCHF) ToMap() map[string]interface{} {
 
 func (j *JCHF) SetMap() {
 	j.avroSet = map[string]interface{}{}
+	j.CustomStr = map[string]string{}
+	j.CustomInt = map[string]int32{}
+	j.CustomBigInt = map[string]int64{}
+	j.CustomMetrics = map[string]MetricInfo{}
+	j.CustomTables = map[string]DeviceTableMetadata{}
+	j.MatchAttr = map[string]*regexp.Regexp{}
 }
 
 func (j *JCHF) SetIFPorts(p IfaceID) *JCHF {
diff --git a/pkg/sinks/http/http.go b/pkg/sinks/http/http.go
index 9e6c6b6..c5fb434 100644
--- a/pkg/sinks/http/http.go
+++ b/pkg/sinks/http/http.go
@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	nurl "net/url"
 	"strings"
+	"time"
 
 	go_metrics "github.com/kentik/go-metrics"
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
@@ -20,11 +22,13 @@ type HttpSink struct {
 	logger.ContextL
 	TargetUrl string
 
-	client   *http.Client
-	tr       *http.Transport
-	registry go_metrics.Registry
-	metrics  *HttpMetric
-	headers  map[string]string
+	client          *http.Client
+	tr              *http.Transport
+	registry        go_metrics.Registry
+	metrics         *HttpMetric
+	headers         map[string]string
+	targetUrls      []string
+	sendMaxDuration time.Duration
 }
 
 type HttpMetric struct {
@@ -44,8 +48,10 @@ func (h *HeaderFlag) Set(value string) error {
 }
 
 var (
-	TargetUrl = flag.String("http_url", "http://localhost:8086/write?db=kentik", "URL to post to")
-	headers   HeaderFlag
+	TargetUrl          = flag.String("http_url", "http://localhost:8086/write?db=kentik", "URL to post to")
+	InsecureSkipVerify = flag.Bool("http_insecure", false, "Allow insecure urls.")
+	TimeoutSec         = flag.Int("http_timeout_sec", 30, "Timeout each request after this long.")
+	headers            HeaderFlag
 )
 
 func NewSink(log logger.Underlying, registry go_metrics.Registry, sink string) (*HttpSink, error) {
@@ -56,7 +62,9 @@ func NewSink(log logger.Underlying, registry go_metrics.Registry, sink string) (
 			DeliveryErr: go_metrics.GetOrRegisterMeter("delivery_errors_http", registry),
 			DeliveryWin: go_metrics.GetOrRegisterMeter("delivery_wins_http", registry),
 		},
-		headers: map[string]string{},
+		headers:         map[string]string{},
+		targetUrls:      []string{},
+		sendMaxDuration: time.Duration(*TimeoutSec) * time.Second,
 	}
 
 	for _, header := range headers {
@@ -82,10 +90,17 @@ func NewSink(log logger.Underlying, registry go_metrics.Registry, sink string) (
 }
 
 func (s *HttpSink) Init(ctx context.Context, format formats.Format, compression kt.Compression, fmtr formats.Formatter) error {
-	s.TargetUrl = *TargetUrl
+	for _, url := range strings.Split(*TargetUrl, ",") {
+		_, err := nurl.Parse(url)
+		if err != nil {
+			return fmt.Errorf("Invalid url for http sink: %s", url)
+		}
+		s.targetUrls = append(s.targetUrls, url)
+		s.Infof("Exporting HTTP to %s", url)
+	}
 
 	s.tr = &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: *InsecureSkipVerify},
 	}
 	s.client = &http.Client{Transport: s.tr}
 
@@ -93,13 +108,17 @@ func (s *HttpSink) Init(ctx context.Context, format formats.Format, compression
 		s.headers["Content-Encoding"] = "GZIP"
 	}
 
-	s.Infof("Exporting to Http at %s", s.TargetUrl)
-
 	return nil
 }
 
 func (s *HttpSink) Send(ctx context.Context, payload *kt.Output) {
-	go s.sendHttp(ctx, payload.Body)
+	for _, url := range s.targetUrls {
+		go func(u string) {
+			ctxC, cancel := context.WithTimeout(ctx, s.sendMaxDuration)
+			defer cancel()
+			s.sendHttp(ctxC, payload.Body, u)
+		}(url)
+	}
 }
 
 func (s *HttpSink) Close() {}
@@ -111,8 +130,8 @@ func (s *HttpSink) HttpInfo() map[string]float64 {
 	}
 }
 
-func (s *HttpSink) sendHttp(ctx context.Context, payload []byte) {
-	req, err := http.NewRequestWithContext(ctx, "POST", s.TargetUrl, bytes.NewBuffer(payload))
+func (s *HttpSink) sendHttp(ctx context.Context, payload []byte, url string) {
+	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload))
 	if err != nil {
 		s.Errorf("There was an error when creating an HTTP request: %v.", err)
 		return
@@ -124,17 +143,17 @@ func (s *HttpSink) sendHttp(ctx context.Context, payload []byte) {
 
 	resp, err := s.client.Do(req)
 	if err != nil {
-		s.Errorf("There was an error when converting to HTTP: %v.", err)
+		s.Errorf("There was an error when sending via HTTP at %s: %v.", url, err)
 		s.client = &http.Client{Transport: s.tr}
 	} else {
 		defer resp.Body.Close()
 		bdy, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			s.Errorf("There was an error when getting the HTTP response body: %v.", err)
+			s.Errorf("There was an error when getting the HTTP at %s response body: %v.", url, err)
 			s.metrics.DeliveryErr.Mark(1)
 		} else {
 			if resp.StatusCode >= 400 {
-				s.Errorf("There was an error when converting to HTTP: %d. Body: %s.", resp.StatusCode, string(bdy))
+				s.Errorf("There was an error when converting to HTTP at %s: %d. Body: %s.", url, resp.StatusCode, string(bdy))
 				s.metrics.DeliveryErr.Mark(1)
 			} else {
 				s.metrics.DeliveryWin.Mark(1)
diff --git a/pkg/util/logger/logger_writer.go b/pkg/util/logger/logger_writer.go
index 0e0cb6e..c75bca7 100644
--- a/pkg/util/logger/logger_writer.go
+++ b/pkg/util/logger/logger_writer.go
@@ -21,6 +21,8 @@ const (
 	LOG_DEBUG   = 1
 	LOG_ERR     = 4
 	LOG_WARNING = 3
+
+	reclaimThreshold int = 5120 // Seems to be around p99
 )
 
 // container for a pending log message
@@ -95,7 +97,11 @@ func SetLogName(p string) (err error) {
 
 // freeMsg releases the message back to be reused
 func freeMsg(msg *logMessage) (err error) {
-	msg.Reset()
+	if msg.Buffer.Cap() > reclaimThreshold {
+		msg.Buffer = *bytes.NewBuffer(make([]byte, 0, reclaimThreshold))
+	} else {
+		msg.Reset()
+	}
 	select {
 	case freeMessages <- msg: // no-op
 	default:
diff --git a/pkg/cat/resolver.go b/pkg/util/resolv/resolver.go
similarity index 99%
rename from pkg/cat/resolver.go
rename to pkg/util/resolv/resolver.go
index c5cd029..b60d73c 100644
--- a/pkg/cat/resolver.go
+++ b/pkg/util/resolv/resolver.go
@@ -1,4 +1,4 @@
-package cat
+package resolv
 
 import (
 	"context"


Test patch:
diff --git a/pkg/inputs/snmp/util/file_test.go b/pkg/inputs/snmp/util/file_test.go
new file mode 100644
index 0000000..04f517e
--- /dev/null
+++ b/pkg/inputs/snmp/util/file_test.go
@@ -0,0 +1,114 @@
+package util
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadFile(t *testing.T) {
+	assert := assert.New(t)
+	content := []byte("aaaa") // Set some test content.
+
+	// Some test server
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, string(content))
+	}))
+	defer svr.Close()
+
+	sMock = &mockS3{
+		lastContent: content,
+	}
+
+	tests := map[string][]byte{
+		":foo":                nil,
+		svr.URL:               content,
+		"s3://foo/bar/a/one":  nil,
+		"S3://foo/bar/a/two":  nil,
+		"s3m://foo/bar/a/one": content,
+	}
+
+	// Save test data to local.
+	file, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.FailNow()
+	}
+	if _, err := file.Write(content); err != nil {
+		t.FailNow()
+	}
+	defer os.Remove(file.Name())
+	tests[file.Name()] = content
+
+	ctx := context.Background()
+	for in, expt := range tests {
+		res, err := LoadFile(ctx, in)
+		if expt == nil {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+			assert.Equal(string(expt), string(res), "failed %s", in)
+		}
+	}
+}
+
+func TestWriteFile(t *testing.T) {
+	assert := assert.New(t)
+	content := []byte("aaaa") // Set some test content.
+
+	fileWeb, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.FailNow()
+	}
+	defer os.Remove(fileWeb.Name())
+
+	// Some test server
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body := r.Body
+		defer body.Close()
+		bb := []byte{}
+		bodyBuffer := bytes.NewBuffer(bb)
+		_, err = bodyBuffer.ReadFrom(body)
+		ioutil.WriteFile(fileWeb.Name(), bodyBuffer.Bytes(), 0644)
+	}))
+	defer svr.Close()
+
+	// Save test data to local.
+	file, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.FailNow()
+	}
+
+	sMock = &mockS3{
+		lastContent: nil,
+	}
+
+	defer os.Remove(file.Name())
+	tests := map[string]string{
+		":foo":                "",
+		file.Name():           file.Name(),
+		svr.URL:               fileWeb.Name(),
+		"s3m://foo/bar/a/one": string(content),
+	}
+
+	ctx := context.Background()
+	for target, local := range tests {
+		err := WriteFile(ctx, target, content, 0644)
+		if local == "" {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+			c, err := ioutil.ReadFile(local) // This one assumes that we're writting locally.
+			if err != nil {
+				c = sMock.lastContent
+			}
+			assert.Equal(string(content), string(c), "failed %s", target)
+		}
+	}
+}