Repository: iter8-tools/iter8
Pull request #1383 (2023-01-05 16:40:24)

modify handler to check readiness simplifying promotion
Signed-off-by: Michael Kalantar <kalantar@us.ibm.com>


Patch:
diff --git a/abn/application/application.go b/abn/application/application.go
index 56ca4dd1..084a2a9a 100644
--- a/abn/application/application.go
+++ b/abn/application/application.go
@@ -49,10 +49,10 @@ func (a *Application) GetVersion(version string, allowNew bool) (*Version, bool)
 	v, ok := a.Versions[version]
 	if !ok {
 		if allowNew {
-			log.Logger.Debugf("GetVersion no data found; returning %+v", v)
 			v = &Version{
 				Metrics: map[string]*summarymetrics.SummaryMetric{},
 			}
+			log.Logger.Debugf("GetVersion no data found; creating %+v", v)
 			a.Versions[version] = v
 			return v, true
 		}
diff --git a/abn/application/applicationmap.go b/abn/application/applicationmap.go
index ba3a09aa..e6cc2991 100644
--- a/abn/application/applicationmap.go
+++ b/abn/application/applicationmap.go
@@ -17,7 +17,7 @@ import (
 const (
 	// secretPostfix is the postfix added to an application name to create a secret name
 	/* #nosec */
-	secretPostfix string = ".iter8abnmetrics"
+	secretPostfix string = "-metrics"
 	// secretKey is the name of the key in the data field of a kubernetes secret in which the application will be written
 	secretKey string = "application.yaml"
 	// defaultBatchWriteInterval is the default value of BatchWriteInterval
@@ -183,20 +183,15 @@ func (m *ThreadSafeApplicationMap) Write(a *Application) error {
 	secretNamespace := namespaceFromKey(a.Name)
 	secretName := secretNameFromKey(a.Name)
 
-	// determine if need to
-	exists := true
+	// get the current secret; it will have been created as part of install
 	secret, err = k8sclient.Client.Typed().CoreV1().Secrets(secretNamespace).Get(context.Background(), secretName, metav1.GetOptions{})
 	if err != nil {
-		exists = false
-		secret = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      secretName,
-				Namespace: secretNamespace,
-			},
-			Data:       map[string][]byte{},
-			StringData: map[string]string{},
-		}
-		log.Logger.Debug("secret does not exist; creating")
+		log.Logger.Error("secret does not exist; no metrics can be recorded")
+		return err
+	}
+
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
 	}
 
 	secret.Data[secretKey] = rawData
@@ -204,21 +199,13 @@ func (m *ThreadSafeApplicationMap) Write(a *Application) error {
 		secret.StringData[secretKey] = string(rawData)
 	}
 
-	// create or update the secret
-	if exists {
-		// TBD do we need to merge what we have?
-		_, err = k8sclient.Client.Typed().CoreV1().Secrets(secretNamespace).Update(
-			context.Background(),
-			secret,
-			metav1.UpdateOptions{},
-		)
-	} else {
-		_, err = k8sclient.Client.Typed().CoreV1().Secrets(secretNamespace).Create(
-			context.Background(),
-			secret,
-			metav1.CreateOptions{},
-		)
-	}
+	// update the secret
+	// TBD do we need to merge what we have?
+	_, err = k8sclient.Client.Typed().CoreV1().Secrets(secretNamespace).Update(
+		context.Background(),
+		secret,
+		metav1.UpdateOptions{},
+	)
 	if err != nil {
 		log.Logger.WithError(err).Warn("unable to persist application")
 		return err
diff --git a/abn/core/config.go b/abn/core/config.go
deleted file mode 100644
index e8b0ec0b..00000000
--- a/abn/core/config.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package core
-
-// config.go - reading of configuration (list of resources/namespaces to watch)
-
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/iter8-tools/iter8/base/log"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-
-	"sigs.k8s.io/yaml"
-)
-
-// config is A/B/n service configuration. The configuration identifies the resources to watch and in which namespaces.
-type config struct {
-	// Namespaces is list of namespaces to watch
-	Namespaces []string `yaml:"namespaces,omitempty"`
-	// Resources is list of resoure types that should be watched in the namespaces
-	Resources []schema.GroupVersionResource `yaml:"resources,omitempty"`
-}
-
-// readConfig reads yaml config file fn and converts to a Config object
-func readConfig(fn string) (c config) {
-	// empty configuration
-	c = config{
-		Namespaces: []string{},
-		Resources:  []schema.GroupVersionResource{},
-	}
-
-	yfile, err := os.ReadFile(filepath.Clean(fn))
-	if err != nil {
-		log.Logger.Warnf("unable to read configuration file %s: %s", fn, err.Error())
-		return c // empty configuration
-	}
-
-	log.Logger.Debugf("read configuration\n%s", string(yfile))
-
-	err = yaml.Unmarshal(yfile, &c)
-	if err != nil {
-		log.Logger.Warnf("invalid configuration file %s: %s", fn, err.Error())
-		return c // empty configuration
-	}
-
-	if len(c.Namespaces) == 0 {
-		log.Logger.Warn("not watching any namespaces - configuration error?")
-	}
-	if len(c.Resources) == 0 {
-		log.Logger.Warn("not watching any resources - configuration error?")
-	}
-
-	return c
-}
diff --git a/abn/core/lookup.go b/abn/core/lookup.go
index a7ef2d60..84e61dca 100644
--- a/abn/core/lookup.go
+++ b/abn/core/lookup.go
@@ -8,6 +8,7 @@ import (
 	"hash/maphash"
 
 	abnapp "github.com/iter8-tools/iter8/abn/application"
+	"github.com/iter8-tools/iter8/base/log"
 )
 
 var versionHasher maphash.Hash
@@ -21,11 +22,8 @@ func lookupInternal(application string, user string) (*abnapp.Application, *stri
 
 	// check that we have a record of the application
 	a, err := abnapp.Applications.Get(application)
-	if err != nil {
-		return nil, nil, fmt.Errorf("application not found: %s", err.Error())
-	}
-	if a == nil {
-		return nil, nil, errors.New("application not found")
+	if err != nil || a == nil {
+		return nil, nil, fmt.Errorf("application %s not found", application)
 	}
 
 	// use rendezvous hash to get track for user, fail if not present
@@ -57,6 +55,7 @@ func rendezvousGet(a *abnapp.Application, user string) string {
 
 	for track, version := range a.Tracks {
 		score := hash(version, user)
+		log.Logger.Debugf("hash(%s,%s) --> %d  --  %d", version, user, score, maxScore)
 		if score > maxScore || (score == maxScore && version > maxVersion) {
 			maxScore = score
 			maxVersion = version
diff --git a/abn/core/service.go b/abn/core/service.go
index 34a801bd..b7ce3f6d 100644
--- a/abn/core/service.go
+++ b/abn/core/service.go
@@ -40,8 +40,7 @@ func Start(port int, stopCh chan struct{}) error {
 	}
 
 	// set up resource watching as defined by config
-	c := readConfig(abnConfigFile)
-	w := watcher.NewIter8Watcher(c.Resources, c.Namespaces)
+	w := watcher.NewIter8Watcher(abnConfigFile)
 
 	// start watchers
 	go w.Start(stopCh)
@@ -82,6 +81,9 @@ func (server *abnServer) Lookup(ctx context.Context, appMsg *pb.Application) (*p
 // WriteMetric identifies the track with which a metric is associated (from user) and
 // writes the metric value (currently only supports summary metrics)
 func (server *abnServer) WriteMetric(ctx context.Context, metricMsg *pb.MetricValue) (*emptypb.Empty, error) {
+	log.Logger.Trace("WriteMetric called")
+	defer log.Logger.Trace("WriteMetric completed")
+
 	err := writeMetricInternal(
 		metricMsg.GetApplication(),
 		metricMsg.GetUser(),
diff --git a/abn/core/write_metric.go b/abn/core/write_metric.go
index 357f42ad..9cd88e58 100644
--- a/abn/core/write_metric.go
+++ b/abn/core/write_metric.go
@@ -11,6 +11,9 @@ import (
 
 // writeMetricInternal is detailed implementation of gRPC method WriteMetric
 func writeMetricInternal(application, user, metric, valueStr string) error {
+	log.Logger.Tracef("writeMetricInternal called for application, user: %s, %s", application, user)
+	defer log.Logger.Trace("writeMetricInternal completed")
+
 	a, track, err := lookupInternal(application, user)
 	if err != nil || track == nil {
 		return err
@@ -25,9 +28,11 @@ func writeMetricInternal(application, user, metric, valueStr string) error {
 	if !ok {
 		return errors.New("track not mapped to version")
 	}
+	log.Logger.Debugf("track %s --> version %s", *track, version)
 
 	v, _ := a.GetVersion(version, false)
 	if v == nil {
+		log.Logger.Warnf("unable to get version %s ", version)
 		return errors.New("unexpected: trying to write metrics for unknown version")
 	}
 
diff --git a/abn/k8sclient/k8sclient.go b/abn/k8sclient/k8sclient.go
index 0079b818..c8da7fdd 100644
--- a/abn/k8sclient/k8sclient.go
+++ b/abn/k8sclient/k8sclient.go
@@ -7,10 +7,11 @@ import (
 	"helm.sh/helm/v3/pkg/cli"
 
 	// Import to initialize client auth plugins.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+
+	// packages for cloud authentication
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 // Client is a global variable. Before use, it must be assigned and initialized
diff --git a/abn/watcher/config.go b/abn/watcher/config.go
new file mode 100644
index 00000000..109da90e
--- /dev/null
+++ b/abn/watcher/config.go
@@ -0,0 +1,73 @@
+package watcher
+
+// config.go - reading of configuration (list of resources/namespaces to watch)
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/iter8-tools/iter8/base/log"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"sigs.k8s.io/yaml"
+)
+
+// serviceConfig is an A/B/n service configuration. It is a map of namespaces to a map of application details
+type serviceConfig map[string]apps
+
+// apps is a map of application names to service configuration details
+type apps map[string]appDetails
+
+// appDetails are the service configuration details for an application: the resource types and the max number of candidates
+type appDetails struct {
+	MaxNumCandidates int            `yaml:"maxNumCandidates,omitempty"`
+	Resources        []resourceInfo `yaml:"resources,omitempty"`
+}
+
+type resourceInfo struct {
+	schema.GroupVersionResource
+	Condition *string `yaml:"condition,omitempty"`
+}
+
+// readServiceConfig reads YAML config file fn and converts to a serviceConfig object
+func readServiceConfig(fn string) (c serviceConfig) {
+	log.Logger.Tracef("readConfig called with config file %s", fn)
+
+	// empty configuration
+	c = serviceConfig{}
+
+	yfile, err := os.ReadFile(filepath.Clean(fn))
+	if err != nil {
+		log.Logger.Warnf("unable to read configuration file %s: %s", fn, err.Error())
+		return c // empty configuration
+	}
+
+	log.Logger.Debugf("read configuration file as:\n%s", string(yfile))
+
+	err = yaml.Unmarshal(yfile, &c)
+	if err != nil {
+		log.Logger.Warnf("invalid configuration file %s: %s", fn, err.Error())
+		return c // empty configuration
+	}
+
+	log.Logger.Tracef("readConfig returning config\n---\n%v\n---", c)
+	return c
+}
+
+// getApplicationConfig extracts the application specific configuration from the service configuration
+func getApplicationConfig(namespace string, application string, c serviceConfig) *appDetails {
+	apps, ok := c[namespace]
+	if !ok {
+		// namespace not found, error
+		log.Logger.Errorf("unable to find application configuration for %s/%s (namespace incorrect?)", namespace, application)
+		return nil
+	}
+
+	appConfig, ok := apps[application]
+	if !ok {
+		log.Logger.Warnf("unable to find application configuration for %s/%s (application incorrect?)", namespace, application)
+		return nil
+	}
+
+	return &appConfig
+}
diff --git a/abn/watcher/doc.go b/abn/watcher/doc.go
index 7be25987..cfbd42cc 100644
--- a/abn/watcher/doc.go
+++ b/abn/watcher/doc.go
@@ -1,15 +1,15 @@
 // Package watcher provides methods to watch resources and build a runtime model of applications
 package watcher
 
-// Configured resource types are watched in a set of namespaces. Each version of an application
-// of interest should be include EXACTLY one watched resource with the following metadata:
-//  - label 'iter8.tools/abn' set to true indicating the resource should be inspected further
-//  - label 'app.kubernetes.io/name' identifying the name of the application to which the resource belongs
-//  - label 'app.kubernetes.io/version' identifying the name of the version  to which the resource belongs
-//  - label 'iter8.tools/track' defining the track label that should be associated with the version (no track when not present)
+// A configuration file defines which resources should be watched. Resources for a given version
+// are expected to be named using the track label. These labels are:
+//  <name>
+//  <name>-candidate-<index>
+// where index is 1, 2, 3, ...
 
-// A track is a user assigned identifier. When the Iter8 A/B/n service is used to lookup versions,
-// the track identifier is returned (instead of a version name). The caller can use the the track
-// identifier to route requests to the appropriate version. To do this, the set of track identifiers
-// should be a (small) fixed set, such as "current" and "candidate", that can be mapped to a set of
-// static routes.
+// Iter8 maintains a mapping of track labels to ready versions. A version is ready if all of the
+// resources that comprise it are present, at least one has the label `app.kubernetes.io/version`
+// (if more than one does, they must all have the same value), and the resources are "ready".
+// "readiness" is determined by the resource type.
+
+// The mapping of track to version changes over time as versions change.
diff --git a/abn/watcher/handler.go b/abn/watcher/handler.go
index 29e1891e..4cf6398f 100644
--- a/abn/watcher/handler.go
+++ b/abn/watcher/handler.go
@@ -1,105 +1,304 @@
 package watcher
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
 	abnapp "github.com/iter8-tools/iter8/abn/application"
+	"github.com/iter8-tools/iter8/abn/k8sclient"
 	"github.com/iter8-tools/iter8/base/log"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/dynamic/dynamicinformer"
 )
 
-// precond is set of preconditions that must hold true before an object is considered.
-// It must have:
-//   - label 'iter8.tools/abn' set to true indicating the resource should be inspected further
-//   - label 'app.kubernetes.io/name' identifying the name of the application to which the resource belongs
-//   - label 'app.kubernetes.io/version' identifying the name of the version  to which the resource belongs
-func precond(w watchedObject) bool {
-	var ok bool
+const (
+	versionLabel   = "app.kubernetes.io/version"
+	iter8Finalizer = "iter8.tools/finalizer"
+)
 
-	if !w.isIter8AbnRelated() {
-		return false
+var (
+	applicationNameRE = regexp.MustCompile(`-candidate-[123456789]\d*$`)
+)
+
+// handle constructs the application object from the objects currently in the cluster
+func handle(obj *unstructured.Unstructured, config serviceConfig, informerFactories map[string]dynamicinformer.DynamicSharedInformerFactory, gvr schema.GroupVersionResource) {
+	log.Logger.Tracef("handle called")
+	defer log.Logger.Trace("handle completed")
+
+	// get object from cluster (even through we have an unstructured.Unstructured, it is really only the metadata; to get the full object we need to fetch it from the cluster)
+	obj, err := getUnstructuredObject(obj, gvr)
+	if err != nil {
+		log.Logger.Debug("unable to fetch object from cluster")
+		return
 	}
 
-	application, ok := w.getNamespacedName()
-	if !ok || application == "" {
-		return false
+	// add finalizer IF object not being deleted AND finalizer not already there
+	if obj.GetDeletionTimestamp() == nil && !containsString(obj.GetFinalizers(), iter8Finalizer) {
+		obj.SetFinalizers(append(obj.GetFinalizers(), iter8Finalizer))
+		log.Logger.Debug("adding Iter8 finalizer")
+		_, err := updateUnstructuredObject(obj, gvr)
+		if err != nil {
+			log.Logger.Warn("unable to add finalizer: ", err.Error())
+		}
+		return // UPDATE action will trigger handle() to do remaining work
 	}
 
-	version, ok := w.getVersion()
-	if !ok || version == "" {
-		return false
+	application := getApplicationNameFromObjectName(obj.GetName())
+	namespace := obj.GetNamespace()
+	version, _ := getVersion(obj)
+	log.Logger.Tracef("handle called for %s/%s (%s))", namespace, application, version)
+
+	// get application configuration
+	appConfig := getApplicationConfig(namespace, application, config)
+	if appConfig == nil {
+		// we found a resource that is not part of an A/B/n test; ignore the object
+		log.Logger.Debugf("object for application %s/%s has no A/B/n configuration", namespace, application)
+		return
 	}
 
-	return true
-}
+	// get the objects related to the application (using the appConfig as a guide)
+	applicationObjs := getApplicationObjects(namespace, application, *appConfig, informerFactories)
+	log.Logger.Debugf("identified %d related objects", len(applicationObjs))
 
-// handle constructs the application object from the objects currently in the cluster
-func handle(w watchedObject, resourceTypes []schema.GroupVersionResource, informerFactories map[string]dynamicinformer.DynamicSharedInformerFactory) {
-	application, _ := w.getNamespacedName()
-	namespace := w.getNamespace()
-	name, _ := w.getName()
-	version, _ := w.getVersion()
-	log.Logger.Tracef("handle called for %s (%s)", application, version)
-
-	applicationObjs := getApplicationObjects(namespace, name, resourceTypes, informerFactories)
-	// there is at least one object (w)
-
-	a, _ := abnapp.Applications.Read(application)
-
-	abnapp.Applications.Lock(application)
-	defer abnapp.Applications.Unlock(application)
-	// clear a.Tracks, a.Versions[*].Track
-	// this is necessary because  we keep old versions in memory
+	// update the application object by updating the mapping of track to version
+	//   get the current application
+	a, _ := abnapp.Applications.Read(namespace + "/" + application)
+
+	abnapp.Applications.Lock(namespace + "/" + application)
+	defer abnapp.Applications.Unlock(namespace + "/" + application)
+
+	//   clear the current mapping
 	a.ClearTracks()
 
-	for _, o := range applicationObjs {
-		version, _ := o.getVersion()
-		a.GetVersion(version, true) // make sure version object created
-		track := o.getTrack()
-		if track != "" {
+	//   for each track, find the version (from cluster objects) and update the mapping
+	for _, track := range getTrackNames(application, *appConfig) {
+		log.Logger.Trace("updateApplication for track ", track)
+		version := isTrackReady(track, applicationObjs[track], len(appConfig.Resources))
+		log.Logger.Trace("updateApplication for track ", track, " found version ", version)
+		if version != "" {
+			a.GetVersion(version, true)
 			a.Tracks[track] = version
 		}
 	}
-}
 
-// getApplicationObjects gets all the objects related to the application based on label app.kubernetes.io/name
-func getApplicationObjects(namespace, name string, gvrs []schema.GroupVersionResource, informerFactories map[string]dynamicinformer.DynamicSharedInformerFactory) []watchedObject {
-	// define selector
-	selector := labels.NewSelector()
-	reqSpec := []struct {
-		key  string
-		op   selection.Operator
-		vals []string
-	}{
-		{key: iter8Label, op: selection.Equals, vals: []string{"true"}},
-		{key: nameLabel, op: selection.Equals, vals: []string{name}},
-		{key: versionLabel, op: selection.Exists, vals: []string{}},
-	}
-	for _, rs := range reqSpec {
-		req, err := labels.NewRequirement(rs.key, rs.op, rs.vals)
+	log.Logger.Debugf("updated application track map: %s/%s --> %v", namespace, application, a.Tracks)
+
+	if obj.GetDeletionTimestamp() != nil && containsString(obj.GetFinalizers(), iter8Finalizer) {
+		// if object is being deleted remove the Iter8 finalizer
+		// do here (at end) after updating the ApplicationsMap
+		log.Logger.Debug("removing Iter8 finalizer")
+		obj.SetFinalizers(removeIter8Finalizer(obj.GetFinalizers()))
+		_, err := updateUnstructuredObject(obj, gvr)
 		if err != nil {
-			log.Logger.Warn(err)
-			return []watchedObject{}
+			log.Logger.Warn("unable to remove finalizer: ", err.Error())
+		}
+	}
+}
+
+func removeIter8Finalizer(finalizers []string) []string {
+	for i, v := range finalizers {
+		if v == iter8Finalizer {
+			return append(finalizers[:i], finalizers[i+1:]...)
 		}
-		selector = selector.Add(*req)
+	}
+	return finalizers
+}
+
+// getApplicationFromObjectName converts the name of the object to the application name
+// it converts a name of the form: app[-candidate-index] to a name of the form: app
+func getApplicationNameFromObjectName(name string) string {
+	locations := applicationNameRE.FindStringIndex(name)
+	if len(locations) == 0 {
+		return name
+	}
+	return name[:locations[0]]
+}
+
+// getVersion gets application version from VERSION_LABEL label on an object
+func getVersion(obj *unstructured.Unstructured) (string, bool) {
+	labels := obj.GetLabels()
+	v, ok := labels[versionLabel]
+	return v, ok
+}
+
+func isObjectReady(obj *unstructured.Unstructured, gvr schema.GroupVersionResource, condition *string) bool {
+	// no condition to check; is ready
+	if condition == nil {
+		return true
+	}
+
+	cs, err := getConditionStatus(obj, *condition)
+	if err != nil {
+		log.Logger.Error("unable to get status: ", err.Error())
+		return false
+	}
+	if strings.EqualFold(*cs, string(corev1.ConditionTrue)) {
+		return true
+	}
+
+	// condition not true
+	return false
+}
+
+// TODO rewrite using NestedStringMap
+func getConditionStatus(obj *unstructured.Unstructured, conditionType string) (*string, error) {
+	if obj == nil {
+		return nil, errors.New("no object")
 	}
 
-	watchedObjects := []watchedObject{}
-	for _, gvr := range gvrs {
-		lister := informerFactories[namespace].ForResource(gvr).Lister()
-		objs, err := lister.List(selector)
+	resultJSON, err := obj.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	resultObj := make(map[string]interface{})
+	err = json.Unmarshal(resultJSON, &resultObj)
+	if err != nil {
+		return nil, err
+	}
+
+	// get object status
+	objStatusInterface, ok := resultObj["status"]
+	if !ok {
+		return nil, errors.New("object does not contain a status")
+	}
+	objStatus := objStatusInterface.(map[string]interface{})
+
+	conditionsInterface, ok := objStatus["conditions"]
+	if !ok {
+		return nil, errors.New("object status does not contain conditions")
+	}
+	conditions := conditionsInterface.([]interface{})
+	for _, conditionInterface := range conditions {
+		condition := conditionInterface.(map[string]interface{})
+		cTypeInterface, ok := condition["type"]
+		if !ok {
+			return nil, errors.New("condition does not have a type")
+		}
+		cType := cTypeInterface.(string)
+		if strings.EqualFold(cType, conditionType) {
+			conditionStatusInterface, ok := condition["status"]
+			if !ok {
+				return nil, fmt.Errorf("condition %s does not have a value", cType)
+			}
+			conditionStatus := conditionStatusInterface.(string)
+			return &conditionStatus, nil
+		}
+	}
+	return nil, errors.New("expected condition not found")
+}
+
+// trackObject is information about an object found to correspond to a track
+type trackObject struct {
+	gvr       schema.GroupVersionResource
+	condition *string
+	object    *unstructured.Unstructured
+}
+
+// getApplicationObjects identifies a list of objects related to application based on the name
+func getApplicationObjects(namespace string, application string, appConfig appDetails, informerFactories map[string]dynamicinformer.DynamicSharedInformerFactory) map[string][]trackObject {
+	// initialize
+	var trackToObjectList = map[string][]trackObject{}
+	tracks := getTrackNames(application, appConfig)
+	for _, track := range tracks {
+		trackToObjectList[track] = []trackObject{}
+	}
+
+	// get objects by resource type
+	for _, r := range appConfig.Resources {
+		lister := informerFactories[namespace].ForResource(r.GroupVersionResource).Lister()
+		objs, err := lister.List(labels.NewSelector())
 		if err != nil {
-			log.Logger.Warn(err)
+			// no such objects; can happen if not deployed
 			continue
 		}
+		// reduce to only those that match expectedObjectNames
+		// all objects are of the same type but for different tracks
 		for _, obj := range objs {
-			wo := watchedObject{Obj: obj.(*unstructured.Unstructured)}
-			if precond(wo) {
-				watchedObjects = append(watchedObjects, wo)
+			ao := obj.(*unstructured.Unstructured)
+			if ao.GetDeletionTimestamp() != nil {
+				// if being deleted, ignore
+				continue
 			}
+			name := ao.GetName()
+			_, ok := trackToObjectList[name]
+			if !ok {
+				// object is not associated with a known track; ignore
+				continue
+			}
+			// create trackObject and add to list of objects for track
+			trackToObjectList[name] = append(trackToObjectList[name], trackObject{gvr: r.GroupVersionResource, condition: r.Condition, object: ao})
 		}
 	}
-	return watchedObjects
+
+	return trackToObjectList
+}
+
+// isTrackReady checks that all expected objects for the track exist, that the version is defined (consistently) and that the objects are ready
+func isTrackReady(track string, trackObjects []trackObject, expectedNumberTrackObjects int) string {
+	// all objects exist
+	if len(trackObjects) != expectedNumberTrackObjects {
+		log.Logger.Debugf("expected %d objects; found %d (track: %s)", expectedNumberTrackObjects, len(trackObjects), track)
+		return ""
+	}
+
+	// single version identified on at least one object
+	var version string
+	for _, to := range trackObjects {
+		v, ok := getVersion(to.object)
+		if ok {
+			if version != "" && version != v {
+				// different versions on resources of the same track
+				log.Logger.Debugf("inconsistent value for label %s (track: %s)", versionLabel, track)
+				return ""
+			}
+			version = v
+		}
+	}
+	if version == "" {
+		log.Logger.Debugf("no value for label %s found (track: %s)", versionLabel, track)
+		return ""
+	}
+
+	// all objects are ready
+	for _, to := range trackObjects {
+		if !isObjectReady(to.object, to.gvr, to.condition) {
+			log.Logger.Debugf("no object found of type %v (track: %s)", to.gvr, track)
+			return ""
+		}
+	}
+
+	return version
+}
+
+func updateUnstructuredObject(uObj *unstructured.Unstructured, gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) {
+	updatedObj, err := k8sclient.Client.Dynamic().
+		Resource(gvr).Namespace(uObj.GetNamespace()).
+		Update(
+			context.TODO(),
+			uObj,
+			metav1.UpdateOptions{},
+		)
+
+	return updatedObj, err
+}
+
+func getUnstructuredObject(uObj *unstructured.Unstructured, gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) {
+	obj, err := k8sclient.Client.Dynamic().
+		Resource(gvr).Namespace(uObj.GetNamespace()).
+		Get(
+			context.TODO(),
+			uObj.GetName(),
+			metav1.GetOptions{},
+		)
+
+	return obj, err
 }
diff --git a/abn/watcher/informer.go b/abn/watcher/informer.go
index c617ee8d..9944af4b 100644
--- a/abn/watcher/informer.go
+++ b/abn/watcher/informer.go
@@ -19,29 +19,49 @@ type Iter8Watcher struct {
 }
 
 // NewIter8Watcher returns a watcher for iter8 related objects
-func NewIter8Watcher(resourceTypes []schema.GroupVersionResource, namespaces []string) *Iter8Watcher {
+func NewIter8Watcher(configFile string) *Iter8Watcher {
+	c := readServiceConfig(configFile)
+
 	w := &Iter8Watcher{
 		factories: map[string]dynamicinformer.DynamicSharedInformerFactory{},
 	}
 
-	handlerFunc := func(obj interface{}) {
-		wo := watchedObject{Obj: obj.(*unstructured.Unstructured)}
-		if precond(wo) {
-			handle(wo, resourceTypes, w.factories)
+	handlerFunc := func(validNames []string, gvr schema.GroupVersionResource) func(obj interface{}) {
+		return func(obj interface{}) {
+			o := obj.(*unstructured.Unstructured)
+			if containsString(validNames, o.GetName()) {
+				handle(o, c, w.factories, gvr)
+			}
 		}
 	}
 
-	// for each namespace, resource type configure Informer
-	for _, ns := range namespaces {
+	// for each namespace:
+	//   identify all resources to watch (create informer for each), and
+	//   names of all resources to watch
+	//   create informerfactory
+	for ns, apps := range c {
+		var byResource map[schema.GroupVersionResource]([]string) = map[schema.GroupVersionResource]([]string){}
 		w.factories[ns] = dynamicinformer.NewFilteredDynamicSharedInformerFactory(k8sclient.Client.Dynamic(), 0, ns, nil)
-		for _, gvr := range resourceTypes {
+
+		// identify resources expected in namespace and list of resource names expected
+		// the resources will be used to create informers
+		// the list of names will be used to filter the objects that trigger the informer
+		for nm, details := range apps {
+			validNames := getValidObjectNames(nm, details.MaxNumCandidates)
+			for _, r := range details.Resources {
+				byResource[r.GroupVersionResource] = append(byResource[r.GroupVersionResource], validNames...)
+			}
+		}
+
+		// create informer for each resource in namespace
+		for gvr, validNames := range byResource {
 			informer := w.factories[ns].ForResource(gvr)
 			_, err := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-				AddFunc: handlerFunc,
+				AddFunc: handlerFunc(validNames, gvr),
 				UpdateFunc: func(oldObj, obj interface{}) {
-					handlerFunc(obj)
+					handlerFunc(validNames, gvr)(obj)
 				},
-				DeleteFunc: handlerFunc,
+				DeleteFunc: handlerFunc(validNames, gvr),
 			})
 
 			if err != nil {
@@ -58,3 +78,32 @@ func (watcher *Iter8Watcher) Start(stopChannel chan struct{}) {
 		f.Start(stopChannel)
 	}
 }
+
+// valid object names for an application
+// based on the assumption that the names are of the form:
+//
+//	app, app-candidate-i for i = 1, 2, ..., maxNumCandidates
+func getValidObjectNames(application string, maxNum int) []string {
+	expectedObjectNames := make([]string, maxNum+1)
+	expectedObjectNames[0] = application
+	for i := 1; i <= maxNum; i++ {
+		expectedObjectNames[i] = fmt.Sprintf("%s-candidate-%d", application, i)
+	}
+	return expectedObjectNames
+}
+
+// getTrackNames creates list of expected getTrackNames
+// based on the assumption that the track names are the same as the object names
+func getTrackNames(application string, appConfig appDetails) []string {
+	return getValidObjectNames(application, appConfig.MaxNumCandidates)
+}
+
+// containsString determines if an array contains a specific string
+func containsString(array []string, s string) bool {
+	for _, v := range array {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
diff --git a/abn/watcher/watchedobject.go b/abn/watcher/watchedobject.go
deleted file mode 100644
index ff369a14..00000000
--- a/abn/watcher/watchedobject.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package watcher
-
-// watchedobject.go - methods to read fields from an unstructured Kubernetes object
-
-import (
-	"strings"
-
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-)
-
-const (
-	nameLabel    = "app.kubernetes.io/name"
-	versionLabel = "app.kubernetes.io/version"
-	trackLabel   = "iter8.tools/track"
-	iter8Label   = "iter8.tools/abn"
-)
-
-// watchedObject is wrapper for object returned by informer
-type watchedObject struct {
-	// Obj is the Kubernetes object
-	Obj *unstructured.Unstructured
-}
-
-// getName gets application name from NAME_LABEL label on watched object
-func (wo watchedObject) getName() (string, bool) {
-	labels := wo.Obj.GetLabels()
-	name, ok := labels[nameLabel]
-	return name, ok
-}
-
-// getNamespace gets namespace of watched object
-func (wo watchedObject) getNamespace() string {
-	return wo.Obj.GetNamespace()
-}
-
-// getNamespacedName returns formatted namespace and (application) name
-func (wo watchedObject) getNamespacedName() (string, bool) {
-	n, ok := wo.getName()
-	return wo.getNamespace() + "/" + n, ok
-}
-
-// getVersion gets application version from VERSION_LABEL label on watched object
-func (wo watchedObject) getVersion() (string, bool) {
-	labels := wo.Obj.GetLabels()
-	v, ok := labels[versionLabel]
-	return v, ok
-}
-
-// getTrack get trace of version from TRACK_ANNOTATION annotation on watched object
-func (wo watchedObject) getTrack() string {
-	labels := wo.Obj.GetLabels()
-	track, ok := labels[trackLabel]
-	if !ok {
-		return ""
-	}
-	return track
-}
-
-func (wo watchedObject) isIter8AbnRelated() bool {
-	labels := wo.Obj.GetLabels()
-	iter8, ok := labels[iter8Label]
-	if !ok {
-		return false
-	}
-	return strings.ToLower(iter8) == "true"
-}


Test patch:
diff --git a/abn/application/application_test.go b/abn/application/application_test.go
index 42a9813b..d6a656bf 100644
--- a/abn/application/application_test.go
+++ b/abn/application/application_test.go
@@ -302,8 +302,10 @@ func setupInMemoryNotInCluster(t *testing.T, applications ...applicationSource)
 
 func writeVerify(t *testing.T, a *Application) *Application {
 	application := a.Name
+	err := ensureSecretCreated(application)
+	assert.NoError(t, err)
 	// write application to cluster (should create the secret, if not present)
-	err := Applications.Write(a)
+	err = Applications.Write(a)
 	assert.NoError(t, err)
 
 	// verify can read it back
diff --git a/abn/application/test_helper.go b/abn/application/test_helper.go
index d05456c7..32bc04bf 100644
--- a/abn/application/test_helper.go
+++ b/abn/application/test_helper.go
@@ -1,6 +1,7 @@
 package application
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -8,16 +9,41 @@ import (
 	"testing"
 	"time"
 
+	"github.com/iter8-tools/iter8/abn/k8sclient"
 	"github.com/iter8-tools/iter8/base/summarymetrics"
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 )
 
 func yamlToSecret(folder, file, name string) error {
+	err := ensureSecretCreated(name)
+	if err != nil {
+		return err
+	}
 	a, _ := yamlToApplication(name, folder, file)
 	return Applications.Write(a)
 }
 
+func ensureSecretCreated(application string) error {
+	namespace := namespaceFromKey(application)
+	name := secretNameFromKey(application)
+	_, err := k8sclient.Client.Typed().CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{})
+	if err == nil {
+		return nil
+	}
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+	_, err = k8sclient.Client.Typed().CoreV1().Secrets(namespace).Create(context.Background(), secret, metav1.CreateOptions{})
+	return err
+}
+
 func readYamlFromFile(folder, file string) ([]byte, error) {
 	_, filename, _, _ := runtime.Caller(1) // one step up the call stack
 	fname := filepath.Join(filepath.Dir(filename), folder, file)
diff --git a/abn/core/config_test.go b/abn/core/config_test.go
deleted file mode 100644
index 946c831f..00000000
--- a/abn/core/config_test.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package core
-
-import (
-	"path/filepath"
-	"runtime"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestReadConfig(t *testing.T) {
-	for _, tt := range []struct {
-		name          string
-		file          string
-		numNamespaces int
-		numResources  int
-	}{
-		{"empty", "config.empty.yaml", 0, 0},
-		{"nonamespaces", "config.nonamespaces.yaml", 0, 1},
-		{"invalid", "config.invalid.yaml", 0, 0},
-		{"garbage", "config.garbage.yaml", 0, 0},
-		{"nofile", "config.nofile.yaml", 0, 0},
-	} {
-		t.Run(tt.name, func(t *testing.T) {
-			c := readConfig(completePath("../../testdata/abninputs", tt.file))
-			assert.Equal(t, tt.numNamespaces, len(c.Namespaces))
-			assert.Equal(t, tt.numResources, len(c.Resources))
-		})
-	}
-}
-
-// utility method
-func completePath(prefix string, suffix string) string {
-	_, filename, _, _ := runtime.Caller(1) // one step up the call stack
-	return filepath.Join(filepath.Dir(filename), prefix, suffix)
-}
diff --git a/abn/core/grpc_service_test.go b/abn/core/grpc_service_test.go
index 1ce5b378..2549b152 100644
--- a/abn/core/grpc_service_test.go
+++ b/abn/core/grpc_service_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -19,6 +20,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"helm.sh/helm/v3/pkg/cli"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 )
 
@@ -36,7 +39,7 @@ type Scenario struct {
 
 func TestLookup(t *testing.T) {
 	testcases := map[string]Scenario{
-		"no application": {application: "default/noapp", user: "foobar", errorSubstring: "application not found", track: ""},
+		"no application": {application: "default/noapp", user: "foobar", errorSubstring: "application default/noapp not found", track: ""},
 		"no user":        {application: "default/application", user: "", errorSubstring: "no user session provided", track: ""},
 		"valid":          {application: "default/application", user: "user", errorSubstring: "", track: "candidate"},
 	}
@@ -168,6 +171,8 @@ func setup(t *testing.T) (*pb.ABNClient, func()) {
 	a, err := yamlToApplication("default/application", "../../testdata", "abninputs/readtest.yaml")
 	assert.NoError(t, err)
 	abnapp.Applications.Put(a)
+	err = ensureSecretCreated(a.Name)
+	assert.NoError(t, err)
 
 	// 49152-65535 are recommended ports; we use a random one for testing
 	/* #nosec */
@@ -244,3 +249,50 @@ func getMetric(a *abnapp.Application, track, metric string) *summarymetrics.Summ
 	m, _ := v.GetMetric(metric, true)
 	return m
 }
+
+func ensureSecretCreated(application string) error {
+	namespace := namespaceFromKey(application)
+	name := secretNameFromKey(application)
+	_, err := k8sclient.Client.Typed().CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{})
+	if err != nil {
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+		}
+		_, err = k8sclient.Client.Typed().CoreV1().Secrets(namespace).Create(context.Background(), secret, metav1.CreateOptions{})
+		return err
+	}
+	return nil
+}
+
+// nameFromKey returns the name from a key of the form "namespace/name"
+func nameFromKey(applicationKey string) string {
+	_, n := splitApplicationKey(applicationKey)
+	return n
+}
+
+// secretNameFromKey returns the name of the secret used to persist an applicatiob
+func secretNameFromKey(applicationKey string) string {
+	return nameFromKey(applicationKey) + "-metrics"
+}
+
+// namespaceFromKey returns the namespace from a key of the form "namespace/name"
+func namespaceFromKey(applicationKey string) string {
+	ns, _ := splitApplicationKey(applicationKey)
+	return ns
+}
+
+// splitApplicationKey is a utility function that returns the name and namespace from a key of the form "namespace/name"
+func splitApplicationKey(applicationKey string) (string, string) {
+	var name, namespace string
+	names := strings.Split(applicationKey, "/")
+	if len(names) > 1 {
+		namespace, name = names[0], names[1]
+	} else {
+		namespace, name = "default", names[0]
+	}
+
+	return namespace, name
+}
diff --git a/abn/k8sclient/test_helper.go b/abn/k8sclient/test_helper.go
index 9358a06b..2da54118 100644
--- a/abn/k8sclient/test_helper.go
+++ b/abn/k8sclient/test_helper.go
@@ -1,11 +1,11 @@
 package k8sclient
 
 import (
+	testing "github.com/iter8-tools/iter8/abn/k8sclient/testing"
 	"helm.sh/helm/v3/pkg/cli"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	dynamicfake "k8s.io/client-go/dynamic/fake"
 	"k8s.io/client-go/kubernetes/fake"
 	ktesting "k8s.io/client-go/testing"
 )
@@ -40,8 +40,7 @@ func NewFakeKubeClient(s *cli.EnvSettings, objects ...runtime.Object) *KubeClien
 	fc.PrependReactor("update", "secrets", secretDataReactor)
 	fakeClient.typedClient = fc
 
-	// fakeClient.dynamicClient = dynamicfake.NewSimpleDynamicClient(runtime.NewScheme())
-	fakeClient.dynamicClient = dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
+	fakeClient.dynamicClient = testing.NewSimpleDynamicClientWithCustomListKinds(
 		runtime.NewScheme(),
 		map[schema.GroupVersionResource]string{
 			{Group: "apps", Version: "v1", Resource: "deployments"}: "DeploymentList",
diff --git a/abn/k8sclient/testing/doc.go b/abn/k8sclient/testing/doc.go
new file mode 100644
index 00000000..dc9610f4
--- /dev/null
+++ b/abn/k8sclient/testing/doc.go
@@ -0,0 +1,6 @@
+// Package testing provides a fake dynamic client that supports finalizers.
+package testing
+
+// The default fake dynamic client does not do so. To add the appropriate behavior (if a finalizer is present,
+// then add a deletion timestamp and update instead). To do so, it is necessary to reimplement the underlying
+// object tracker (fixture.go).
diff --git a/abn/k8sclient/testing/fixture.go b/abn/k8sclient/testing/fixture.go
new file mode 100644
index 00000000..e222a6f0
--- /dev/null
+++ b/abn/k8sclient/testing/fixture.go
@@ -0,0 +1,332 @@
+package testing
+
+// Mostly copied from https://github.com/kubernetes/client-go/blob/e7cd4ba474b5efc2882e377362c9aa8b407428d9/testing/fixture.go
+// Changes were made in Delete() method to handle finalizers; that is, to change delete to update if a finalizer exists
+// Finalizers are not handled in original
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+
+	ktesting "k8s.io/client-go/testing"
+)
+
+type tracker struct {
+	scheme  ktesting.ObjectScheme
+	decoder runtime.Decoder
+	lock    sync.RWMutex
+	objects map[schema.GroupVersionResource]map[types.NamespacedName]runtime.Object
+	// The value type of watchers is a map of which the key is either a namespace or
+	// all/non namespace aka "" and its value is list of fake watchers.
+	// Manipulations on resources will broadcast the notification events into the
+	// watchers' channel. Note that too many unhandled events (currently 100,
+	// see apimachinery/pkg/watch.DefaultChanSize) will cause a panic.
+	watchers map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher
+}
+
+var _ ktesting.ObjectTracker = &tracker{}
+
+// NewObjectTracker returns an ObjectTracker that can be used to keep track
+// of objects for the fake clientset. Mostly useful for unit tests.
+func NewObjectTracker(scheme ktesting.ObjectScheme, decoder runtime.Decoder) ktesting.ObjectTracker {
+	return &tracker{
+		scheme:   scheme,
+		decoder:  decoder,
+		objects:  make(map[schema.GroupVersionResource]map[types.NamespacedName]runtime.Object),
+		watchers: make(map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher),
+	}
+}
+
+func (t *tracker) List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string) (runtime.Object, error) {
+	// Heuristic for list kind: original kind + List suffix. Might
+	// not always be true but this tracker has a pretty limited
+	// understanding of the actual API model.
+	listGVK := gvk
+	listGVK.Kind = listGVK.Kind + "List"
+	// GVK does have the concept of "internal version". The scheme recognizes
+	// the runtime.APIVersionInternal, but not the empty string.
+	if listGVK.Version == "" {
+		listGVK.Version = runtime.APIVersionInternal
+	}
+
+	list, err := t.scheme.New(listGVK)
+	if err != nil {
+		return nil, err
+	}
+
+	if !meta.IsListType(list) {
+		return nil, fmt.Errorf("%q is not a list type", listGVK.Kind)
+	}
+
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	objs, ok := t.objects[gvr]
+	if !ok {
+		return list, nil
+	}
+
+	matchingObjs, err := filterByNamespace(objs, ns)
+	if err != nil {
+		return nil, err
+	}
+	if err := meta.SetList(list, matchingObjs); err != nil {
+		return nil, err
+	}
+	return list.DeepCopyObject(), nil
+}
+
+func (t *tracker) Watch(gvr schema.GroupVersionResource, ns string) (watch.Interface, error) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	fakewatcher := watch.NewRaceFreeFake()
+
+	if _, exists := t.watchers[gvr]; !exists {
+		t.watchers[gvr] = make(map[string][]*watch.RaceFreeFakeWatcher)
+	}
+	t.watchers[gvr][ns] = append(t.watchers[gvr][ns], fakewatcher)
+	return fakewatcher, nil
+}
+
+func (t *tracker) Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error) {
+	errNotFound := errors.NewNotFound(gvr.GroupResource(), name)
+
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	objs, ok := t.objects[gvr]
+	if !ok {
+		return nil, errNotFound
+	}
+
+	matchingObj, ok := objs[types.NamespacedName{Namespace: ns, Name: name}]
+	if !ok {
+		return nil, errNotFound
+	}
+
+	// Only one object should match in the tracker if it works
+	// correctly, as Add/Update methods enforce kind/namespace/name
+	// uniqueness.
+	obj := matchingObj.DeepCopyObject()
+	if status, ok := obj.(*metav1.Status); ok {
+		if status.Status != metav1.StatusSuccess {
+			return nil, &errors.StatusError{ErrStatus: *status}
+		}
+	}
+
+	return obj, nil
+}
+
+func (t *tracker) Add(obj runtime.Object) error {
+	if meta.IsListType(obj) {
+		return t.addList(obj, false)
+	}
+	objMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	gvks, _, err := t.scheme.ObjectKinds(obj)
+	if err != nil {
+		return err
+	}
+
+	if partial, ok := obj.(*metav1.PartialObjectMetadata); ok && len(partial.TypeMeta.APIVersion) > 0 {
+		gvks = []schema.GroupVersionKind{partial.TypeMeta.GroupVersionKind()}
+	}
+
+	if len(gvks) == 0 {
+		return fmt.Errorf("no registered kinds for %v", obj)
+	}
+	for _, gvk := range gvks {
+		// NOTE: UnsafeGuessKindToResource is a heuristic and default match. The
+		// actual registration in apiserver can specify arbitrary route for a
+		// gvk. If a test uses such objects, it cannot preset the tracker with
+		// objects via Add(). Instead, it should trigger the Create() function
+		// of the tracker, where an arbitrary gvr can be specified.
+		gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+		// Resource doesn't have the concept of "__internal" version, just set it to "".
+		if gvr.Version == runtime.APIVersionInternal {
+			gvr.Version = ""
+		}
+
+		err := t.add(gvr, obj, objMeta.GetNamespace(), false)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	return t.add(gvr, obj, ns, false)
+}
+
+func (t *tracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	return t.add(gvr, obj, ns, true)
+}
+
+func (t *tracker) getWatches(gvr schema.GroupVersionResource, ns string) []*watch.RaceFreeFakeWatcher {
+	watches := []*watch.RaceFreeFakeWatcher{}
+	if t.watchers[gvr] != nil {
+		if w := t.watchers[gvr][ns]; w != nil {
+			watches = append(watches, w...)
+		}
+		if ns != metav1.NamespaceAll {
+			if w := t.watchers[gvr][metav1.NamespaceAll]; w != nil {
+				watches = append(watches, w...)
+			}
+		}
+	}
+	return watches
+}
+
+func (t *tracker) add(gvr schema.GroupVersionResource, obj runtime.Object, ns string, replaceExisting bool) error {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	gr := gvr.GroupResource()
+
+	// To avoid the object from being accidentally modified by caller
+	// after it's been added to the tracker, we always store the deep
+	// copy.
+	obj = obj.DeepCopyObject()
+
+	newMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	// Propagate namespace to the new object if hasn't already been set.
+	if len(newMeta.GetNamespace()) == 0 {
+		newMeta.SetNamespace(ns)
+	}
+
+	if ns != newMeta.GetNamespace() {
+		msg := fmt.Sprintf("request namespace does not match object namespace, request: %q object: %q", ns, newMeta.GetNamespace())
+		return errors.NewBadRequest(msg)
+	}
+
+	_, ok := t.objects[gvr]
+	if !ok {
+		t.objects[gvr] = make(map[types.NamespacedName]runtime.Object)
+	}
+
+	namespacedName := types.NamespacedName{Namespace: newMeta.GetNamespace(), Name: newMeta.GetName()}
+	if _, ok = t.objects[gvr][namespacedName]; ok {
+		if replaceExisting {
+			for _, w := range t.getWatches(gvr, ns) {
+				// To avoid the object from being accidentally modified by watcher
+				w.Modify(obj.DeepCopyObject())
+			}
+			t.objects[gvr][namespacedName] = obj
+			return nil
+		}
+		return errors.NewAlreadyExists(gr, newMeta.GetName())
+	}
+
+	if replaceExisting {
+		// Tried to update but no matching object was found.
+		return errors.NewNotFound(gr, newMeta.GetName())
+	}
+
+	t.objects[gvr][namespacedName] = obj
+
+	for _, w := range t.getWatches(gvr, ns) {
+		// To avoid the object from being accidentally modified by watcher
+		w.Add(obj.DeepCopyObject())
+	}
+
+	return nil
+}
+
+func (t *tracker) addList(obj runtime.Object, replaceExisting bool) error {
+	list, err := meta.ExtractList(obj)
+	if err != nil {
+		return err
+	}
+	errs := runtime.DecodeList(list, t.decoder)
+	if len(errs) > 0 {
+		return errs[0]
+	}
+	for _, obj := range list {
+		if err := t.Add(obj); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tracker) Delete(gvr schema.GroupVersionResource, ns, name string) error {
+	t.lock.Lock()
+	// defer t.lock.Unlock()
+
+	objs, ok := t.objects[gvr]
+	if !ok {
+		t.lock.Unlock()
+		return errors.NewNotFound(gvr.GroupResource(), name)
+	}
+
+	namespacedName := types.NamespacedName{Namespace: ns, Name: name}
+	obj, ok := objs[namespacedName]
+	if !ok {
+		t.lock.Unlock()
+		return errors.NewNotFound(gvr.GroupResource(), name)
+	}
+
+	// additions for finalizers
+	// if finalizers, set DeletionTimestamp and convert to update, else delete
+	uObj := obj.(*unstructured.Unstructured)
+	if len(uObj.GetFinalizers()) > 0 {
+		now := metav1.Now()
+		uObj.SetDeletionTimestamp(&now)
+		t.lock.Unlock()
+		return t.Update(gvr, uObj, ns)
+	}
+
+	delete(objs, namespacedName)
+	for _, w := range t.getWatches(gvr, ns) {
+		w.Delete(obj.DeepCopyObject())
+	}
+	t.lock.Unlock()
+	return nil
+}
+
+// filterByNamespace returns all objects in the collection that
+// match provided namespace. Empty namespace matches
+// non-namespaced objects.
+func filterByNamespace(objs map[types.NamespacedName]runtime.Object, ns string) ([]runtime.Object, error) {
+	var res []runtime.Object
+
+	for _, obj := range objs {
+		acc, err := meta.Accessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		if ns != "" && acc.GetNamespace() != ns {
+			continue
+		}
+		res = append(res, obj)
+	}
+
+	// Sort res to get deterministic order.
+	sort.Slice(res, func(i, j int) bool {
+		acc1, _ := meta.Accessor(res[i])
+		acc2, _ := meta.Accessor(res[j])
+		if acc1.GetNamespace() != acc2.GetNamespace() {
+			return acc1.GetNamespace() < acc2.GetNamespace()
+		}
+		return acc1.GetName() < acc2.GetName()
+	})
+	return res, nil
+}
diff --git a/abn/k8sclient/testing/simple.go b/abn/k8sclient/testing/simple.go
new file mode 100644
index 00000000..57d65e29
--- /dev/null
+++ b/abn/k8sclient/testing/simple.go
@@ -0,0 +1,460 @@
+package testing
+
+// Mostly copied from https://github.com/kubernetes/client-go/blob/e7cd4ba474b5efc2882e377362c9aa8b407428d9/dynamic/fake/simple.go
+// Some unused methods not included
+// Primary change is use of locally defined method NewObjectTracker(). See fixture.go
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/testing"
+)
+
+// NewSimpleDynamicClientWithCustomListKinds try not to use this.  In general you want to have the scheme have the List types registered
+// and allow the default guessing for resources match.  Sometimes that doesn't work, so you can specify a custom mapping here.
+func NewSimpleDynamicClientWithCustomListKinds(scheme *runtime.Scheme, gvrToListKind map[schema.GroupVersionResource]string, objects ...runtime.Object) *FakeDynamicClient {
+	// In order to use List with this client, you have to have your lists registered so that the object tracker will find them
+	// in the scheme to support the t.scheme.New(listGVK) call when it's building the return value.
+	// Since the base fake client needs the listGVK passed through the action (in cases where there are no instances, it
+	// cannot look up the actual hits), we need to know a mapping of GVR to listGVK here.  For GETs and other types of calls,
+	// there is no return value that contains a GVK, so it doesn't have to know the mapping in advance.
+
+	// first we attempt to invert known List types from the scheme to auto guess the resource with unsafe guesses
+	// this covers common usage of registering types in scheme and passing them
+	completeGVRToListKind := map[schema.GroupVersionResource]string{}
+	for listGVK := range scheme.AllKnownTypes() {
+		if !strings.HasSuffix(listGVK.Kind, "List") {
+			continue
+		}
+		nonListGVK := listGVK.GroupVersion().WithKind(listGVK.Kind[:len(listGVK.Kind)-4])
+		plural, _ := meta.UnsafeGuessKindToResource(nonListGVK)
+		completeGVRToListKind[plural] = listGVK.Kind
+	}
+
+	for gvr, listKind := range gvrToListKind {
+		if !strings.HasSuffix(listKind, "List") {
+			panic("coding error, listGVK must end in List or this fake client doesn't work right")
+		}
+		listGVK := gvr.GroupVersion().WithKind(listKind)
+
+		// if we already have this type registered, just skip it
+		if _, err := scheme.New(listGVK); err == nil {
+			completeGVRToListKind[gvr] = listKind
+			continue
+		}
+
+		scheme.AddKnownTypeWithName(listGVK, &unstructured.UnstructuredList{})
+		completeGVRToListKind[gvr] = listKind
+	}
+
+	codecs := serializer.NewCodecFactory(scheme)
+	o := NewObjectTracker(scheme, codecs.UniversalDecoder())
+	for _, obj := range objects {
+		if err := o.Add(obj); err != nil {
+			panic(err)
+		}
+	}
+
+	cs := &FakeDynamicClient{scheme: scheme, gvrToListKind: completeGVRToListKind, tracker: o}
+	cs.AddReactor("*", "*", testing.ObjectReaction(o))
+	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
+		gvr := action.GetResource()
+		ns := action.GetNamespace()
+		watch, err := o.Watch(gvr, ns)
+		if err != nil {
+			return false, nil, err
+		}
+		return true, watch, nil
+	})
+
+	return cs
+}
+
+// FakeDynamicClient ...
+// Clientset implements clientset.Interface. Meant to be embedded into a
+// struct to get a default implementation. This makes faking out just the method
+// you want to test easier.
+type FakeDynamicClient struct {
+	testing.Fake
+	scheme        *runtime.Scheme
+	gvrToListKind map[schema.GroupVersionResource]string
+	tracker       testing.ObjectTracker
+}
+
+type dynamicResourceClient struct {
+	client    *FakeDynamicClient
+	namespace string
+	resource  schema.GroupVersionResource
+	listKind  string
+}
+
+var (
+	_ dynamic.Interface  = &FakeDynamicClient{}
+	_ testing.FakeClient = &FakeDynamicClient{}
+)
+
+// Tracker ...
+func (c *FakeDynamicClient) Tracker() testing.ObjectTracker {
+	return c.tracker
+}
+
+// Resource ...
+func (c *FakeDynamicClient) Resource(resource schema.GroupVersionResource) dynamic.NamespaceableResourceInterface {
+	return &dynamicResourceClient{client: c, resource: resource, listKind: c.gvrToListKind[resource]}
+}
+
+func (c *dynamicResourceClient) Namespace(ns string) dynamic.ResourceInterface {
+	ret := *c
+	ret.namespace = ns
+	return &ret
+}
+
+func (c *dynamicResourceClient) Create(ctx context.Context, obj *unstructured.Unstructured, opts metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
+	var uncastRet runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootCreateAction(c.resource, obj), obj)
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		var accessor metav1.Object // avoid shadowing err
+		accessor, err = meta.Accessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		name := accessor.GetName()
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), obj), obj)
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewCreateAction(c.resource, c.namespace, obj), obj)
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		var accessor metav1.Object // avoid shadowing err
+		accessor, err = meta.Accessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		name := accessor.GetName()
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), c.namespace, obj), obj)
+
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, err
+}
+
+func (c *dynamicResourceClient) Update(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) {
+	var uncastRet runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootUpdateAction(c.resource, obj), obj)
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), obj), obj)
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewUpdateAction(c.resource, c.namespace, obj), obj)
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, obj), obj)
+
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, err
+}
+
+func (c *dynamicResourceClient) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) {
+	var uncastRet runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootUpdateSubresourceAction(c.resource, "status", obj), obj)
+
+	case len(c.namespace) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewUpdateSubresourceAction(c.resource, "status", c.namespace, obj), obj)
+
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, err
+}
+
+func (c *dynamicResourceClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error {
+	var err error
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		_, err = c.client.Fake.
+			Invokes(testing.NewRootDeleteAction(c.resource, name), &metav1.Status{Status: "dynamic delete fail"})
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		_, err = c.client.Fake.
+			Invokes(testing.NewRootDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic delete fail"})
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		_, err = c.client.Fake.
+			Invokes(testing.NewDeleteAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic delete fail"})
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		_, err = c.client.Fake.
+			Invokes(testing.NewDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, name), &metav1.Status{Status: "dynamic delete fail"})
+	}
+
+	return err
+}
+
+func (c *dynamicResourceClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOptions metav1.ListOptions) error {
+	var err error
+	switch {
+	case len(c.namespace) == 0:
+		action := testing.NewRootDeleteCollectionAction(c.resource, listOptions)
+		_, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"})
+
+	case len(c.namespace) > 0:
+		action := testing.NewDeleteCollectionAction(c.resource, c.namespace, listOptions)
+		_, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"})
+
+	}
+
+	return err
+}
+
+func (c *dynamicResourceClient) Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
+	var uncastRet runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootGetAction(c.resource, name), &metav1.Status{Status: "dynamic get fail"})
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootGetSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"})
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewGetAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic get fail"})
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewGetSubresourceAction(c.resource, c.namespace, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"})
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, err
+}
+
+func (c *dynamicResourceClient) List(ctx context.Context, opts metav1.ListOptions) (*unstructured.UnstructuredList, error) {
+	if len(c.listKind) == 0 {
+		panic(fmt.Sprintf("coding error: you must register resource to list kind for every resource you're going to LIST when creating the client.  See NewSimpleDynamicClientWithCustomListKinds or register the list into the scheme: %v out of %v", c.resource, c.client.gvrToListKind))
+	}
+	listGVK := c.resource.GroupVersion().WithKind(c.listKind)
+	listForFakeClientGVK := c.resource.GroupVersion().WithKind(c.listKind[:len(c.listKind)-4]) /*base library appends List*/
+
+	var obj runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0:
+		obj, err = c.client.Fake.
+			Invokes(testing.NewRootListAction(c.resource, listForFakeClientGVK, opts), &metav1.Status{Status: "dynamic list fail"})
+
+	case len(c.namespace) > 0:
+		obj, err = c.client.Fake.
+			Invokes(testing.NewListAction(c.resource, listForFakeClientGVK, c.namespace, opts), &metav1.Status{Status: "dynamic list fail"})
+
+	}
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+
+	retUnstructured := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(obj, retUnstructured, nil); err != nil {
+		return nil, err
+	}
+	entireList, err := retUnstructured.ToList()
+	if err != nil {
+		return nil, err
+	}
+
+	list := &unstructured.UnstructuredList{}
+	list.SetRemainingItemCount(entireList.GetRemainingItemCount())
+	list.SetResourceVersion(entireList.GetResourceVersion())
+	list.SetContinue(entireList.GetContinue())
+	list.GetObjectKind().SetGroupVersionKind(listGVK)
+	for i := range entireList.Items {
+		item := &entireList.Items[i]
+		metadata, err := meta.Accessor(item)
+		if err != nil {
+			return nil, err
+		}
+		if label.Matches(labels.Set(metadata.GetLabels())) {
+			list.Items = append(list.Items, *item)
+		}
+	}
+	return list, nil
+}
+
+func (c *dynamicResourceClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+	switch {
+	case len(c.namespace) == 0:
+		return c.client.Fake.
+			InvokesWatch(testing.NewRootWatchAction(c.resource, opts))
+
+	case len(c.namespace) > 0:
+		return c.client.Fake.
+			InvokesWatch(testing.NewWatchAction(c.resource, c.namespace, opts))
+
+	}
+
+	panic("math broke")
+}
+
+// TODO: opts are currently ignored.
+func (c *dynamicResourceClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
+	var uncastRet runtime.Object
+	var err error
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootPatchAction(c.resource, name, pt, data), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootPatchSubresourceAction(c.resource, name, pt, data, subresources...), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewPatchAction(c.resource, c.namespace, name, pt, data), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewPatchSubresourceAction(c.resource, c.namespace, name, pt, data, subresources...), &metav1.Status{Status: "dynamic patch fail"})
+
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, err
+}
+
+// TODO: opts are currently ignored.
+func (c *dynamicResourceClient) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) {
+	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
+	if err != nil {
+		return nil, err
+	}
+	var uncastRet runtime.Object
+	switch {
+	case len(c.namespace) == 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootPatchAction(c.resource, name, types.ApplyPatchType, outBytes), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) == 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewRootPatchSubresourceAction(c.resource, name, types.ApplyPatchType, outBytes, subresources...), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) > 0 && len(subresources) == 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewPatchAction(c.resource, c.namespace, name, types.ApplyPatchType, outBytes), &metav1.Status{Status: "dynamic patch fail"})
+
+	case len(c.namespace) > 0 && len(subresources) > 0:
+		uncastRet, err = c.client.Fake.
+			Invokes(testing.NewPatchSubresourceAction(c.resource, c.namespace, name, types.ApplyPatchType, outBytes, subresources...), &metav1.Status{Status: "dynamic patch fail"})
+
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if uncastRet == nil {
+		return nil, err
+	}
+
+	ret := &unstructured.Unstructured{}
+	if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func (c *dynamicResourceClient) ApplyStatus(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions) (*unstructured.Unstructured, error) {
+	return c.Apply(ctx, name, obj, options, "status")
+}
diff --git a/abn/watcher/config_test.go b/abn/watcher/config_test.go
new file mode 100644
index 00000000..9aacae66
--- /dev/null
+++ b/abn/watcher/config_test.go
@@ -0,0 +1,64 @@
+package watcher
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestReadServiceConfig tests invalid/problem cases.
+// The good case is actually tested in TestGetApplicationConfig below.
+func TestReadServiceConfig(t *testing.T) {
+	for _, tt := range []struct {
+		name          string
+		file          string
+		numNamespaces int
+		numResources  int
+	}{
+		{"empty", "config.empty.yaml", 0, 0},
+		{"garbage", "config.garbage.yaml", 0, 0},
+		{"nofile", "config.nofile.yaml", 0, 0},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			c := readServiceConfig(completePath("../../testdata/abninputs", tt.file))
+			assert.Equal(t, tt.numNamespaces, len(c))
+		})
+	}
+}
+
+// TestGetApplicationConfig reads a good service configurationa
+// each application config is then extracted
+func TestGetApplicationConfig(t *testing.T) {
+	for _, tt := range []struct {
+		file             string
+		namespace        string
+		name             string
+		maxNumCandidates int
+		numResources     int
+	}{
+		{"config.yaml", "default", "backend", 2, 2},
+		{"config.yaml", "default", "frontend", 1, 1},
+		{"config.yaml", "test", "iter8", 1, 1},
+		{"config.yaml", "empty", "frontend", 0, 0},
+		{"config.yaml", "default", "notpresent", 0, 0},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			svcConfig := readServiceConfig(completePath("../../testdata/abninputs", tt.file))
+			assert.Contains(t, svcConfig, tt.namespace)
+			appConfig := getApplicationConfig(tt.namespace, tt.name, svcConfig)
+			if tt.maxNumCandidates != 0 {
+				assert.NotNil(t, appConfig)
+				assert.Equal(t, tt.maxNumCandidates, appConfig.MaxNumCandidates)
+				assert.Equal(t, tt.numResources, len(appConfig.Resources))
+			}
+		})
+	}
+}
+
+// utility method
+func completePath(prefix string, suffix string) string {
+	_, filename, _, _ := runtime.Caller(1) // one step up the call stack
+	return filepath.Join(filepath.Dir(filename), prefix, suffix)
+}
diff --git a/abn/watcher/handler_test.go b/abn/watcher/handler_test.go
new file mode 100644
index 00000000..5b725a6d
--- /dev/null
+++ b/abn/watcher/handler_test.go
@@ -0,0 +1,96 @@
+package watcher
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func TestGetApplicationNameFromObjectName(t *testing.T) {
+	for _, tt := range []struct {
+		inputname          string
+		expectedoutputname string
+	}{
+		{"hello", "hello"},
+		{"hello-candidate-1", "hello"},
+		{"hello-candidate-2", "hello"},
+		{"hello-candidate-0", "hello-candidate-0"},
+	} {
+		t.Run(tt.inputname, func(t *testing.T) {
+			n := getApplicationNameFromObjectName(tt.inputname)
+			assert.Equal(t, tt.expectedoutputname, n)
+		})
+	}
+}
+
+func TestGetVersion(t *testing.T) {
+	for _, tt := range []struct {
+		testLabel  string
+		hasLabel   bool
+		labelValue string
+	}{
+		{"no label", false, ""},
+		{"has label", true, "version"},
+	} {
+		t.Run(tt.testLabel, func(t *testing.T) {
+			v, ok := getVersion(newUnstructuredDeployment("default", "name", tt.labelValue))
+			if tt.hasLabel {
+				assert.True(t, ok)
+				assert.Equal(t, tt.labelValue, v)
+			}
+		})
+	}
+
+}
+
+func newUnstructuredDeployment(namespace, name, version string) *unstructured.Unstructured {
+	labels := map[string]string{
+		versionLabel: version,
+	}
+
+	deployment := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "apps/v1",
+			Kind:       "Deployment",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+	}
+
+	condition := appsv1.DeploymentCondition{Type: "Available", Status: "True"}
+	deployment.Status.Conditions = append(deployment.Status.Conditions, condition)
+
+	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(deployment)
+	if err != nil {
+		return nil
+	}
+	return &unstructured.Unstructured{Object: o}
+}
+
+func newUnstructuredService(namespace, name, version string) *unstructured.Unstructured {
+	labels := map[string]interface{}{}
+	if version != "" {
+		labels[versionLabel] = version
+	}
+
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Service",
+			"metadata": map[string]interface{}{
+				"namespace": namespace,
+				"name":      name,
+				"labels":    labels,
+			},
+			"spec": name,
+		},
+	}
+}
diff --git a/abn/watcher/informer_test.go b/abn/watcher/informer_test.go
index 34e5738f..0f511ee8 100644
--- a/abn/watcher/informer_test.go
+++ b/abn/watcher/informer_test.go
@@ -2,7 +2,7 @@ package watcher
 
 import (
 	"context"
-	"strings"
+	"fmt"
 	"testing"
 	"time"
 
@@ -16,179 +16,143 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
+// If sufficient objects created to match config, then Applications should contain suitable entry
+// Probably could test various combinations
 func TestAdd(t *testing.T) {
-	scenarios := map[string]struct {
-		iter8related string
-		namespace    string
-		application  string
-		version      string
-		track        string
-	}{
-		"iter8 not set":  {iter8related: "", namespace: "namespace", application: "name", version: "version", track: "track"},
-		"iter8 not true": {iter8related: "false", namespace: "namespace", application: "name", version: "version", track: "track"},
-		"no application": {iter8related: "true", namespace: "namespace", application: "", version: "version", track: "track"},
-		"no version":     {iter8related: "true", namespace: "namespace", application: "name", version: "", track: "track"},
-		"no track":       {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: ""},
-		"all":            {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: "track"},
-	}
+	var svcConfigFile = "../../testdata/abninputs/config.yaml"
+	var namespace = "default"
+	var application = "backend"
+	var track = application
+	var version = "v1"
+
+	// set up - initialize channel
+	done := make(chan struct{})
+	defer close(done)
+
+	// set up - Applications and informers
+	setup(t, svcConfigFile, done)
+
+	// create objects in cluster
+	createObject(t, newUnstructuredDeployment(namespace, application, version))
+	createObject(t, newUnstructuredService(namespace, application, version))
+
+	// creation of these objects should trigger handler which will add application to Applications map
+	assert.Eventually(t, func() bool {
+		return assertApplicationExists(t, namespace, application, []string{track}, []string{version})
+	}, 10*time.Second, 100*time.Millisecond)
+}
 
-	for label, s := range scenarios {
-		t.Run(label, func(t *testing.T) {
-			gvr := schema.GroupVersionResource{
-				Group:    "apps",
-				Version:  "v1",
-				Resource: "deployments",
-			}
+// update version associated with track
+func TestUpdate(t *testing.T) {
+	var svcConfigFile = "../../testdata/abninputs/config.yaml"
+	var namespace = "default"
+	var application = "backend"
+	var track = application
+	var version = "v1"
+	var version2 = "v2"
+
+	// set up - initialize channel
+	done := make(chan struct{})
+	defer close(done)
+
+	// set up - Applications and informers
+	setup(t, svcConfigFile, done)
+
+	// create objects in cluster
+	deployObj := createObject(t, newUnstructuredDeployment(namespace, application, version))
+	svcObj := createObject(t, newUnstructuredService(namespace, application, version))
+
+	// creation of these objects should trigger handler which will add application to Applications map
+	assert.Eventually(t, func() bool {
+		return assertApplicationExists(t, namespace, application, []string{track}, []string{version})
+	}, 10*time.Second, 100*time.Millisecond)
+
+	assertV(t, version, deployObj)
+	assertV(t, version, svcObj)
+
+	// update version on first object (so mismatch)
+	updateLabel(deployObj, versionLabel, version2)
+	assertV(t, version2, deployObj)
+	updateObject(t, deployObj)
+
+	// eventually track map cleared
+	assert.Eventually(t, func() bool { return assertApplicationExists(t, namespace, application, []string{}, []string{version}) }, 10*time.Second, 100*time.Millisecond)
+
+	// update version on second object (so match)
+	updateLabel(svcObj, versionLabel, version2)
+	assertV(t, version2, svcObj)
+	updateObject(t, svcObj)
+
+	// eventually track reset
+	assert.Eventually(t, func() bool {
+		return assertApplicationExists(t, namespace, application, []string{track}, []string{version, version2})
+	}, 10*time.Second, 100*time.Millisecond)
+}
 
-			done := make(chan struct{})
-			defer close(done)
-			setup(t, gvr, s.namespace, done)
-
-			createObject(t, gvr, s.iter8related, s.namespace, s.application, s.version, s.track)
-
-			assert.Eventually(
-				t,
-				func() bool {
-					r := true
-					// if any preconditions are not met, then no application added
-					if strings.ToLower(s.iter8related) != "true" ||
-						s.application == "" ||
-						s.version == "" {
-						r = r && abnapp.NumApplications(t, 0)
-						return r
-					}
-
-					// otherwise application created
-
-					a, err := abnapp.Applications.Get(s.namespace + "/" + s.application)
-					r = r && assert.NoError(t, err)
-					r = r && assert.NotNil(t, a)
-
-					tracks := []string{}
-					if s.track != "" {
-						tracks = []string{s.track}
-					}
-					r = r && assertApplication(t, a, applicationAssertion{
-						namespace: s.namespace,
-						name:      s.application,
-						tracks:    tracks,
-						versions:  []string{s.version},
-					})
-					return r
-				},
-				10*time.Second,
-				100*time.Millisecond,
-			)
-		})
-	}
+func assertV(t *testing.T, expected string, obj *unstructured.Unstructured) {
+	assert.NotNil(t, obj)
+	v, ok := getVersion(obj)
+	assert.True(t, ok)
+	assert.Equal(t, expected, v)
 }
 
-func TestUpdate(t *testing.T) {
-	scenarios := map[string]struct {
-		iter8related string
-		namespace    string
-		application  string
-		version      string
-		track        string
-	}{
-		"no application": {iter8related: "true", namespace: "namespace", application: "", version: "version", track: "track"},
-		"no version":     {iter8related: "true", namespace: "namespace", application: "name", version: "", track: "track"},
-		"no track":       {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: ""},
-		"all":            {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: "track"},
-	}
+// test deletion of application object does not remove it from Applications; BUT the track map should be cleared
+func TestDelete(t *testing.T) {
+	var svcConfigFile = "../../testdata/abninputs/config.yaml"
+	var namespace = "default"
+	var application = "backend"
+	var track = application
+	var version = "v1"
 
-	for label, s := range scenarios {
-		t.Run(label, func(t *testing.T) {
-			gvr := schema.GroupVersionResource{
-				Group:    "apps",
-				Version:  "v1",
-				Resource: "deployments",
-			}
+	// set up - initialize channel
+	done := make(chan struct{})
+	defer close(done)
 
-			done := make(chan struct{})
-			defer close(done)
-			setup(t, gvr, s.namespace, done)
-			existingObj := createObject(t, gvr, "true", "namespace", "name", "version", "")
-			// verify that existing object is as expected
-			assert.Eventually(
-				t,
-				func() bool {
-					r := true
-					a, err := abnapp.Applications.Get("namespace/name")
-					r = r && assert.NoError(t, err)
-					r = r && assert.NotNil(t, a)
-					r = r && assertApplication(t, a, applicationAssertion{
-						namespace: "namespace",
-						name:      "name",
-						tracks:    []string{},
-						versions:  []string{"version"},
-					})
-					return r
-				},
-				10*time.Second,
-				100*time.Millisecond,
-			)
-
-			//update object labels
-			updateLabel(existingObj, iter8Label, s.iter8related)
-			updateLabel(existingObj, nameLabel, s.application)
-			updateLabel(existingObj, versionLabel, s.version)
-			updateLabel(existingObj, trackLabel, s.track)
-
-			updatedObj, err := k8sclient.Client.Dynamic().
-				Resource(gvr).Namespace(s.namespace).
-				Update(
-					context.TODO(),
-					existingObj,
-					metav1.UpdateOptions{},
-				)
-			assert.NoError(t, err)
-			assert.NotNil(t, updatedObj)
-
-			assert.Eventually(
-				t,
-				func() bool {
-					r := true
-					// if any preconditions are not met, then no change was made to applications
-					if strings.ToLower(s.iter8related) != "true" ||
-						s.application == "" ||
-						s.version == "" {
-						// abnapp.NumApplications(t, 1)
-						a, err := abnapp.Applications.Get("namespace/name")
-						r = r && assert.NoError(t, err)
-						r = r && assert.NotNil(t, a)
-						r = r && assertApplication(t, a, applicationAssertion{
-							namespace: "namespace",
-							name:      "name",
-							tracks:    []string{},
-							versions:  []string{"version"},
-						})
-						return r
-					}
-
-					// otherwise application was possibility modified in some way
-					// at least this application exists since it was preexisting
-					a, err := abnapp.Applications.Get(s.namespace + "/" + s.application)
-					r = r && assert.NoError(t, err)
-					r = r && assert.NotNil(t, a)
-
-					tracks := []string{}
-					if s.track != "" {
-						tracks = []string{s.track}
-					}
-					r = r && assertApplication(t, a, applicationAssertion{
-						namespace: s.namespace,
-						name:      s.application,
-						tracks:    tracks,
-						versions:  []string{s.version},
-					})
-					return r
-				},
-				10*time.Second,
-				100*time.Millisecond,
-			)
-		})
-	}
+	// set up - Applications and informers
+	setup(t, svcConfigFile, done)
+
+	// create objects in cluster
+	createObject(t, newUnstructuredDeployment(namespace, application, version))
+	svcObj := createObject(t, newUnstructuredService(namespace, application, version))
+
+	// creation of these objects should trigger handler which will add application to Applications map
+	assert.Eventually(t, func() bool {
+		return assertApplicationExists(t, namespace, application, []string{track}, []string{version})
+	}, 10*time.Second, 100*time.Millisecond)
+
+	// delete one of the objects
+	deleteObject(t, svcObj)
+
+	// eventually track map cleared
+	assert.Eventually(t, func() bool { return assertApplicationExists(t, namespace, application, []string{}, []string{version}) }, 10*time.Second, 100*time.Millisecond)
+}
+
+func setup(t *testing.T, svcConfigFile string, done chan struct{}) {
+	// set up - clear ApplicationsMap
+	abnapp.Applications.Clear()
+
+	// set up - define watcher and start informers
+	k8sclient.Client = *k8sclient.NewFakeKubeClient(cli.New())
+	w := NewIter8Watcher(svcConfigFile)
+	assert.NotNil(t, w)
+	w.Start(done)
+}
+
+func assertApplicationExists(t *testing.T, namespace, application string, tracks []string, versions []string) bool {
+	r := true
+
+	a, err := abnapp.Applications.Get(namespace + "/" + application)
+	r = r && assert.NoError(t, err)
+	r = r && assert.NotNil(t, a)
+
+	r = r && assertApplication(t, a, applicationAssertion{
+		namespace: namespace,
+		name:      application,
+		tracks:    tracks,
+		versions:  versions,
+	})
+
+	return r
 }
 
 func updateLabel(obj *unstructured.Unstructured, key string, value string) {
@@ -197,163 +161,94 @@ func updateLabel(obj *unstructured.Unstructured, key string, value string) {
 	} else {
 		(obj.Object["metadata"].(map[string]interface{}))["labels"].(map[string]interface{})[key] = value
 	}
-
 }
 
-func TestDelete(t *testing.T) {
-	scenarios := map[string]struct {
-		iter8related string
-		namespace    string
-		application  string
-		version      string
-		track        string
-	}{
-		"no track, ready": {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: ""},
-		"track, ready":    {iter8related: "true", namespace: "namespace", application: "name", version: "version", track: "track"},
-	}
-
-	for label, s := range scenarios {
-		t.Run(label, func(t *testing.T) {
-			gvr := schema.GroupVersionResource{
-				Group:    "apps",
-				Version:  "v1",
-				Resource: "deployments",
-			}
-
-			done := make(chan struct{})
-			defer close(done)
-			setup(t, gvr, s.namespace, done)
-
-			// add existing object (to delete)
-			createObject(t, gvr, "true", "namespace", "name", "version", "track")
-			// verify that existing object is as expected
-			assert.Eventually(
-				t,
-				func() bool {
-					r := true
-					a, err := abnapp.Applications.Get("namespace/name")
-					r = r && assert.NoError(t, err)
-					r = r && assert.NotNil(t, a)
-					r = r && assertApplication(t, a, applicationAssertion{
-						namespace: "namespace",
-						name:      "name",
-						tracks:    []string{"track"},
-						versions:  []string{"version"},
-					})
-					return r
-				},
-				10*time.Second,
-				100*time.Millisecond,
-			)
-
-			// delete object --> no track anymore
-			err := k8sclient.Client.Dynamic().
-				Resource(gvr).Namespace(s.namespace).
-				Delete(
-					context.TODO(),
-					s.application,
-					metav1.DeleteOptions{},
-				)
-			assert.NoError(t, err)
-
-			assert.Eventually(
-				t,
-				func() bool {
-					r := true
-					// if any preconditions are not met, then no change was made to applications
-					if strings.ToLower(s.iter8related) != "true" ||
-						s.application == "" ||
-						s.version == "" {
-						a, err := abnapp.Applications.Get("namespace/name")
-						r = r && assert.NoError(t, err)
-						r = r && assert.NotNil(t, a)
-						r = r && assertApplication(t, a, applicationAssertion{
-							namespace: "namespace",
-							name:      "name",
-							tracks:    []string{"track"},
-							versions:  []string{"version"},
-						})
-						return r
-					}
-
-					// otherwise application was possibility modified in some way
-					// at least this application exists since it was preexisting
-					a, err := abnapp.Applications.Get(s.namespace + "/" + s.application)
-					r = r && assert.NoError(t, err)
-					r = r && assert.NotNil(t, a)
-
-					r = r && assertApplication(t, a, applicationAssertion{
-						namespace: s.namespace,
-						name:      s.application,
-						tracks:    []string{},
-						versions:  []string{s.version},
-					})
-
-					return r
-				},
-				10*time.Second,
-				100*time.Millisecond,
-			)
-		})
+// for testing, way to get GVR from object
+func getGVR(uObj *unstructured.Unstructured) *schema.GroupVersionResource {
+	switch uObj.GetKind() {
+	case "Service":
+		return &schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}
+	case "Deployment":
+		return &schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
+	default:
+		return nil
 	}
 }
 
-func setup(t *testing.T, gvr schema.GroupVersionResource, namespace string, done chan struct{}) {
-	// ensure global record of applications is empty
-	abnapp.Applications.Clear()
+func createObject(t *testing.T, uObj *unstructured.Unstructured) *unstructured.Unstructured {
 
-	// define watcher
-	k8sclient.Client = *k8sclient.NewFakeKubeClient(cli.New())
-	w := NewIter8Watcher(
-		[]schema.GroupVersionResource{gvr},
-		[]string{namespace},
-	)
-	assert.NotNil(t, w)
+	gvr := getGVR(uObj)
+	assert.NotNil(t, gvr)
 
-	// start informers
-	w.Start(done)
-}
-
-func createObject(t *testing.T, gvr schema.GroupVersionResource, iter8related, namespace, application, version, track string) *unstructured.Unstructured {
-	// create object; no track defined
 	createdObj, err := k8sclient.Client.Dynamic().
-		Resource(gvr).Namespace(namespace).
+		Resource(*gvr).Namespace(uObj.GetNamespace()).
 		Create(
 			context.TODO(),
-			newUnstructuredDeployment(iter8related, namespace, application, version, track),
+			uObj,
 			metav1.CreateOptions{},
 		)
 	assert.NoError(t, err)
 	assert.NotNil(t, createdObj)
+
 	return createdObj
 }
 
-func newUnstructuredDeployment(iter8related, namespace, application, version, track string) *unstructured.Unstructured {
-	labels := map[string]interface{}{}
-	if application != "" {
-		labels[nameLabel] = application
-	}
-	if version != "" {
-		labels[versionLabel] = version
-	}
-	if iter8related != "" {
-		labels[iter8Label] = iter8related
-	}
+func updateObject(t *testing.T, uObj *unstructured.Unstructured) *unstructured.Unstructured {
+	gvr := getGVR(uObj)
+	assert.NotNil(t, gvr)
 
-	if track != "" {
-		labels[trackLabel] = track
-	}
+	updatedObj, err := k8sclient.Client.Dynamic().
+		Resource(*gvr).Namespace(uObj.GetNamespace()).
+		Update(
+			context.TODO(),
+			uObj,
+			metav1.UpdateOptions{},
+		)
+	assert.NoError(t, err)
+	assert.NotNil(t, updatedObj)
+
+	return updatedObj
+}
+
+func deleteObject(t *testing.T, uObj *unstructured.Unstructured) *unstructured.Unstructured {
+	gvr := getGVR(uObj)
+	assert.NotNil(t, gvr)
+
+	err := k8sclient.Client.Dynamic().
+		Resource(*gvr).Namespace(uObj.GetNamespace()).
+		Delete(
+			context.TODO(),
+			uObj.GetName(),
+			metav1.DeleteOptions{},
+		)
+	assert.NoError(t, err)
+
+	return nil
+}
 
-	return &unstructured.Unstructured{
-		Object: map[string]interface{}{
-			"apiVersion": "apps/v1",
-			"kind":       "Deployment",
-			"metadata": map[string]interface{}{
-				"namespace": namespace,
-				"name":      application,
-				"labels":    labels,
-			},
-			"spec": application,
-		},
+func TestTrackNames(t *testing.T) {
+	// also tests validObjectNames()
+	for _, tt := range []struct {
+		name          string
+		numCandidates int
+	}{
+		{"zero", 0},
+		{"one", 1},
+		{"two", 2},
+		{"ten", 10},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			names := getValidObjectNames(tt.name, tt.numCandidates)
+			assert.Equal(t, tt.numCandidates+1, len(names))
+			assert.Contains(t, names, tt.name)
+			if tt.numCandidates > 0 {
+				assert.Contains(t, names, fmt.Sprintf("%s-candidate-%d", tt.name, tt.numCandidates))
+			}
+		})
 	}
 }
+
+func TestContainsString(t *testing.T) {
+	assert.True(t, containsString([]string{"one", "two", "three"}, "one"))
+	assert.False(t, containsString([]string{"one", "two", "three"}, "four"))
+}
diff --git a/abn/watcher/watchedobject_test.go b/abn/watcher/watchedobject_test.go
deleted file mode 100644
index 5792aee4..00000000
--- a/abn/watcher/watchedobject_test.go
+++ /dev/null
@@ -1,96 +0,0 @@
-package watcher
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-)
-
-func TestHasName(t *testing.T) {
-	objName := "myobj"
-	objNamespace := "mynamespace"
-	appName := "myapp"
-	appVersion := "myversion"
-	appTrack := "myTrack"
-
-	p := corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      objName,
-			Namespace: objNamespace,
-			Labels: map[string]string{
-				nameLabel:    appName,
-				versionLabel: appVersion,
-				trackLabel:   appTrack,
-			},
-		},
-		Spec: corev1.PodSpec{},
-	}
-	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&p)
-	assert.NoError(t, err)
-	wo := watchedObject{Obj: &unstructured.Unstructured{Object: obj}}
-
-	n, ok := wo.getName()
-	assert.True(t, ok)
-	assert.Equal(t, appName, n)
-
-	assert.Equal(t, objNamespace, wo.getNamespace())
-
-	nn, ok := wo.getNamespacedName()
-	assert.True(t, ok)
-	assert.Equal(t, objNamespace+"/"+appName, nn)
-
-	v, ok := wo.getVersion()
-	assert.True(t, ok)
-	assert.Equal(t, appVersion, v)
-
-	assert.Equal(t, appTrack, wo.getTrack())
-}
-
-func TestHasNoName(t *testing.T) {
-	objName := "myobj"
-	objNamespace := "mynamespace"
-
-	p := corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      objName,
-			Namespace: objNamespace,
-		},
-		Spec: corev1.PodSpec{},
-	}
-	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&p)
-	assert.NoError(t, err)
-	wo := watchedObject{Obj: &unstructured.Unstructured{Object: obj}}
-
-	n, ok := wo.getName()
-	assert.False(t, ok)
-	assert.Empty(t, n)
-}
-
-func TestHasNoTrack(t *testing.T) {
-	objName := "myobj"
-	objNamespace := "mynamespace"
-	appName := "myapp"
-	appVersion := "myversion"
-
-	p := corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      objName,
-			Namespace: objNamespace,
-			Labels: map[string]string{
-				nameLabel:    appName,
-				versionLabel: appVersion,
-			},
-		},
-		Spec: corev1.PodSpec{},
-	}
-	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&p)
-	assert.NoError(t, err)
-	wo := watchedObject{Obj: &unstructured.Unstructured{Object: obj}}
-
-	assert.Equal(t, "", wo.getTrack())
-}
diff --git a/testdata/abninputs/config.yaml b/testdata/abninputs/config.yaml
index 938d28b1..a13987e5 100644
--- a/testdata/abninputs/config.yaml
+++ b/testdata/abninputs/config.yaml
@@ -1,8 +1,24 @@
-resources:
-- group: apps
-  version: v1
-  resource: deployments
-- version: v1
-  resource: services
-namespaces:
-- default
\ No newline at end of file
+default:
+  backend:
+    maxNumCandidates: 2
+    resources:
+    - group: apps
+      version: v1
+      resource: deployments
+      condition: Available
+    - version: v1
+      resource: services
+  frontend:
+    maxNumCandidates: 1
+    resources:
+    - version: v1
+      resource: services
+test:
+  iter8:
+    maxNumCandidates: 1
+    resources:
+    - group: apps
+      version: v1
+      resource: deployments
+      condition: Available
+empty:
\ No newline at end of file
diff --git a/testdata/abninputs/config_copy.yaml b/testdata/abninputs/config_copy.yaml
new file mode 100644
index 00000000..174022bb
--- /dev/null
+++ b/testdata/abninputs/config_copy.yaml
@@ -0,0 +1,14 @@
+backend:
+  maxNumCandidates: 2
+  resources:
+  - group: apps
+    version: v1
+    resource: deployments
+  - version: v1
+    resource: services
+frontend:
+  maxNumCandidates: 1
+  resources:
+  - group: serving.knative.dev
+    version: v1
+    resource: services