Repository: VictoriaMetrics/operator

Issue:
Refactoring broke selector discovery logic
 Bug introduced at commit https://github.com/VictoriaMetrics/operator/commit/a4aeda3fb7f64df370c5715d4d128b6824924c00


 Due to change from switch case

```
	switch {
	case len(watchNS) > 0:
		// perform match only for watched namespaces
		// filters by namespace is disabled, since operator cannot access cluster-wide APIs
		// this case could be improved to additionally filter by namespace name - metadata.name label
		namespaces = append(namespaces, watchNS...)
	case objectSelector != nil && nsSelector == nil:
		// in single namespace mode, return object ns
		namespaces = append(namespaces, objNamespace)
	case nsSelector != nil:
		// perform a cluster wide request for namespaces with given filters
		nsSelector, err := metav1.LabelSelectorAsSelector(nsSelector)
		if err != nil {
			return fmt.Errorf("cannot convert selector: %w", err)
		}
		namespaces, err = SelectNamespaces(ctx, rclient, nsSelector)
		if err != nil {
			return fmt.Errorf("cannot select namespaces for  match: %w", err)
		}
		// if nsSelector is specified and no match, return directly
		if len(namespaces) == 0 {
			return nil
		}
	}
```

 to else if:
```
	if s.NamespaceSelector == nil && s.ObjectSelector != nil {
		// in single namespace mode, return object ns
		if len(namespaces) == 0 || slices.Contains(namespaces, s.DefaultNamespace) {
			namespaces = append(namespaces[:0], s.DefaultNamespace)
		}
	} else if len(namespaces) == 0 {
		// perform a cluster wide request for namespaces with given filters
		// filters by namespace is disabled, when WATCH_NAMESPACES is defined,
		// since operator cannot access cluster-wide APIs this case could be improved
		// to additionally filter by namespace name - metadata.name label
		opts := &client.ListOptions{}
		if s.NamespaceSelector != nil {
			selector, err := metav1.LabelSelectorAsSelector(s.NamespaceSelector)
			if err != nil {
				return nil, fmt.Errorf("cannot convert selector: %w", err)
			}
			opts.LabelSelector = selector
		}
		l := &corev1.NamespaceList{}
		if err := rclient.List(ctx, l, opts); err != nil {
			return nil, fmt.Errorf("cannot select namespaces for  match: %w", err)
		}
		for _, n := range l.Items {
			namespaces = append(namespaces, n.Name)
		}
	}
```

 This path cannot trigger anymore:


```
	if len(nss) == 0 {
		if err := rclient.List(ctx, dst, opts...); err != nil {
			return fmt.Errorf("cannot list objects at cluster scope: %w", err)
		}
		collect(dst)
		return nil
	}
```

 It results into per-namespace get requests and additional memory for per-namespace caching. 

Discussion:
@f41gh7:  The bugfix was included into [v0.61.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.61.1) release

Solution patch:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 68135b13..cd23904d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -14,6 +14,7 @@ aliases:
 ## tip
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): properly add TLS configuration for scrape configuration. Previously, tls options were applied to the root of scrape configuration, which caused an error at `vmagent` startup. Bug was introduced in v0.60.0 release.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/), [vmalert](https://docs.victoriametrics.com/operator/resources/vmalert/), [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/) and [vmalertmanager](https://docs.victoriametrics.com/operator/resources/vmalertmanager/): reduce Kubernetes API-server and operator resource usage for objects discovery with `NamespaceSelector: {}`. See this [1468](https://github.com/VictoriaMetrics/operator/issues/1468) issue for details.
 
 ## [v0.61.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.61.0)
 
diff --git a/internal/controller/operator/factory/k8stools/selectors.go b/internal/controller/operator/factory/k8stools/selectors.go
index d3c03e1b..d9aae7c2 100644
--- a/internal/controller/operator/factory/k8stools/selectors.go
+++ b/internal/controller/operator/factory/k8stools/selectors.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"slices"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -12,6 +13,8 @@ import (
 	"github.com/VictoriaMetrics/operator/internal/config"
 )
 
+var getWatchNamespaces = config.MustGetWatchNamespaces
+
 type listing[L any] interface {
 	client.ObjectList
 	*L
@@ -36,7 +39,7 @@ func VisitSelected[T any, PT listing[T]](ctx context.Context, rclient client.Cli
 	if s.isUnmanaged() {
 		return nil
 	}
-	nss, err := discoverNamespaces(ctx, rclient, s)
+	dnsr, err := discoverNamespaces(ctx, rclient, s)
 	if err != nil {
 		return err
 	}
@@ -49,38 +52,61 @@ func VisitSelected[T any, PT listing[T]](ctx context.Context, rclient client.Cli
 		}
 		opts.LabelSelector = selector
 	}
+	if dnsr == nil {
+		// special case, nil value must be treated as selectors match nothing
+		return nil
+	}
+	nss := dnsr.namespaces
 	// namespaces could still be empty and it's ok
 	return ListObjectsByNamespace(ctx, rclient, nss, cb, opts)
 }
 
+type discoverNamespacesResponse struct {
+	namespaces []string
+}
+
 // discoverNamespaces select namespaces by given label selector
-func discoverNamespaces(ctx context.Context, rclient client.Client, s *SelectorOpts) ([]string, error) {
-	namespaces := config.MustGetWatchNamespaces()
-	if s.NamespaceSelector == nil && s.ObjectSelector != nil {
+func discoverNamespaces(ctx context.Context, rclient client.Client, s *SelectorOpts) (*discoverNamespacesResponse, error) {
+	watchNS := getWatchNamespaces()
+
+	var namespaces []string
+	switch {
+	case len(watchNS) > 0:
+		// perform match only for watched namespaces
+		// filters by namespace is disabled, since operator cannot access cluster-wide APIs
+		// this case could be improved to additionally filter by namespace name - metadata.name label
+
+		// impossible case
+		if !slices.Contains(watchNS, s.DefaultNamespace) {
+			panic(fmt.Sprintf("BUG: watch namespaces: %s must contain namespace for the current reconcile object: %s", strings.Join(watchNS, ","), s.DefaultNamespace))
+		}
+		namespaces = append(namespaces, watchNS...)
+
+	case s.ObjectSelector != nil && s.NamespaceSelector == nil:
 		// in single namespace mode, return object ns
-		if len(namespaces) == 0 || slices.Contains(namespaces, s.DefaultNamespace) {
-			namespaces = append(namespaces[:0], s.DefaultNamespace)
+		namespaces = append(namespaces, s.DefaultNamespace)
+	case s.NamespaceSelector != nil:
+		if len(s.NamespaceSelector.MatchExpressions) == 0 && len(s.NamespaceSelector.MatchLabels) == 0 {
+			// fast path, match everything
+			return &discoverNamespacesResponse{namespaces: namespaces}, nil
 		}
-	} else if len(namespaces) == 0 {
 		// perform a cluster wide request for namespaces with given filters
-		// filters by namespace is disabled, when WATCH_NAMESPACES is defined,
-		// since operator cannot access cluster-wide APIs this case could be improved
-		// to additionally filter by namespace name - metadata.name label
 		opts := &client.ListOptions{}
-		if s.NamespaceSelector != nil {
-			selector, err := metav1.LabelSelectorAsSelector(s.NamespaceSelector)
-			if err != nil {
-				return nil, fmt.Errorf("cannot convert selector: %w", err)
-			}
-			opts.LabelSelector = selector
+		selector, err := metav1.LabelSelectorAsSelector(s.NamespaceSelector)
+		if err != nil {
+			return nil, fmt.Errorf("cannot convert selector: %w", err)
 		}
+		opts.LabelSelector = selector
 		l := &corev1.NamespaceList{}
 		if err := rclient.List(ctx, l, opts); err != nil {
 			return nil, fmt.Errorf("cannot select namespaces for  match: %w", err)
 		}
+		if len(l.Items) == 0 {
+			return nil, nil
+		}
 		for _, n := range l.Items {
 			namespaces = append(namespaces, n.Name)
 		}
 	}
-	return namespaces, nil
+	return &discoverNamespacesResponse{namespaces: namespaces}, nil
 }



Test patch:
diff --git a/internal/controller/operator/factory/k8stools/selectors_test.go b/internal/controller/operator/factory/k8stools/selectors_test.go
index 14b7fc85..b786a5e2 100644
--- a/internal/controller/operator/factory/k8stools/selectors_test.go
+++ b/internal/controller/operator/factory/k8stools/selectors_test.go
@@ -2,58 +2,386 @@ package k8stools
 
 import (
 	"context"
-	"reflect"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-func Test_discoverNamespaces(t *testing.T) {
-	tests := []struct {
-		name         string
-		selectorOpts *SelectorOpts
-		predefinedNs []runtime.Object
-		want         []string
-		wantErr      bool
-	}{
-		{
-			name:         "select 1 ns",
-			selectorOpts: &SelectorOpts{},
-			predefinedNs: []runtime.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}}},
-			want:         []string{"ns1"},
-			wantErr:      false,
-		},
-		{
-			name: "select 1 ns with label selector",
-			selectorOpts: &SelectorOpts{
-				NamespaceSelector: &metav1.LabelSelector{
-					MatchLabels: map[string]string{
-						"name": "kube-system",
-					},
+func Test_discoverNamespacesOk(t *testing.T) {
+	type opts struct {
+		selectorOpts      SelectorOpts
+		predefinedObjects []runtime.Object
+		want              *discoverNamespacesResponse
+	}
+	f := func(opts opts) {
+		t.Helper()
+		fclient := GetTestClientWithObjects(opts.predefinedObjects)
+		got, err := discoverNamespaces(context.TODO(), fclient, &opts.selectorOpts)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if d := cmp.Diff(got, opts.want, cmp.AllowUnexported(discoverNamespacesResponse{})); len(d) > 0 {
+			t.Fatalf("unexpected diff: %s", d)
+		}
+	}
+
+	// match nothing on namespace selector mismatch
+	o := opts{
+		selectorOpts: SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"env": "dev"},
+			},
+		},
+		predefinedObjects: []runtime.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}}},
+	}
+	f(o)
+
+	// match everything - non-nil want
+	o = opts{
+		predefinedObjects: []runtime.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}}},
+		want:              &discoverNamespacesResponse{},
+	}
+	f(o)
+
+	// select 1 ns with label selector
+	o = opts{
+		selectorOpts: SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"name": "kube-system",
 				},
-				SelectAll: true,
 			},
-			predefinedNs: []runtime.Object{
-				&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}},
-				&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system", Labels: map[string]string{"name": "kube-system"}}},
+		},
+		predefinedObjects: []runtime.Object{
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system", Labels: map[string]string{"name": "kube-system"}}},
+		},
+		want: &discoverNamespacesResponse{namespaces: []string{"kube-system"}},
+	}
+	f(o)
+	// match for object NS only
+	o = opts{
+		selectorOpts: SelectorOpts{
+			DefaultNamespace: "default",
+			ObjectSelector: &metav1.LabelSelector{
+
+				MatchLabels: map[string]string{
+					"name": "kube-system",
+				},
 			},
-			want:    []string{"kube-system"},
-			wantErr: false,
 		},
+		predefinedObjects: []runtime.Object{
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default", Labels: map[string]string{"name": "default-1"}}},
+		},
+		want: &discoverNamespacesResponse{namespaces: []string{"default"}},
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			fclient := GetTestClientWithObjects(tt.predefinedNs)
-			got, err := discoverNamespaces(context.TODO(), fclient, tt.selectorOpts)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("discoverNamespaces() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("discoverNamespaces() got = %v, want %v", got, tt.want)
+	f(o)
+}
+
+func TestVisitSelected(t *testing.T) {
+	type opts struct {
+		so                *SelectorOpts
+		watchNamespaces   []string
+		wantPods          []corev1.Pod
+		predefinedObjects []runtime.Object
+	}
+
+	podFromNameNs := func(name string, namespace string) *corev1.Pod {
+		return &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+		}
+	}
+	ignoreDiffOpts := cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion")
+
+	f := func(opts opts) {
+		if len(opts.watchNamespaces) > 0 {
+			origin := getWatchNamespaces
+			getWatchNamespaces = func() []string {
+				return opts.watchNamespaces
 			}
+			defer func() {
+				getWatchNamespaces = origin
+			}()
+		}
+		t.Helper()
+		ctx := context.Background()
+		fclient := GetTestClientWithObjects(opts.predefinedObjects)
+		var gotPods []corev1.Pod
+		err := VisitSelected(ctx, fclient, opts.so, func(pl *corev1.PodList) {
+			gotPods = append(gotPods, pl.Items...)
 		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if d := cmp.Diff(opts.wantPods, gotPods, ignoreDiffOpts); len(d) > 0 {
+			t.Fatalf("unexpected diff: %s", d)
+		}
+	}
+	// empty objects
+	o := opts{
+		so: &SelectorOpts{
+			DefaultNamespace: "default",
+		},
+	}
+	f(o)
+
+	// all objects at single namespace
+	o = opts{
+		so: &SelectorOpts{
+			ObjectSelector:   &metav1.LabelSelector{},
+			DefaultNamespace: "default",
+		},
+		wantPods: []corev1.Pod{
+			*podFromNameNs("pod-1", "default"),
+			*podFromNameNs("pod-2", "default"),
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-2"}},
+		},
+	}
+	f(o)
+
+	// all objects at any namespace
+	o = opts{
+		so: &SelectorOpts{
+			SelectAll:        true,
+			DefaultNamespace: "default",
+		},
+		wantPods: []corev1.Pod{
+			*podFromNameNs("pod-1", "default"),
+			*podFromNameNs("pod-2", "default"),
+			*podFromNameNs("pod-3", "default-1"),
+			*podFromNameNs("pod-3", "default-2"),
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-2"}},
+		},
+	}
+	f(o)
+
+	// all objects at any namespace with non-nil selectors
+	o = opts{
+		so: &SelectorOpts{
+			ObjectSelector:    &metav1.LabelSelector{},
+			NamespaceSelector: &metav1.LabelSelector{},
+			DefaultNamespace:  "default",
+		},
+		wantPods: []corev1.Pod{
+			*podFromNameNs("pod-1", "default"),
+			*podFromNameNs("pod-2", "default"),
+			*podFromNameNs("pod-3", "default-1"),
+			*podFromNameNs("pod-3", "default-2"),
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-2"}},
+		},
 	}
+	f(o)
+
+	// objects matched selectors at single namespace
+	o = opts{
+		so: &SelectorOpts{
+			ObjectSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"env": "prod", "v": "v1"},
+			},
+			DefaultNamespace: "default-5",
+		},
+		wantPods: []corev1.Pod{
+			{
+				ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar"}}},
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-5"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-2"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// all objects at multiple namespaces matched selectors
+	o = opts{
+		so: &SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "env", "foo": "bar"}},
+			DefaultNamespace:  "default-5",
+		},
+		wantPods: []corev1.Pod{
+			*podFromNameNs("pod-3", "default-1"),
+			*podFromNameNs("pod-1", "default-3"),
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default-3"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// objects matched selectors at multiple namespaces matched selectors
+	o = opts{
+		so: &SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "env", "foo": "bar"}},
+			ObjectSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"env": "prod", "v": "v1"},
+			},
+			DefaultNamespace: "default-5",
+		},
+		wantPods: []corev1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-1", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar", "baz": "bar"}}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod-5", Namespace: "default-3", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar"}}},
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-0", "default-3"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-4", "default-1"),
+			podFromNameNs("pod-6", "default-2"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-1", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar", "baz": "bar"}}},
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-5", Namespace: "default-3", Labels: map[string]string{"env": "prod", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// match nothing due to namespace selectors mismatch
+	o = opts{
+		so: &SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "prod", "bar": "baz"}},
+			DefaultNamespace:  "default-5",
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default-3"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "d", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// match nothing due to object selectors mismatch
+	o = opts{
+		so: &SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "env", "foo": "bar"}},
+			ObjectSelector:    &metav1.LabelSelector{MatchLabels: map[string]string{"bar": "baz"}},
+			DefaultNamespace:  "default-5",
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default-3"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "default-2"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "d", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// watch namespace is set for single ns
+	o = opts{
+		watchNamespaces: []string{"dev"},
+		so: &SelectorOpts{
+			NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "env", "foo": "bar"}},
+			ObjectSelector:    &metav1.LabelSelector{},
+			DefaultNamespace:  "dev",
+		},
+		wantPods: []corev1.Pod{
+			*podFromNameNs("pod-3", "dev"),
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default-3"),
+			podFromNameNs("pod-2", "default"),
+			podFromNameNs("pod-3", "default-1"),
+			podFromNameNs("pod-3", "dev"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default-5", Labels: map[string]string{"env": "d", "v": "v1", "foo": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dev"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
+	// watch namespace is set for multiple namespace with object selector
+	o = opts{
+		watchNamespaces: []string{"dev", "prod"},
+		so: &SelectorOpts{
+			ObjectSelector:   &metav1.LabelSelector{MatchLabels: map[string]string{"dev": "env", "foo": "bar"}},
+			DefaultNamespace: "dev",
+		},
+		wantPods: []corev1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "dev", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod-5", Namespace: "prod", Labels: map[string]string{"dev": "env", "foo": "bar", "baz": "bar"}}},
+		},
+		predefinedObjects: []runtime.Object{
+			podFromNameNs("pod-1", "default-3"),
+			podFromNameNs("pod-2", "dev"),
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "dev", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-5", Namespace: "prod", Labels: map[string]string{"dev": "env", "foo": "bar", "baz": "bar"}}},
+
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dev"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "prod"}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-1", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-3", Labels: map[string]string{"dev": "env", "foo": "bar"}}},
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default-5"}},
+		},
+	}
+	f(o)
+
 }