Repository: istio/istio (go)

Issue:
Removing the default ingress gateway causes runaway logs
**Description**

After installing Istio 1.9.1 with the default profile using istioctl, and subsequently disabling the default istio-ingressgateway with an IstioOperator file and istioctl, the istiod pod/discovery container outputs huge amounts of logs in the format:

```
2021-03-09T07:06:33.010074Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.010260Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.010449Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.010768Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.010970Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.011142Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.011280Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
2021-03-09T07:06:33.011464Z info Work item handle failed (service "istio-ingressgateway" not found), retry after delay 1s
```

When I first applied the change, it was outputing about 2000 logs lines/minute.  This linearly increased over 4 hours to 18000 log lines/minute.  I then restarted the `istiod` deployment and the issue stopped.

If left unnoticed, this could cause very large amounts of logs to be generated (and stored, costing much money).

**Affected product area (please put an X in all that apply)**

[ ] Docs
[x] Installation
[ ] Networking
[ ] Performance and Scalability
[ ] Extensions and Telemetry
[ ] Security
[ ] Test and Release
[ ] User Experience
[ ] Developer Infrastructure
[ ] Upgrade

**Affected features (please put an X in all that apply)**

[ ] Multi Cluster
[ ] Virtual Machine
[ ] Multi Control Plane

**Expected behavior**

No work should have been attempted on `istio-ingressgateway` as it was removed.  and/or, less log lines should have been generated.

**Steps to reproduce the bug**

Using GKE:

1. Create a new GKE cluster (I used v1.18.16-gke.300)
2. Install Istio 1.9.1 using `istioctl install`
3. Disable the default ingress gateway by applying this config using `istioctl install -f config.yaml`

config.yaml:
```yaml
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  components:
    ingressGateways:
      - name: istio-ingressgateway
        enabled: false
```

4. istiod pod, discovery container starts outputing many logs to stdout

**Version** (include the output of `istioctl version --remote` and `kubectl version --short` and `helm version --short` if you used Helm)

client version: 1.9.1
control plane version: 1.9.1
data plane version: 1.9.1 (3 proxies)

Client Version: v1.16.6-beta.0
Server Version: v1.18.16-gke.300

**How was Istio installed?**

`istioctl install`

**Environment where the bug was observed (cloud vendor, OS, etc)**

Google Cloud / GKE

Pull request description:
Fix runaway ingress status errors
Fixes #31336, probably want to cherry-pick back through 1.8

Still unclear to me how there could be 2000-18000 logs per minute with the queue rate limiting the errors without multiple status controller queues, which there don't seem to be from goroutine dump

[ ] Configuration Infrastructure
[ ] Docs
[ ] Installation
[x] Networking
[ ] Performance and Scalability
[ ] Policies and Telemetry
[ ] Security
[ ] Test and Release
[ ] User Experience
[ ] Developer Infrastructure


Pull Request Attributes

Please check any characteristics that apply to this pull request. 

[ ] Does not have any changes that may affect Istio users.


Patch:
diff --git a/operator/pkg/helmreconciler/prune.go b/operator/pkg/helmreconciler/prune.go
index be50996b22..bea44242de 100644
--- a/operator/pkg/helmreconciler/prune.go
+++ b/operator/pkg/helmreconciler/prune.go
@@ -211,8 +211,23 @@ func (h *HelmReconciler) GetPrunedResources(revision string, includeClusterResou
 			err = h.client.List(context.TODO(), objects,
 				client.MatchingLabelsSelector{Selector: s.Add(*componentRequirement)})
 		} else {
-			err = h.client.List(context.TODO(), objects,
-				client.MatchingLabelsSelector{Selector: selector.Add(*componentRequirement)})
+			// do not prune base components or unknown components
+			includeCN := []string{
+				string(name.PilotComponentName), string(name.IstiodRemoteComponentName),
+				string(name.IngressComponentName), string(name.EgressComponentName),
+				string(name.CNIComponentName), string(name.IstioOperatorComponentName),
+			}
+			includeRequirement, err := klabels.NewRequirement(IstioComponentLabelStr, selection.In, includeCN)
+			if err != nil {
+				return usList, err
+			}
+			if err = h.client.List(context.TODO(), objects,
+				client.MatchingLabelsSelector{
+					Selector: selector.Add(*includeRequirement, *componentRequirement),
+				},
+			); err != nil {
+				continue
+			}
 		}
 		if err != nil {
 			continue
diff --git a/pilot/pkg/config/kube/ingress/status.go b/pilot/pkg/config/kube/ingress/status.go
index 8ded605de0..5a7982698a 100644
--- a/pilot/pkg/config/kube/ingress/status.go
+++ b/pilot/pkg/config/kube/ingress/status.go
@@ -46,10 +46,6 @@ type StatusSyncer struct {
 	meshHolder mesh.Holder
 	client     kubernetes.Interface
 
-	// Name of service (istio-ingressgateway default) to find the IP
-	ingressService  string
-	ingressSelector string
-
 	queue              queue.Instance
 	ingressLister      listerv1beta1.IngressLister
 	podLister          listerv1.PodLister
@@ -73,7 +69,7 @@ func NewStatusSyncer(meshHolder mesh.Holder, client kubelib.Client) *StatusSynce
 	}
 
 	// queue requires a time duration for a retry delay after a handler error
-	q := queue.NewQueue(1 * time.Second)
+	q := queue.NewQueue(5 * time.Second)
 
 	return &StatusSyncer{
 		meshHolder:         meshHolder,
@@ -84,14 +80,15 @@ func NewStatusSyncer(meshHolder mesh.Holder, client kubelib.Client) *StatusSynce
 		nodeLister:         client.KubeInformer().Core().V1().Nodes().Lister(),
 		ingressClassLister: ingressClassLister,
 		queue:              q,
-		ingressService:     meshHolder.Mesh().IngressService,
-		ingressSelector:    meshHolder.Mesh().IngressSelector,
 	}
 }
 
 func (s *StatusSyncer) onEvent() error {
 	addrs, err := s.runningAddresses(ingressNamespace)
 	if err != nil {
+		if kerrors.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
 
@@ -161,9 +158,11 @@ func (s *StatusSyncer) updateStatus(status []coreV1.LoadBalancerIngress) error {
 // where the ingress controller is currently running
 func (s *StatusSyncer) runningAddresses(ingressNs string) ([]string, error) {
 	addrs := make([]string, 0)
+	ingressService := s.meshHolder.Mesh().IngressService
+	ingressSelector := s.meshHolder.Mesh().IngressSelector
 
-	if s.ingressService != "" {
-		svc, err := s.serviceLister.Services(ingressNs).Get(s.ingressService)
+	if ingressService != "" {
+		svc, err := s.serviceLister.Services(ingressNs).Get(ingressService)
 		if err != nil {
 			return nil, err
 		}
@@ -186,7 +185,7 @@ func (s *StatusSyncer) runningAddresses(ingressNs string) ([]string, error) {
 	}
 
 	// get all pods acting as ingress gateways
-	igSelector := getIngressGatewaySelector(s.ingressSelector, s.ingressService)
+	igSelector := getIngressGatewaySelector(ingressSelector, ingressService)
 	igPods, err := s.podLister.Pods(ingressNamespace).List(labels.SelectorFromSet(igSelector))
 	if err != nil {
 		return nil, err
diff --git a/pilot/pkg/config/kube/ingressv1/status.go b/pilot/pkg/config/kube/ingressv1/status.go
index cea212e9bd..6fff2a9ba8 100644
--- a/pilot/pkg/config/kube/ingressv1/status.go
+++ b/pilot/pkg/config/kube/ingressv1/status.go
@@ -46,10 +46,6 @@ type StatusSyncer struct {
 	meshHolder mesh.Holder
 	client     kubernetes.Interface
 
-	// Name of service (istio-ingressgateway default) to find the IP
-	ingressService  string
-	ingressSelector string
-
 	queue              queue.Instance
 	ingressLister      ingresslister.IngressLister
 	podLister          listerv1.PodLister
@@ -67,7 +63,7 @@ func (s *StatusSyncer) Run(stopCh <-chan struct{}) {
 // NewStatusSyncer creates a new instance
 func NewStatusSyncer(meshHolder mesh.Holder, client kubelib.Client) *StatusSyncer {
 	// queue requires a time duration for a retry delay after a handler error
-	q := queue.NewQueue(1 * time.Second)
+	q := queue.NewQueue(5 * time.Second)
 
 	return &StatusSyncer{
 		meshHolder:         meshHolder,
@@ -78,14 +74,15 @@ func NewStatusSyncer(meshHolder mesh.Holder, client kubelib.Client) *StatusSynce
 		nodeLister:         client.KubeInformer().Core().V1().Nodes().Lister(),
 		ingressClassLister: client.KubeInformer().Networking().V1().IngressClasses().Lister(),
 		queue:              q,
-		ingressService:     meshHolder.Mesh().IngressService,
-		ingressSelector:    meshHolder.Mesh().IngressSelector,
 	}
 }
 
 func (s *StatusSyncer) onEvent() error {
 	addrs, err := s.runningAddresses(ingressNamespace)
 	if err != nil {
+		if kerrors.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
 
@@ -155,9 +152,11 @@ func (s *StatusSyncer) updateStatus(status []coreV1.LoadBalancerIngress) error {
 // where the ingress controller is currently running
 func (s *StatusSyncer) runningAddresses(ingressNs string) ([]string, error) {
 	addrs := make([]string, 0)
+	ingressService := s.meshHolder.Mesh().IngressService
+	ingressSelector := s.meshHolder.Mesh().IngressSelector
 
-	if s.ingressService != "" {
-		svc, err := s.serviceLister.Services(ingressNs).Get(s.ingressService)
+	if ingressService != "" {
+		svc, err := s.serviceLister.Services(ingressNs).Get(ingressService)
 		if err != nil {
 			return nil, err
 		}
@@ -180,7 +179,7 @@ func (s *StatusSyncer) runningAddresses(ingressNs string) ([]string, error) {
 	}
 
 	// get all pods acting as ingress gateways
-	igSelector := getIngressGatewaySelector(s.ingressSelector, s.ingressService)
+	igSelector := getIngressGatewaySelector(ingressSelector, ingressService)
 	igPods, err := s.podLister.Pods(ingressNamespace).List(labels.SelectorFromSet(igSelector))
 	if err != nil {
 		return nil, err
diff --git a/releasenotes/notes/31336.yaml b/releasenotes/notes/31336.yaml
new file mode 100644
index 0000000000..dec81fb931
--- /dev/null
+++ b/releasenotes/notes/31336.yaml
@@ -0,0 +1,8 @@
+apiVersion: release-notes/v2
+kind: bug
+area: networking
+issue:
+  - 31336
+releaseNotes:
+  - |
+    **Fixed** a bug causing runaway logs in `istiod` after disabling the default ingress controller.


Test patch:
diff --git a/pilot/pkg/config/kube/ingress/status_test.go b/pilot/pkg/config/kube/ingress/status_test.go
index 2f87896a42..c426079d9a 100644
--- a/pilot/pkg/config/kube/ingress/status_test.go
+++ b/pilot/pkg/config/kube/ingress/status_test.go
@@ -102,9 +102,9 @@ func setupFake(t *testing.T, client kubelib.Client) {
 	}
 }
 
-func fakeMeshHolder() mesh.Holder {
+func fakeMeshHolder(ingressService string) mesh.Holder {
 	config := mesh.DefaultMeshConfig()
-	config.IngressService = "istio-ingress"
+	config.IngressService = ingressService
 	return mesh.NewFixedWatcher(&config)
 }
 
@@ -115,7 +115,7 @@ func makeStatusSyncer(t *testing.T) *StatusSyncer {
 
 	client := kubelib.NewFakeClient()
 	setupFake(t, client)
-	sync := NewStatusSyncer(fakeMeshHolder(), client)
+	sync := NewStatusSyncer(fakeMeshHolder("istio-ingress"), client)
 	stop := make(chan struct{})
 	client.RunAndWait(stop)
 	t.Cleanup(func() {
@@ -156,7 +156,7 @@ func testRunningAddressesWithService(t *testing.T) {
 
 func testRunningAddressesWithHostname(t *testing.T) {
 	syncer := makeStatusSyncer(t)
-	syncer.ingressService = "istio-ingress-hostname"
+	syncer.meshHolder = fakeMeshHolder("istio-ingress-hostname")
 
 	address, err := syncer.runningAddresses(testNamespace)
 	if err != nil {
@@ -171,7 +171,7 @@ func testRunningAddressesWithHostname(t *testing.T) {
 func TestRunningAddressesWithPod(t *testing.T) {
 	ingressNamespace = "istio-system" // it is set in real pilot on newController.
 	syncer := makeStatusSyncer(t)
-	syncer.ingressService = ""
+	syncer.meshHolder = fakeMeshHolder("")
 
 	address, err := syncer.runningAddresses(ingressNamespace)
 	if err != nil {
diff --git a/pilot/pkg/config/kube/ingressv1/status_test.go b/pilot/pkg/config/kube/ingressv1/status_test.go
index 2f87896a42..c426079d9a 100644
--- a/pilot/pkg/config/kube/ingressv1/status_test.go
+++ b/pilot/pkg/config/kube/ingressv1/status_test.go
@@ -102,9 +102,9 @@ func setupFake(t *testing.T, client kubelib.Client) {
 	}
 }
 
-func fakeMeshHolder() mesh.Holder {
+func fakeMeshHolder(ingressService string) mesh.Holder {
 	config := mesh.DefaultMeshConfig()
-	config.IngressService = "istio-ingress"
+	config.IngressService = ingressService
 	return mesh.NewFixedWatcher(&config)
 }
 
@@ -115,7 +115,7 @@ func makeStatusSyncer(t *testing.T) *StatusSyncer {
 
 	client := kubelib.NewFakeClient()
 	setupFake(t, client)
-	sync := NewStatusSyncer(fakeMeshHolder(), client)
+	sync := NewStatusSyncer(fakeMeshHolder("istio-ingress"), client)
 	stop := make(chan struct{})
 	client.RunAndWait(stop)
 	t.Cleanup(func() {
@@ -156,7 +156,7 @@ func testRunningAddressesWithService(t *testing.T) {
 
 func testRunningAddressesWithHostname(t *testing.T) {
 	syncer := makeStatusSyncer(t)
-	syncer.ingressService = "istio-ingress-hostname"
+	syncer.meshHolder = fakeMeshHolder("istio-ingress-hostname")
 
 	address, err := syncer.runningAddresses(testNamespace)
 	if err != nil {
@@ -171,7 +171,7 @@ func testRunningAddressesWithHostname(t *testing.T) {
 func TestRunningAddressesWithPod(t *testing.T) {
 	ingressNamespace = "istio-system" // it is set in real pilot on newController.
 	syncer := makeStatusSyncer(t)
-	syncer.ingressService = ""
+	syncer.meshHolder = fakeMeshHolder("")
 
 	address, err := syncer.runningAddresses(ingressNamespace)
 	if err != nil {
diff --git a/pkg/test/framework/components/environment/kube/kube.go b/pkg/test/framework/components/environment/kube/kube.go
index bd2cc96331..2ca449a4a2 100644
--- a/pkg/test/framework/components/environment/kube/kube.go
+++ b/pkg/test/framework/components/environment/kube/kube.go
@@ -33,7 +33,16 @@ type Environment struct {
 var _ resource.Environment = &Environment{}
 
 // New returns a new Kubernetes environment
-func New(ctx resource.Context, s *Settings) (resource.Environment, error) {
+func New(ctx resource.Context, s *Settings) (env resource.Environment, err error) {
+	defer func() {
+		if err != nil && !ctx.Settings().CIMode {
+			scopes.Framework.Infof(`
+There was an error while setting up the Kubernetes test environment.
+Check the test framework wiki for details on running the tests locally:
+    https://github.com/istio/istio/wiki/Istio-Test-Framework
+`)
+		}
+	}()
 	scopes.Framework.Infof("Test Framework Kubernetes environment Settings:\n%s", s)
 	e := &Environment{
 		ctx: ctx,
diff --git a/tests/integration/pilot/revisions/uninstall_test.go b/tests/integration/pilot/revisions/uninstall_test.go
index e70d4c0c13..2793f0c391 100644
--- a/tests/integration/pilot/revisions/uninstall_test.go
+++ b/tests/integration/pilot/revisions/uninstall_test.go
@@ -18,20 +18,17 @@ package revisions
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	kubeApiMeta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"istio.io/istio/operator/pkg/helmreconciler"
-	"istio.io/istio/operator/pkg/manifest"
 	"istio.io/istio/operator/pkg/name"
-	"istio.io/istio/operator/pkg/object"
 	"istio.io/istio/pkg/test"
 	"istio.io/istio/pkg/test/env"
 	"istio.io/istio/pkg/test/framework"
@@ -42,7 +39,9 @@ import (
 )
 
 const (
-	stableRevision = "stable"
+	stableRevision       = "stable"
+	checkResourceTimeout = time.Second * 120
+	checkResourceDelay   = time.Millisecond * 100
 )
 
 var ManifestPath = filepath.Join(env.IstioSrc, "manifests")
@@ -64,7 +63,7 @@ func TestUninstallByRevision(t *testing.T) {
 				}
 				cs := t.Clusters().Default()
 				ls := fmt.Sprintf("istio.io/rev=%s", stableRevision)
-				checkCPResourcesUninstalled(t, cs, append(helmreconciler.NamespacedResources, helmreconciler.ClusterCPResources...), ls)
+				checkCPResourcesUninstalled(t, cs, append(helmreconciler.NamespacedResources, helmreconciler.ClusterCPResources...), ls, false)
 			})
 		})
 }
@@ -86,63 +85,11 @@ func TestUninstallWithSetFlag(t *testing.T) {
 				}
 				cs := t.Clusters().Default()
 				ls := fmt.Sprintf("istio.io/rev=%s", stableRevision)
-				checkCPResourcesUninstalled(t, cs, append(helmreconciler.NamespacedResources, helmreconciler.ClusterCPResources...), ls)
+				checkCPResourcesUninstalled(t, cs, append(helmreconciler.NamespacedResources, helmreconciler.ClusterCPResources...), ls, false)
 			})
 		})
 }
 
-func TestUninstallByManifest(t *testing.T) {
-	framework.
-		NewTest(t).
-		Features("installation.istioctl.uninstall_manifest").
-		Run(func(t framework.TestContext) {
-			workDir, err := t.CreateTmpDirectory("uninstall-test")
-			if err != nil {
-				t.Fatal("failed to create test directory")
-			}
-			istioCtl := istioctl.NewOrFail(t, t, istioctl.Config{})
-			iopFile := filepath.Join(workDir, "iop.yaml")
-			iopYAML := `
-apiVersion: install.istio.io/v1alpha1
-kind: IstioOperator
-spec:
-  revision: %s
-  installPackagePath: %s
-`
-			iopYAML = fmt.Sprintf(iopYAML, stableRevision, ManifestPath)
-			if err := ioutil.WriteFile(iopFile, []byte(iopYAML), os.ModePerm); err != nil {
-				t.Fatalf("failed to write iop cr file: %v", err)
-			}
-			uninstallCmd := []string{
-				"x", "uninstall",
-				"--filename=" + iopFile, "--skip-confirmation",
-			}
-			istioCtl.InvokeOrFail(t, uninstallCmd)
-			cs := t.Clusters().Default()
-			retry.UntilSuccessOrFail(t, func() error {
-				manifestMap, _, err := manifest.GenManifests([]string{iopFile}, []string{}, true, nil, nil)
-				if err != nil {
-					t.Fatalf("failed to generate manifest: %v", err)
-				}
-				manifests := manifestMap[name.PilotComponentName]
-				objects, err := object.ParseK8sObjectsFromYAMLManifest(strings.Join(manifests, "---"))
-				if err != nil {
-					t.Fatalf("failed parse k8s objects from yaml: %v", err)
-				}
-				objMap := objects.ToMap()
-				for _, obj := range objMap {
-					resources := strings.ToLower(obj.Kind) + "s"
-					gvr := schema.GroupVersionResource{Group: obj.Group, Version: obj.Version(), Resource: resources}
-					ls := fmt.Sprintf("istio.io/rev=%s", stableRevision)
-					if err := checkResourcesNotInCluster(cs, gvr, ls); err != nil {
-						return err
-					}
-				}
-				return nil
-			}, retry.Delay(time.Millisecond*100), retry.Timeout(time.Second*120))
-		})
-}
-
 func TestUninstallPurge(t *testing.T) {
 	framework.
 		NewTest(t).
@@ -156,43 +103,69 @@ func TestUninstallPurge(t *testing.T) {
 			istioCtl.InvokeOrFail(t, uninstallCmd)
 			cs := t.Clusters().Default()
 			checkCPResourcesUninstalled(t, cs, append(helmreconciler.NamespacedResources, helmreconciler.AllClusterResources...),
-				helmreconciler.IstioComponentLabelStr)
+				helmreconciler.IstioComponentLabelStr, true)
 		})
 }
 
 // checkCPResourcesUninstalled is a helper function to check list of gvk resources matched with label are uninstalled
-func checkCPResourcesUninstalled(t test.Failer, cs cluster.Cluster, gvkResources []schema.GroupVersionKind, label string) {
+// If purge is set to true, we expect all resources are removed.
+// Otherwise we expect only selected resources from control plane are removed, resources from base and the legacy addon installation would not be touched.
+func checkCPResourcesUninstalled(t test.Failer, cs cluster.Cluster, gvkResources []schema.GroupVersionKind, label string, purge bool) {
 	retry.UntilSuccessOrFail(t, func() error {
+		var reStrlist []string
+		var reItemList []unstructured.Unstructured
 		for _, gvk := range gvkResources {
 			resources := strings.ToLower(gvk.Kind) + "s"
 			gvr := schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: resources}
-
-			if err := checkResourcesNotInCluster(cs, gvr, label); err != nil {
-				return err
-			}
+			reList, reStr := getRemainingResourcesCluster(cs, gvr, label)
+			reItemList = append(reItemList, reList...)
+			reStrlist = append(reStrlist, reStr...)
 		}
-		return nil
-	}, retry.Delay(time.Millisecond*100), retry.Timeout(time.Second*120))
+		return inspectRemainingResources(reItemList, reStrlist, purge)
+	}, retry.Delay(checkResourceDelay), retry.Timeout(checkResourceTimeout))
 }
 
-func checkResourcesNotInCluster(cs cluster.Cluster, gvr schema.GroupVersionResource, ls string) error {
+// getRemainingResourcesCluster get specific resources from the cluster
+func getRemainingResourcesCluster(cs cluster.Cluster, gvr schema.GroupVersionResource, ls string) ([]unstructured.Unstructured, []string) {
 	usList, _ := cs.Dynamic().Resource(gvr).List(context.TODO(), kubeApiMeta.ListOptions{LabelSelector: ls})
+	var remainingResources []unstructured.Unstructured
+	var staleList []string
 	if usList != nil && len(usList.Items) != 0 {
-		var stalelist []string
 		for _, item := range usList.Items {
 			// ignore IstioOperator CRD because the operator CR is not in the pruning list
 			if item.GetName() == "istiooperators.install.istio.io" {
 				continue
 			}
-			stalelist = append(stalelist, item.GroupVersionKind().String()+"/"+item.GetName())
+			remainingResources = append(remainingResources, item)
+			staleList = append(staleList, item.GroupVersionKind().String()+"/"+item.GetName())
 		}
-		if len(stalelist) == 0 {
-			return nil
+	}
+	return remainingResources, staleList
+}
+
+func inspectRemainingResources(reItemList []unstructured.Unstructured, reStrList []string, purge bool) error {
+	// for purge case we expect all resources removed
+	if purge {
+		if len(reStrList) != 0 {
+			msg := fmt.Sprintf("resources expected to be pruned but still exist in the cluster: %s",
+				strings.Join(reStrList, " "))
+			scopes.Framework.Warnf(msg)
+			return fmt.Errorf(msg)
+		}
+		return nil
+	}
+	// for other cases, we expect base component resources to be kept.
+	if len(reStrList) != 0 {
+		for _, remaining := range reItemList {
+			labels := remaining.GetLabels()
+			cn, ok := labels["operator.istio.io/component"]
+			// we don't need to check the legacy addons here because we would not install that in test anymore.
+			if ok && cn != string(name.IstioBaseComponentName) {
+				return fmt.Errorf("expect only base component resources still exist")
+			}
 		}
-		msg := fmt.Sprintf("resources expected to be pruned but still exist in the cluster: %s",
-			strings.Join(stalelist, " "))
-		scopes.Framework.Warnf(msg)
-		return fmt.Errorf(msg)
+	} else {
+		return fmt.Errorf("expect base component resources to exist but they were removed")
 	}
 	return nil
 }