Repository: minio/operator

Issue:
Disable the CRDs installation via Helm Chart configuration
**Is your feature request related to a problem? Please describe.**
I created a Helm Chart to manage the CRDs for all the services I use, and I want to disable the installation of CRDs from the MinIO Operator Helm Chart.

**Describe the solution you'd like**

Add a new flag to the MinIO Operator Helm Chart to control CRD installation, with the default value set to `true`.

```
operator:
    crds:
      enabled: true
```

**Additional context**
The K8S cluster is managed by ArgoCD


Discussion:
@ramondeklein: I don't see a valid use-case for splitting the CRDs. The operator Helm chart doesn't instantiate these CRDs, so there should be no issue with registering/updating the CRDs.@alita1991: The MinIO Helm chart includes CRDs, and if they're present in the templates directory, Helm will always install them.

Currently, one solution which Helm recommends is the following:

- When CRDs are placed in the templates folder, a feature gate (flag) is typically used to control whether they are installed. (our scenario)

https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#install-a-crd-declaration-before-using-the-resource
@stale: This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 21 days if no further activity occurs. Thank you for your contributions.

Solution patch:
diff --git a/README.md b/README.md
index 65ef1b33..479df45f 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # MinIO Operator
 
-![build](https://github.com/minio/operator/workflows/Go/badge.svg) ![license](https://img.shields.io/badge/license-AGPL%20V3-blue)
+[![build](https://img.shields.io/badge/build-passing-green.svg)](https://github.com/minio/operator/actions) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/operator/blob/master/LICENSE)
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml
index f4cf809d..93f1fdd4 100644
--- a/helm/operator/templates/minio.min.io_tenants.yaml
+++ b/helm/operator/templates/minio.min.io_tenants.yaml
@@ -1,3 +1,4 @@
+{{ if .Values.operator.installCRDs }}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -5754,3 +5755,4 @@ spec:
     storage: true
     subresources:
       status: {}
+{{ end }}
diff --git a/helm/operator/templates/sts.min.io_policybindings.yaml b/helm/operator/templates/sts.min.io_policybindings.yaml
index 36099645..93a83e3e 100644
--- a/helm/operator/templates/sts.min.io_policybindings.yaml
+++ b/helm/operator/templates/sts.min.io_policybindings.yaml
@@ -1,3 +1,4 @@
+{{ if .Values.operator.installCRDs }}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -131,3 +132,4 @@ spec:
     storage: true
     subresources:
       status: {}
+{{ end }}
diff --git a/helm/operator/values.yaml b/helm/operator/values.yaml
index 77a6f750..39e1c8a1 100644
--- a/helm/operator/values.yaml
+++ b/helm/operator/values.yaml
@@ -75,6 +75,8 @@ operator:
   #      pullPolicy: IfNotPresent
   #
   sidecarImage: {}
+  # Install custom resource definitions
+  installCRDs: true
   ###
   #
   # An array of Kubernetes secrets to use for pulling images from a private ``image.repository``.
diff --git a/pkg/apis/minio.min.io/v2/names.go b/pkg/apis/minio.min.io/v2/names.go
index b0761f91..b65a667b 100644
--- a/pkg/apis/minio.min.io/v2/names.go
+++ b/pkg/apis/minio.min.io/v2/names.go
@@ -207,7 +207,7 @@ func (t *Tenant) PrometheusConfigJobName() string {
 // PrometheusOperatorAddlConfigJobName returns the name of the prometheus job
 // when prometheus operator is enabled
 func (t *Tenant) PrometheusOperatorAddlConfigJobName() string {
-	return fmt.Sprintf("%s-minio-job", t.Name)
+	return fmt.Sprintf("%s-%s-minio-job", t.Name, t.Namespace)
 }
 
 // PrometheusConfigMapName returns name of the config map for Prometheus.
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 2b60fa7e..0dda2213 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -15,6 +15,7 @@
 package controller
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -79,8 +80,10 @@ func StartOperator(kubeconfig string) {
 	_ = stsv1beta1.AddToScheme(scheme.Scheme)
 	_ = stsv1alpha1.AddToScheme(scheme.Scheme)
 	klog.Info("Starting MinIO Operator")
+
 	// set up signals, so we handle the first shutdown signal gracefully
-	stopCh := setupSignalHandler()
+	ctx, cancel := setupSignalHandler(context.Background())
+	defer cancel()
 
 	flag.Parse()
 
@@ -170,15 +173,15 @@ func StartOperator(kubeconfig string) {
 		kubeInformerFactoryInOperatorNamespace,
 	)
 
-	go kubeInformerFactory.Start(stopCh)
-	go minioInformerFactory.Start(stopCh)
-	go kubeInformerFactoryInOperatorNamespace.Start(stopCh)
+	go kubeInformerFactory.Start(ctx.Done())
+	go minioInformerFactory.Start(ctx.Done())
+	go kubeInformerFactoryInOperatorNamespace.Start(ctx.Done())
 
-	if err = mainController.Start(2, stopCh); err != nil {
+	if err = mainController.Start(ctx, cancel, 2); err != nil {
 		klog.Fatalf("Error running mainController: %s", err.Error())
 	}
 
-	<-stopCh
+	<-ctx.Done()
 	klog.Info("Shutting down the MinIO Operator")
 	mainController.Stop()
 }
@@ -186,22 +189,11 @@ func StartOperator(kubeconfig string) {
 // setupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
 // which is closed on one of these signals. If a second signal is caught, the program
 // is terminated with exit code 1.
-func setupSignalHandler() (stopCh <-chan struct{}) {
+func setupSignalHandler(ctx context.Context) (context.Context, context.CancelFunc) {
 	// panics when called twice
 	close(onlyOneSignalHandler)
 
-	stop := make(chan struct{})
-	c := make(chan os.Signal, 2)
-	signal.Notify(c, shutdownSignals...)
-	go func() {
-		<-c
-		close(stop)
-		<-c
-		// second signal. Exit directly.
-		os.Exit(1)
-	}()
-
-	return stop
+	return signal.NotifyContext(ctx, shutdownSignals...)
 }
 
 // Result contains the result of a sync invocation.
diff --git a/pkg/controller/csr.go b/pkg/controller/csr.go
index e627a187..c6763fc3 100644
--- a/pkg/controller/csr.go
+++ b/pkg/controller/csr.go
@@ -24,7 +24,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"os/signal"
 	"syscall"
 	"time"
@@ -188,15 +187,14 @@ func (c *Controller) fetchCertificate(ctx context.Context, csrName string) ([]by
 
 	timeout := time.NewTimer(miniov2.DefaultQueryTimeout)
 
-	ch := make(chan os.Signal, 1) // should be always un-buffered SA1017
-	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
-	defer signal.Stop(ch)
+	ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
 
 	for {
 		select {
-		case s := <-ch:
-			klog.Infof("Signal %s received, exiting ...", s.String())
-			return nil, fmt.Errorf("%s", s.String())
+		case <-ctx.Done():
+			klog.Infof("Termination signal received, exiting.")
+			return nil, errors.New("Termination signal received, exiting.")
 
 		case <-tick.C:
 			if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 {
diff --git a/pkg/controller/decomission.go b/pkg/controller/decomission.go
index 926b07a2..c487b120 100644
--- a/pkg/controller/decomission.go
+++ b/pkg/controller/decomission.go
@@ -79,6 +79,9 @@ func (c *Controller) checkForPoolDecommission(ctx context.Context, key string, t
 		}
 		if noDecomCommon {
 			klog.Warningf("%s Detected we are removing a pool but spec.Pool[].Name's are duplicated - disallowing removal", key)
+			if _, err = c.updateTenantStatus(ctx, tenant, StatusDecommissioningNotAllowed, 0); err != nil {
+				return nil, err
+			}
 			return nil, errors.New("removing pool not allowed")
 		}
 
diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go
index 4f8b70f6..c7f1758e 100644
--- a/pkg/controller/main-controller.go
+++ b/pkg/controller/main-controller.go
@@ -21,10 +21,7 @@ import (
 	"fmt"
 	"maps"
 	"net/http"
-	"os"
-	"os/signal"
 	"strings"
-	"syscall"
 	"time"
 
 	kubeinformers "k8s.io/client-go/informers"
@@ -304,6 +301,9 @@ func NewController(
 			}
 			controller.enqueueTenant(newObj)
 		},
+		// Enqueue tenant to perform some delete handling actions
+		// during reconciliation
+		DeleteFunc: controller.enqueueTenant,
 	})
 
 	// Set up an event handler for when StatefulSet resources change. This
@@ -376,8 +376,8 @@ func NewController(
 }
 
 // StartPodInformer runs PodInformer
-func (c *Controller) StartPodInformer(stopCh <-chan struct{}) {
-	c.podInformer.Run(stopCh)
+func (c *Controller) StartPodInformer(ctx context.Context) {
+	c.podInformer.Run(ctx.Done())
 }
 
 // startUpgradeServer Starts the Upgrade tenant API server and notifies the start and stop via notificationChannel returned
@@ -421,7 +421,7 @@ func (c *Controller) startSTSAPIServer(ctx context.Context, notificationChannel
 
 // leaderRun start the Controller and the API's
 // When a new leader is elected this function is ran in the pod
-func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-chan struct{}, notificationChannel chan *EventNotification) {
+func leaderRun(ctx context.Context, c *Controller, threadiness int, notificationChannel chan *EventNotification) {
 	// we declate the channel to communicate on servers errors
 	var upgradeServerChannel <-chan error
 
@@ -433,21 +433,21 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 
 	// Wait for the caches to be synced before starting workers
 	klog.Info("Waiting for informer caches to sync")
-	if ok := cache.WaitForCacheSync(stopCh, c.statefulSetListerSynced, c.deploymentListerSynced, c.tenantsSynced, c.policyBindingListerSynced, c.secretListerSynced); !ok {
+	if ok := cache.WaitForCacheSync(ctx.Done(), c.statefulSetListerSynced, c.deploymentListerSynced, c.tenantsSynced, c.policyBindingListerSynced, c.secretListerSynced); !ok {
 		panic("failed to wait for caches to sync")
 	}
 
 	// Launch two workers to process Job resources
 	for i := 0; i < threadiness; i++ {
-		go wait.Until(c.runWorker, time.Second, stopCh)
+		go wait.Until(c.runWorker, time.Second, ctx.Done())
 	}
 
 	// Launch a single worker for Health Check reacting to Pod Changes
-	go wait.Until(c.runHealthCheckWorker, time.Second, stopCh)
+	go wait.Until(c.runHealthCheckWorker, time.Second, ctx.Done())
 
 	// Launch a goroutine to monitor all Tenants
-	go c.recurrentTenantStatusMonitor(stopCh)
-	go c.StartPodInformer(stopCh)
+	go c.recurrentTenantStatusMonitor(ctx)
+	go c.StartPodInformer(ctx)
 
 	// 2) we need to make sure we have STS API certificates (if enabled)
 	if IsSTSEnabled() {
@@ -475,7 +475,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 			}
 			// webserver was instructed to stop, do not attempt to restart
 			continue
-		case <-stopCh:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -485,23 +485,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 // as syncing informer caches and starting workers. It will block until stopCh
 // is closed, at which point it will shutdown the workqueue and wait for
 // workers to finish processing their current work items.
-func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-ch
-		klog.Info("Received termination, signaling shutdown")
-		cancel()
-	}()
-
+func (c *Controller) Start(ctx context.Context, cancel context.CancelFunc, threadiness int) error {
 	leaseLockName := "minio-operator-lock"
 	leaseLockNamespace := miniov2.GetNSFromFile()
 
@@ -553,7 +537,7 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				// start the controller + API code
-				leaderRun(ctx, c, threadiness, stopCh, notificationChannel)
+				leaderRun(ctx, c, threadiness, notificationChannel)
 			},
 			OnStoppedLeading: func() {
 				klog.Infof("leader lost, removing any leader labels that I '%s' might have", c.podName)
@@ -568,7 +552,6 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 				} else {
 					c.kubeClientSet.CoreV1().Pods(leaseLockNamespace).Patch(ctx, c.podName, types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
 				}
-				c.Stop()
 				cancel()
 			},
 			OnNewLeader: func(identity string) {
diff --git a/pkg/controller/monitoring.go b/pkg/controller/monitoring.go
index 884c3eb1..abf3fe35 100644
--- a/pkg/controller/monitoring.go
+++ b/pkg/controller/monitoring.go
@@ -40,7 +40,7 @@ const (
 )
 
 // recurrentTenantStatusMonitor loop that checks every N minutes for tenants health
-func (c *Controller) recurrentTenantStatusMonitor(stopCh <-chan struct{}) {
+func (c *Controller) recurrentTenantStatusMonitor(ctx context.Context) {
 	// How often will this function run
 	interval := miniov2.GetMonitoringInterval()
 	ticker := time.NewTicker(time.Duration(interval) * time.Minute)
@@ -53,7 +53,7 @@ func (c *Controller) recurrentTenantStatusMonitor(stopCh <-chan struct{}) {
 			if err := c.tenantsHealthMonitor(); err != nil {
 				klog.Infof("%v", err)
 			}
-		case <-stopCh:
+		case <-ctx.Done():
 			ticker.Stop()
 			return
 		}
diff --git a/pkg/controller/prometheus.go b/pkg/controller/prometheus.go
index 7af822c7..249d2356 100644
--- a/pkg/controller/prometheus.go
+++ b/pkg/controller/prometheus.go
@@ -140,11 +140,13 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 	}
 
 	promCfg := configmaps.GetPrometheusConfig(tenant, accessKey, secretKey)
+	minIOScrapeConfigs := make([]configmaps.ScrapeConfig, len(promCfg.ScrapeConfigs))
+	copy(minIOScrapeConfigs, promCfg.ScrapeConfigs)
 
 	// If the secret is not found, create the secret
 	if k8serrors.IsNotFound(err) {
 		klog.Infof("Adding MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name)
-		scrapeCfgYaml, err := yaml.Marshal(&promCfg.ScrapeConfigs)
+		scrapeCfgYaml, err := yaml.Marshal(&minIOScrapeConfigs)
 		if err != nil {
 			return err
 		}
@@ -168,33 +170,47 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 		if err != nil {
 			return err
 		}
-		// get other scrape configs
+		// Get all non-MinIO scrape configs, as well as MinIO scrape configs for other tenants
 		for _, sc := range scrapeConfigs {
 			if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
 				expectedScrapeConfigs = append(expectedScrapeConfigs, sc)
 			}
 		}
-		ignoreScrapeConfigsIndex := len(expectedScrapeConfigs)
-		expectedScrapeConfigs = append(expectedScrapeConfigs, promCfg.ScrapeConfigs...)
+
+		// Add the expected MinIO scrape configs for the current tenant
+		expectedScrapeConfigs = append(expectedScrapeConfigs, minIOScrapeConfigs...)
+
+		// Verify that all expected scrape configs are present in the installed configs
 		updateScrapeConfig := false
 		if len(scrapeConfigs) != len(expectedScrapeConfigs) {
 			updateScrapeConfig = true
 		} else {
-			for i := range scrapeConfigs {
-				// can't compare that is generated by operator
-				if i < ignoreScrapeConfigsIndex {
-					continue
-				}
-				if scrapeConfigs[i].JobName != expectedScrapeConfigs[i].JobName ||
-					scrapeConfigs[i].MetricsPath != expectedScrapeConfigs[i].MetricsPath ||
-					scrapeConfigs[i].Scheme != expectedScrapeConfigs[i].Scheme ||
-					!reflect.DeepEqual(scrapeConfigs[i].TLSConfig, expectedScrapeConfigs[i].TLSConfig) ||
-					!reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, expectedScrapeConfigs[i].StaticConfigs) {
-					updateScrapeConfig = true
-					break
+			// Verify that all MinIO scrape configs for the current tenant
+			// are included in the installed configs
+			for _, minIOConfig := range minIOScrapeConfigs {
+				configMatched := false
+				for _, config := range scrapeConfigs {
+					if minIOConfig.JobName == config.JobName {
+						if config.MetricsPath != minIOConfig.MetricsPath ||
+							config.Scheme != minIOConfig.Scheme ||
+							!reflect.DeepEqual(config.TLSConfig, minIOConfig.TLSConfig) ||
+							!reflect.DeepEqual(config.StaticConfigs, minIOConfig.StaticConfigs) {
+							updateScrapeConfig = true
+							break
+						}
+						accKey, err := miniov2.GetAccessKeyFromBearerToken(config.BearerToken, secretKey)
+						if err != nil || accKey != accessKey {
+							updateScrapeConfig = true
+							break
+						}
+
+						configMatched = true
+						break
+					}
 				}
-				accKey, _ := miniov2.GetAccessKeyFromBearerToken(scrapeConfigs[i].BearerToken, secretKey)
-				if accKey != accessKey {
+				// Update the installed scrape configs if any MinIO configs for
+				// the current tenant are missing
+				if !configMatched {
 					updateScrapeConfig = true
 					break
 				}
@@ -255,6 +271,7 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min
 		return err
 	}
 	for _, sc := range scrapeConfigs {
+		// Get all non-MinIO scrape configs, as well as MinIO scrape configs for other tenants
 		if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
 			exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc)
 		}
diff --git a/pkg/controller/tls.go b/pkg/controller/tls.go
index b491e2b7..9241379a 100644
--- a/pkg/controller/tls.go
+++ b/pkg/controller/tls.go
@@ -123,9 +123,7 @@ func (c *Controller) generateTLSCertificateForService(serviceName, secretName, d
 		tlsCertSecret, err := c.getCertificateSecret(ctx, namespace, secretName)
 		if err != nil {
 			if k8serrors.IsNotFound(err) {
-				if k8serrors.IsNotFound(err) {
-					klog.Infof("%s TLS secret not found: %v", secretName, err)
-				}
+				klog.Infof("%s TLS secret not found: %v", secretName, err)
 				if err = c.checkAndCreateCSR(ctx, operatorDeployment, serviceName, csrName, secretName); err != nil {
 					klog.Infof("Waiting for the %s certificates to be issued %v", serviceName, err.Error())
 					time.Sleep(time.Second * 10)
diff --git a/resources/base/cluster-role.yaml b/resources/base/cluster-role.yaml
index 84bc6a06..dce73c50 100644
--- a/resources/base/cluster-role.yaml
+++ b/resources/base/cluster-role.yaml
@@ -154,8 +154,9 @@ rules:
       - monitoring.coreos.com
     resources:
       - prometheuses
+      - prometheusagents
     verbs:
-      - '*'
+      - "*"
   - apiGroups:
       - "coordination.k8s.io"
     resources:



Test patch:
diff --git a/pkg/controller/prometheus_test.go b/pkg/controller/prometheus_test.go
index c934e90d..d327adef 100644
--- a/pkg/controller/prometheus_test.go
+++ b/pkg/controller/prometheus_test.go
@@ -16,6 +16,7 @@ package controller
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
@@ -38,26 +39,38 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		changedTokenNumber int
 	}
 	type arg struct {
-		accessKey     string
-		secretKey     string
-		beforeScrapes []configmaps.ScrapeConfig
-		paths         []string
-		expects       expect
+		accessKey       string
+		secretKey       string
+		tenantName      string
+		tenantNamespace string
+		beforeScrapes   []configmaps.ScrapeConfig
+		paths           []string
+		expects         expect
 	}
 	type args struct {
 		name string
 		arg  arg
 	}
+
+	tenantAName := "tenantA"
+	tenantANamespace := "tenantA-ns"
+	tenantBName := "tenantB"
+	tenantBNamespace := "tenantB-ns"
+
+	// Next is to emulate a tenant with same name as tenantA (different namespace)
+	tenantCName := tenantAName
+	tenantCNamespace := "tenantC-ns"
+
 	testRunCheckAndCreatePrometheusAddlConfig := func(t *testing.T, arg arg) {
 		beforeScrapesBytes, err := yaml.Marshal(arg.beforeScrapes)
 		if err != nil {
 			t.Fatal(err)
 		}
-		testNS := "default"
+		testNS := arg.tenantNamespace
 		prometheus := &v1.Prometheus{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "testPrometheus",
-				Namespace: testNS,
+				Namespace: "default",
 			},
 			Spec: v1.PrometheusSpec{
 				CommonPrometheusFields: v1.CommonPrometheusFields{
@@ -73,7 +86,7 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		beforeSecret := &v3.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      miniov2.PrometheusAddlScrapeConfigSecret,
-				Namespace: testNS,
+				Namespace: "default",
 			},
 			Data: map[string][]byte{
 				miniov2.PrometheusAddlScrapeConfigKey: beforeScrapesBytes,
@@ -86,7 +99,7 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		}
 		tenant := &miniov2.Tenant{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      "testTenant",
+				Name:      arg.tenantName,
 				Namespace: testNS,
 			},
 			Spec: miniov2.TenantSpec{
@@ -99,7 +112,7 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		afterSecret, err := kubeclient.CoreV1().Secrets(testNS).Get(context.Background(), miniov2.PrometheusAddlScrapeConfigSecret, metav1.GetOptions{})
+		afterSecret, err := kubeclient.CoreV1().Secrets("default").Get(context.Background(), miniov2.PrometheusAddlScrapeConfigSecret, metav1.GetOptions{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -138,13 +151,17 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		{
 			name: "testDefault",
 			arg: arg{
-				accessKey:     "accessKey",
-				secretKey:     "secretKey",
-				beforeScrapes: []configmaps.ScrapeConfig{},
+				accessKey:       "accessKey",
+				secretKey:       "secretKey",
+				beforeScrapes:   []configmaps.ScrapeConfig{},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
 				expects: expect{
-					afterScrapeNumber:  1,
-					afterScrapePath:    []string{"/minio/v2/metrics/cluster"},
-					afterScrapeName:    []string{"testTenant-minio-job-0"},
+					afterScrapeNumber: 1,
+					afterScrapePath:   []string{"/minio/v2/metrics/cluster"},
+					afterScrapeName: []string{
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+					},
 					changedTokenNumber: 0,
 				},
 			},
@@ -152,14 +169,19 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 		{
 			name: "testFirst",
 			arg: arg{
-				accessKey:     "accessKey",
-				secretKey:     "secretKey",
-				paths:         []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
-				beforeScrapes: []configmaps.ScrapeConfig{},
+				accessKey:       "accessKey",
+				secretKey:       "secretKey",
+				paths:           []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+				beforeScrapes:   []configmaps.ScrapeConfig{},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
 				expects: expect{
-					afterScrapeNumber:  2,
-					afterScrapePath:    []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
-					afterScrapeName:    []string{"testTenant-minio-job-0", "testTenant-minio-job-1"},
+					afterScrapeNumber: 2,
+					afterScrapePath:   []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+					},
 					changedTokenNumber: 0,
 				},
 			},
@@ -180,10 +202,16 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 					},
 				},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
 				expects: expect{
-					afterScrapeNumber:  3,
-					afterScrapePath:    []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
-					afterScrapeName:    []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"},
+					afterScrapeNumber: 3,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+					},
 					changedTokenNumber: 0,
 				},
 			},
@@ -204,7 +232,7 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 					},
 					{
-						JobName:     "testTenant-minio-job-0",
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
 						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
 						MetricsPath: "/minio/v2/metrics/cluster",
 						Scheme:      "https",
@@ -213,12 +241,12 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 						StaticConfigs: []configmaps.StaticConfig{
 							{
-								Targets: []string{"minio.default.svc.cluster.local:443"},
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
 							},
 						},
 					},
 					{
-						JobName:     "testTenant-minio-job-1",
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
 						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
 						MetricsPath: "/minio/metrics/v3/api",
 						Scheme:      "https",
@@ -227,15 +255,21 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 						StaticConfigs: []configmaps.StaticConfig{
 							{
-								Targets: []string{"minio.default.svc.cluster.local:443"},
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
 							},
 						},
 					},
 				},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
 				expects: expect{
-					afterScrapeNumber:  3,
-					afterScrapePath:    []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
-					afterScrapeName:    []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"},
+					afterScrapeNumber: 3,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+					},
 					changedTokenNumber: 0,
 				},
 			},
@@ -256,7 +290,7 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 					},
 					{
-						JobName:     "testTenant-minio-job-0",
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
 						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
 						MetricsPath: "/minio/v2/metrics/cluster",
 						Scheme:      "https",
@@ -265,12 +299,12 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 						StaticConfigs: []configmaps.StaticConfig{
 							{
-								Targets: []string{"minio.default.svc.cluster.local:443"},
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
 							},
 						},
 					},
 					{
-						JobName:     "testTenant-minio-job-1",
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
 						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
 						MetricsPath: "/minio/metrics/v3/api",
 						Scheme:      "https",
@@ -279,19 +313,233 @@ func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) {
 						},
 						StaticConfigs: []configmaps.StaticConfig{
 							{
-								Targets: []string{"minio.default.svc.cluster.local:443"},
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
 							},
 						},
 					},
 				},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
 				expects: expect{
-					afterScrapeNumber:  3,
-					afterScrapePath:    []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
-					afterScrapeName:    []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"},
+					afterScrapeNumber: 3,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+					},
 					changedTokenNumber: 2,
 				},
 			},
 		},
+		{
+			name: "testMultiTenantDifferentName",
+			arg: arg{
+				accessKey: "accessKey",
+				secretKey: "secretKey",
+				paths:     []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+				beforeScrapes: []configmaps.ScrapeConfig{
+					{
+						JobName: "testTarget",
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{"testTarget"},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/v2/metrics/cluster",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/metrics/v3/api",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+				},
+				tenantName:      tenantBName,
+				tenantNamespace: tenantBNamespace,
+				expects: expect{
+					afterScrapeNumber: 5,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-0", tenantBName, tenantBNamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantBName, tenantBNamespace),
+					},
+					changedTokenNumber: 0,
+				},
+			},
+		},
+		{
+			name: "testMultiTenantSameName",
+			arg: arg{
+				accessKey: "accessKey",
+				secretKey: "secretKey",
+				paths:     []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+				beforeScrapes: []configmaps.ScrapeConfig{
+					{
+						JobName: "testTarget",
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{"testTarget"},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/v2/metrics/cluster",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/metrics/v3/api",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+				},
+				tenantName:      tenantCName,
+				tenantNamespace: tenantCNamespace,
+				expects: expect{
+					afterScrapeNumber: 5,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-0", tenantCName, tenantCNamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantCName, tenantCNamespace),
+					},
+					changedTokenNumber: 0,
+				},
+			},
+		},
+		{
+			name: "testMultiTenantSameNameNoChange",
+			arg: arg{
+				accessKey: "accessKey",
+				secretKey: "secretKey",
+				paths:     []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+				beforeScrapes: []configmaps.ScrapeConfig{
+					{
+						JobName: "testTarget",
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{"testTarget"},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/v2/metrics/cluster",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/metrics/v3/api",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantANamespace)},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-0", tenantCName, tenantCNamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/v2/metrics/cluster",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantCNamespace)},
+							},
+						},
+					},
+					{
+						JobName:     fmt.Sprintf("%s-%s-minio-job-1", tenantCName, tenantCNamespace),
+						BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"),
+						MetricsPath: "/minio/metrics/v3/api",
+						Scheme:      "https",
+						TLSConfig: configmaps.TLSConfig{
+							CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+						},
+						StaticConfigs: []configmaps.StaticConfig{
+							{
+								Targets: []string{fmt.Sprintf("minio.%s.svc.cluster.local:443", tenantCNamespace)},
+							},
+						},
+					},
+				},
+				tenantName:      tenantAName,
+				tenantNamespace: tenantANamespace,
+				expects: expect{
+					afterScrapeNumber: 5,
+					afterScrapePath:   []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"},
+					afterScrapeName: []string{
+						"testTarget",
+						fmt.Sprintf("%s-%s-minio-job-0", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantAName, tenantANamespace),
+						fmt.Sprintf("%s-%s-minio-job-0", tenantCName, tenantCNamespace),
+						fmt.Sprintf("%s-%s-minio-job-1", tenantCName, tenantCNamespace),
+					},
+					changedTokenNumber: 0,
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {