Repository: kumahq/kuma
Pull request #12862 (2025-02-17 10:19:58)

feat(kumactl): update install observability components
## Motivation

I know we plan to remove `kumactl install observability` eventually, but I think we should update its components for now, especially if it’s easy to do. Once the versions are more current, Renovate can take care of updates going forward.

## Implementation information

- Updated Prometheus server image to the latest available and reordered arguments for better grouping.
- Added missing resources to kube-state-metrics roles to prevent RBAC errors.
- Improved Loki configuration:
  - Added startupProbe and adjusted other probes for faster startup.
  - Reconfigured volumes to align with Grafana documentation.
  - Provided Loki config secret as stringData instead of data to avoid manual base64 encoding.
  - Applied minor configuration improvements.

## Supporting documentation

Related to: https://github.com/kumahq/kuma/issues/11693

github-actions[bot]: ## Reviewer Checklist
:mag: Each of these sections need to be checked by the reviewer of the PR :mag::
If something doesn't apply please check the box and add a justification if the reason is non obvious.
- [ ] Is the PR title satisfactory? Is this part of a larger feature and should be grouped using `> Changelog`?
- [ ] PR description is clear and complete. It [Links to relevant issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) as well as docs and UI issues
- [ ] This will not break child repos: it doesn't hardcode values (.e.g "kumahq" as an image registry)
- [ ] IPv6 is taken into account (.e.g: no string concatenation of host port)
- [ ] Tests (Unit test, E2E tests, manual test on universal and k8s)
    - Don't forget `ci/` labels to run additional/fewer tests
- [ ] Does this contain a change that needs to be notified to users? In this case, [`UPGRADE.md`](../blob/master/UPGRADE.md) should be updated.
- [ ] Does it need to be backported according to the [backporting policy](../blob/master/CONTRIBUTING.md#backporting)? ([this](https://github.com/kumahq/kuma/actions/workflows/auto-backport.yaml) GH action will add "backport" label based on these [file globs](https://github.com/kumahq/kuma/blob/master/.github/workflows/auto-backport.yaml#L6), if you want to prevent it from adding the "backport" label use [no-backport-autolabel](https://github.com/kumahq/kuma/blob/master/.github/workflows/auto-backport.yaml#L8) label)
<!-- Sticky Pull Request CommentPR reviewer checklist -->

Patch:
diff --git a/api/system/v1alpha1/secret.pb.go b/api/system/v1alpha1/secret.pb.go
index e6ee67919..98e56b70f 100644
--- a/api/system/v1alpha1/secret.pb.go
+++ b/api/system/v1alpha1/secret.pb.go
@@ -81,20 +81,20 @@ var file_api_system_v1alpha1_secret_proto_rawDesc = []byte{
 	0x73, 0x68, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
 	0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
 	0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x22, 0x99, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x64,
+	0x22, 0xa3, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x64,
 	0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
 	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x79, 0x74, 0x65,
-	0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x5e, 0xaa, 0x8c,
+	0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x68, 0xaa, 0x8c,
 	0x89, 0xa6, 0x01, 0x10, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f,
 	0x75, 0x72, 0x63, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x08, 0x12, 0x06, 0x53, 0x65, 0x63, 0x72,
 	0x65, 0x74, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x08, 0x22, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
 	0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x60, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02,
-	0x10, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0a, 0x3a, 0x08, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72,
-	0x65, 0x74, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x3a, 0x02, 0x20, 0x01, 0x42, 0x2c, 0x5a, 0x2a,
-	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68,
-	0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x79, 0x73, 0x74, 0x65,
-	0x6d, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x33,
+	0x10, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x08, 0x00, 0xaa, 0x8c, 0x89, 0xa6,
+	0x01, 0x0a, 0x3a, 0x08, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0xaa, 0x8c, 0x89, 0xa6,
+	0x01, 0x04, 0x3a, 0x02, 0x20, 0x01, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61,
+	0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2f, 0x76, 0x31, 0x61, 0x6c,
+	0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/api/system/v1alpha1/secret.proto b/api/system/v1alpha1/secret.proto
index e239b43df..c6b28a365 100644
--- a/api/system/v1alpha1/secret.proto
+++ b/api/system/v1alpha1/secret.proto
@@ -15,6 +15,7 @@ message Secret {
   option (kuma.mesh.resource).package = "system";
   option (kuma.mesh.resource).skip_kubernetes_wrappers = true;
   option (kuma.mesh.resource).kds.send_to_zone = true;
+  option (kuma.mesh.resource).kds.send_to_global = false;
   option (kuma.mesh.resource).ws.name = "secret";
   option (kuma.mesh.resource).ws.admin_only = true;
 
diff --git a/app/kumactl/data/install/k8s/logging/loki/loki.yaml b/app/kumactl/data/install/k8s/logging/loki/loki.yaml
index 1b3d09f3d..c452f2879 100644
--- a/app/kumactl/data/install/k8s/logging/loki/loki.yaml
+++ b/app/kumactl/data/install/k8s/logging/loki/loki.yaml
@@ -7,8 +7,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -33,6 +75,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.{{ .Namespace }}:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -508,18 +552,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.{{ .Namespace }}:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -558,12 +600,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -610,7 +652,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -620,16 +661,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/app/kumactl/data/install/k8s/metrics/prometheus/kube-stats-metrics.yaml b/app/kumactl/data/install/k8s/metrics/prometheus/kube-stats-metrics.yaml
index 0d206f7c2..ee3c65053 100644
--- a/app/kumactl/data/install/k8s/metrics/prometheus/kube-stats-metrics.yaml
+++ b/app/kumactl/data/install/k8s/metrics/prometheus/kube-stats-metrics.yaml
@@ -35,6 +35,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -97,6 +104,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
diff --git a/app/kumactl/data/install/k8s/metrics/prometheus/server.yaml b/app/kumactl/data/install/k8s/metrics/prometheus/server.yaml
index 2591eb08c..7da6e9b8a 100644
--- a/app/kumactl/data/install/k8s/metrics/prometheus/server.yaml
+++ b/app/kumactl/data/install/k8s/metrics/prometheus/server.yaml
@@ -423,7 +423,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -435,16 +435,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
diff --git a/pkg/core/resources/store/store.go b/pkg/core/resources/store/store.go
index 3edd4ba1f..a8d541a10 100644
--- a/pkg/core/resources/store/store.go
+++ b/pkg/core/resources/store/store.go
@@ -149,6 +149,10 @@ func IsResourceNotFound(err error) bool {
 	return err != nil && strings.HasPrefix(err.Error(), "Resource not found")
 }
 
+func IsResourceAlreadyExists(err error) bool {
+	return err != nil && strings.HasPrefix(err.Error(), "resource already exists")
+}
+
 // AssertionError
 type AssertionError struct {
 	msg string
diff --git a/pkg/kds/v2/client/kds_client.go b/pkg/kds/v2/client/kds_client.go
index aef018037..f8fa1d918 100644
--- a/pkg/kds/v2/client/kds_client.go
+++ b/pkg/kds/v2/client/kds_client.go
@@ -36,7 +36,7 @@ func (u *UpstreamResponse) Validate() error {
 }
 
 type Callbacks struct {
-	OnResourcesReceived func(upstream UpstreamResponse) error
+	OnResourcesReceived func(upstream UpstreamResponse) (error, error)
 }
 
 // All methods other than Receive() are non-blocking. It does not wait until the peer CP receives the message.
@@ -114,13 +114,14 @@ func (s *kdsSyncClient) Receive() error {
 			}
 			continue
 		}
-		err = s.callbacks.OnResourcesReceived(received)
+		err, nackError := s.callbacks.OnResourcesReceived(received)
 		if err != nil {
 			return errors.Wrapf(err, "failed to store %s resources", received.Type)
 		}
-		if validationErrors != nil {
-			s.log.Info("received resource is invalid, sending NACK", "err", validationErrors)
-			if err := s.kdsStream.NACK(received.Type, validationErrors); err != nil {
+		if nackError != nil || validationErrors != nil {
+			combinedErrors := std_errors.Join(nackError, validationErrors)
+			s.log.Info("received resource is invalid, sending NACK", "err", combinedErrors)
+			if err := s.kdsStream.NACK(received.Type, combinedErrors); err != nil {
 				if err == io.EOF {
 					return nil
 				}
diff --git a/pkg/kds/v2/store/sync.go b/pkg/kds/v2/store/sync.go
index c8c602e5b..a87fe96a7 100644
--- a/pkg/kds/v2/store/sync.go
+++ b/pkg/kds/v2/store/sync.go
@@ -2,6 +2,7 @@ package store
 
 import (
 	"context"
+	std_errors "errors"
 	"fmt"
 	"maps"
 	"strings"
@@ -41,12 +42,20 @@ type ResourceSyncer interface {
 	//
 	// Sync takes into account only 'Name' and 'Mesh' when it comes to upstream's Meta.
 	// 'Version', 'CreationTime' and 'ModificationTime' are managed by downstream store.
-	Sync(ctx context.Context, upstream client_v2.UpstreamResponse, fs ...SyncOptionFunc) error
+	// We return now 2 errors:
+	// The first error cancels the connection, forcing the client to re-establish it.
+	// This can happen due to a database error.
+	//
+	// The second error is related to non-critical issues, such as `ResourceAlreadyExists`,
+	// which shouldn't prevent other resources from being stored.
+	// Instead, we return a NACK message with information for the user.
+	Sync(ctx context.Context, upstream client_v2.UpstreamResponse, fs ...SyncOptionFunc) (error, error)
 }
 
 type SyncOption struct {
-	Predicate func(r core_model.Resource) bool
-	Zone      string
+	Predicate            func(r core_model.Resource) bool
+	SkipConflictResource bool
+	Zone                 string
 }
 
 type SyncOptionFunc func(*SyncOption)
@@ -65,6 +74,12 @@ func Zone(name string) SyncOptionFunc {
 	}
 }
 
+func SkipConflictResource() SyncOptionFunc {
+	return func(opts *SyncOption) {
+		opts.SkipConflictResource = true
+	}
+}
+
 func PrefilterBy(predicate func(r core_model.Resource) bool) SyncOptionFunc {
 	return func(opts *SyncOption) {
 		opts.Predicate = predicate
@@ -107,7 +122,7 @@ type OnUpdate struct {
 	opts []store.UpdateOptionsFunc
 }
 
-func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse client_v2.UpstreamResponse, fs ...SyncOptionFunc) error {
+func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse client_v2.UpstreamResponse, fs ...SyncOptionFunc) (error, error) {
 	now := core.Now()
 	defer func() {
 		s.metric.Observe(float64(time.Since(now).Milliseconds()) / 1000)
@@ -120,16 +135,16 @@ func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse clien
 	downstream, err := registry.Global().NewList(upstreamResponse.Type)
 	indexedInvalidResources := model.IndexKeys(upstreamResponse.InvalidResourcesKey)
 	if err != nil {
-		return err
+		return err, nil
 	}
 	if upstreamResponse.IsInitialRequest {
 		if err := s.resourceStore.List(ctx, downstream); err != nil {
-			return err
+			return err, nil
 		}
 	} else {
 		upstreamChangeKeys := append(core_model.ResourceListToResourceKeys(upstream), upstreamResponse.RemovedResourcesKey...)
 		if err := s.resourceStore.List(ctx, downstream, store.ListByResourceKeys(upstreamChangeKeys)); err != nil {
-			return err
+			return err, nil
 		}
 	}
 
@@ -156,10 +171,10 @@ func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse clien
 	log.V(1).Info("before filtering", "downstream", downstream, "upstream", upstream)
 
 	if downstream, err = filterResources(downstream, predicate); err != nil {
-		return err
+		return err, nil
 	}
 	if upstream, err = filterResources(upstream, predicate); err != nil {
-		return err
+		return err, nil
 	}
 
 	log.V(1).Info("after filtering", "downstream", downstream, "upstream", upstream)
@@ -216,10 +231,11 @@ func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse clien
 	zone := system.NewZoneResource()
 	if opts.Zone != "" && len(onCreate) > 0 {
 		if err := s.resourceStore.Get(ctx, zone, store.GetByKey(opts.Zone, core_model.NoMesh)); err != nil {
-			return err
+			return err, nil
 		}
 	}
-	return store.InTx(ctx, s.transactions, func(ctx context.Context) error {
+	var nackError error
+	err = store.InTx(ctx, s.transactions, func(ctx context.Context) error {
 		for _, r := range onCreate {
 			rk := core_model.MetaToResourceKey(r.GetMeta())
 			log.Info("creating a new resource from upstream", "name", r.GetMeta().GetName(), "mesh", r.GetMeta().GetMesh())
@@ -237,7 +253,11 @@ func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse clien
 			r.SetMeta(nil)
 
 			if err := s.resourceStore.Create(ctx, r, createOpts...); err != nil {
-				return err
+				if opts.SkipConflictResource && store.IsResourceAlreadyExists(err) {
+					nackError = std_errors.Join(nackError, err)
+				} else {
+					return err
+				}
 			}
 		}
 
@@ -261,6 +281,7 @@ func (s *syncResourceStore) Sync(syncCtx context.Context, upstreamResponse clien
 		}
 		return nil
 	})
+	return err, nackError
 }
 
 func filter(rs core_model.ResourceList, predicate func(r core_model.Resource) bool) (core_model.ResourceList, error) {
@@ -280,10 +301,10 @@ func filter(rs core_model.ResourceList, predicate func(r core_model.Resource) bo
 
 func ZoneSyncCallback(ctx context.Context, configToSync map[string]bool, syncer ResourceSyncer, k8sStore bool, localZone string, kubeFactory resources_k8s.KubeFactory, systemNamespace string) *client_v2.Callbacks {
 	return &client_v2.Callbacks{
-		OnResourcesReceived: func(upstream client_v2.UpstreamResponse) error {
+		OnResourcesReceived: func(upstream client_v2.UpstreamResponse) (error, error) {
 			if k8sStore && upstream.Type != system.ConfigType && upstream.Type != system.SecretType && upstream.Type != system.GlobalSecretType {
 				if err := addNamespaceSuffix(kubeFactory, upstream, systemNamespace); err != nil {
-					return err
+					return err, nil
 				}
 			}
 
@@ -300,6 +321,11 @@ func ZoneSyncCallback(ctx context.Context, configToSync map[string]bool, syncer
 						zone_tokens.SigningPublicKeyPrefix,
 					)
 				}))
+
+			case upstream.Type == system.SecretType:
+				return syncer.Sync(ctx, upstream, PrefilterBy(func(r core_model.Resource) bool {
+					return !core_model.IsLocallyOriginated(config_core.Zone, r.GetMeta().GetLabels())
+				}), SkipConflictResource())
 			}
 
 			return syncer.Sync(ctx, upstream, PrefilterBy(func(r core_model.Resource) bool {
@@ -334,7 +360,7 @@ func GlobalSyncCallback(
 	supportsHashSuffixes := kds.ContextHasFeature(ctx, kds.FeatureHashSuffix)
 
 	return &client_v2.Callbacks{
-		OnResourcesReceived: func(upstream client_v2.UpstreamResponse) error {
+		OnResourcesReceived: func(upstream client_v2.UpstreamResponse) (error, error) {
 			if !supportsHashSuffixes {
 				// todo: remove in 2 releases after 2.6.x
 				upstream.RemovedResourcesKey = util.AddPrefixToResourceKeyNames(upstream.RemovedResourcesKey, upstream.ControlPlaneId)
@@ -351,7 +377,7 @@ func GlobalSyncCallback(
 
 			if k8sStore {
 				if err := addNamespaceSuffix(kubeFactory, upstream, systemNamespace); err != nil {
-					return err
+					return err, nil
 				}
 			}
 


Test patch:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.defaults.golden.yaml
index 0d11456db..8e9c849b4 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.defaults.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.defaults.golden.yaml
@@ -11714,6 +11714,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -11776,6 +11783,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
@@ -12141,7 +12149,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -12153,16 +12161,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
@@ -12214,8 +12222,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -12240,6 +12290,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.mesh-observability:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -12695,18 +12747,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.mesh-observability:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -12745,12 +12795,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -12797,7 +12847,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -12807,16 +12856,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.no-grafana.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.no-grafana.golden.yaml
index 96730e31c..734ad07d8 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.no-grafana.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.no-grafana.golden.yaml
@@ -368,6 +368,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -430,6 +437,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
@@ -584,7 +592,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -596,16 +604,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
@@ -692,8 +700,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -718,6 +768,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.mesh-observability:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -1173,18 +1225,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.mesh-observability:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -1223,12 +1273,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -1275,7 +1325,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -1285,16 +1334,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.no-jaeger.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.no-jaeger.golden.yaml
index cb18b8fcb..53061a48a 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.no-jaeger.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.no-jaeger.golden.yaml
@@ -11714,6 +11714,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -11776,6 +11783,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
@@ -12141,7 +12149,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -12153,16 +12161,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
@@ -12214,8 +12222,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -12240,6 +12290,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.mesh-observability:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -12695,18 +12747,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.mesh-observability:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -12745,12 +12795,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -12797,7 +12847,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -12807,16 +12856,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.no-loki.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.no-loki.golden.yaml
index 1963d8cb0..f3f26fcab 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.no-loki.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.no-loki.golden.yaml
@@ -11714,6 +11714,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -11776,6 +11783,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
@@ -12141,7 +12149,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -12153,16 +12161,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.no-prometheus.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.no-prometheus.golden.yaml
index e551e4805..eb03e93ae 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.no-prometheus.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.no-prometheus.golden.yaml
@@ -11558,8 +11558,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -11584,6 +11626,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.mesh-observability:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -12039,18 +12083,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.mesh-observability:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -12089,12 +12131,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -12141,7 +12183,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -12151,16 +12192,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/app/kumactl/cmd/install/testdata/install-observability.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-observability.overrides.golden.yaml
index 25ed17830..59a3efaf7 100644
--- a/app/kumactl/cmd/install/testdata/install-observability.overrides.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-observability.overrides.golden.yaml
@@ -11714,6 +11714,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
     resources:
@@ -11776,6 +11783,7 @@ rules:
       - networking.k8s.io
     resources:
       - networkpolicies
+      - ingresses
     verbs:
       - list
       - watch
@@ -12141,7 +12149,7 @@ spec:
       serviceAccountName: prometheus-server
       containers:
         - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.6.1"
+          image: "jimmidyson/configmap-reload:v0.9.0"
           imagePullPolicy: "IfNotPresent"
           args:
             - --volume-dir=/etc/config
@@ -12153,16 +12161,16 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: prometheus-server
-          image: "prom/prometheus:v2.35.0"
+          image: "prom/prometheus:v3.1.0"
           imagePullPolicy: "IfNotPresent"
           args:
-            - --storage.tsdb.retention.time=15d
             - --config.file=/etc/config/prometheus.yml
             - --storage.tsdb.path=/data
+            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.size=7GB
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --web.enable-lifecycle
-            - --storage.tsdb.retention.size=7GB
           ports:
             - containerPort: 9090
           resources:
@@ -12214,8 +12222,50 @@ metadata:
   labels:
     app: loki
     release: loki
-data:
-  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpxdWVyaWVyOgogIGVuZ2luZToKICAgIG1heF9sb29rX2JhY2tfcGVyaW9kOiAwcwppbmdlc3RlcjoKICBjaHVua19ibG9ja19zaXplOiAyNjIxNDQKICBjaHVua19pZGxlX3BlcmlvZDogM20KICBjaHVua19yZXRhaW5fcGVyaW9kOiAxbQogIGxpZmVjeWNsZXI6CiAgICByaW5nOgogICAgICBrdnN0b3JlOgogICAgICAgIHN0b3JlOiBpbm1lbW9yeQogICAgICByZXBsaWNhdGlvbl9mYWN0b3I6IDEKICB3YWw6CiAgICBlbmFibGVkOiBmYWxzZQpjb21wYWN0b3I6CiAgcmV0ZW50aW9uX2VuYWJsZWQ6IGZhbHNlCiAgd29ya2luZ19kaXJlY3Rvcnk6IC9kYXRhL2xva2kvcmV0ZW50aW9uCiAgY29tcGFjdGlvbl9pbnRlcnZhbDogMTBtCiAgZGVsZXRlX3JlcXVlc3Rfc3RvcmU6IHRzZGIKc2NoZW1hX2NvbmZpZzoKICBjb25maWdzOgogICAgLSBmcm9tOiAyMDIzLTEwLTIwCiAgICAgIHN0b3JlOiB0c2RiCiAgICAgIG9iamVjdF9zdG9yZTogZmlsZXN5c3RlbQogICAgICBzY2hlbWE6IHYxMwogICAgICBpbmRleDoKICAgICAgICBwcmVmaXg6IGluZGV4XwogICAgICAgIHBlcmlvZDogMjRoCnN0b3JhZ2VfY29uZmlnOgogIHRzZGJfc2hpcHBlcjoKICAgIGFjdGl2ZV9pbmRleF9kaXJlY3Rvcnk6IC9kYXRhL2xva2kvaW5kZXgKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2NhY2hlCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBzCg==
+stringData:
+  loki.yaml: |-
+    auth_enabled: false
+    querier:
+      engine:
+        max_look_back_period: 0s
+    ingester:
+      chunk_block_size: 262144
+      chunk_idle_period: 3m
+      chunk_retain_period: 1m
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+      wal:
+        enabled: false
+      shutdown_marker_path: /data/liki/shutdown
+    pattern_ingester:
+      enabled: true
+      lifecycler:
+        ring:
+          kvstore:
+            store: inmemory
+    compactor:
+      retention_enabled: false
+      working_directory: /data/loki/retention
+      compaction_interval: 10m
+      delete_request_store: tsdb
+    schema_config:
+      configs:
+        - from: 2023-10-20
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      tsdb_shipper:
+        active_index_directory: /data/loki/index
+        cache_location: /data/loki/cache
+      filesystem:
+        directory: /data/loki/chunks
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -12240,6 +12290,8 @@ data:
       filename: /run/promtail/positions.yaml
     server:
       http_listen_port: 3101
+    clients:
+      - url: http://loki.kuma:3100/loki/api/v1/push
     target_config:
       sync_period: 10s
     scrape_configs:
@@ -12695,18 +12747,16 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/promtail/promtail.yaml"
-            - "-client.url=http://loki.kuma:3100/loki/api/v1/push"
           volumeMounts:
             - name: config
               mountPath: /etc/promtail
             - name: run
               mountPath: /run/promtail
-            - mountPath: /var/lib/docker/containers
-              name: docker
-              readOnly: true
-            - mountPath: /var/log/pods
-              name: pods
+            - name: docker
+              mountPath: /var/lib/docker/containers
               readOnly: true
+            - name: logs
+              mountPath: /var/log
           env:
             - name: HOSTNAME
               valueFrom:
@@ -12745,12 +12795,12 @@ spec:
         - name: run
           hostPath:
             path: /run/promtail
-        - hostPath:
+        - name: docker
+          hostPath:
             path: /var/lib/docker/containers
-          name: docker
-        - hostPath:
-            path: /var/log/pods
-          name: pods
+        - name: logs
+          hostPath:
+            path: /var/log
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -12797,7 +12847,6 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "-config.file=/etc/loki/loki.yaml"
-            - "-target=all,table-manager"
           volumeMounts:
             - name: config
               mountPath: /etc/loki
@@ -12807,16 +12856,20 @@ spec:
             - name: http-metrics
               containerPort: 3100
               protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: http-metrics
+            failureThreshold: 60
+            periodSeconds: 10
           livenessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           readinessProbe:
             httpGet:
               path: /ready
               port: http-metrics
-            initialDelaySeconds: 45
           resources:
             {}
           securityContext:
diff --git a/pkg/kds/v2/store/sync_test.go b/pkg/kds/v2/store/sync_test.go
index 0908e6218..26af86b90 100644
--- a/pkg/kds/v2/store/sync_test.go
+++ b/pkg/kds/v2/store/sync_test.go
@@ -65,8 +65,9 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 		upstreamResponse.Type = upstream.GetItemType()
 		upstreamResponse.AddedResources = upstream
 
-		err := syncer.Sync(context.Background(), upstreamResponse)
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse)
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 
 		actual := &mesh.MeshResourceList{}
 		err = resourceStore.List(context.Background(), actual)
@@ -88,8 +89,9 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 		upstreamResponse.AddedResources = upstream
 		upstreamResponse.RemovedResourcesKey = removedResources
 
-		err := syncer.Sync(context.Background(), upstreamResponse)
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse)
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 
 		actual := &mesh.MeshResourceList{}
 		err = resourceStore.List(context.Background(), actual)
@@ -125,8 +127,9 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 			model.WithoutMesh("mesh-10"),
 		}
 
-		err := syncer.Sync(context.Background(), upstreamResponse)
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse)
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 
 		actual := &mesh.MeshResourceList{}
 		err = resourceStore.List(context.Background(), actual)
@@ -156,8 +159,9 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 		upstreamResponse.AddedResources = upstream
 		upstreamResponse.IsInitialRequest = true
 
-		err := syncer.Sync(context.Background(), upstreamResponse)
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse)
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 
 		actual := &mesh.MeshResourceList{}
 		err = resourceStore.List(context.Background(), actual)
@@ -177,17 +181,62 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 		upstreamResponse.AddedResources = upstream
 
 		// when
-		err := syncer.Sync(context.Background(), upstreamResponse, sync_store.PrefilterBy(func(r model.Resource) bool {
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse, sync_store.PrefilterBy(func(r model.Resource) bool {
 			return r.GetMeta().GetName() != "mesh-1"
 		}))
 
 		// then
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 		actual := &mesh.MeshResourceList{}
 		Expect(resourceStore.List(context.Background(), actual)).To(Succeed())
 		Expect(actual.GetItems()).To(BeEmpty())
 	})
 
+	It("should add all resources and skip the conflict one", func() {
+		mesh1 := meshBuilder(1)
+		mesh2 := meshBuilder(2)
+		mesh3 := meshBuilder(3)
+		Expect(resourceStore.Create(
+			context.Background(),
+			mesh2,
+			store.CreateBy(model.MetaToResourceKey(mesh2.GetMeta())),
+			store.CreateWithLabels(map[string]string{mesh_proto.ResourceOriginLabel: "zone"}),
+		)).ToNot(HaveOccurred())
+
+		// given
+		upstream := &mesh.MeshResourceList{}
+
+		// try to add resource without the label
+		mesh2 = meshBuilder(2)
+		mesh2.Spec.MeshServices = &mesh_proto.Mesh_MeshServices{
+			Mode: mesh_proto.Mesh_MeshServices_Exclusive,
+		}
+		Expect(upstream.AddItem(mesh1)).To(Succeed())
+		Expect(upstream.AddItem(mesh2)).To(Succeed())
+		Expect(upstream.AddItem(mesh3)).To(Succeed())
+		upstreamResponse := client_v2.UpstreamResponse{}
+		upstreamResponse.Type = upstream.GetItemType()
+		upstreamResponse.AddedResources = upstream
+
+		// when
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse, sync_store.PrefilterBy(func(r model.Resource) bool {
+			return r.GetMeta().GetLabels()[mesh_proto.ResourceOriginLabel] != "zone"
+		}), sync_store.SkipConflictResource())
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).To(HaveOccurred())
+
+		actual := &mesh.MeshResourceList{}
+		Expect(resourceStore.List(context.Background(), actual)).To(Succeed())
+		Expect(actual.GetItems()).To(HaveLen(3))
+		Expect(actual.GetItems()[0].GetSpec()).To(MatchProto(meshBuilder(2).GetSpec()))
+		Expect(actual.GetItems()[1].GetSpec()).To(MatchProto(mesh1.GetSpec()))
+		// should not update resource since mesh-2 already exists
+		Expect(actual.GetItems()[2].GetSpec()).To(MatchProto(mesh3.GetSpec()))
+	})
+
 	It("should ignore invalid resource from upstream and add only valid", func() {
 		// given
 		upstream := &mesh.MeshResourceList{}
@@ -203,10 +252,11 @@ var _ = Describe("SyncResourceStoreDelta", func() {
 		upstreamResponse.InvalidResourcesKey = []model.ResourceKey{model.MetaToResourceKey(mesh2.GetMeta())}
 
 		// when
-		err := syncer.Sync(context.Background(), upstreamResponse)
+		err, nackError := syncer.Sync(context.Background(), upstreamResponse)
 
 		// then
 		Expect(err).ToNot(HaveOccurred())
+		Expect(nackError).ToNot(HaveOccurred())
 		actual := &mesh.MeshResourceList{}
 		Expect(resourceStore.List(context.Background(), actual)).To(Succeed())
 		Expect(actual.GetItems()).To(HaveLen(2))
diff --git a/test/e2e_env/multizone/sync/sync.go b/test/e2e_env/multizone/sync/sync.go
index 421167c2e..60f8524a8 100644
--- a/test/e2e_env/multizone/sync/sync.go
+++ b/test/e2e_env/multizone/sync/sync.go
@@ -136,6 +136,32 @@ spec:
 				g.Expect(strings.Count(out, "argocd.argoproj.io")).To(Equal(0))
 			}, "30s", "1s").Should(Succeed())
 		})
+
+		It("should not sync secret from zone to global", func() {
+			zoneSecret := fmt.Sprintf(`
+apiVersion: v1
+kind: Secret
+metadata:
+  name: zone-k8s-secret
+  namespace: %s
+  labels:
+    kuma.io/mesh: %s
+data:
+  value: dGVzdAo=
+type: system.kuma.io/secret `, Config.KumaNamespace, meshName)
+			Expect(multizone.KubeZone1.Install(YamlK8s(zoneSecret))).To(Succeed())
+			Eventually(func(g Gomega) {
+				out, err := multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "kuma.io/display-name: zone-k8s-secret")).To(Equal(1))
+			}, "30s", "1s").Should(Succeed())
+			// should not be synced to global
+			Consistently(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "zone-k8s-secret")).To(Equal(0))
+			}, "10s", "1s").Should(Succeed())
+		})
 	})
 
 	Context("from Global to Zone", func() {
@@ -187,6 +213,84 @@ spec:
 			}, "30s", "1s").Should(ContainSubstring(name))
 		}
 
+		It("should not sync secret from zone to global", func() {
+			secretName := "global-and-zone-secret"
+			zoneSecret := fmt.Sprintf(`
+apiVersion: v1
+kind: Secret
+metadata:
+  name: %s
+  namespace: %s
+  labels:
+    kuma.io/mesh: %s
+data:
+  value: dGVzdAo=
+type: system.kuma.io/secret `, secretName, Config.KumaNamespace, meshName)
+			globalSecret := fmt.Sprintf(`
+type: Secret
+name: %s
+mesh: %s
+data: Z2xvYmFsCg==`, secretName, meshName)
+			newGlobalSecret := fmt.Sprintf(`
+type: Secret
+name: new-global-secret
+mesh: %s
+data: bmV3Z2xvYmFsCg==`, meshName)
+			Expect(multizone.KubeZone1.Install(YamlK8s(zoneSecret))).To(Succeed())
+			Eventually(func(g Gomega) {
+				out, err := multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, fmt.Sprintf("kuma.io/display-name: %s", secretName))).To(Equal(1))
+			}, "30s", "1s").Should(Succeed())
+			// should not be synced to global
+			Consistently(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, fmt.Sprintf("kuma.io/display-name: %s", secretName))).To(Equal(0))
+			}, "10s", "1s").Should(Succeed())
+
+			Expect(multizone.Global.Install(YamlUniversal(globalSecret))).To(Succeed())
+			// should not override existing in zone
+			Eventually(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, secretName)).To(Equal(1))
+				g.Expect(strings.Count(out, "Z2xvYmFsCg==")).To(Equal(1))
+			}, "30s", "1s").Should(Succeed())
+			Consistently(func(g Gomega) {
+				out, err := multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, fmt.Sprintf("kuma.io/display-name: %s", secretName))).To(Equal(1))
+				g.Expect(strings.Count(out, "Z2xvYmFsCg==")).To(Equal(0))
+			}, "10s", "1s").Should(Succeed())
+
+			Expect(multizone.Global.Install(YamlUniversal(newGlobalSecret))).To(Succeed())
+			// should create a new one
+			Eventually(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "new-global-secret")).To(Equal(1))
+			}, "30s", "1s").Should(Succeed())
+			// should sync resource to the zone even if one is invalid
+			Eventually(func(g Gomega) {
+				out, err := multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "kuma.io/display-name: new-global-secret")).To(Equal(1))
+			}, "30s", "1s").Should(Succeed())
+			Expect(multizone.Global.GetKumactlOptions().RunKumactl("delete", "secret", secretName, "--mesh", meshName)).To(Succeed())
+			Eventually(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, secretName)).To(Equal(0))
+			}, "30s", "1s").Should(Succeed())
+			// should not remove zone secret with the same name
+			Consistently(func(g Gomega) {
+				out, err := multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "secrets", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, fmt.Sprintf("kuma.io/display-name: %s", secretName))).To(Equal(1))
+			}, "10s", "1s").Should(Succeed())
+		})
+
 		It("should sync policy creation", func() {
 			// given
 			name := "tr-synced"