Repository: open-telemetry/opentelemetry-operator
Pull request #3991 (2025-05-13 15:43:14)

enhancement: set default target allocator grace period to 30s
**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
enhancement - update default target allocator grace period from 0 to 30s

**Link to tracking Issue(s):** <Issue number if applicable>

- Resolves: #3989 

**Testing:** <Describe what testing was performed and which tests were added.>
updated unit-test to have CollectorNotReadyGracePeriod defaulted to 30s 

**Documentation:** <Describe the documentation added.>


carsontham: hi @swiatekm, I have fixed it in the latest commit, thanks!
swiatekm: Looks good to me, but we need to fix the one failing e2e test. You need to add `collector_not_ready_grace_period: 30s` to the ConfigMap here: https://github.com/open-telemetry/opentelemetry-operator/blob/c2b2f136d4c9dfeb0fd455b66e7228d72c03d8db/tests/e2e-targetallocator-cr/01-assert.yaml#L89.
carsontham: hi @swiatekm, i have fixed it. thank you for your guidance!

Patch:
diff --git a/.chloggen/ct_update-default-grace-period-to-30s.yaml b/.chloggen/ct_update-default-grace-period-to-30s.yaml
new file mode 100755
index 00000000..6002dcd1
--- /dev/null
+++ b/.chloggen/ct_update-default-grace-period-to-30s.yaml
@@ -0,0 +1,17 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: target allocator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: set default target allocator grace period to 30 seconds
+
+# One or more tracking issues related to the change
+issues: [3989]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The config option collectorNotReadyGracePeriod have been set to 30s by default. The target allocator now waits for 30 seconds before reallocating targets from a collector which isn't Ready.
+  Setting this value to 0 will restore previous behaviour.
\ No newline at end of file
diff --git a/apis/v1alpha1/targetallocator_types.go b/apis/v1alpha1/targetallocator_types.go
index aaf11a54..5ff59162 100644
--- a/apis/v1alpha1/targetallocator_types.go
+++ b/apis/v1alpha1/targetallocator_types.go
@@ -87,8 +87,10 @@ type TargetAllocatorSpec struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Observability"
 	Observability v1beta1.ObservabilitySpec `json:"observability,omitempty"`
 	// CollectorNotReadyGracePeriod defines the grace period after which a TargetAllocator stops considering a collector is target assignable.
-	// The default is 0s, which means that all collectors can be assigned targets irrespective of their readiness.
+	// The default is 30s, which means that if a collector becomes not Ready, the target allocator will wait for 30 seconds before reassigning its targets. The assumption is that the state is temporary, and an expensive target reallocation should be avoided if possible.
 	//
 	// +optional
+	// +kubebuilder:default:="30s"
+	// +kubebuilder:validation:Format:=duration
 	CollectorNotReadyGracePeriod *metav1.Duration `json:"collectorNotReadyGracePeriod,omitempty"`
 }
diff --git a/apis/v1beta1/opentelemetrycollector_types.go b/apis/v1beta1/opentelemetrycollector_types.go
index f788faee..1f4b489d 100644
--- a/apis/v1beta1/opentelemetrycollector_types.go
+++ b/apis/v1beta1/opentelemetrycollector_types.go
@@ -211,9 +211,11 @@ type TargetAllocatorEmbedded struct {
 	// +optional
 	PodDisruptionBudget *PodDisruptionBudgetSpec `json:"podDisruptionBudget,omitempty"`
 	// CollectorNotReadyGracePeriod defines the grace period after which a TargetAllocator stops considering a collector is target assignable.
-	// The default is 0s, which means that all collectors can be assigned targets irrespective of their readiness.
+	// The default is 30s, which means that if a collector becomes not Ready, the target allocator will wait for 30 seconds before reassigning its targets. The assumption is that the state is temporary, and an expensive target reallocation should be avoided if possible.
 	//
 	// +optional
+	// +kubebuilder:default:="30s"
+	// +kubebuilder:validation:Format:=duration
 	CollectorNotReadyGracePeriod *metav1.Duration `json:"collectorNotReadyGracePeriod,omitempty"`
 }
 
diff --git a/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml
index fa2e349a..78a10ac0 100644
--- a/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml
+++ b/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml
@@ -99,7 +99,7 @@ metadata:
     categories: Logging & Tracing,Monitoring
     certified: "false"
     containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator
-    createdAt: "2025-05-15T09:49:54Z"
+    createdAt: "2025-05-21T14:41:11Z"
     description: Provides the OpenTelemetry components, including the Collector
     operators.operatorframework.io/builder: operator-sdk-v1.29.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
diff --git a/bundle/community/manifests/opentelemetry.io_opentelemetrycollectors.yaml b/bundle/community/manifests/opentelemetry.io_opentelemetrycollectors.yaml
index 8ce05e81..fc9c453d 100644
--- a/bundle/community/manifests/opentelemetry.io_opentelemetrycollectors.yaml
+++ b/bundle/community/manifests/opentelemetry.io_opentelemetrycollectors.yaml
@@ -7706,6 +7706,8 @@ spec:
                     - per-node
                     type: string
                   collectorNotReadyGracePeriod:
+                    default: 30s
+                    format: duration
                     type: string
                   enabled:
                     type: boolean
diff --git a/bundle/community/manifests/opentelemetry.io_targetallocators.yaml b/bundle/community/manifests/opentelemetry.io_targetallocators.yaml
index 5e1da99a..9947acbb 100644
--- a/bundle/community/manifests/opentelemetry.io_targetallocators.yaml
+++ b/bundle/community/manifests/opentelemetry.io_targetallocators.yaml
@@ -1171,6 +1171,8 @@ spec:
                   type: string
                 type: object
               collectorNotReadyGracePeriod:
+                default: 30s
+                format: duration
                 type: string
               env:
                 items:
diff --git a/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml
index ec23f2df..08a9a023 100644
--- a/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml
+++ b/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml
@@ -99,7 +99,7 @@ metadata:
     categories: Logging & Tracing,Monitoring
     certified: "false"
     containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator
-    createdAt: "2025-05-15T09:49:54Z"
+    createdAt: "2025-05-21T14:41:12Z"
     description: Provides the OpenTelemetry components, including the Collector
     operators.operatorframework.io/builder: operator-sdk-v1.29.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
diff --git a/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml b/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml
index 8ce05e81..fc9c453d 100644
--- a/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml
+++ b/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml
@@ -7706,6 +7706,8 @@ spec:
                     - per-node
                     type: string
                   collectorNotReadyGracePeriod:
+                    default: 30s
+                    format: duration
                     type: string
                   enabled:
                     type: boolean
diff --git a/bundle/openshift/manifests/opentelemetry.io_targetallocators.yaml b/bundle/openshift/manifests/opentelemetry.io_targetallocators.yaml
index 5e1da99a..9947acbb 100644
--- a/bundle/openshift/manifests/opentelemetry.io_targetallocators.yaml
+++ b/bundle/openshift/manifests/opentelemetry.io_targetallocators.yaml
@@ -1171,6 +1171,8 @@ spec:
                   type: string
                 type: object
               collectorNotReadyGracePeriod:
+                default: 30s
+                format: duration
                 type: string
               env:
                 items:
diff --git a/cmd/otel-allocator/internal/config/config.go b/cmd/otel-allocator/internal/config/config.go
index 9cd974cf..409fdac3 100644
--- a/cmd/otel-allocator/internal/config/config.go
+++ b/cmd/otel-allocator/internal/config/config.go
@@ -42,7 +42,7 @@ const (
 	DefaultCRScrapeInterval             model.Duration = model.Duration(time.Second * 30)
 	DefaultAllocationStrategy                          = "consistent-hashing"
 	DefaultFilterStrategy                              = "relabel-config"
-	DefaultCollectorNotReadyGracePeriod                = 0 * time.Second
+	DefaultCollectorNotReadyGracePeriod                = 30 * time.Second
 )
 
 var (
diff --git a/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml b/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml
index 75ddf5cb..8bc83cce 100644
--- a/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml
+++ b/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml
@@ -7692,6 +7692,8 @@ spec:
                     - per-node
                     type: string
                   collectorNotReadyGracePeriod:
+                    default: 30s
+                    format: duration
                     type: string
                   enabled:
                     type: boolean
diff --git a/config/crd/bases/opentelemetry.io_targetallocators.yaml b/config/crd/bases/opentelemetry.io_targetallocators.yaml
index 86fccaa1..905c523b 100644
--- a/config/crd/bases/opentelemetry.io_targetallocators.yaml
+++ b/config/crd/bases/opentelemetry.io_targetallocators.yaml
@@ -1168,6 +1168,8 @@ spec:
                   type: string
                 type: object
               collectorNotReadyGracePeriod:
+                default: 30s
+                format: duration
                 type: string
               env:
                 items:
diff --git a/docs/api/opentelemetrycollectors.md b/docs/api/opentelemetrycollectors.md
index f2bd3203..f580ad2f 100644
--- a/docs/api/opentelemetrycollectors.md
+++ b/docs/api/opentelemetrycollectors.md
@@ -30719,7 +30719,10 @@ WARNING: The per-node strategy currently ignores targets without a Node, like co
         <td>string</td>
         <td>
           CollectorNotReadyGracePeriod defines the grace period after which a TargetAllocator stops considering a collector is target assignable.
-The default is 0s, which means that all collectors can be assigned targets irrespective of their readiness.<br/>
+The default is 30s, which means that if a collector becomes not Ready, the target allocator will wait for 30 seconds before reassigning its targets. The assumption is that the state is temporary, and an expensive target reallocation should be avoided if possible.<br/>
+          <br/>
+            <i>Format</i>: duration<br/>
+            <i>Default</i>: 30s<br/>
         </td>
         <td>false</td>
       </tr><tr>
diff --git a/docs/api/targetallocators.md b/docs/api/targetallocators.md
index b1706057..50c385b7 100644
--- a/docs/api/targetallocators.md
+++ b/docs/api/targetallocators.md
@@ -142,7 +142,10 @@ WARNING: The per-node strategy currently ignores targets without a Node, like co
         <td>string</td>
         <td>
           CollectorNotReadyGracePeriod defines the grace period after which a TargetAllocator stops considering a collector is target assignable.
-The default is 0s, which means that all collectors can be assigned targets irrespective of their readiness.<br/>
+The default is 30s, which means that if a collector becomes not Ready, the target allocator will wait for 30 seconds before reassigning its targets. The assumption is that the state is temporary, and an expensive target reallocation should be avoided if possible.<br/>
+          <br/>
+            <i>Format</i>: duration<br/>
+            <i>Default</i>: 30s<br/>
         </td>
         <td>false</td>
       </tr><tr>


Test patch:
diff --git a/cmd/otel-allocator/internal/config/config_test.go b/cmd/otel-allocator/internal/config/config_test.go
index 381f9070..ed97b57a 100644
--- a/cmd/otel-allocator/internal/config/config_test.go
+++ b/cmd/otel-allocator/internal/config/config_test.go
@@ -279,6 +279,7 @@ func TestLoadFromFile(t *testing.T) {
 						},
 					},
 				},
+				CollectorNotReadyGracePeriod: 30 * time.Second,
 			},
 			wantErr: assert.NoError,
 		},
@@ -360,6 +361,7 @@ func TestLoadFromFile(t *testing.T) {
 						},
 					},
 				},
+				CollectorNotReadyGracePeriod: 30 * time.Second,
 			},
 			wantErr: assert.NoError,
 		},
@@ -465,6 +467,7 @@ func TestLoadFromFile(t *testing.T) {
 						},
 					},
 				},
+				CollectorNotReadyGracePeriod: 30 * time.Second,
 			},
 			wantErr: assert.NoError,
 		},
@@ -570,6 +573,7 @@ func TestLoadFromFile(t *testing.T) {
 						},
 					},
 				},
+				CollectorNotReadyGracePeriod: 30 * time.Second,
 			},
 			wantErr: assert.NoError,
 		},
diff --git a/tests/e2e-targetallocator-cr/00-assert.yaml b/tests/e2e-targetallocator-cr/00-assert.yaml
index 45476aa1..c4e744c6 100644
--- a/tests/e2e-targetallocator-cr/00-assert.yaml
+++ b/tests/e2e-targetallocator-cr/00-assert.yaml
@@ -84,6 +84,7 @@ apiVersion: v1
 data:
   targetallocator.yaml: |
     allocation_strategy: consistent-hashing
+    collector_not_ready_grace_period: 30s
     collector_selector: null
     config:
       scrape_configs:
diff --git a/tests/e2e-targetallocator-cr/01-assert.yaml b/tests/e2e-targetallocator-cr/01-assert.yaml
index 94ea3fc3..d670ba64 100644
--- a/tests/e2e-targetallocator-cr/01-assert.yaml
+++ b/tests/e2e-targetallocator-cr/01-assert.yaml
@@ -86,6 +86,7 @@ data:
   # Note: The content of this file must match exactly (careful about the order of keys) the content of the file in the ConfigMap
   targetallocator.yaml: |
     allocation_strategy: consistent-hashing
+    collector_not_ready_grace_period: 30s
     collector_selector: null
     filter_strategy: relabel-config
     prometheus_cr: