Repository: open-telemetry/opentelemetry-collector (Go)

Issue:
[otelcol] Collector.Shutdown may be a no-op during config reload
### Component(s)

_No response_

### What happened?

**Describe the bug**

If the Collector.Shutdown method is called during a config reload, the collector's state will briefly be "closing" and the Shutdown method will be a no-op.

This explains https://github.com/open-telemetry/opentelemetry-collector/actions/runs/17367754316/job/49297665087?pr=13739

**Steps to reproduce**

This diff demonstrates the issue:

```diff
```yaml
diff --git a/otelcol/collector.go b/otelcol/collector.go
index 12a27685c..2730b17da 100644
--- a/otelcol/collector.go
+++ b/otelcol/collector.go
@@ -14,6 +14,7 @@ import (
 	"os/signal"
 	"sync/atomic"
 	"syscall"
+	"time"
 
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
@@ -250,6 +251,7 @@ func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
 func (col *Collector) reloadConfiguration(ctx context.Context) error {
 	col.service.Logger().Warn("Config updated, restart service")
 	col.setCollectorState(StateClosing)
+	time.Sleep(time.Second)
 
 	if err := col.service.Shutdown(ctx); err != nil {
 		return fmt.Errorf("failed to shutdown the retiring config: %w", err)
diff --git a/otelcol/collector_test.go b/otelcol/collector_test.go
index f4063b2b5..377fe6c39 100644
--- a/otelcol/collector_test.go
+++ b/otelcol/collector_test.go
@@ -107,7 +107,7 @@ func TestCollectorStateAfterConfigChange(t *testing.T) {
 	watcher(&confmap.ChangeEvent{})
 
 	assert.Eventually(t, func() bool {
-		return StateRunning == col.GetState()
+		return StateClosing == col.GetState()
 	}, 2*time.Second, 200*time.Millisecond)
```

**What did you expect to see?**

I expect the test to pass reliably.

**What did you see instead?**

Flaky test.

This is not just a flaky test though, the bug could affect an actual real life collector. Specifically on Windows, the service logic does involve calling the Shutdown method which means the Run method may not return.

### Collector version

76862b85509925c999361f505281c7e28c344cc8

### Additional context

_No response_

### Tip

<sub>[React](https://github.blog/news-insights/product-news/add-reactions-to-pull-requests-issues-and-comments/) with 👍 to help prioritize this issue. Please use comments to provide useful context, avoiding `+1` or `me too`, to help us triage it. Learn more [here](https://opentelemetry.io/community/end-user/issue-participation/).</sub>


Discussion:

Patch:
diff --git a/.chloggen/collector-shutdown.yaml b/.chloggen/collector-shutdown.yaml
new file mode 100644
index 00000000000..38c65c555c7
--- /dev/null
+++ b/.chloggen/collector-shutdown.yaml
@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: otelcol
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix a potential deadlock during collector shutdown.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13740]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/otelcol/collector.go b/otelcol/collector.go
index 12a27685c48..d29d0e6c01f 100644
--- a/otelcol/collector.go
+++ b/otelcol/collector.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"sync"
 	"sync/atomic"
 	"syscall"
 
@@ -105,6 +106,8 @@ type Collector struct {
 
 	// shutdownChan is used to terminate the collector.
 	shutdownChan chan struct{}
+	shutdownOnce sync.Once
+
 	// signalsChannel is used to receive termination signals from the OS.
 	signalsChannel chan os.Signal
 	// asyncErrorChannel is used to signal a fatal error from any component.
@@ -150,14 +153,9 @@ func (col *Collector) GetState() State {
 
 // Shutdown shuts down the collector server.
 func (col *Collector) Shutdown() {
-	// Only shutdown if we're in a Running or Starting State else noop
-	state := col.GetState()
-	if state == StateRunning || state == StateStarting {
-		defer func() {
-			_ = recover()
-		}()
+	col.shutdownOnce.Do(func() {
 		close(col.shutdownChan)
-	}
+	})
 }
 
 func buildModuleInfo(m map[component.Type]string) map[component.Type]service.ModuleInfo {


Test patch:
diff --git a/otelcol/collector_test.go b/otelcol/collector_test.go
index f4063b2b575..30379887391 100644
--- a/otelcol/collector_test.go
+++ b/otelcol/collector_test.go
@@ -7,6 +7,8 @@ package otelcol
 import (
 	"context"
 	"errors"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"sync"
@@ -77,10 +79,46 @@ func TestCollectorCancelContext(t *testing.T) {
 }
 
 func TestCollectorStateAfterConfigChange(t *testing.T) {
+	metricsPushRequests := make(chan chan struct{})
+	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		unblock := make(chan struct{})
+		select {
+		case <-r.Context().Done():
+		case metricsPushRequests <- unblock:
+			select {
+			case <-unblock:
+			case <-r.Context().Done():
+			}
+		}
+	}))
+	defer srv.Close()
+
 	var watcher confmap.WatcherFunc
 	fileProvider := newFakeProvider("file", func(_ context.Context, uri string, w confmap.WatcherFunc) (*confmap.Retrieved, error) {
 		watcher = w
-		return confmap.NewRetrieved(newConfFromFile(t, uri[5:]))
+		conf := newConfFromFile(t, uri[5:])
+		conf["service"].(map[string]any)["telemetry"] = map[string]any{
+			"metrics": map[string]any{
+				"level": "basic",
+				"readers": []any{
+					map[string]any{
+						// Use a periodic metric reader with a long period,
+						// so we should only see an export on shutdown.
+						"periodic": map[string]any{
+							"interval": 30000, // 30s
+							"exporter": map[string]any{
+								"otlp": map[string]any{
+									"endpoint": srv.URL,
+									"insecure": true,
+									"protocol": "http/protobuf",
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+		return confmap.NewRetrieved(conf)
 	})
 	set := ConfigProviderSettings{
 		ResolverSettings: confmap.ResolverSettings{
@@ -95,23 +133,36 @@ func TestCollectorStateAfterConfigChange(t *testing.T) {
 		Factories:              nopFactories,
 		ConfigProviderSettings: set,
 	})
-
 	require.NoError(t, err)
 
 	wg := startCollector(context.Background(), t, col)
-
 	assert.Eventually(t, func() bool {
 		return StateRunning == col.GetState()
-	}, 2*time.Second, 200*time.Millisecond)
+	}, 10*time.Second, 10*time.Millisecond)
 
+	// On config change, the collector will internally close
+	// and recreate the service. The metrics reader will try to
+	// push to the OTLP endpoint. We block the request to check
+	// the state of the collector during the config change event.
 	watcher(&confmap.ChangeEvent{})
-
+	unblock := <-metricsPushRequests
+	assert.Equal(t, StateClosing, col.GetState())
+	close(unblock)
 	assert.Eventually(t, func() bool {
 		return StateRunning == col.GetState()
-	}, 2*time.Second, 200*time.Millisecond)
+	}, 10*time.Second, 10*time.Millisecond)
 
+	// Do it again, but this time call Shutdown during the
+	// config change to make sure the internal service shutdown
+	// does not influence collector shutdown.
+	watcher(&confmap.ChangeEvent{})
+	unblock = <-metricsPushRequests
+	assert.Equal(t, StateClosing, col.GetState())
 	col.Shutdown()
+	close(unblock)
 
+	// After the config reload, the final shutdown should occur.
+	close(<-metricsPushRequests)
 	wg.Wait()
 	assert.Equal(t, StateClosed, col.GetState())
 }