Repository: signalfx/splunk-otel-java
Pull request #2233: Fix Flaky Test
Author: login: tduncan
Created: 2025-03-06T22:06:47Z | Merged: 2025-03-17T07:45:33Z

Use a scheduler to stagger the simulated concurrent start span operations in the test that verifies only a single thread is profiled at a time.


Diff:
diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
index 9175b0043..c9345c5af 100644
--- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
+++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
@@ -57,17 +57,17 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
 
   @Override
   public void start(SpanContext spanContext) {
-    if (samplers.containsKey(spanContext.getTraceId())) {
-      return;
-    }
-
-    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
-    samplers.put(spanContext.getTraceId(), scheduler);
-    scheduler.scheduleAtFixedRate(
-        new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
-        SCHEDULER_INITIAL_DELAY,
-        samplingPeriod.toMillis(),
-        TimeUnit.MILLISECONDS);
+    samplers.computeIfAbsent(
+        spanContext.getTraceId(),
+        traceId -> {
+          ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
+          scheduler.scheduleAtFixedRate(
+              new StackTraceGatherer(traceId, Thread.currentThread().getId()),
+              SCHEDULER_INITIAL_DELAY,
+              samplingPeriod.toMillis(),
+              TimeUnit.MILLISECONDS);
+          return scheduler;
+        });
   }
 
   @Override
diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStagingArea.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStagingArea.java
index be2a2922f..2aa0629c7 100644
--- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStagingArea.java
+++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStagingArea.java
@@ -16,10 +16,10 @@
 
 package com.splunk.opentelemetry.profiler.snapshot;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.stream.Collectors;
 
 /**
@@ -27,7 +27,7 @@
  * the stockpiled {@link StackTrace}s. Intended for testing use only.
  */
 class InMemoryStagingArea implements StagingArea {
-  private final Map<String, List<StackTrace>> stackTraces = new ConcurrentHashMap<>();
+  private final ConcurrentMap<String, List<StackTrace>> stackTraces = new ConcurrentHashMap<>();
 
   @Override
   public void stage(String traceId, StackTrace stackTrace) {
@@ -35,7 +35,7 @@ public void stage(String traceId, StackTrace stackTrace) {
         traceId,
         (id, stackTraces) -> {
           if (stackTraces == null) {
-            stackTraces = new ArrayList<>();
+            stackTraces = new CopyOnWriteArrayList<>();
           }
           stackTraces.add(stackTrace);
           return stackTraces;
diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
index 75c4bf61f..c5c7e3098 100644
--- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
+++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
@@ -16,7 +16,6 @@
 
 package com.splunk.opentelemetry.profiler.snapshot;
 
-import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -27,6 +26,7 @@
 import java.time.Duration;
 import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 
@@ -81,32 +81,44 @@ void emptyStagingAreaAfterSamplingStops() {
 
   @Test
   void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
-    var latch = new CountDownLatch(1);
+    var executor = Executors.newFixedThreadPool(2);
+    var startSpanLatch = new CountDownLatch(1);
+    var shutdownLatch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
+    var spanContext2 = randomSpanContext(traceId);
+    var spanContext1 = randomSpanContext(traceId);
 
-    threadOne.start();
-    threadTwo.start();
-    await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
-    latch.countDown();
+    executor.submit(startSampling(spanContext1, startSpanLatch, shutdownLatch));
+    executor.submit(startSampling(spanContext2, startSpanLatch, shutdownLatch));
 
-    var threadIds =
-        staging.allStackTraces().stream().map(StackTrace::getThreadId).collect(Collectors.toSet());
-    assertThat(threadIds).containsOnly(threadOne.getId());
+    try {
+      startSpanLatch.countDown();
+      await().until(() -> staging.allStackTraces().size() > 5);
+      shutdownLatch.countDown();
+
+      var threadIds =
+          staging.allStackTraces().stream()
+              .map(StackTrace::getThreadId)
+              .collect(Collectors.toSet());
+      assertEquals(1, threadIds.size());
+    } finally {
+      executor.shutdownNow();
+      sampler.stop(spanContext1);
+      sampler.stop(spanContext2);
+    }
   }
 
-  private Thread startAndStopSampler(SpanContext spanContext, CountDownLatch latch) {
-    return new Thread(
-        () -> {
-          try {
-            sampler.start(spanContext);
-            latch.await();
-            sampler.stop(spanContext);
-          } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-          }
-        });
+  private Runnable startSampling(
+      SpanContext spanContext, CountDownLatch startSpanLatch, CountDownLatch shutdownLatch) {
+    return (() -> {
+      try {
+        startSpanLatch.await();
+        sampler.start(spanContext);
+        shutdownLatch.await();
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+    });
   }
 
   private SpanContext randomSpanContext() {


Reviews:
user: login: robsunday
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T08:47:18Z
user: login: laurit
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-07T14:34:09Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T17:09:35Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T17:10:54Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T18:57:10Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T18:58:27Z
user: login: robsunday
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T12:43:53Z
user: login: robsunday
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T12:51:09Z
user: login: laurit
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T11:10:01Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T16:32:42Z
user: login: laurit
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T18:03:47Z
user: login: tduncan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T18:42:33Z
user: login: laurit
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-17T07:45:27Z

Review comments:
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: robsunday
body: For better test isolation `idGenerator`, `staging` and `sampler` should be instantiated in `@BeforeEach` annotated  method.

created_at: 2025-03-07T08:40:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
     latch.countDown();
 
     var threadIds =
         staging.allStackTraces().stream().map(StackTrace::getThreadId).collect(Collectors.toSet());
-    assertThat(threadIds).containsOnly(threadOne.getId());
+    assertThat(threadIds).containsOnly(threadOne.get().getId());
   }
 
-  private Thread startAndStopSampler(SpanContext spanContext, CountDownLatch latch) {
-    return new Thread(
-        () -> {
-          try {
-            sampler.start(spanContext);
-            latch.await();
-            sampler.stop(spanContext);
-          } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-          }
-        });
+  private Callable<Thread> startSampling(SpanContext spanContext, CountDownLatch latch) {
+    return (() -> {
+      try {
+        System.out.println(
+            "Starting thread "
+                + Thread.currentThread().getName()
+                + ", id "
+                + Thread.currentThread().getId());
+        sampler.start(spanContext);
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: robsunday
body: sampler.stop(spanContext) is never called
created_at: 2025-03-07T08:41:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: robsunday
body: Why 5?
created_at: 2025-03-07T08:43:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
     latch.countDown();
 
     var threadIds =
         staging.allStackTraces().stream().map(StackTrace::getThreadId).collect(Collectors.toSet());
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: robsunday
body: `staging` may not be initially empty in this test because its instance is reused among all test cases. That's why `staging` should be instantiated in `@BeforeEach` as I already mentioned
created_at: 2025-03-07T08:46:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: laurit
body: Firstly I would advice agains using short timeouts like `HALF_SECOND`, these will often make the tests flaky. The machine executing the tests may experience high load and tests may execute significantly slower than what you'd expect. I'd just go with the default 10s timeout that awaitility has.
Secondly set a breakpoint on this line `await().atMo...`, in breakpoint properties set that it should suspend only the current thread. Similarly set a breakpoint to https://github.com/signalfx/splunk-otel-java/blob/e0b4f4db0e67b8aded6ea3c9c530afc4b7d18547/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java#L65 Make it also suspend only the current thread. Run the test in debug mode. First resume the 2 threads in `ScheduledExecutorStackTraceSampler`, after that the main test thread. You should observe test failure.
created_at: 2025-03-07T14:34:09Z
author_association: COLLABORATOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: tduncan
body: There isn't anything special about the number 5, per say. The intent here is for multiple samples to be taken and for each thread to be running at the same time. Since the threads would potentially be on different sampling schedules it needs to be more than 2, but also not so high that the test must wait a long time to be certain the 2nd thread isn't being sampled. 5 seemed OK. I don't think a smaller number is a good idea but happy to increase it a bit.
created_at: 2025-03-07T17:09:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: tduncan
body: Sure, I can increase the wait time.
created_at: 2025-03-07T17:10:54Z
author_association: CONTRIBUTOR
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: tduncan
body: This is a misunderstanding about how JUnit isolates tests. By default a new instance of the test class is instantiated  for each test method (see [JUnit 5's Test Lifecycle](https://junit.org/junit5/docs/current/user-guide/#writing-tests-test-instance-lifecycle)) so we can lean on this behavior to ensure that new instances of all test dependencies are also instantiated. We do have the option of changing the test lifecycle but we need to explicitly use the `@TestInstance(Lifecycle.PER_CLASS)` annotation. This is of course assuming we haven't overridden the default using the `junit.jupiter.testinstance.lifecycle.default` property which I don't see in the `build` check.

`@BeforeEach` setup methods are really best used to prime external resources (e.g. insert data in a database) that apply to every (or at least most) tests in the test class. Obviously we can instantiate test fields in setup methods but it doesn't add additional safety, only boilerplate.

If you've used TestNG in the past it explicitly does the opposite by reusing the test class instance therefore requiring the use of before and after test callback methods to manage state.

```java
public class LifecycleTest {
  private final List<UUID> uuids = new ArrayList<>();

  @Test
  void testOne() {
    uuids.add(UUID.randomUUID());
    System.out.println(uuids);
  }

  @Test
  void testTwo() {
    uuids.add(UUID.randomUUID());
    System.out.println(uuids);
  }
}

Output:
[3195ee77-46e6-4f79-ab3a-efc3f9816b47]
[75990fbf-23ae-4377-a0aa-da91fd5fa430]
```

```java
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class LifecycleTest {
  private final List<UUID> uuids = new ArrayList<>();

  @Test
  void testOne() {
    uuids.add(UUID.randomUUID());
    System.out.println(uuids);
  }

  @Test
  void testTwo() {
    uuids.add(UUID.randomUUID());
    System.out.println(uuids);
  }
}

Output:
[f903466c-6037-4bb0-8232-f64b7995eb65]
[f903466c-6037-4bb0-8232-f64b7995eb65, 08203264-a72b-48f8-9de9-35327da0d3ff]
```
created_at: 2025-03-07T18:57:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,33 +83,39 @@ void emptyStagingAreaAfterSamplingStops() {
   }
 
   @Test
-  void onlyTakeStackTraceSamplesForOneThreadPerTrace() {
+  void onlyTakeStackTraceSamplesForOneThreadPerTrace() throws Exception {
+    var scheduler = Executors.newScheduledThreadPool(2);
     var latch = new CountDownLatch(1);
     var traceId = idGenerator.generateTraceId();
-    var threadOne = startAndStopSampler(randomSpanContext(traceId), latch);
-    var threadTwo = startAndStopSampler(randomSpanContext(traceId), latch);
 
-    threadOne.start();
-    threadTwo.start();
+    var threadOne =
+        scheduler.schedule(
+            startSampling(randomSpanContext(traceId), latch), 0, TimeUnit.MILLISECONDS);
+    scheduler.schedule(startSampling(randomSpanContext(traceId), latch), 25, TimeUnit.MILLISECONDS);
+
     await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() > 5);
     latch.countDown();
 
     var threadIds =
         staging.allStackTraces().stream().map(StackTrace::getThreadId).collect(Collectors.toSet());
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: tduncan
body: See https://github.com/signalfx/splunk-otel-java/pull/2233#discussion_r1985548449
created_at: 2025-03-07T18:58:27Z
author_association: CONTRIBUTOR
path: profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java
user: login: robsunday
body: Yeah, you are right. Too long break with Java coding...
created_at: 2025-03-13T12:43:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,17 +57,18 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
 
   @Override
   public void start(SpanContext spanContext) {
-    if (samplers.containsKey(spanContext.getTraceId())) {
-      return;
-    }
-
     ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
-    samplers.put(spanContext.getTraceId(), scheduler);
-    scheduler.scheduleAtFixedRate(
-        new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
-        SCHEDULER_INITIAL_DELAY,
-        samplingPeriod.toMillis(),
-        TimeUnit.MILLISECONDS);
+    ScheduledExecutorService mappedScheduler =
+        samplers.computeIfAbsent(spanContext.getTraceId(), id -> scheduler);
+    if (scheduler == mappedScheduler) {
+      scheduler.scheduleAtFixedRate(
+          new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
+          SCHEDULER_INITIAL_DELAY,
+          samplingPeriod.toMillis(),
+          TimeUnit.MILLISECONDS);
+    } else {
+      scheduler.shutdown();
path: profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
user: login: laurit
body: why is this needed?
created_at: 2025-03-14T11:10:01Z
author_association: COLLABORATOR
diff_hunk: @@ -57,17 +57,18 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
 
   @Override
   public void start(SpanContext spanContext) {
-    if (samplers.containsKey(spanContext.getTraceId())) {
-      return;
-    }
-
     ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
-    samplers.put(spanContext.getTraceId(), scheduler);
-    scheduler.scheduleAtFixedRate(
-        new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
-        SCHEDULER_INITIAL_DELAY,
-        samplingPeriod.toMillis(),
-        TimeUnit.MILLISECONDS);
+    ScheduledExecutorService mappedScheduler =
+        samplers.computeIfAbsent(spanContext.getTraceId(), id -> scheduler);
+    if (scheduler == mappedScheduler) {
+      scheduler.scheduleAtFixedRate(
+          new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
+          SCHEDULER_INITIAL_DELAY,
+          samplingPeriod.toMillis(),
+          TimeUnit.MILLISECONDS);
+    } else {
+      scheduler.shutdown();
path: profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
user: login: tduncan
body: To ensure the thread allocated to this unused `ScheduledExecutorService` is released back to the JVM. 
created_at: 2025-03-14T16:32:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,17 +57,18 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
 
   @Override
   public void start(SpanContext spanContext) {
-    if (samplers.containsKey(spanContext.getTraceId())) {
-      return;
-    }
-
     ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
-    samplers.put(spanContext.getTraceId(), scheduler);
-    scheduler.scheduleAtFixedRate(
-        new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
-        SCHEDULER_INITIAL_DELAY,
-        samplingPeriod.toMillis(),
-        TimeUnit.MILLISECONDS);
+    ScheduledExecutorService mappedScheduler =
+        samplers.computeIfAbsent(spanContext.getTraceId(), id -> scheduler);
+    if (scheduler == mappedScheduler) {
+      scheduler.scheduleAtFixedRate(
+          new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
+          SCHEDULER_INITIAL_DELAY,
+          samplingPeriod.toMillis(),
+          TimeUnit.MILLISECONDS);
+    } else {
+      scheduler.shutdown();
path: profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
user: login: laurit
body: ok, makes sense. For some reason though that `scheduler` was what was already in the map. Did you consider writing this as
```java
    samplers.computeIfAbsent(spanContext.getTraceId(), traceId -> {
      ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
      scheduler.scheduleAtFixedRate(
          new StackTraceGatherer(traceId, Thread.currentThread().getId()),
          SCHEDULER_INITIAL_DELAY,
          samplingPeriod.toMillis(),
          TimeUnit.MILLISECONDS);
      return scheduler;
    });
```
created_at: 2025-03-14T18:03:47Z
author_association: COLLABORATOR
diff_hunk: @@ -57,17 +57,18 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
 
   @Override
   public void start(SpanContext spanContext) {
-    if (samplers.containsKey(spanContext.getTraceId())) {
-      return;
-    }
-
     ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
-    samplers.put(spanContext.getTraceId(), scheduler);
-    scheduler.scheduleAtFixedRate(
-        new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
-        SCHEDULER_INITIAL_DELAY,
-        samplingPeriod.toMillis(),
-        TimeUnit.MILLISECONDS);
+    ScheduledExecutorService mappedScheduler =
+        samplers.computeIfAbsent(spanContext.getTraceId(), id -> scheduler);
+    if (scheduler == mappedScheduler) {
+      scheduler.scheduleAtFixedRate(
+          new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()),
+          SCHEDULER_INITIAL_DELAY,
+          samplingPeriod.toMillis(),
+          TimeUnit.MILLISECONDS);
+    } else {
+      scheduler.shutdown();
path: profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java
user: login: tduncan
body: That is cleaner. Updated.
created_at: 2025-03-14T18:42:33Z
author_association: CONTRIBUTOR

PR comments:
user: login: breedx-splk
created_at: 2025-03-06T22:28:39Z
body: Resolves #2229 
author_association: CONTRIBUTOR
user: login: laurit
created_at: 2025-03-13T13:03:37Z
body: Test failure is still easily reproducible using a debugger.
author_association: COLLABORATOR
user: login: tduncan
created_at: 2025-03-13T22:39:54Z
body: > Test failure is still easily reproducible using a debugger.

I've updated the map writing to use `computeIfAbsent` which should protect against this race condition. It's worth noting, however, that I was unable to reproduce the failure without using a debugger despite running thousands of iterations.
author_association: CONTRIBUTOR