Repository: floci-io/floci (2026-07-28T02:40:30Z)

Issue:
[BUG] CloudFormation DeleteStack OOMs after persistent restart with ELBv2 resources
## Service

CloudFormation / ELBv2

## AWS API Action

`DeleteStack`

## Expected behavior

After restarting Floci with the same persistent storage directory, a
`CREATE_COMPLETE` stack containing an Application Load Balancer, target group,
and listener should delete successfully. Memory usage should remain bounded and
the Floci process should remain available.

## Actual behavior

After the persistent restart, `DescribeStacks` still reports
`CREATE_COMPLETE`. Calling `DeleteStack` then causes Floci's memory usage to
grow rapidly until the container is killed by its memory limit:

```text
sample_utc=2026-07-20T13:35:32Z mem=182.7MiB / 2GiB mem_pct=8.92% cpu=69.88%
sample_utc=2026-07-20T13:35:36Z mem=939.5MiB / 2GiB mem_pct=45.87% cpu=101.47%
sample_utc=2026-07-20T13:35:40Z mem=1.005GiB / 2GiB mem_pct=50.23% cpu=99.87%
status=exited exit=137 oom=true
```

The CLI eventually reports:

```text
Could not connect to the endpoint URL: "http://localhost:4570/"
```

Immediately before the OOM, Floci repeatedly logs loads of the persistent
ELBv2 state files, including `elbv2-load-balancers.json`,
`elbv2-target-groups.json`, `elbv2-listeners.json`, `elbv2-rules.json`, and
`elbv2-tags.json`.

## Reproduction

```bash
set -euo pipefail

IMAGE='floci/floci@sha256:0fb28cbebe2bb3a11e2b1336d8eaf2e9bf2feb411550b3ebe89ede88ae29965c'
DATA="$PWD/floci-persistent-data"
ENDPOINT='http://localhost:4570'

rm -rf "$DATA"
mkdir -p "$DATA"

cat > /tmp/floci-elbv2-stack.json <<'JSON'
{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Resources": {
    "Vpc": {
      "Type": "AWS::EC2::VPC",
      "Properties": {"CidrBlock": "10.42.0.0/16"}
    },
    "SubnetA": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": {"Ref": "Vpc"},
        "CidrBlock": "10.42.1.0/24",
        "AvailabilityZone": "us-east-2a"
      }
    },
    "SubnetB": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": {"Ref": "Vpc"},
        "CidrBlock": "10.42.2.0/24",
        "AvailabilityZone": "us-east-2b"
      }
    },
    "SecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "generic repro",
        "VpcId": {"Ref": "Vpc"}
      }
    },
    "LoadBalancer": {
      "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
      "Properties": {
        "Scheme": "internal",
        "Subnets": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}],
        "SecurityGroups": [{"Ref": "SecurityGroup"}]
      }
    },
    "TargetGroup": {
      "Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
      "Properties": {
        "VpcId": {"Ref": "Vpc"},
        "Port": 80,
        "Protocol": "HTTP",
        "TargetType": "ip"
      }
    },
    "Listener": {
      "Type": "AWS::ElasticLoadBalancingV2::Listener",
      "Properties": {
        "LoadBalancerArn": {"Ref": "LoadBalancer"},
        "Port": 80,
        "Protocol": "HTTP",
        "DefaultActions": [{
          "Type": "forward",
          "TargetGroupArn": {"Ref": "TargetGroup"}
        }]
      }
    }
  }
}
JSON

start_floci() {
  docker run -d --name floci-delete-repro --memory 2g -p 4570:4566 \
    -e FLOCI_DEFAULT_ACCOUNT_ID=000000000000 \
    -e FLOCI_DEFAULT_REGION=us-east-2 \
    -e FLOCI_STORAGE_MODE=persistent \
    -e FLOCI_STORAGE_PERSISTENT_PATH=/app/data \
    -e FLOCI_STORAGE_HOST_PERSISTENT_PATH="$DATA" \
    -e FLOCI_TLS_ENABLED=false \
    -v "$DATA:/app/data" \
    "$IMAGE"

  until curl -sf "$ENDPOINT/_localstack/health" >/dev/null; do sleep 1; done
}

export AWS_ACCESS_KEY_ID=test
export AWS_SECRET_ACCESS_KEY=test
export AWS_DEFAULT_REGION=us-east-2
export AWS_PAGER=''

start_floci
aws --endpoint-url "$ENDPOINT" cloudformation create-stack \
  --stack-name generic-elbv2-persistent-delete \
  --template-body file:///tmp/floci-elbv2-stack.json
aws --endpoint-url "$ENDPOINT" cloudformation wait stack-create-complete \
  --stack-name generic-elbv2-persistent-delete

docker stop --time 30 floci-delete-repro
docker rm floci-delete-repro
start_floci

aws --endpoint-url "$ENDPOINT" cloudformation describe-stacks \
  --stack-name generic-elbv2-persistent-delete \
  --query 'Stacks[0].StackStatus' --output text
# CREATE_COMPLETE

aws --endpoint-url "$ENDPOINT" cloudformation delete-stack \
  --stack-name generic-elbv2-persistent-delete

docker inspect floci-delete-repro \
  --format 'status={{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}'
# status=exited exit=137 oom=true
```

## Environment

- Floci version / image tag: `nightly-07202026`, `floci/floci:nightly`, digest `sha256:0fb28cbebe2bb3a11e2b1336d8eaf2e9bf2feb411550b3ebe89ede88ae29965c`
- Java SDK version (if applicable): N/A (AWS CLI reproduction; Floci reports Quarkus 3.36.3)
- How you're running Floci (Docker / native / `mvn quarkus:dev`): Docker on Amazon Linux 2023, persistent storage mode, 2 GiB container memory limit



Discussion:

Patch:
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 9d554a8629..58af767a5c 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -22,6 +22,7 @@
 import io.github.hectorvent.floci.services.neptune.proxy.NeptuneProxyManager;
 import io.github.hectorvent.floci.services.pipes.PipesService;
 import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;
+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
 import io.github.hectorvent.floci.services.rds.RdsService;
 import io.github.hectorvent.floci.services.memorydb.container.MemoryDbContainerManager;
 import io.github.hectorvent.floci.services.memorydb.proxy.MemoryDbProxyManager;
@@ -75,6 +76,7 @@ public class EmulatorLifecycle {
     private final NeptuneProxyManager neptuneProxyManager;
     private final RabbitMqManager rabbitMqManager;
     private final RdsService rdsService;
+    private final ElbV2Service elbV2Service;
     private final InitializationHooksRunner initializationHooksRunner;
     private final SqsEventSourcePoller sqsPoller;
     private final KinesisEventSourcePoller kinesisPoller;
@@ -103,6 +105,7 @@ public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceR
                              NeptuneProxyManager neptuneProxyManager,
                              RabbitMqManager rabbitMqManager,
                              RdsService rdsService,
+                             ElbV2Service elbV2Service,
                              InitializationHooksRunner initializationHooksRunner,
                              SqsEventSourcePoller sqsPoller,
                              KinesisEventSourcePoller kinesisPoller,
@@ -130,6 +133,7 @@ public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceR
         this.neptuneProxyManager = neptuneProxyManager;
         this.rabbitMqManager = rabbitMqManager;
         this.rdsService = rdsService;
+        this.elbV2Service = elbV2Service;
         this.initializationHooksRunner = initializationHooksRunner;
         this.sqsPoller = sqsPoller;
         this.kinesisPoller = kinesisPoller;
@@ -173,6 +177,9 @@ void onStart(@Observes StartupEvent ignored) {
         dynamodbStreamsPoller.startPersistedPollers();
         pipesService.startPersistedPollers();
         rdsService.restorePersistedRuntime();
+        if (config.services().elbv2().enabled()) {
+            elbV2Service.restorePersistedRuntime();
+        }
 
         if (config.services().ec2().enabled() && !config.services().ec2().mock()) {
             ec2MetadataServer.start().exceptionally(ex -> {
diff --git a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
index b32a2479e1..6936ccd34b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java
@@ -101,6 +101,13 @@ private <V> void persistRegion(Map<String, Map<String, V>> resources, String reg
         }
     }
 
+    /**
+     * Rebuilds the in-memory indexes from the storage-backed maps. Pure and in-process on purpose:
+     * nothing reachable from {@link #initializeStorage()} may call an injected collaborator, because
+     * {@link ElbV2DataPlane} calls back into this service and CDI has not registered the instance in
+     * the ApplicationScoped context yet — re-entering bean creation instead (issue #1913). Runtime
+     * side effects belong in {@link #restorePersistedRuntime()}.
+     */
     private void rebuildIndexes()
     {
         lbToListeners.clear();
@@ -123,7 +130,20 @@ private void rebuildIndexes()
             for (TargetGroup targetGroup : regionEntry.getValue().values()) {
                 tgToLbs.computeIfAbsent(targetGroup.getTargetGroupArn(), k -> ConcurrentHashMap.newKeySet())
                         .addAll(targetGroup.getLoadBalancerArns());
-                if (healthChecker != null) {
+            }
+        }
+    }
+
+    /**
+     * Brings the data plane and health checks back up for state restored from disk. Invoked from
+     * {@code EmulatorLifecycle} once the bean is fully constructed, alongside the other services that
+     * restore persisted runtime after {@code storageFactory.loadAll()}.
+     */
+    public void restorePersistedRuntime()
+    {
+        if (healthChecker != null) {
+            for (Map<String, TargetGroup> regionTargetGroups : targetGroups.values()) {
+                for (TargetGroup targetGroup : regionTargetGroups.values()) {
                     healthChecker.startMonitoring(targetGroup);
                     if (!targetGroup.getTargets().isEmpty()) {
                         healthChecker.addTargets(targetGroup.getTargetGroupArn(), targetGroup.getTargets(), targetGroup);
@@ -131,10 +151,10 @@ private void rebuildIndexes()
                 }
             }
         }
-        for (Map.Entry<String, Map<String, Listener>> regionEntry : listeners.entrySet()) {
-            String region = regionEntry.getKey();
-            for (Listener listener : regionEntry.getValue().values()) {
-                if (dataPlane != null) {
+        if (dataPlane != null) {
+            for (Map.Entry<String, Map<String, Listener>> regionEntry : listeners.entrySet()) {
+                String region = regionEntry.getKey();
+                for (Listener listener : regionEntry.getValue().values()) {
                     dataPlane.startListener(listener, region, getListenerRules(region, listener.getListenerArn()));
                 }
             }


Test patch:
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index ea3473abdd..9193228408 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -57,6 +57,7 @@ class EmulatorLifecycleTest {
     @Mock private EmulatorConfig.StorageConfig storageConfig;
     @Mock private EmulatorConfig.ServicesConfig servicesConfig;
     @Mock private EmulatorConfig.Ec2ServiceConfig ec2ServiceConfig;
+    @Mock private EmulatorConfig.ElbV2ServiceConfig elbv2ServiceConfig;
     @Mock private ElastiCacheContainerManager elastiCacheContainerManager;
     @Mock private ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;
     @Mock private ElastiCacheProxyManager elastiCacheProxyManager;
@@ -69,6 +70,7 @@ class EmulatorLifecycleTest {
     @Mock private NeptuneProxyManager neptuneProxyManager;
     @Mock private io.github.hectorvent.floci.services.amazonmq.container.RabbitMqManager rabbitMqManager;
     @Mock private RdsService rdsService;
+    @Mock private io.github.hectorvent.floci.services.elbv2.ElbV2Service elbV2Service;
     @Mock private InitializationHooksRunner initializationHooksRunner;
     @Mock private SqsEventSourcePoller sqsPoller;
     @Mock private KinesisEventSourcePoller kinesisPoller;
@@ -90,6 +92,8 @@ void setUp() {
         Mockito.lenient().when(config.services()).thenReturn(servicesConfig);
         Mockito.lenient().when(servicesConfig.ec2()).thenReturn(ec2ServiceConfig);
         Mockito.lenient().when(ec2ServiceConfig.enabled()).thenReturn(false);
+        Mockito.lenient().when(servicesConfig.elbv2()).thenReturn(elbv2ServiceConfig);
+        Mockito.lenient().when(elbv2ServiceConfig.enabled()).thenReturn(false);
         Mockito.lenient().when(config.tls()).thenReturn(tlsConfig);
         Mockito.lenient().when(tlsConfig.enabled()).thenReturn(false);
 
@@ -99,7 +103,7 @@ void setUp() {
                 elastiCacheProxyManager, rdsContainerManager, rdsProxyManager,
                 memoryDbContainerManager, memoryDbProxyManager,
                 docDbContainerManager, neptuneContainerManager, neptuneProxyManager,
-                rabbitMqManager, rdsService,
+                rabbitMqManager, rdsService, elbV2Service,
                 initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,
                 pipesService, ec2MetadataServer, ecrRegistryManager, flociUiManager, initLifecycleState,
                 schemaCreationWorker, containerTeardowns, persistentPathValidator);
@@ -129,6 +133,36 @@ void shouldRunBootHooksBeforeStorageLoad() throws IOException, InterruptedExcept
         inOrder.verify(rdsService).restorePersistedRuntime();
     }
 
+    @Test
+    @DisplayName("Should restore ELBv2 persisted runtime after loading storage when elbv2 is enabled")
+    void shouldRestoreElbV2PersistedRuntimeAfterStorageLoad() {
+        // The restore has to happen after the bean is constructed, not from @PostConstruct, or
+        // ElbV2DataPlane's callback re-enters bean creation (#1913). Pin the ordering so the call
+        // cannot be dropped or moved back into construction unnoticed.
+        stubStorageConfig();
+        when(elbv2ServiceConfig.enabled()).thenReturn(true);
+        when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+        when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+        emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+        var inOrder = Mockito.inOrder(storageFactory, elbV2Service);
+        inOrder.verify(storageFactory).loadAll();
+        inOrder.verify(elbV2Service).restorePersistedRuntime();
+    }
+
+    @Test
+    @DisplayName("Should not restore ELBv2 persisted runtime when elbv2 is disabled")
+    void shouldNotRestoreElbV2PersistedRuntimeWhenDisabled() {
+        stubStorageConfig();
+        when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+        when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+        emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+        Mockito.verify(elbV2Service, Mockito.never()).restorePersistedRuntime();
+    }
+
     @Test
     @DisplayName("Should validate the persistent path after BOOT hooks and before loading storage")
     void shouldValidatePersistentPathBeforeStorageLoad() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
index da6e479553..2d7d316b83 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java
@@ -40,6 +40,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
@@ -179,6 +180,9 @@ void initializeStorageReloadsPersistedResourcesAndRebuildsIndexes() {
         assertTrue(reloadedRules.stream().anyMatch(candidate -> "10".equals(candidate.getPriority())));
         assertEquals("test", reloaded.describeTags(List.of(lbArn)).get(lbArn).get("env"));
         assertThrows(AwsException.class, () -> reloaded.deleteTargetGroup(REGION, tgArn));
+
+        // Data-plane and health-check startup is deliberately not part of construction (#1913).
+        reloaded.restorePersistedRuntime();
         verify(reloadedHealthChecker).startMonitoring(any(TargetGroup.class));
         ArgumentCaptor<List<TargetDescription>> targetsCaptor = ArgumentCaptor.captor();
         verify(reloadedHealthChecker).addTargets(eq(tgArn), targetsCaptor.capture(), any(TargetGroup.class));
@@ -193,7 +197,41 @@ void initializeStorageReloadsPersistedResourcesAndRebuildsIndexes() {
     }
 
     @Test
-    void initializeStorageStartsPersistedListenersWithoutTargetGroups() {
+    void initializeStorageDoesNotTouchCollaborators() {
+        // #1913: ElbV2DataPlane.binding() calls back into this service, and during @PostConstruct the
+        // ApplicationScoped context has no instance yet — so CDI re-enters bean creation and recurses
+        // until the stack (or, before the storage dedupe in #1931, the heap) is gone. Nothing reachable
+        // from initializeStorage() may call an injected collaborator. A mock cannot reproduce the
+        // recursion, since it never calls back, so assert the absence of the interaction instead.
+        SharedStorageFactory storageFactory = new SharedStorageFactory();
+        ElbV2Service first = serviceWithStorage(storageFactory, mock(ElbV2DataPlane.class), mock(ElbV2HealthChecker.class));
+        String lbArn = first.createLoadBalancer(
+                REGION, "recursion-lb", "internal", "application", "ipv4",
+                ALB_SUBNETS, List.of("sg-a"), Map.of()).getLoadBalancerArn();
+        String tgArn = first.createTargetGroup(
+                REGION, "recursion-tg", "HTTP", "HTTP1", 8080, "vpc-a", "instance",
+                "HTTP", "traffic-port", true, "/health", 15, 5, 3, 2, "200",
+                "ipv4", Map.of()).getTargetGroupArn();
+        String listenerArn = first.createListener(
+                REGION, lbArn, "HTTP", 9090, null, List.of(),
+                List.of(forwardAction(tgArn)), List.of(), Map.of()).getListenerArn();
+
+        ElbV2DataPlane reloadedDataPlane = mock(ElbV2DataPlane.class);
+        ElbV2HealthChecker reloadedHealthChecker = mock(ElbV2HealthChecker.class);
+        ElbV2Service reloaded = serviceWithStorage(storageFactory, reloadedDataPlane, reloadedHealthChecker);
+
+        verifyNoInteractions(reloadedDataPlane, reloadedHealthChecker);
+
+        reloaded.restorePersistedRuntime();
+
+        ArgumentCaptor<Listener> listenerCaptor = ArgumentCaptor.captor();
+        verify(reloadedDataPlane).startListener(listenerCaptor.capture(), eq(REGION), anyList());
+        assertEquals(listenerArn, listenerCaptor.getValue().getListenerArn());
+        verify(reloadedHealthChecker).startMonitoring(any(TargetGroup.class));
+    }
+
+    @Test
+    void restorePersistedRuntimeStartsPersistedListenersWithoutTargetGroups() {
         SharedStorageFactory storageFactory = new SharedStorageFactory();
         ElbV2Service first = serviceWithStorage(storageFactory, mock(ElbV2DataPlane.class), mock(ElbV2HealthChecker.class));
         String lbArn = first.createLoadBalancer(
@@ -204,7 +242,8 @@ void initializeStorageStartsPersistedListenersWithoutTargetGroups() {
                 List.of(), List.of(), Map.of()).getListenerArn();
 
         ElbV2DataPlane reloadedDataPlane = mock(ElbV2DataPlane.class);
-        serviceWithStorage(storageFactory, reloadedDataPlane, mock(ElbV2HealthChecker.class));
+        ElbV2Service reloaded = serviceWithStorage(storageFactory, reloadedDataPlane, mock(ElbV2HealthChecker.class));
+        reloaded.restorePersistedRuntime();
 
         ArgumentCaptor<Listener> listenerCaptor = ArgumentCaptor.captor();
         verify(reloadedDataPlane).startListener(listenerCaptor.capture(), eq(REGION), anyList());