Repository: apache/accumulo
Pull request #5399: Make garbage collecting wal and recovery files faster
Author: login: dlmarion
Created: 2025-03-11T21:08:06Z | Merged: 2025-03-13T20:25:58Z

Uses the same property and technique used for deleting RFiles to delete wal and recovery files.

Closes #5397

Linked issue #5397: Garbage Collecting Write Ahead Logs can be slow for a large number of logs
**Describe the bug**
GarbageCollectWriteAheadLogs deletes walog files serially, where rfiles are deleted using a thread pool.

**Versions (OS, Maven, Java, and others, as appropriate):**
 - Affected version(s) of this project: 2.1.3


Diff:
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 942dfeea456..ff0cad4baba 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -940,8 +940,10 @@ public enum Property {
       "1.3.5"),
   GC_PORT("gc.port.client", "9998", PropertyType.PORT,
       "The listening port for the garbage collector's monitor service.", "1.3.5"),
+  GC_DELETE_WAL_THREADS("gc.threads.delete.wal", "4", PropertyType.COUNT,
+      "The number of threads used to delete write-ahead logs and recovery files.", "2.1.4"),
   GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT,
-      "The number of threads used to delete RFiles and write-ahead logs.", "1.3.5"),
+      "The number of threads used to delete RFiles.", "1.3.5"),
   @Experimental
   GC_REMOVE_IN_USE_CANDIDATES("gc.remove.in.use.candidates", "false", PropertyType.BOOLEAN,
       "GC will remove deletion candidates that are in-use from the metadata location. "
diff --git a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPoolNames.java b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPoolNames.java
index 43a4a24b70b..861a6ee89e6 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPoolNames.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPoolNames.java
@@ -36,6 +36,7 @@ public enum ThreadPoolNames {
   COORDINATOR_FINALIZER_BACKGROUND_POOL("accumulo.pool.compaction.finalizer.background.pool"),
   COORDINATOR_FINALIZER_NOTIFIER_POOL("accumulo.pool.compaction.coordinator.compaction.finalizer"),
   GC_DELETE_POOL("accumulo.pool.gc.threads.delete"),
+  GC_WAL_DELETE_POOL("accumulo.pool.gc.threads.delete.wal"),
   GENERAL_SERVER_POOL("accumulo.pool.general.server"),
   GENERAL_SERVER_SIMPLETIMER_POOL("accumulo.pool.general.server.simpletimer"),
   IMPORT_TABLE_RENAME_POOL("accumulo.pool.import.table.rename"),
diff --git a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
index 240a41459cb..136065beb4d 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
@@ -23,6 +23,7 @@
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.accumulo.core.util.threads.ThreadPoolNames.ACCUMULO_POOL_PREFIX;
 import static org.apache.accumulo.core.util.threads.ThreadPoolNames.GC_DELETE_POOL;
+import static org.apache.accumulo.core.util.threads.ThreadPoolNames.GC_WAL_DELETE_POOL;
 import static org.apache.accumulo.core.util.threads.ThreadPoolNames.GENERAL_SERVER_POOL;
 import static org.apache.accumulo.core.util.threads.ThreadPoolNames.GENERAL_SERVER_SIMPLETIMER_POOL;
 import static org.apache.accumulo.core.util.threads.ThreadPoolNames.MANAGER_BULK_IMPORT_POOL;
@@ -375,6 +376,8 @@ public ThreadPoolExecutor createExecutorService(final AccumuloConfiguration conf
           builder.enableThreadPoolMetrics();
         }
         return builder.build();
+      case GC_DELETE_WAL_THREADS:
+        return getPoolBuilder(GC_WAL_DELETE_POOL).numCoreThreads(conf.getCount(p)).build();
       case GC_DELETE_THREADS:
         return getPoolBuilder(GC_DELETE_POOL).numCoreThreads(conf.getCount(p)).build();
       case REPLICATION_WORKER_THREADS:
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 0fe4843ad19..4d9c4e745a6 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -23,16 +23,22 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Stream;
 
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.gc.thrift.GCStatus;
@@ -46,6 +52,7 @@
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.trace.TraceUtil;
 import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.threads.ThreadPools;
 import org.apache.accumulo.server.ServerContext;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.log.WalStateManager;
@@ -266,37 +273,102 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
-    for (Pair<WalState,Path> stateFile : collection) {
-      Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
+    try {
+      for (Pair<WalState,Path> stateFile : collection) {
+        Path path = stateFile.getSecond();
+        futures.put(path, removeFile(deleteThreadPool, path, counter,
+            "Removing " + stateFile.getFirst() + " WAL " + path));
+      }
+
+      while (!futures.isEmpty()) {
+        Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+        while (iter.hasNext()) {
+          Entry<Path,Future<?>> f = iter.next();
+          if (f.getValue().isDone()) {
+            try {
+              iter.remove();
+              f.getValue().get();
+            } catch (InterruptedException | ExecutionException e) {
+              throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+            }
+          }
+        }
+        try {
+          Thread.sleep(500);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          throw new RuntimeException("Interrupted while sleeping", e);
+        }
+      }
+    } finally {
+      deleteThreadPool.shutdownNow();
     }
-    return status.currentLog.deleted;
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
-    for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
+    try {
+      for (Path path : values) {
+        futures.put(path,
+            removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+      }
+
+      while (!futures.isEmpty()) {
+        Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+        while (iter.hasNext()) {
+          Entry<Path,Future<?>> f = iter.next();
+          if (f.getValue().isDone()) {
+            try {
+              iter.remove();
+              f.getValue().get();
+            } catch (InterruptedException | ExecutionException e) {
+              throw new RuntimeException(
+                  "Uncaught exception deleting recovery log file" + f.getKey(), e);
+            }
+          }
+        }
+        try {
+          Thread.sleep(500);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          throw new RuntimeException("Interrupted while sleeping", e);
+        }
+      }
+    } finally {
+      deleteThreadPool.shutdownNow();
     }
-    return count;
+    return counter.get();
   }
 
   private UUID path2uuid(Path path) {
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
index 544407b495a..1cb8cd61f6c 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
@@ -28,6 +28,8 @@
 import java.util.stream.Stream;
 
 import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -92,6 +94,7 @@ public class GarbageCollectWriteAheadLogsTest {
 
   @Test
   public void testRemoveUnusedLog() throws Exception {
+    AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
     ServerContext context = EasyMock.createMock(ServerContext.class);
     VolumeManager fs = EasyMock.createMock(VolumeManager.class);
     WalStateManager marker = EasyMock.createMock(WalStateManager.class);
@@ -99,6 +102,8 @@ public void testRemoveUnusedLog() throws Exception {
 
     GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
 
+    EasyMock.expect(context.getConfiguration()).andReturn(conf).times(2);
+    EasyMock.expect(conf.getCount(Property.GC_DELETE_WAL_THREADS)).andReturn(8).anyTimes();
     tserverSet.scanServers();
     EasyMock.expectLastCall();
     EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
@@ -108,7 +113,7 @@ public void testRemoveUnusedLog() throws Exception {
     EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
     marker.removeWalMarker(server1, id);
     EasyMock.expectLastCall().once();
-    EasyMock.replay(context, fs, marker, tserverSet);
+    EasyMock.replay(conf, context, fs, marker, tserverSet);
     var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, false, marker,
         tabletOnServer1List) {
       @Override
@@ -123,11 +128,12 @@ protected Map<UUID,Path> getSortedWALogs() {
       }
     };
     gc.collect(status);
-    EasyMock.verify(context, fs, marker, tserverSet);
+    EasyMock.verify(conf, context, fs, marker, tserverSet);
   }
 
   @Test
   public void testKeepClosedLog() throws Exception {
+    AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
     ServerContext context = EasyMock.createMock(ServerContext.class);
     VolumeManager fs = EasyMock.createMock(VolumeManager.class);
     WalStateManager marker = EasyMock.createMock(WalStateManager.class);
@@ -135,13 +141,15 @@ public void testKeepClosedLog() throws Exception {
 
     GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
 
+    EasyMock.expect(context.getConfiguration()).andReturn(conf).times(2);
+    EasyMock.expect(conf.getCount(Property.GC_DELETE_WAL_THREADS)).andReturn(8).anyTimes();
     tserverSet.scanServers();
     EasyMock.expectLastCall();
     EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
 
     EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once();
     EasyMock.expect(marker.state(server1, id)).andReturn(new Pair<>(WalState.CLOSED, path));
-    EasyMock.replay(context, marker, tserverSet, fs);
+    EasyMock.replay(conf, context, marker, tserverSet, fs);
     var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, false, marker,
         tabletOnServer1List) {
       @Override
@@ -156,11 +164,12 @@ protected Map<UUID,Path> getSortedWALogs() {
       }
     };
     gc.collect(status);
-    EasyMock.verify(context, marker, tserverSet, fs);
+    EasyMock.verify(conf, context, marker, tserverSet, fs);
   }
 
   @Test
   public void deleteUnreferencedLogOnDeadServer() throws Exception {
+    AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
     ServerContext context = EasyMock.createMock(ServerContext.class);
     VolumeManager fs = EasyMock.createMock(VolumeManager.class);
     WalStateManager marker = EasyMock.createMock(WalStateManager.class);
@@ -170,6 +179,8 @@ public void deleteUnreferencedLogOnDeadServer() throws Exception {
 
     GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
 
+    EasyMock.expect(context.getConfiguration()).andReturn(conf).times(2);
+    EasyMock.expect(conf.getCount(Property.GC_DELETE_WAL_THREADS)).andReturn(8).anyTimes();
     tserverSet.scanServers();
     EasyMock.expectLastCall();
     EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
@@ -195,7 +206,7 @@ public void deleteUnreferencedLogOnDeadServer() throws Exception {
     EasyMock.expectLastCall().once();
     marker.forget(server2);
     EasyMock.expectLastCall().once();
-    EasyMock.replay(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.replay(conf, context, fs, marker, tserverSet, rscanner, mscanner);
     var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, false, marker,
         tabletOnServer1List) {
       @Override
@@ -204,11 +215,12 @@ protected Map<UUID,Path> getSortedWALogs() {
       }
     };
     gc.collect(status);
-    EasyMock.verify(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.verify(conf, context, fs, marker, tserverSet, rscanner, mscanner);
   }
 
   @Test
   public void ignoreReferenceLogOnDeadServer() throws Exception {
+    AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
     ServerContext context = EasyMock.createMock(ServerContext.class);
     VolumeManager fs = EasyMock.createMock(VolumeManager.class);
     WalStateManager marker = EasyMock.createMock(WalStateManager.class);
@@ -218,6 +230,8 @@ public void ignoreReferenceLogOnDeadServer() throws Exception {
 
     GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
 
+    EasyMock.expect(context.getConfiguration()).andReturn(conf).times(2);
+    EasyMock.expect(conf.getCount(Property.GC_DELETE_WAL_THREADS)).andReturn(8).anyTimes();
     tserverSet.scanServers();
     EasyMock.expectLastCall();
     EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
@@ -238,7 +252,7 @@ public void ignoreReferenceLogOnDeadServer() throws Exception {
     mscanner.setRange(ReplicationSection.getRange());
     EasyMock.expectLastCall().once();
     EasyMock.expect(mscanner.iterator()).andReturn(emptyKV);
-    EasyMock.replay(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.replay(conf, context, fs, marker, tserverSet, rscanner, mscanner);
     var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, false, marker,
         tabletOnServer2List) {
       @Override
@@ -247,11 +261,12 @@ protected Map<UUID,Path> getSortedWALogs() {
       }
     };
     gc.collect(status);
-    EasyMock.verify(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.verify(conf, context, fs, marker, tserverSet, rscanner, mscanner);
   }
 
   @Test
   public void replicationDelaysFileCollection() throws Exception {
+    AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
     ServerContext context = EasyMock.createMock(ServerContext.class);
     VolumeManager fs = EasyMock.createMock(VolumeManager.class);
     WalStateManager marker = EasyMock.createMock(WalStateManager.class);
@@ -266,6 +281,8 @@ public void replicationDelaysFileCollection() throws Exception {
 
     GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
 
+    EasyMock.expect(context.getConfiguration()).andReturn(conf).times(2);
+    EasyMock.expect(conf.getCount(Property.GC_DELETE_WAL_THREADS)).andReturn(8).anyTimes();
     tserverSet.scanServers();
     EasyMock.expectLastCall();
     EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
@@ -286,7 +303,7 @@ public void replicationDelaysFileCollection() throws Exception {
     mscanner.setRange(ReplicationSection.getRange());
     EasyMock.expectLastCall().once();
     EasyMock.expect(mscanner.iterator()).andReturn(replicationWork.entrySet().iterator());
-    EasyMock.replay(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.replay(conf, context, fs, marker, tserverSet, rscanner, mscanner);
     var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, false, marker,
         tabletOnServer1List) {
       @Override
@@ -295,6 +312,6 @@ protected Map<UUID,Path> getSortedWALogs() {
       }
     };
     gc.collect(status);
-    EasyMock.verify(context, fs, marker, tserverSet, rscanner, mscanner);
+    EasyMock.verify(conf, context, fs, marker, tserverSet, rscanner, mscanner);
   }
 }


Reviews:
user: login: ddanielr
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T15:14:17Z
user: login: ddanielr
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T15:38:21Z
user: login: ddanielr
body: Changes look good to me.

The only suggestion I have is using a separate `GC_WAL_DELETE_THREADS` property.

In the current code, user table activity will actively delay WAL cleanup processing as the wals are processed at the end of a GC run cycle before flushing the metadata and root tables.

If the goal is to speed up GC of the wals and recovery files, then I don't see a reason why the WAL logs have to be handled at the end of the GC run cycle vs just always be running as a separate task thread.

`GC_DELETE_THREADS` is most likely going to be set higher than needed for the WALs since it's based on file activity in the system while the upper limit of WALs should be based on the amount of tservers * `tserver.wal.max.referenced` and tserver churn (dead/recovered/etc).

If we move to having the WALs GC'd in a separate thread or just pulled into a different GC process, then using a separate property would make it easier to avoid exceeding the max amount of available threads on the server.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T16:07:11Z
user: login: keith-turner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T18:43:41Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:25:23Z
user: login: cshannon
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:26:52Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:28:26Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:35:12Z
user: login: cshannon
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:41:29Z
user: login: keith-turner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T20:23:36Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T20:51:07Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T20:51:18Z
user: login: keith-turner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T21:35:42Z
user: login: keith-turner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T21:39:13Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T21:40:45Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:49:19Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:49:34Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:50:01Z
user: login: keith-turner
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T19:37:58Z
user: login: dlmarion
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T20:07:38Z

Review comments:
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: ddanielr
body: When deleting rfiles, the threadpool is created and shutdown for each delete pass to ensure that all rfiles are deleted before we remove the gcCandidates from their respective metadata location.

For WALs, the GC looks at each wal directory location to find files, then compares the file against the live tservers set, recovery operations, or dead tserver list to see if the file can be removed. 

Since we aren't processing gcCandidates, do we need to create and delete the threadpool on each removeFiles run?

If we made a longer lived pool then we could enable metrics on the threadpool and get more details about how many threads might be utilized.
created_at: 2025-03-12T15:14:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: ddanielr
body: Started looking at the WALStateManager code and realized my understanding of the WAL log process was incorrect. While the GC does look at recovery directory locations, the rest of the WAL information is stored in ZK. 
After the WALs are processed, the WALStateManager is used to update ZK references to the WAL files.

So the existing threadpool start/shutdown pattern makes sense here.
created_at: 2025-03-12T15:38:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path);
     }
-    return status.currentLog.deleted;
+
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    return counter.get();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: The code was incrementing this deleted counter, should it still do that?

```suggestion
    status.currentLog.deleted += counter.get();
    return counter.get();
```
created_at: 2025-03-12T18:43:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path);
     }
-    return status.currentLog.deleted;
+
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    return counter.get();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Yep, I forgot to increment the counter. I created it though, so I was half way there.
created_at: 2025-03-12T19:25:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: cshannon
body: I was wondering if we need to handle any other exception types here as an uncaught exception can bubble up and kill the thead pool. It's pretty common to catch all exceptions as a catch all.
created_at: 2025-03-12T19:26:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path);
     }
-    return status.currentLog.deleted;
+
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    return counter.get();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: On second look, counter is being passed to removeFile, and incremented there.
created_at: 2025-03-12T19:28:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: The previous code just bubbled the RuntimeExceptions up the call stack. I could call `submit` instead, test the futures, and return any errors up the stack.
created_at: 2025-03-12T19:35:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: cshannon
body: If you used futures that might be better, you could wait for them to finish instead of relying on the thread pool to shutdown for completion. Either way could work, I just figured I'd mention it as I'm not too familiar with this code but I noticed we were only catching those specific exceptions so was curious if a random runtime exception would cause issues with the new thread pool.
created_at: 2025-03-12T19:41:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path);
     }
-    return status.currentLog.deleted;
+
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    return counter.get();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: yeah counter is being incrementing removeFile, but the old code used to increment `status.currentLog.deleted` which may be used for display on the monitor and logging.  Seems like that is no longer incremented.
created_at: 2025-03-12T20:23:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Updated to use futures in 7c54495
created_at: 2025-03-12T20:51:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +271,62 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    deleteThreadPool.execute(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete wal {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path);
     }
-    return status.currentLog.deleted;
+
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    return counter.get();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Updated in 7c54495
created_at: 2025-03-12T20:51:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -283,42 +286,77 @@ private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong
       } catch (FileNotFoundException ex) {
         // ignored
       } catch (IOException ex) {
-        log.error("Unable to delete wal {}", path, ex);
+        log.error("Unable to delete {}", path, ex);
       }
     });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
 
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      removeFile(deleteThreadPool, path, counter,
-          "Removing " + stateFile.getFirst() + " WAL " + path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
     }
 
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
+    }
     deleteThreadPool.shutdown();
     try {
       while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
       }
     } catch (InterruptedException e1) {
       log.error("{}", e1.getMessage(), e1);
     }
+    status.currentLog.deleted += counter.get();
     return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
+
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Path path : values) {
-      removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
     }
 
+    while (!futures.isEmpty()) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: The get() method in a future waits until its done or failed, so should be able to loop through all futures once calling get.

```java
futures.forEach((path,future)->{
             try {
            future.get();
          } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException("Uncaught exception deleting recovery log file" + path,
                e);
          }
});
```
created_at: 2025-03-12T21:35:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +274,97 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
     }
-    return status.currentLog.deleted;
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting recovery log file" + f.getKey(),
+                e);
+          }
+        }
+      }
+    }
+    deleteThreadPool.shutdown();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: If an exception is throw the pool would not be shut down.
created_at: 2025-03-12T21:38:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +274,97 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
     }
-    return status.currentLog.deleted;
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting recovery log file" + f.getKey(),
+                e);
+          }
+        }
+      }
+    }
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: Could remove this code now that futures are being waited on.
created_at: 2025-03-12T21:39:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -283,42 +286,77 @@ private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong
       } catch (FileNotFoundException ex) {
         // ignored
       } catch (IOException ex) {
-        log.error("Unable to delete wal {}", path, ex);
+        log.error("Unable to delete {}", path, ex);
       }
     });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
 
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      removeFile(deleteThreadPool, path, counter,
-          "Removing " + stateFile.getFirst() + " WAL " + path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
     }
 
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
+    }
     deleteThreadPool.shutdown();
     try {
       while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
       }
     } catch (InterruptedException e1) {
       log.error("{}", e1.getMessage(), e1);
     }
+    status.currentLog.deleted += counter.get();
     return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
+
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Path path : values) {
-      removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
     }
 
+    while (!futures.isEmpty()) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: I was thinking the faster I process the ones that are complete and remove from the map, the faster I'm giving back memory to the VM. I probably need to put a wait at the bottom of the loop though. If the first one the iterator returns is the last one submitted, then I would be waiting until they are all done.
created_at: 2025-03-12T21:40:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +274,97 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
     }
-    return status.currentLog.deleted;
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting recovery log file" + f.getKey(),
+                e);
+          }
+        }
+      }
+    }
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Fixed in 93f5c6d
created_at: 2025-03-13T11:49:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +274,97 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
     }
-    return status.currentLog.deleted;
+    deleteThreadPool.shutdown();
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
+      }
+    } catch (InterruptedException e1) {
+      log.error("{}", e1.getMessage(), e1);
+    }
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting recovery log file" + f.getKey(),
+                e);
+          }
+        }
+      }
+    }
+    deleteThreadPool.shutdown();
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Fixed this in 93f5c6d
created_at: 2025-03-13T11:49:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -283,42 +286,77 @@ private void removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong
       } catch (FileNotFoundException ex) {
         // ignored
       } catch (IOException ex) {
-        log.error("Unable to delete wal {}", path, ex);
+        log.error("Unable to delete {}", path, ex);
       }
     });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
 
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Pair<WalState,Path> stateFile : collection) {
       Path path = stateFile.getSecond();
-      removeFile(deleteThreadPool, path, counter,
-          "Removing " + stateFile.getFirst() + " WAL " + path);
+      futures.put(path, removeFile(deleteThreadPool, path, counter,
+          "Removing " + stateFile.getFirst() + " WAL " + path));
     }
 
+    while (!futures.isEmpty()) {
+      Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<Path,Future<?>> f = iter.next();
+        if (f.getValue().isDone()) {
+          try {
+            iter.remove();
+            f.getValue().get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+          }
+        }
+      }
+    }
     deleteThreadPool.shutdown();
     try {
       while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) { // empty
       }
     } catch (InterruptedException e1) {
       log.error("{}", e1.getMessage(), e1);
     }
+    status.currentLog.deleted += counter.get();
     return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
+
     final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
-        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_THREADS);
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
     final AtomicLong counter = new AtomicLong();
 
     for (Path path : values) {
-      removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
     }
 
+    while (!futures.isEmpty()) {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: Added a sleep at the bottom of the while loop in 93f5c6d.
created_at: 2025-03-13T11:50:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +273,102 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
-    for (Pair<WalState,Path> stateFile : collection) {
-      Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
+    try {
+      for (Pair<WalState,Path> stateFile : collection) {
+        Path path = stateFile.getSecond();
+        futures.put(path, removeFile(deleteThreadPool, path, counter,
+            "Removing " + stateFile.getFirst() + " WAL " + path));
+      }
+
+      while (!futures.isEmpty()) {
+        Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+        while (iter.hasNext()) {
+          Entry<Path,Future<?>> f = iter.next();
+          if (f.getValue().isDone()) {
+            try {
+              iter.remove();
+              f.getValue().get();
+            } catch (InterruptedException | ExecutionException e) {
+              throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+            }
+          }
+        }
+        try {
+          Thread.sleep(500);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          throw new RuntimeException("Interrupted while sleeping", e);
+        }
+      }
+    } finally {
+      deleteThreadPool.shutdownNow();
     }
-    return status.currentLog.deleted;
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    try {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: keith-turner
body: could move this try up to be immediately after the creation of deleteThreadPool
created_at: 2025-03-13T19:33:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -266,37 +273,102 @@ private long removeTabletServerMarkers(Map<UUID,TServerInstance> uidMap,
     return result;
   }
 
-  private long removeFile(Path path) {
-    try {
-      if (!useTrash || !fs.moveToTrash(path)) {
-        fs.deleteRecursively(path);
+  private Future<?> removeFile(ExecutorService deleteThreadPool, Path path, AtomicLong counter,
+      String msg) {
+    return deleteThreadPool.submit(() -> {
+      try {
+        log.debug(msg);
+        if (!useTrash || !fs.moveToTrash(path)) {
+          fs.deleteRecursively(path);
+        }
+        counter.incrementAndGet();
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ex) {
+        log.error("Unable to delete {}", path, ex);
       }
-      return 1;
-    } catch (FileNotFoundException ex) {
-      // ignored
-    } catch (IOException ex) {
-      log.error("Unable to delete wal {}", path, ex);
-    }
-
-    return 0;
+    });
   }
 
   private long removeFiles(Collection<Pair<WalState,Path>> collection, final GCStatus status) {
-    for (Pair<WalState,Path> stateFile : collection) {
-      Path path = stateFile.getSecond();
-      log.debug("Removing {} WAL {}", stateFile.getFirst(), path);
-      status.currentLog.deleted += removeFile(path);
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(collection.size());
+    final AtomicLong counter = new AtomicLong();
+
+    try {
+      for (Pair<WalState,Path> stateFile : collection) {
+        Path path = stateFile.getSecond();
+        futures.put(path, removeFile(deleteThreadPool, path, counter,
+            "Removing " + stateFile.getFirst() + " WAL " + path));
+      }
+
+      while (!futures.isEmpty()) {
+        Iterator<Entry<Path,Future<?>>> iter = futures.entrySet().iterator();
+        while (iter.hasNext()) {
+          Entry<Path,Future<?>> f = iter.next();
+          if (f.getValue().isDone()) {
+            try {
+              iter.remove();
+              f.getValue().get();
+            } catch (InterruptedException | ExecutionException e) {
+              throw new RuntimeException("Uncaught exception deleting wal file" + f.getKey(), e);
+            }
+          }
+        }
+        try {
+          Thread.sleep(500);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          throw new RuntimeException("Interrupted while sleeping", e);
+        }
+      }
+    } finally {
+      deleteThreadPool.shutdownNow();
     }
-    return status.currentLog.deleted;
+    status.currentLog.deleted += counter.get();
+    return counter.get();
   }
 
   private long removeFiles(Collection<Path> values) {
-    long count = 0;
+
+    final ExecutorService deleteThreadPool = ThreadPools.getServerThreadPools()
+        .createExecutorService(context.getConfiguration(), Property.GC_DELETE_WAL_THREADS);
+    final Map<Path,Future<?>> futures = new HashMap<>(values.size());
+    final AtomicLong counter = new AtomicLong();
+
     for (Path path : values) {
-      log.debug("Removing recovery log {}", path);
-      count += removeFile(path);
+      futures.put(path,
+          removeFile(deleteThreadPool, path, counter, "Removing recovery log " + path));
+    }
+
+    try {
path: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
user: login: dlmarion
body: smh, I did that on the other method. I'll fix.
created_at: 2025-03-13T20:07:38Z
author_association: CONTRIBUTOR

PR comments:
user: login: dlmarion
created_at: 2025-03-11T21:24:52Z
body: Kicked off IT build
author_association: CONTRIBUTOR
user: login: dlmarion
created_at: 2025-03-12T12:08:45Z
body: Full IT build successful
author_association: CONTRIBUTOR
user: login: dlmarion
created_at: 2025-03-12T20:51:40Z
body: > 
> The only suggestion I have is using a separate `GC_WAL_DELETE_THREADS` property.
> 

Added in 7c54495
author_association: CONTRIBUTOR