Repository: apache/ozone
Pull request #8085: HDDS-12557. Added progress indicator for checkpoint tarball in leader OM
Author: login: SaketaChalamchala
Created: 2025-03-15T01:29:24Z | Merged: 2025-03-20T11:52:46Z

## What changes were proposed in this pull request?

No logs to indicate checkpoint tarball in leader OM. 
Added logs to provide an initial estimate of the amount of data, number of files and snapshots to be transferred in the tarball stream.
Added progress indicator logs every 30s detailing # sst files and size of data trasferred per request. 

## What is the link to the Apache JIRA

https://issues.apache.org/jira/browse/HDDS-12557

## How was this patch tested?

Manually tested with docker with a progress log interval of `500ms`
Leader OM: 
```
INFO om.OMDBCheckpointServlet: Estimates for Checkpoint Tarball Stream - Data size: 64190 KB, SST files: 48317, snapshots: 300
INFO om.OMDBCheckpointServlet: Transferred 91266 KB, #files 2774 to checkpoint tarball stream... // Will print every 30s. 
INFO om.OMDBCheckpointServlet: Completed transfer of 106103 KB, #files 3183 to checkpoint tarball stream. Checkpoint tarball is complete. // excluding hard links.
```

As usual the follower tracks the amount of data transferred
Follower OM:
```
INFO ratis_snapshot.OmRatisSnapshotProvider: Downloading latest checkpoint from Leader OM om1. Checkpoint: om.db-om1-1742344843952.tar URL: http://om1:9874/dbCheckpoint?includeSnapshotData=true&flushBeforeCheckpoint=true
INFO ratis_snapshot.OmRatisSnapshotProvider: Download completed for 'om.db-om1-1742344843952.tar'. Total size: 116045 KB
```




Diff:
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
index 5d04f2060f59..8d2c5881e8f0 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
@@ -620,18 +620,20 @@ public static void writeDBCheckpointToStream(
     }
   }
 
-  public static void includeFile(File file, String entryName,
+  public static long includeFile(File file, String entryName,
                                  ArchiveOutputStream archiveOutputStream)
       throws IOException {
     ArchiveEntry archiveEntry =
         archiveOutputStream.createArchiveEntry(file, entryName);
     archiveOutputStream.putArchiveEntry(archiveEntry);
+    long bytesWritten;
     try (InputStream fis = Files.newInputStream(file.toPath())) {
-      IOUtils.copy(fis, archiveOutputStream);
+      bytesWritten = IOUtils.copy(fis, archiveOutputStream);
       archiveOutputStream.flush();
     } finally {
       archiveOutputStream.closeArchiveEntry();
     }
+    return bytesWritten;
   }
 
   // Mark tarball completed.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
index 7c02df4bebff..e7df608aed1e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.om;
 
+import static org.apache.commons.io.filefilter.TrueFileFilter.TRUE;
 import static org.apache.hadoop.hdds.utils.HddsServerUtil.includeFile;
 import static org.apache.hadoop.hdds.utils.HddsServerUtil.includeRatisSnapshotCompleteFlag;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR;
@@ -56,6 +57,11 @@
 import org.apache.commons.compress.archivers.ArchiveOutputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOCase;
+import org.apache.commons.io.file.Counters;
+import org.apache.commons.io.file.CountingPathVisitor;
+import org.apache.commons.io.file.PathFilter;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.recon.ReconConfig;
 import org.apache.hadoop.hdds.utils.DBCheckpointServlet;
@@ -69,6 +75,7 @@
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
 import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -93,6 +100,8 @@ public class OMDBCheckpointServlet extends DBCheckpointServlet {
   private static final long serialVersionUID = 1L;
   private transient BootstrapStateHandler.Lock lock;
   private long maxTotalSstSize = 0;
+  private static final PathFilter SST_FILE_FILTER =
+      new SuffixFileFilter(ROCKSDB_SST_SUFFIX, IOCase.INSENSITIVE);
 
   @Override
   public void init() throws ServletException {
@@ -167,6 +176,7 @@ public void writeDbDataToStream(DBCheckpoint checkpoint,
       // Files to be excluded from tarball
       Map<String, Map<Path, Path>> sstFilesToExclude = normalizeExcludeList(toExcludeList,
           checkpoint.getCheckpointLocation(), sstBackupDir);
+
       boolean completed = getFilesForArchive(checkpoint, copyFiles,
           hardLinkFiles, sstFilesToExclude, includeSnapshotData(request),
           excludedList, sstBackupDir, compactionLogDir);
@@ -270,13 +280,13 @@ public File getTmpDir() {
 
   @SuppressWarnings("checkstyle:ParameterNumber")
   private boolean getFilesForArchive(DBCheckpoint checkpoint,
-                                  Map<String, Map<Path, Path>> copyFiles,
-                                  Map<Path, Path> hardLinkFiles,
-                                  Map<String, Map<Path, Path>> sstFilesToExclude,
-                                  boolean includeSnapshotData,
-                                  List<String> excluded,
-                                  DirectoryData sstBackupDir,
-                                  DirectoryData compactionLogDir)
+      Map<String, Map<Path, Path>> copyFiles,
+      Map<Path, Path> hardLinkFiles,
+      Map<String, Map<Path, Path>> sstFilesToExclude,
+      boolean includeSnapshotData,
+      List<String> excluded,
+      DirectoryData sstBackupDir,
+      DirectoryData compactionLogDir)
       throws IOException {
 
     maxTotalSstSize = getConf().getLong(
@@ -290,6 +300,12 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     }
 
     AtomicLong copySize = new AtomicLong(0L);
+
+    // Log estimated total data transferred on first request.
+    if (sstFilesToExclude.isEmpty()) {
+      logEstimatedTarballSize(checkpoint, includeSnapshotData);
+    }
+
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
     if (!processDir(dir, copyFiles, hardLinkFiles, sstFilesToExclude,
@@ -302,7 +318,7 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     }
 
     // Get the snapshot files.
-    Set<Path> snapshotPaths = waitForSnapshotDirs(checkpoint);
+    Set<Path> snapshotPaths = getSnapshotDirs(checkpoint, true);
     Path snapshotDir = getSnapshotDir();
     if (!processDir(snapshotDir, copyFiles, hardLinkFiles, sstFilesToExclude,
         snapshotPaths, excluded, copySize, null)) {
@@ -320,16 +336,40 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         hardLinkFiles, sstFilesToExclude,
         new HashSet<>(), excluded, copySize,
         compactionLogDir.getOriginalDir().toPath());
+  }
 
+  private void logEstimatedTarballSize(
+      DBCheckpoint checkpoint, boolean includeSnapshotData) {
+    try {
+      Counters.PathCounters counters = Counters.longPathCounters();
+      CountingPathVisitor visitor = new CountingPathVisitor(
+          counters, SST_FILE_FILTER, TRUE);
+      Files.walkFileTree(checkpoint.getCheckpointLocation(), visitor);
+      int totalSnapshots = 0;
+      if (includeSnapshotData) {
+        Set<Path> snapshotPaths = getSnapshotDirs(checkpoint, false);
+        totalSnapshots = snapshotPaths.size();
+        for (Path snapshotDir: snapshotPaths) {
+          Files.walkFileTree(snapshotDir, visitor);
+        }
+      }
+      LOG.info("Estimates for Checkpoint Tarball Stream - Data size: {} KB, " + "SST files: {}{}",
+          counters.getByteCounter().get() / (1024),
+          counters.getFileCounter().get(),
+          (includeSnapshotData ? ", snapshots: " + totalSnapshots : ""));
+    } catch (Exception e) {
+      LOG.error("Could not estimate size of transfer to Checkpoint Tarball Stream.", e);
+    }
   }
 
   /**
    * The snapshotInfo table may contain a snapshot that
    * doesn't yet exist on the fs, so wait a few seconds for it.
    * @param checkpoint Checkpoint containing snapshot entries expected.
+   * @param waitForDir Wait for dir to exist on fs.
    * @return Set of expected snapshot dirs.
    */
-  private Set<Path> waitForSnapshotDirs(DBCheckpoint checkpoint)
+  private Set<Path> getSnapshotDirs(DBCheckpoint checkpoint, boolean waitForDir)
       throws IOException {
 
     OzoneConfiguration conf = getConf();
@@ -348,7 +388,9 @@ private Set<Path> waitForSnapshotDirs(DBCheckpoint checkpoint)
       while (iterator.hasNext()) {
         Table.KeyValue<String, SnapshotInfo> entry = iterator.next();
         Path path = Paths.get(getSnapshotPath(conf, entry.getValue()));
-        waitForDirToExist(path);
+        if (waitForDir) {
+          waitForDirToExist(path);
+        }
         snapshotPaths.add(path);
       }
     } finally {
@@ -552,17 +594,21 @@ private void writeFilesToArchive(
         e.getKey().getFileName().toString().toLowerCase().endsWith(".sst")).
         collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
+    long bytesWritten = 0L;
+    int filesWritten = 0;
+    long lastLoggedTime = Time.monotonicNow();
+
     // Go through each of the files to be copied and add to archive.
     for (Map.Entry<Path, Path> entry : filteredCopyFiles.entrySet()) {
-      Path file = entry.getValue();
+      Path path = entry.getValue();
 
       // Confirm the data is in the right place.
-      if (!file.toString().startsWith(metaDirPath.toString())) {
+      if (!path.toString().startsWith(metaDirPath.toString())) {
         throw new IOException("tarball file not in metadata dir: "
-            + file + ": " + metaDirPath);
+            + path + ": " + metaDirPath);
       }
 
-      String fixedFile = truncateFileName(truncateLength, file);
+      String fixedFile = truncateFileName(truncateLength, path);
       if (fixedFile.startsWith(OM_CHECKPOINT_DIR)) {
         // checkpoint files go to root of tarball
         Path f = Paths.get(fixedFile).getFileName();
@@ -570,7 +616,17 @@ private void writeFilesToArchive(
           fixedFile = f.toString();
         }
       }
-      includeFile(entry.getKey().toFile(), fixedFile, archiveOutputStream);
+      File file = entry.getKey().toFile();
+      if (!file.isDirectory()) {
+        filesWritten++;
+      }
+      bytesWritten += includeFile(file, fixedFile, archiveOutputStream);
+      // Log progress every 30 seconds
+      if (Time.monotonicNow() - lastLoggedTime >= 30000) {
+        LOG.info("Transferred {} KB, #files {} to checkpoint tarball stream...",
+            bytesWritten / (1024), filesWritten);
+        lastLoggedTime = Time.monotonicNow();
+      }
     }
 
     if (completed) {
@@ -595,6 +651,10 @@ private void writeFilesToArchive(
       // Mark tarball completed.
       includeRatisSnapshotCompleteFlag(archiveOutputStream);
     }
+    LOG.info("Completed transfer of {} KB, #files {} " +
+            "to checkpoint tarball stream.{}",
+        bytesWritten / (1024), filesWritten, (completed) ?
+            " Checkpoint tarball is complete." : "");
   }
 
   @Nonnull


Reviews:
user: login: adoroszlai
body: Thanks @SaketaChalamchala for the patch.
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-15T06:04:13Z
user: login: adoroszlai
body: Thanks @SaketaChalamchala for updating the patch, LGTM.  Let's see if anyone else would like to review.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T08:07:25Z
user: login: hemantk-12
body: Thanks, @SaketaChalamchala for this improvement. 
Left some minor comments, otherwise looks good to me. 
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T21:21:22Z
user: login: SaketaChalamchala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T21:55:00Z
user: login: hemantk-12
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T23:03:24Z
user: login: adoroszlai
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T09:23:59Z
user: login: adoroszlai
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T11:53:07Z

Review comments:
diff_hunk: @@ -292,6 +304,29 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     AtomicLong copySize = new AtomicLong(0L);
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
+
+    // Log estimated total data transferred on first request.
+    if (logTotals) {
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: New parameter `logTotals` is not needed.

```suggestion
    if (sstFilesToExclude.isEmpty()) {
```
created_at: 2025-03-15T05:39:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -292,6 +304,29 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     AtomicLong copySize = new AtomicLong(0L);
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
+
+    // Log estimated total data transferred on first request.
+    if (logTotals) {
+      try {
+        estimateTotals(dir, totalFiles, totalSize);
+        if (includeSnapshotData) {
+          Set<Path> snapshotPaths = getSnapshotDirs(checkpoint, false);
+          totalSnapshots = snapshotPaths.size();
+          for (Path snapshotDir: snapshotPaths) {
+            estimateTotals(snapshotDir, totalFiles, totalSize);
+          }
+        }
+        LOG.info("Transfer estimates to Checkpoint Tarball Stream - " +
+                "Estimated Data size: {} MB, " +
+                "Estimated number of SST files: {}, " +
+                "Estimated number of Snapshots: {}",
+            totalSize.get() / (1024 * 1024), totalFiles.get(), totalSnapshots);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: - We can use Commons IO for getting recursive file count and total size.  `estimateTotals` can be removed.
- Declare variables like `totalSnapshots` close to their usage.
- Log size in same unit as in other places.

```java
        Counters.PathCounters counters = Counters.longPathCounters();
        CountingPathVisitor visitor = new CountingPathVisitor(counters, SST_FILE_FILTER, TRUE);
        Files.walkFileTree(dir, visitor);
        int totalSnapshots = 0;
        if (includeSnapshotData) {
          Set<Path> snapshotPaths = getSnapshotDirs(checkpoint, false);
          totalSnapshots = snapshotPaths.size();
          for (Path snapshotDir: snapshotPaths) {
            Files.walkFileTree(snapshotDir, visitor);
          }
        }
        LOG.info("Estimates for checkpoint tarball stream: size: {} KB, SST files: {}{}",
            counters.getByteCounter().get() / 1024, counters.getFileCounter().get(),
            (includeSnapshotData ? ", snapshots: " + totalSnapshots : "")
        );
```

with:

```java
  private static final PathFilter SST_FILE_FILTER = new SuffixFileFilter(ROCKSDB_SST_SUFFIX, IOCase.INSENSITIVE);
```

and imports:

```java
import static org.apache.commons.io.filefilter.TrueFileFilter.TRUE;

import org.apache.commons.io.IOCase;
import org.apache.commons.io.file.Counters;
import org.apache.commons.io.file.CountingPathVisitor;
import org.apache.commons.io.file.PathFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
```
created_at: 2025-03-15T05:45:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -570,7 +639,18 @@ private void writeFilesToArchive(
           fixedFile = f.toString();
         }
       }
-      includeFile(entry.getKey().toFile(), fixedFile, archiveOutputStream);
+      Path f = entry.getKey();
+      if (!f.toFile().isDirectory()) {
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: Please rename to `path` and store result of `toFile()` in another variable.
created_at: 2025-03-15T05:49:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -570,7 +639,18 @@ private void writeFilesToArchive(
           fixedFile = f.toString();
         }
       }
-      includeFile(entry.getKey().toFile(), fixedFile, archiveOutputStream);
+      Path f = entry.getKey();
+      if (!f.toFile().isDirectory()) {
+        bytesWritten += Files.size(f);
+        filesWritten++;
+      }
+      includeFile(f.toFile(), fixedFile, archiveOutputStream);
+      // Log progress every 30 seconds
+      if (System.currentTimeMillis() - lastLoggedTime >= 30000) {
+        LOG.info("Transferred {} KB, #files {} to checkpoint tarball stream...",
+            bytesWritten / (1024), filesWritten);
+        lastLoggedTime = System.currentTimeMillis();
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: Please replace with `System.currentTimeMillis` with `Time.monotonicNow()`.
created_at: 2025-03-15T05:51:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -594,7 +674,11 @@ private void writeFilesToArchive(
       }
       // Mark tarball completed.
       includeRatisSnapshotCompleteFlag(archiveOutputStream);
+      LOG.info("Completed checkpoint tarball transfer.");
     }
+    LOG.info("Completed transfer of {} KB, #files {} " +
+            "to checkpoint tarball stream.",
+        bytesWritten / (1024), filesWritten);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: We could indicate in the `Completed transfer of ...` message whether transfer is really `completed` or not, and then `Completed checkpoint tarball transfer` is not needed.
created_at: 2025-03-15T05:56:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -570,7 +639,18 @@ private void writeFilesToArchive(
           fixedFile = f.toString();
         }
       }
-      includeFile(entry.getKey().toFile(), fixedFile, archiveOutputStream);
+      Path f = entry.getKey();
+      if (!f.toFile().isDirectory()) {
+        bytesWritten += Files.size(f);
+        filesWritten++;
+      }
+      includeFile(f.toFile(), fixedFile, archiveOutputStream);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: We can avoid `Files.size` lookup by changing `includeFile` to return the number of bytes it copies.
created_at: 2025-03-15T05:59:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,16 +355,44 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         hardLinkFiles, sstFilesToExclude,
         new HashSet<>(), excluded, copySize,
         compactionLogDir.getOriginalDir().toPath());
+  }
+
+  private void estimateTotals(Path dir,
+     AtomicInteger totalFiles,
+     AtomicLong totalSize) throws IOException {
+    List<Path> subDirs = new ArrayList<>();
+    Stream<Path> paths = Files.list(dir);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: The stream returned by `Files.list` should be closed, so best using it in `try (...)`.

(but `estimateTotals` can be completely removed, so not applicable now)
created_at: 2025-03-15T06:02:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,16 +355,44 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         hardLinkFiles, sstFilesToExclude,
         new HashSet<>(), excluded, copySize,
         compactionLogDir.getOriginalDir().toPath());
+  }
+
+  private void estimateTotals(Path dir,
+     AtomicInteger totalFiles,
+     AtomicLong totalSize) throws IOException {
+    List<Path> subDirs = new ArrayList<>();
+    Stream<Path> paths = Files.list(dir);
+    totalFiles.addAndGet(paths
+        .filter(p -> {
+          File f = p.toFile();
+          if (f.isDirectory()) {
+            subDirs.add(p);
+            return false;
+          }
+          return f.getName().toLowerCase().endsWith(".sst");
+        })
+        .map(f -> {
+          try {
+            totalSize.addAndGet(Files.size(f));
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return f.toString();
+        }).collect(Collectors.toList()).size());
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: Collecting into a list just to get its size is not needed, `Stream` has `count()`.

(but `estimateTotals` can be completely removed, so not applicable now)
created_at: 2025-03-15T06:03:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -292,6 +302,33 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     AtomicLong copySize = new AtomicLong(0L);
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
+
+    // Log estimated total data transferred on first request.
+    if (sstFilesToExclude.isEmpty()) {
+      try {
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: hemantk-12
body: nit: `getFilesForArchive` is growing larger, and since we can't avoid calling `getSnapshotDirs()` while keeping the code clean, I suggest moving this logic to a helper function.
created_at: 2025-03-19T20:57:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -292,6 +302,33 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     AtomicLong copySize = new AtomicLong(0L);
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
+
+    // Log estimated total data transferred on first request.
+    if (sstFilesToExclude.isEmpty()) {
+      try {
+        Counters.PathCounters counters = Counters.longPathCounters();
+        CountingPathVisitor visitor = new CountingPathVisitor(
+            counters, SST_FILE_FILTER, TRUE);
+        Files.walkFileTree(dir, visitor);
+        int totalSnapshots = 0;
+        if (includeSnapshotData) {
+          Set<Path> snapshotPaths = getSnapshotDirs(checkpoint, false);
+          totalSnapshots = snapshotPaths.size();
+          for (Path snapshotDir: snapshotPaths) {
+            Files.walkFileTree(snapshotDir, visitor);
+          }
+        }
+        LOG.info("Estimates for Checkpoint Tarball Stream - " +
+                "Data size: {} KB, " + "SST files: {}{}",
+            counters.getByteCounter().get() / (1024),
+            counters.getFileCounter().get(),
+            (includeSnapshotData ? ", snapshots: " + totalSnapshots : ""));
+      } catch (Exception e) {
+        LOG.error("Could not determine estimated size of transfer to " +
+            "Checkpoint Tarball Stream", e);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: hemantk-12
body: ```suggestion
        LOG.error("Could not estimate size of transfer to Checkpoint Tarball Stream.", e);
```
created_at: 2025-03-19T20:59:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -292,6 +302,33 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     AtomicLong copySize = new AtomicLong(0L);
     // Get the active fs files.
     Path dir = checkpoint.getCheckpointLocation();
+
+    // Log estimated total data transferred on first request.
+    if (sstFilesToExclude.isEmpty()) {
+      try {
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: SaketaChalamchala
body: I had a generic `estimateTotals(dir, ..)` in a previous commit. I guess I could do `estimateTotals(checkpoint)` and walk AOS and snapshots in that function.
created_at: 2025-03-19T21:55:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -358,6 +338,30 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         compactionLogDir.getOriginalDir().toPath());
   }
 
+  private void logTotals(
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: hemantk-12
body: nit: rename it to `logEstimatedTarballSize`
created_at: 2025-03-19T23:03:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -358,6 +338,30 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         compactionLogDir.getOriginalDir().toPath());
   }
 
+  private void logTotals(
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: ```suggestion
  private void logEstimatedTarballSize(
```
created_at: 2025-03-20T09:23:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -290,6 +300,12 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
     }
 
     AtomicLong copySize = new AtomicLong(0L);
+
+    // Log estimated total data transferred on first request.
+    if (sstFilesToExclude.isEmpty()) {
+      logTotals(checkpoint, includeSnapshotData);
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: ```suggestion
      logEstimatedTarballSize(checkpoint, includeSnapshotData);
```
created_at: 2025-03-20T09:23:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -358,6 +338,30 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
         compactionLogDir.getOriginalDir().toPath());
   }
 
+  private void logTotals(
path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
user: login: adoroszlai
body: Updated.
created_at: 2025-03-20T11:53:07Z
author_association: CONTRIBUTOR

PR comments:
user: login: SaketaChalamchala
created_at: 2025-03-15T01:30:08Z
body: @hemantk-12 @swamirishi could you please take a look.
author_association: CONTRIBUTOR
user: login: SaketaChalamchala
created_at: 2025-03-19T01:10:13Z
body: Thanks for the review @adoroszlai. Please review the updated patch.
author_association: CONTRIBUTOR
user: login: adoroszlai
created_at: 2025-03-20T11:52:58Z
body: Thanks @SaketaChalamchala for the patch, @hemantk-12 for the review.
author_association: CONTRIBUTOR