Repository: apache/iceberg
Pull request #12647: Spark 3.4 : Use correct statistics file in SparkScan::estimateStatistics(Snapshot)
Author: login: jeesou
Labels: name: spark
Created: 2025-03-26T06:59:43Z | Merged: 2025-03-27T10:37:20Z



Diff:
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
index 019f3919dc57..106b296de098 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
@@ -21,6 +21,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import org.apache.iceberg.BlobMetadata;
@@ -195,8 +196,10 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
     if (readConf.reportColumnStats() && cboEnabled) {
       colStatsMap = Maps.newHashMap();
       List<StatisticsFile> files = table.statisticsFiles();
-      if (!files.isEmpty()) {
-        List<BlobMetadata> metadataList = (files.get(0)).blobMetadata();
+      Optional<StatisticsFile> file =
+          files.stream().filter(f -> f.snapshotId() == snapshot.snapshotId()).findFirst();
+      if (file.isPresent()) {
+        List<BlobMetadata> metadataList = file.get().blobMetadata();
 
         Map<Integer, List<BlobMetadata>> groupedByField =
             metadataList.stream()
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java
index d2426613889e..399619c8a75d 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java
@@ -234,17 +234,6 @@ public void testTableWithOneColStats() throws NoSuchTableException {
     Table table = validationCatalog.loadTable(tableIdent);
     long snapshotId = table.currentSnapshot().snapshotId();
 
-    SparkScanBuilder scanBuilder =
-        new SparkScanBuilder(spark, table, CaseInsensitiveStringMap.empty());
-    SparkScan scan = (SparkScan) scanBuilder.build();
-
-    Map<String, String> reportColStatsDisabled =
-        ImmutableMap.of(
-            SQLConf.CBO_ENABLED().key(), "true", SparkSQLProperties.REPORT_COLUMN_STATS, "false");
-
-    Map<String, String> reportColStatsEnabled =
-        ImmutableMap.of(SQLConf.CBO_ENABLED().key(), "true");
-
     GenericStatisticsFile statisticsFile =
         new GenericStatisticsFile(
             snapshotId,
@@ -261,12 +250,50 @@ public void testTableWithOneColStats() throws NoSuchTableException {
 
     table.updateStatistics().setStatistics(statisticsFile).commit();
 
-    checkColStatisticsNotReported(scan, 4L);
-    withSQLConf(reportColStatsDisabled, () -> checkColStatisticsNotReported(scan, 4L));
+    List<SimpleRecord> newRecords =
+        Lists.newArrayList(new SimpleRecord(5, "a"), new SimpleRecord(6, "b"));
+    spark
+        .createDataset(newRecords, Encoders.bean(SimpleRecord.class))
+        .coalesce(1)
+        .writeTo(tableName)
+        .append();
 
-    Map<String, Long> expectedOneNDV = Maps.newHashMap();
-    expectedOneNDV.put("id", 4L);
-    withSQLConf(reportColStatsEnabled, () -> checkColStatisticsReported(scan, 4L, expectedOneNDV));
+    table.refresh();
+    long snapshotId2 = table.currentSnapshot().snapshotId();
+
+    GenericStatisticsFile statisticsFile2 =
+        new GenericStatisticsFile(
+            snapshotId2,
+            "/test/statistics/file2.puffin",
+            100,
+            42,
+            ImmutableList.of(
+                new GenericBlobMetadata(
+                    APACHE_DATASKETCHES_THETA_V1,
+                    snapshotId2,
+                    2,
+                    ImmutableList.of(1),
+                    ImmutableMap.of("ndv", "6"))));
+
+    table.updateStatistics().setStatistics(statisticsFile2).commit();
+
+    SparkScanBuilder scanBuilder =
+        new SparkScanBuilder(spark, table, CaseInsensitiveStringMap.empty());
+    SparkScan scan = (SparkScan) scanBuilder.build();
+
+    Map<String, String> reportColStatsDisabled =
+        ImmutableMap.of(
+            SQLConf.CBO_ENABLED().key(), "true", SparkSQLProperties.REPORT_COLUMN_STATS, "false");
+
+    checkColStatisticsNotReported(scan, 6L);
+    withSQLConf(reportColStatsDisabled, () -> checkColStatisticsNotReported(scan, 6L));
+
+    Map<String, String> reportColStatsEnabled =
+        ImmutableMap.of(SQLConf.CBO_ENABLED().key(), "true");
+
+    Map<String, Long> expectedNDV = Maps.newHashMap();
+    expectedNDV.put("id", 6L);
+    withSQLConf(reportColStatsEnabled, () -> checkColStatisticsReported(scan, 6L, expectedNDV));
   }
 
   @Test


Reviews:
user: login: pvary
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T05:55:39Z
user: login: ajantha-bhat
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T06:00:00Z
user: login: pvary
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T10:19:09Z
user: login: pvary
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T10:35:08Z
user: login: wypoon
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T17:47:00Z

Review comments:
diff_hunk: @@ -195,8 +196,10 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
     if (readConf.reportColumnStats() && cboEnabled) {
       colStatsMap = Maps.newHashMap();
       List<StatisticsFile> files = table.statisticsFiles();
-      if (!files.isEmpty()) {
-        List<BlobMetadata> metadataList = (files.get(0)).blobMetadata();
+      Optional<StatisticsFile> file =
+          files.stream().filter(f -> f.snapshotId() == snapshot.snapshotId()).findFirst();
path: spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
user: login: pvary
body: Could we have multiple files with the same snapshotId? Or `findAny` is enough?
created_at: 2025-03-27T05:55:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -195,8 +196,10 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
     if (readConf.reportColumnStats() && cboEnabled) {
       colStatsMap = Maps.newHashMap();
       List<StatisticsFile> files = table.statisticsFiles();
-      if (!files.isEmpty()) {
-        List<BlobMetadata> metadataList = (files.get(0)).blobMetadata();
+      Optional<StatisticsFile> file =
+          files.stream().filter(f -> f.snapshotId() == snapshot.snapshotId()).findFirst();
path: spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
user: login: ajantha-bhat
body: As per java implementation, it is always a single stats file per snapshot id. 

https://github.com/apache/iceberg/blob/695374d8bfbd22f4347cfe43b54a93c41b88f166/core/src/main/java/org/apache/iceberg/TableMetadata.java#L1369

I followed similar design while working on partition stats too. 
created_at: 2025-03-27T06:00:00Z
author_association: MEMBER
diff_hunk: @@ -195,8 +196,10 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
     if (readConf.reportColumnStats() && cboEnabled) {
       colStatsMap = Maps.newHashMap();
       List<StatisticsFile> files = table.statisticsFiles();
-      if (!files.isEmpty()) {
-        List<BlobMetadata> metadataList = (files.get(0)).blobMetadata();
+      Optional<StatisticsFile> file =
+          files.stream().filter(f -> f.snapshotId() == snapshot.snapshotId()).findFirst();
path: spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
user: login: pvary
body: So we could have a theoretical performance boost from `findAny`
created_at: 2025-03-27T10:19:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -195,8 +196,10 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
     if (readConf.reportColumnStats() && cboEnabled) {
       colStatsMap = Maps.newHashMap();
       List<StatisticsFile> files = table.statisticsFiles();
-      if (!files.isEmpty()) {
-        List<BlobMetadata> metadataList = (files.get(0)).blobMetadata();
+      Optional<StatisticsFile> file =
+          files.stream().filter(f -> f.snapshotId() == snapshot.snapshotId()).findFirst();
path: spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
user: login: wypoon
body: The spec doesn't actually say that there should only be *one* statistics file per snapshot. This happens to be how it is implemented in Java. The spec simply allows for multiple statistics files.
I was thinking about the problem of tracking orphaned statistics files when they are recomputed. One idea I had was to keep replaced statistics files (for a snapshot) still in the list (as long as the files are tracked in metadata we can clean up unused ones), but to keep the newest one before others. Hence `findFirst`. It was just an idea (and honestly not one I'm seriously considering).
In any case, I do not think that `findAny` is faster than `findFirst` here.
created_at: 2025-03-27T17:46:59Z
author_association: CONTRIBUTOR

PR comments:
user: login: jeesou
created_at: 2025-03-26T08:03:00Z
body: This is the backport changes from https://github.com/apache/iceberg/pull/12482

@wypoon @findepi 

Kindly help review and merge
author_association: CONTRIBUTOR
user: login: wypoon
created_at: 2025-03-26T17:45:28Z
body: This is a clean backport of my fix.
@jeesou can you please use the same title as the original fix, prefixed by "Spark 3.4" --
"Spark 3.4: Use correct statistics file in SparkScan::estimateStatistics(Snapshot)".
Thanks.
(Btw, it is customary to acknowledge the author of the original fix if you're doing a backport.)
author_association: CONTRIBUTOR
user: login: jeesou
created_at: 2025-03-27T04:30:21Z
body: Yes thanks @wypoon for this fix, and for reviewing it.
author_association: CONTRIBUTOR
user: login: pvary
created_at: 2025-03-27T10:22:24Z
body: Why is this PR against Spark 3.4?
In Flink we usually create PRs against the current version
author_association: CONTRIBUTOR
user: login: ajantha-bhat
created_at: 2025-03-27T10:23:51Z
body: @pvary: This PR is looks like just a backport of https://github.com/apache/iceberg/pull/12482. 

He has not updated the PR descriptions properly (looks like new contributor).
author_association: MEMBER
user: login: pvary
created_at: 2025-03-27T10:38:33Z
body: Thanks @ajantha-bhat for catching that this is a backport!

@jeesou: Please when backporting tell it in the PR description. Also highlight any changes needed compared to the original PR, so the reviewers could have easier time.
author_association: CONTRIBUTOR
user: login: wypoon
created_at: 2025-03-27T17:35:10Z
body: @pvary, @ajantha-bhat I already reviewed this a day before you and if you read my comment, I mentioned that this is a clean **backport** of my fix.

@jeesou as Peter stated, you should state in the **description** that this is a backport of https://github.com/apache/iceberg/pull/12482 (and that I am the author of the fix). I implied it but did not state it explicitly.
author_association: CONTRIBUTOR