Repository: StarRocks/starrocks
Pull request #56385: [Feature]Support iceberg remove orphan files
Author: login: Max-Cheng
Labels: name: version:3.5
Created: 2025-02-27T13:13:30Z | Merged: 2025-03-11T05:50:58Z

## Why I'm doing:
#49811
## What I'm doing:
syntax:
- remove_orphan_files
  - ALTER TABLE table_name EXECUTE remove_orphan_files();
    - Default minimal retention period for the orphan file is `now - 7d`
  - ALTER TABLE table_name EXECUTE remove_orphan_files('2025-02-27 00:00:00');
 
#49811
## What type of PR is this:

- [ ] BugFix
- [x] Feature
- [ ] Enhancement
- [ ] Refactor
- [ ] UT
- [ ] Doc
- [ ] Tool

Does this PR entail a change in behavior?

- [ ] Yes, this PR will result in a change in behavior.
- [x] No, this PR will not result in a change in behavior.

If yes, please specify the type of change:

- [ ] Interface/UI changes: syntax, type conversion, expression evaluation, display information
- [ ] Parameter changes: default values, similar parameters but with different default values
- [ ] Policy changes: use new policy to replace old one, functionality automatically enabled
- [ ] Feature removed
- [ ] Miscellaneous: upgrade & downgrade compatibility, etc.

## Checklist:

- [x] I have added test cases for my bug fix or my new feature
- [x] This pr needs user documentation (for new or modified features or behaviors)
  - [ ] I have added documentation for my new feature or new function
- [ ] This is a backport pr

## Bugfix cherry-pick branch check:
- [x] I have checked the version labels which the pr will be auto-backported to the target branch
  - [ ] 3.4
  - [ ] 3.3
  - [ ] 3.2
  - [ ] 3.1
  - [ ] 3.0


Diff:
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
index c8894afdd695a9..c97813cda3f0e0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
@@ -14,6 +14,7 @@
 
 package com.starrocks.connector.iceberg;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.starrocks.analysis.ColumnPosition;
 import com.starrocks.catalog.Column;
 import com.starrocks.catalog.Type;
@@ -21,6 +22,7 @@
 import com.starrocks.common.util.TimeUtils;
 import com.starrocks.connector.BranchOptions;
 import com.starrocks.connector.ConnectorAlterTableExecutor;
+import com.starrocks.connector.HdfsEnvironment;
 import com.starrocks.connector.TagOptions;
 import com.starrocks.connector.exception.StarRocksConnectorException;
 import com.starrocks.qe.ConnectContext;
@@ -41,23 +43,41 @@
 import com.starrocks.sql.ast.ModifyTablePropertiesClause;
 import com.starrocks.sql.ast.TableRenameClause;
 import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.iceberg.ContentFile;
 import org.apache.iceberg.ExpireSnapshots;
 import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.ManageSnapshots;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
 import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.Transaction;
 import org.apache.iceberg.UpdateProperties;
 import org.apache.iceberg.UpdateSchema;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.net.URI;
 import java.time.Duration;
 import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.starrocks.analysis.OutFileClause.PARQUET_COMPRESSION_TYPE_MAP;
@@ -66,17 +86,31 @@
 import static com.starrocks.connector.iceberg.IcebergMetadata.COMPRESSION_CODEC;
 import static com.starrocks.connector.iceberg.IcebergMetadata.FILE_FORMAT;
 import static com.starrocks.connector.iceberg.IcebergMetadata.LOCATION_PROPERTY;
+import static com.starrocks.connector.iceberg.IcebergUtil.fileName;
 import static com.starrocks.sql.common.UnsupportedException.unsupportedException;
+import static org.apache.iceberg.ReachableFileUtil.metadataFileLocations;
+import static org.apache.iceberg.ReachableFileUtil.statisticsFilesLocations;
 
 public class IcebergAlterTableExecutor extends ConnectorAlterTableExecutor {
-    private org.apache.iceberg.Table table;
+    private static final Logger LOGGER = LoggerFactory.getLogger(IcebergAlterTableExecutor.class);
+    private Table table;
     private IcebergCatalog icebergCatalog;
     private Transaction transaction;
+    private HdfsEnvironment hdfsEnvironment;
 
-    public IcebergAlterTableExecutor(AlterTableStmt stmt, org.apache.iceberg.Table table, IcebergCatalog icebergCatalog) {
+    private static final int DELETE_BATCH_SIZE = 1000;
+
+    // TODO:Support using session to set default retention_threshold.
+    private static final Duration DEFAULT_RETENTION_THRESHOLD = Duration.ofDays(7);
+
+    public IcebergAlterTableExecutor(AlterTableStmt stmt,
+            Table table,
+            IcebergCatalog icebergCatalog,
+            HdfsEnvironment hdfsEnvironment) {
         super(stmt);
         this.table = table;
         this.icebergCatalog = icebergCatalog;
+        this.hdfsEnvironment = hdfsEnvironment;
     }
 
     @Override
@@ -430,6 +464,9 @@ public Void visitAlterTableOperationClause(AlterTableOperationClause clause, Con
             case EXPIRE_SNAPSHOTS:
                 expireSnapshots(args);
                 break;
+            case REMOVE_ORPHAN_FILES:
+                removeOrphanFiles(args);
+                break;
             default:
                 throw new StarRocksConnectorException("Unsupported table operation %s", op);
         }
@@ -495,4 +532,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
+            return;
+        }
+
+        Set<String> processedManifestFilePaths = new HashSet<>();
+        Set<String> validFileNames = new HashSet<>();
+
+        for (Snapshot snapshot : table.snapshots()) {
+            if (snapshot.manifestListLocation() != null) {
+                validFileNames.add(fileName(snapshot.manifestListLocation()));
+            }
+
+            for (ManifestFile manifest : snapshot.allManifests(table.io())) {
+                if (!processedManifestFilePaths.add(manifest.path())) {
+                    continue;
+                }
+
+                validFileNames.add(fileName(manifest.path()));
+                try (ManifestReader<? extends ContentFile<?>> manifestReader = readerForManifest(table, manifest)) {
+                    for (ContentFile<?> contentFile : manifestReader) {
+                        validFileNames.add(fileName(contentFile.location()));
+                    }
+                } catch (IOException e) {
+                    throw new StarRocksConnectorException("Unable to list manifest file content from " + manifest.path(), e);
+                }
+            }
+        }
+
+        metadataFileLocations(table, false).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        statisticsFilesLocations(table).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        validFileNames.add("version-hint.text");
+
+        actions.add(() -> scanAndDeleteInvalidFiles(table.location(), olderThanMillis, validFileNames));
+    }
+
+    private static ManifestReader<? extends ContentFile<?>> readerForManifest(Table table, ManifestFile manifest) {
+        return switch (manifest.content()) {
+            case DATA -> ManifestFiles.read(manifest, table.io());
+            case DELETES -> ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs());
+        };
+    }
+
+    private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Set<String> validFiles) {
+        try {
+            URI uri = new Path(tableLocation).toUri();
+            FileSystem fileSystem = FileSystem.get(uri, hdfsEnvironment.getConfiguration());
+            RemoteIterator<LocatedFileStatus> allFiles = fileSystem.listFiles(new Path(tableLocation), true);
+            List<Path> filesToDelete = new ArrayList<>();
+            while (allFiles.hasNext()) {
+                LocatedFileStatus entry = allFiles.next();
+                FileStatus status = fileSystem.getFileStatus(entry.getPath());
+                if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
+                    filesToDelete.add(entry.getPath());
+                    if (filesToDelete.size() >= DELETE_BATCH_SIZE) {
+                        deleteFiles(fileSystem, filesToDelete);
+                        filesToDelete.clear();
+                    }
+                }
+            }
+            if (!filesToDelete.isEmpty()) {
+                deleteFiles(fileSystem, filesToDelete);
+                filesToDelete.clear();
+            }
+        } catch (IOException e) {
+            throw new StarRocksConnectorException("Failed accessing data: ", e);
+        }
+    }
+
+    // TODO:implement deleteFiles in FsUtils
+    @VisibleForTesting
+    public static void deleteFiles(FileSystem fs, List<Path> files) {
+        files.forEach(file -> {
+            try {
+                fs.delete(file, false);
+                LOGGER.debug("Deleted file {}", file);
+            } catch (IOException e) {
+                LOGGER.error("Failed to delete file {}", file, e);
+                throw new StarRocksConnectorException("Failed to delete file " + file, e);
+            }
+        });
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java
index be4636b7bb7bee..fe10a10f6d43c9 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergMetadata.java
@@ -343,7 +343,7 @@ public void alterTable(ConnectContext context, AlterTableStmt stmt) throws StarR
                     "Failed to load iceberg table: " + stmt.getTbl().toString());
         }
 
-        IcebergAlterTableExecutor executor = new IcebergAlterTableExecutor(stmt, table, icebergCatalog);
+        IcebergAlterTableExecutor executor = new IcebergAlterTableExecutor(stmt, table, icebergCatalog, hdfsEnvironment);
         executor.execute();
 
         synchronized (this) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergTableOperation.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergTableOperation.java
index 72d51d964cef60..9812f9af368d6a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergTableOperation.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergTableOperation.java
@@ -18,6 +18,7 @@ public enum IcebergTableOperation {
     EXPIRE_SNAPSHOTS,
     FAST_FORWARD,
     CHERRYPICK_SNAPSHOT,
+    REMOVE_ORPHAN_FILES,
     UNKNOWN;
 
     public static IcebergTableOperation fromString(String opStr) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergUtil.java b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergUtil.java
new file mode 100644
index 00000000000000..aa1db3cec48f88
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergUtil.java
@@ -0,0 +1,23 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.connector.iceberg;
+
+public final class IcebergUtil {
+    private IcebergUtil() {}
+
+    public static String fileName(String path) {
+        return path.substring(path.lastIndexOf('/') + 1);
+    }
+}
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
index 26beee91dea2e6..75218e17b0af0a 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
@@ -68,6 +68,7 @@
 import com.starrocks.sql.ast.AddColumnsClause;
 import com.starrocks.sql.ast.AlterClause;
 import com.starrocks.sql.ast.AlterTableCommentClause;
+import com.starrocks.sql.ast.AlterTableOperationClause;
 import com.starrocks.sql.ast.AlterTableStmt;
 import com.starrocks.sql.ast.ColumnDef;
 import com.starrocks.sql.ast.ColumnRenameClause;
@@ -106,6 +107,7 @@
 import org.apache.iceberg.DataFiles;
 import org.apache.iceberg.FileScanTask;
 import org.apache.iceberg.MetricsModes;
+import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.TableMetadata;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.TableScan;
@@ -143,6 +145,7 @@
 import static com.starrocks.connector.iceberg.IcebergMetadata.COMPRESSION_CODEC;
 import static com.starrocks.connector.iceberg.IcebergMetadata.FILE_FORMAT;
 import static com.starrocks.connector.iceberg.IcebergMetadata.LOCATION_PROPERTY;
+import static com.starrocks.connector.iceberg.IcebergTableOperation.REMOVE_ORPHAN_FILES;
 
 public class IcebergMetadataTest extends TableTestBase {
     private static final String CATALOG_NAME = "iceberg_catalog";
@@ -1519,4 +1522,102 @@ public String uuid() {
         Assert.assertEquals(1, icebergTable.getTableIdentifier().split(":").length);
         Assert.assertEquals(3, icebergTable.getUUID().split("\\.").length);
     }
+
+    @Test
+    public void testAlterTableExecuteRemoveOrphanFiles(@Mocked IcebergHiveCatalog icebergHiveCatalog,
+                                                       @Mocked Snapshot snapshot) throws Exception {
+        new MockUp<IcebergMetadata>() {
+            @Mock
+            public Database getDb(String dbName) {
+                return new Database(1, "db");
+            }
+        };
+
+        new MockUp<IcebergHiveCatalog>() {
+            @Mock
+            org.apache.iceberg.Table getTable(String dbName, String tableName) throws StarRocksConnectorException {
+                return mockedNativeTableA;
+            }
+
+            @Mock
+            boolean tableExists(String dbName, String tableName) {
+                return true;
+            }
+        };
+
+
+        // Normalize Date
+        TableName tableName = new TableName(CATALOG_NAME, "db", "table");
+        AlterTableOperationClause clause = new AlterTableOperationClause(NodePosition.ZERO,
+                REMOVE_ORPHAN_FILES.toString(), List.of());
+        clause.setArgs(List.of(ConstantOperator.createChar("2024-01-01 00:00:00")));
+
+        IcebergAlterTableExecutor executor = new IcebergAlterTableExecutor(new AlterTableStmt(
+                tableName,
+                List.of(clause)),
+                icebergHiveCatalog.getTable(tableName.getDb(), tableName.getTbl()), icebergHiveCatalog, HDFS_ENVIRONMENT);
+        executor.execute();
+
+        // Illegal date
+        tableName = new TableName(CATALOG_NAME, "db", "table");
+        clause = new AlterTableOperationClause(NodePosition.ZERO, REMOVE_ORPHAN_FILES.toString(), List.of());
+        clause.setArgs(List.of(ConstantOperator.createChar("illegal date")));
+
+        executor = new IcebergAlterTableExecutor(new AlterTableStmt(
+                tableName,
+                List.of(clause)),
+                icebergHiveCatalog.getTable(tableName.getDb(), tableName.getTbl()), icebergHiveCatalog, HDFS_ENVIRONMENT);
+        IcebergAlterTableExecutor finalExecutor = executor;
+        Assert.assertThrows(DdlException.class, finalExecutor::execute);
+
+        // Default retention interval
+        tableName = new TableName(CATALOG_NAME, "db", "table");
+        clause = new AlterTableOperationClause(NodePosition.ZERO, REMOVE_ORPHAN_FILES.toString(), List.of());
+        clause.setArgs(List.of(ConstantOperator.createChar("")));
+
+        executor = new IcebergAlterTableExecutor(new AlterTableStmt(
+                tableName,
+                List.of(clause)),
+                icebergHiveCatalog.getTable(tableName.getDb(), tableName.getTbl()), icebergHiveCatalog, HDFS_ENVIRONMENT);
+        finalExecutor = executor;
+        Assert.assertThrows(DdlException.class, finalExecutor::execute);
+
+        // Mock snapshot behavior
+        new Expectations() {{
+                snapshot.snapshotId();
+                result = 1L;
+                minTimes = 0;
+
+                snapshot.timestampMillis();
+                result = System.currentTimeMillis();
+                minTimes = 0;
+
+                snapshot.parentId();
+                result = 0L;
+                minTimes = 0;
+            }};
+
+        new MockUp<org.apache.iceberg.BaseTable>() {
+            @Mock
+            public org.apache.iceberg.Snapshot currentSnapshot() {
+                return snapshot;
+            }
+
+            @Mock
+            public Iterable<org.apache.iceberg.Snapshot> snapshots() {
+                return List.of(snapshot, snapshot, snapshot);
+            }
+        };
+
+        // inject snapshot
+        tableName = new TableName(CATALOG_NAME, "db", "table");
+        clause = new AlterTableOperationClause(NodePosition.ZERO, REMOVE_ORPHAN_FILES.toString(), List.of());
+        clause.setArgs(List.of(ConstantOperator.createChar("2024-01-01 00:00:00")));
+
+        executor = new IcebergAlterTableExecutor(new AlterTableStmt(
+                tableName,
+                List.of(clause)),
+                icebergHiveCatalog.getTable(tableName.getDb(), tableName.getTbl()), icebergHiveCatalog, HDFS_ENVIRONMENT);
+        executor.execute();
+    }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergUtilTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergUtilTest.java
new file mode 100644
index 00000000000000..c488e364cbe690
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergUtilTest.java
@@ -0,0 +1,30 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.connector.iceberg;
+
+import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class IcebergUtilTest {
+
+    @Test
+    public void testFileName() {
+        assertEquals("1.orc", IcebergUtil.fileName("hdfs://tans/user/hive/warehouse/max-test/1.orc"));
+        assertEquals("2.orc", IcebergUtil.fileName("cos://tans/user/hive/warehouse/max-test/2.orc"));
+        assertEquals("3.orc", IcebergUtil.fileName("s3://tans/user/hive/warehouse/max-test/3.orc"));
+        assertEquals("4.orc", IcebergUtil.fileName("gs://tans/user/hive/warehouse/max-test/4.orc"));
+    }
+}
\ No newline at end of file
diff --git a/test/sql/test_iceberg/R/test_iceberg_remove_orphan_files b/test/sql/test_iceberg/R/test_iceberg_remove_orphan_files
new file mode 100644
index 00000000000000..9de46c9640d2f3
--- /dev/null
+++ b/test/sql/test_iceberg/R/test_iceberg_remove_orphan_files
@@ -0,0 +1,49 @@
+-- name: test_iceberg_remove_orphan_files
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+-- result:
+-- !result
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+create table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0}(k1 int) properties('location' = 'oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}');
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 1;
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 2;
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 3;
+-- result:
+-- !result
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+-- result:
+3
+-- !result
+insert overwrite iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} SELECT 4;
+-- result:
+-- !result
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+-- result:
+4
+-- !result
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute expire_snapshots(now());
+-- result:
+-- !result
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute remove_orphan_files(now());
+-- result:
+-- !result
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+-- result:
+1
+-- !result
+drop table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} force;
+-- result:
+-- !result
+drop database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+drop catalog iceberg_sql_test_${uuid0};
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_iceberg/T/test_iceberg_remove_orphan_files b/test/sql/test_iceberg/T/test_iceberg_remove_orphan_files
new file mode 100644
index 00000000000000..33d1e30a71ca00
--- /dev/null
+++ b/test/sql/test_iceberg/T/test_iceberg_remove_orphan_files
@@ -0,0 +1,20 @@
+-- name: test_iceberg_remove_orphan_files
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+create table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0}(k1 int) properties('location' = 'oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}');
+
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 1;
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 2;
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 3;
+
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+insert overwrite iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} SELECT 4;
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute expire_snapshots(now());
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute remove_orphan_files(now());
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+
+drop table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} force;
+drop database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+drop catalog iceberg_sql_test_${uuid0};
\ No newline at end of file


Reviews:
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-02T15:39:18Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T01:59:47Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T02:32:22Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T02:33:01Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T07:44:30Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T07:50:43Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T07:54:21Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T11:43:34Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T11:49:22Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T02:12:23Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T02:18:37Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T03:26:20Z
user: login: Youngwb
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T04:00:56Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T04:50:47Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T05:39:39Z
user: login: DorianZheng
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T05:53:54Z
user: login: Youngwb
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T06:17:08Z
user: login: dirtysalt
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T23:49:25Z
user: login: Max-Cheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T10:19:36Z
user: login: Youngwb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T11:47:26Z
user: login: Youngwb
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T05:47:05Z
user: login: DorianZheng
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T05:50:46Z

Review comments:
diff_hunk: @@ -495,4 +522,107 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: what's the meaning if argments are empty?
created_at: 2025-03-02T14:15:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +522,107 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            olderThanMillis = -1L;
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
+            return;
+        }
+
+        Set<String> processedManifestFilePaths = new HashSet<>();
+        Set<String> validFileNames = new HashSet<>();
+
+        for (Snapshot snapshot : table.snapshots()) {
+            if (snapshot.manifestListLocation() != null) {
+                validFileNames.add(fileName(snapshot.manifestListLocation()));
+            }
+
+            for (ManifestFile manifest : snapshot.allManifests(table.io())) {
+                if (!processedManifestFilePaths.add(manifest.path())) {
+                    continue;
+                }
+
+                validFileNames.add(fileName(manifest.path()));
+                try (ManifestReader<? extends ContentFile<?>> manifestReader = readerForManifest(table, manifest)) {
+                    for (ContentFile<?> contentFile : manifestReader) {
+                        validFileNames.add(fileName(contentFile.location()));
+                    }
+                } catch (IOException e) {
+                    throw new StarRocksConnectorException("Unable to list manifest file content from " + manifest.path(), e);
+                }
+            }
+        }
+
+        metadataFileLocations(table, false).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        statisticsFilesLocations(table).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        validFileNames.add("version-hint.text");
+
+        actions.add(() -> scanAndDeleteInvalidFiles(table.location(), olderThanMillis, validFileNames));
+    }
+
+    private static ManifestReader<? extends ContentFile<?>> readerForManifest(Table table, ManifestFile manifest) {
+        return switch (manifest.content()) {
+            case DATA -> ManifestFiles.read(manifest, table.io());
+            case DELETES -> ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs());
+        };
+    }
+
+    private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Set<String> validFiles) {
+        try {
+            URI uri = new Path(tableLocation).toUri();
+            FileSystem fileSystem = FileSystem.get(uri, hdfsEnvironment.getConfiguration());
+            RemoteIterator<LocatedFileStatus> allFiles = fileSystem.listFiles(new Path(tableLocation), true);
+            List<Path> filesToDelete = new ArrayList<>();
+            while (allFiles.hasNext()) {
+                LocatedFileStatus entry = allFiles.next();
+                FileStatus status = fileSystem.getFileStatus(entry.getPath());
+                if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
+                    filesToDelete.add(entry.getPath());
+                    if (filesToDelete.size() >= 5000) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: I think 1000 is enough
created_at: 2025-03-02T15:39:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +522,107 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            olderThanMillis = -1L;
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
+            return;
+        }
+
+        Set<String> processedManifestFilePaths = new HashSet<>();
+        Set<String> validFileNames = new HashSet<>();
+
+        for (Snapshot snapshot : table.snapshots()) {
+            if (snapshot.manifestListLocation() != null) {
+                validFileNames.add(fileName(snapshot.manifestListLocation()));
+            }
+
+            for (ManifestFile manifest : snapshot.allManifests(table.io())) {
+                if (!processedManifestFilePaths.add(manifest.path())) {
+                    continue;
+                }
+
+                validFileNames.add(fileName(manifest.path()));
+                try (ManifestReader<? extends ContentFile<?>> manifestReader = readerForManifest(table, manifest)) {
+                    for (ContentFile<?> contentFile : manifestReader) {
+                        validFileNames.add(fileName(contentFile.location()));
+                    }
+                } catch (IOException e) {
+                    throw new StarRocksConnectorException("Unable to list manifest file content from " + manifest.path(), e);
+                }
+            }
+        }
+
+        metadataFileLocations(table, false).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        statisticsFilesLocations(table).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        validFileNames.add("version-hint.text");
+
+        actions.add(() -> scanAndDeleteInvalidFiles(table.location(), olderThanMillis, validFileNames));
+    }
+
+    private static ManifestReader<? extends ContentFile<?>> readerForManifest(Table table, ManifestFile manifest) {
+        return switch (manifest.content()) {
+            case DATA -> ManifestFiles.read(manifest, table.io());
+            case DELETES -> ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs());
+        };
+    }
+
+    private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Set<String> validFiles) {
+        try {
+            URI uri = new Path(tableLocation).toUri();
+            FileSystem fileSystem = FileSystem.get(uri, hdfsEnvironment.getConfiguration());
+            RemoteIterator<LocatedFileStatus> allFiles = fileSystem.listFiles(new Path(tableLocation), true);
+            List<Path> filesToDelete = new ArrayList<>();
+            while (allFiles.hasNext()) {
+                LocatedFileStatus entry = allFiles.next();
+                FileStatus status = fileSystem.getFileStatus(entry.getPath());
+                if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
+                    filesToDelete.add(entry.getPath());
+                    if (filesToDelete.size() >= 5000) {
+                        filesToDelete.forEach(path -> {
+                            try {
+                                fileSystem.delete(path, false);
+                            } catch (IOException e) {
+                                throw new RuntimeException(e);
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: if throw RuntimeException,  the outermost   try   block of this function cannot catch it.
created_at: 2025-03-03T01:59:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +522,107 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: Because I see that `expire_snapshot` doesn't indicate the default retention time, that's my fault.
created_at: 2025-03-03T02:32:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +522,107 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            olderThanMillis = -1L;
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
+            return;
+        }
+
+        Set<String> processedManifestFilePaths = new HashSet<>();
+        Set<String> validFileNames = new HashSet<>();
+
+        for (Snapshot snapshot : table.snapshots()) {
+            if (snapshot.manifestListLocation() != null) {
+                validFileNames.add(fileName(snapshot.manifestListLocation()));
+            }
+
+            for (ManifestFile manifest : snapshot.allManifests(table.io())) {
+                if (!processedManifestFilePaths.add(manifest.path())) {
+                    continue;
+                }
+
+                validFileNames.add(fileName(manifest.path()));
+                try (ManifestReader<? extends ContentFile<?>> manifestReader = readerForManifest(table, manifest)) {
+                    for (ContentFile<?> contentFile : manifestReader) {
+                        validFileNames.add(fileName(contentFile.location()));
+                    }
+                } catch (IOException e) {
+                    throw new StarRocksConnectorException("Unable to list manifest file content from " + manifest.path(), e);
+                }
+            }
+        }
+
+        metadataFileLocations(table, false).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        statisticsFilesLocations(table).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        validFileNames.add("version-hint.text");
+
+        actions.add(() -> scanAndDeleteInvalidFiles(table.location(), olderThanMillis, validFileNames));
+    }
+
+    private static ManifestReader<? extends ContentFile<?>> readerForManifest(Table table, ManifestFile manifest) {
+        return switch (manifest.content()) {
+            case DATA -> ManifestFiles.read(manifest, table.io());
+            case DELETES -> ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs());
+        };
+    }
+
+    private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Set<String> validFiles) {
+        try {
+            URI uri = new Path(tableLocation).toUri();
+            FileSystem fileSystem = FileSystem.get(uri, hdfsEnvironment.getConfiguration());
+            RemoteIterator<LocatedFileStatus> allFiles = fileSystem.listFiles(new Path(tableLocation), true);
+            List<Path> filesToDelete = new ArrayList<>();
+            while (allFiles.hasNext()) {
+                LocatedFileStatus entry = allFiles.next();
+                FileStatus status = fileSystem.getFileStatus(entry.getPath());
+                if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
+                    filesToDelete.add(entry.getPath());
+                    if (filesToDelete.size() >= 5000) {
+                        filesToDelete.forEach(path -> {
+                            try {
+                                fileSystem.delete(path, false);
+                            } catch (IOException e) {
+                                throw new RuntimeException(e);
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: Now we using logger to replace.
created_at: 2025-03-03T02:33:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: I think this is useless, if I drop a iceberg table, the metadata/data files would still exists, and I recreate a iceberg table using the same location, the new create iceberg table snapshot could be null, then run removeOrphanFiles should remove the metadata/data which belone to the old iceberg table
created_at: 2025-03-03T07:44:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,49 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.connector.iceberg;
+
+import org.apache.iceberg.types.Type.PrimitiveType;
+import org.apache.iceberg.types.Types.NestedField;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Stream;
+
+public final class IcebergUtil {
+    private IcebergUtil() {}
+
+    private static Stream<Entry<Integer, PrimitiveType>> primitiveFieldTypes(List<NestedField> nestedFields) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergUtil.java
user: login: Youngwb
body: These look like unused
created_at: 2025-03-03T07:50:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
+            return;
+        }
+
+        Set<String> processedManifestFilePaths = new HashSet<>();
+        Set<String> validFileNames = new HashSet<>();
+
+        for (Snapshot snapshot : table.snapshots()) {
+            if (snapshot.manifestListLocation() != null) {
+                validFileNames.add(fileName(snapshot.manifestListLocation()));
+            }
+
+            for (ManifestFile manifest : snapshot.allManifests(table.io())) {
+                if (!processedManifestFilePaths.add(manifest.path())) {
+                    continue;
+                }
+
+                validFileNames.add(fileName(manifest.path()));
+                try (ManifestReader<? extends ContentFile<?>> manifestReader = readerForManifest(table, manifest)) {
+                    for (ContentFile<?> contentFile : manifestReader) {
+                        validFileNames.add(fileName(contentFile.location()));
+                    }
+                } catch (IOException e) {
+                    throw new StarRocksConnectorException("Unable to list manifest file content from " + manifest.path(), e);
+                }
+            }
+        }
+
+        metadataFileLocations(table, false).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        statisticsFilesLocations(table).stream()
+                .map(IcebergUtil::fileName)
+                .forEach(validFileNames::add);
+
+        validFileNames.add("version-hint.text");
+
+        actions.add(() -> scanAndDeleteInvalidFiles(table.location(), olderThanMillis, validFileNames));
+    }
+
+    private static ManifestReader<? extends ContentFile<?>> readerForManifest(Table table, ManifestFile manifest) {
+        return switch (manifest.content()) {
+            case DATA -> ManifestFiles.read(manifest, table.io());
+            case DELETES -> ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs());
+        };
+    }
+
+    private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Set<String> validFiles) {
+        try {
+            URI uri = new Path(tableLocation).toUri();
+            FileSystem fileSystem = FileSystem.get(uri, hdfsEnvironment.getConfiguration());
+            RemoteIterator<LocatedFileStatus> allFiles = fileSystem.listFiles(new Path(tableLocation), true);
+            List<Path> filesToDelete = new ArrayList<>();
+            while (allFiles.hasNext()) {
+                LocatedFileStatus entry = allFiles.next();
+                FileStatus status = fileSystem.getFileStatus(entry.getPath());
+                if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
+                    filesToDelete.add(entry.getPath());
+                    if (filesToDelete.size() >= DELETE_BATCH_SIZE) {
+                        filesToDelete.forEach(path -> {
+                            try {
+                                fileSystem.delete(path, false);
+                                LOGGER.debug("Deleted file {}", path);
+                            } catch (IOException e) {
+                                LOGGER.error("Failed to delete file {}", path, e);
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: This exceprtion would only pring error in log, and wil not catch by the outermoset try-catch
created_at: 2025-03-03T07:54:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: This is a valid concern. When a table is dropped and recreated using the same location, there's a potential issue with orphan file cleanup procedures. The procedure needs to correctly identify which files are truly orphaned versus which ones should be part of the newly created table.
Link to trino about this behavior discuss https://github.com/trinodb/trino/issues/12573
created_at: 2025-03-03T11:43:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: Also, I've noticed that Starrocks drop table default behavior won't delete phyical file. If user create new snapshot and we remove orphan file. maybe those behavior will be conflict?
created_at: 2025-03-03T11:49:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: Yes, starrocks drop table xxx can not delete the file in file system, you can use `delete table xxx force` to delete them. spark/trino remove orphan file also delete the old table files, I think it's ok 
created_at: 2025-03-04T02:12:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: It's a good idea to prevent create iceberg table on non-empty location, we can support this later
created_at: 2025-03-04T02:18:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,31 +610,30 @@ private void scanAndDeleteInvalidFiles(String tableLocation, long expiration, Se
                 if (status.getModificationTime() < expiration && !validFiles.contains(entry.getPath().getName())) {
                     filesToDelete.add(entry.getPath());
                     if (filesToDelete.size() >= DELETE_BATCH_SIZE) {
-                        filesToDelete.forEach(path -> {
-                            try {
-                                fileSystem.delete(path, false);
-                                LOGGER.debug("Deleted file {}", path);
-                            } catch (IOException e) {
-                                LOGGER.error("Failed to delete file {}", path, e);
-                            }
-                        });
+                        deleteFiles(fileSystem, filesToDelete);
                         filesToDelete.clear();
                     }
                 }
             }
             if (!filesToDelete.isEmpty()) {
-                filesToDelete.forEach(path -> {
-                    try {
-                        fileSystem.delete(path, false);
-                        LOGGER.debug("Deleted file {}", path);
-                    } catch (IOException e) {
-                        LOGGER.error("Failed to delete file {}", path, e);
-                    }
-                });
+                deleteFiles(fileSystem, filesToDelete);
                 filesToDelete.clear();
             }
         } catch (IOException e) {
             throw new StarRocksConnectorException("Failed accessing data: ", e);
         }
     }
+
+    // TODO:implement deleteFiles in FsUtils
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: I believe Starrocks needs to implement its upper FileSystem class like Trino did.
created_at: 2025-03-04T03:26:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Max-Cheng
body: If you can, please assign it to me.
created_at: 2025-03-04T04:50:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -495,4 +531,110 @@ private void expireSnapshots(List<ConstantOperator> args) {
             expireSnapshots.commit();
         });
     }
+
+    private void removeOrphanFiles(List<ConstantOperator> args) {
+        if (args.size() > 1) {
+            throw new StarRocksConnectorException("invalid args. only support " +
+                    "`older_than` in the remove orphan files operation");
+        }
+
+        long olderThanMillis;
+        if (args.isEmpty()) {
+            LocalDateTime time = LocalDateTime.now(TimeUtils.getTimeZone().toZoneId());
+            olderThanMillis = time.minus(DEFAULT_RETENTION_THRESHOLD).toInstant(ZoneOffset.UTC).toEpochMilli();
+        } else {
+            LocalDateTime time = Optional.ofNullable(args.get(0))
+                    .flatMap(arg -> arg.castTo(Type.DATETIME)
+                    .map(ConstantOperator::getDatetime))
+                    .orElseThrow(() -> new StarRocksConnectorException("invalid arg %s", args.get(0)));
+            olderThanMillis = Duration.ofSeconds(time.atZone(TimeUtils.getTimeZone().toZoneId()).toEpochSecond()).toMillis();
+        }
+
+        if (table.currentSnapshot() == null) {
path: fe/fe-core/src/main/java/com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java
user: login: Youngwb
body: you can create an issue, and assign to yourself
created_at: 2025-03-04T05:39:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,48 @@
+-- name: test_iceberg_remove_orphan_files
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+-- result:
+-- !result
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+create table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0}(k1 int);
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 1;
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 2;
+-- result:
+-- !result
+insert into iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} select 3;
+-- result:
+-- !result
+select count(1) from files ("path"="${warehouse_path}/test_2/data","list_files_only" = "true");
+-- result:
+-- !result
+insert overwrite iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} SELECT 4;
+-- result:
+-- !result
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
+-- result:
+4
+-- !result
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute expire_snapshots(now());
+-- result:
+-- !result
+alter table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0} execute remove_orphan_files(now());
+-- result:
+-- !result
+select count(1) from files ("path"="oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}/data","list_files_only" = "true");
path: test/sql/test_iceberg/R/test_iceberg_remove_orphan_files
user: login: Max-Cheng
body: @stephen-shelby I'm not sure this is the right path.
created_at: 2025-03-07T10:19:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,20 @@
+-- name: test_iceberg_remove_orphan_files
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+create table iceberg_sql_test_${uuid0}.iceberg_db_${uuid0}.ice_tbl_${uuid0}(k1 int);
path: test/sql/test_iceberg/T/test_iceberg_remove_orphan_files
user: login: Youngwb
body: If you want specify the table location, you need to use table property, like 
```
create table ice_tbl_2 (k1 int) properties('location' = 'oss://starrocks-ci-test/iceberg_db_${uuid0}/ice_tbl_${uuid0}')
```
created_at: 2025-03-10T11:47:26Z
author_association: CONTRIBUTOR

PR comments:
user: login: CLAassistant
created_at: 2025-02-27T13:26:10Z
body: [![CLA assistant check](https://cla-assistant.io/pull/badge/signed)](https://cla-assistant.io/StarRocks/starrocks?pullRequest=56385) <br/>All committers have signed the CLA.
author_association: NONE
user: login: stephen-shelby
created_at: 2025-03-03T01:14:00Z
body: please add some tests
author_association: CONTRIBUTOR
user: login: Youngwb
created_at: 2025-03-03T07:56:51Z
body: @Max-Cheng plz add some example to explain the function arguments in the PR description
author_association: CONTRIBUTOR
user: login: Max-Cheng
created_at: 2025-03-04T04:52:16Z
body: > please add some tests

I am looking for some place to write integration tests, should I include `expire_snapshot` in this MR?
author_association: CONTRIBUTOR
user: login: stephen-shelby
created_at: 2025-03-06T12:41:48Z
body: > > please add some tests
> 
> I am looking for some place to write integration tests, should I include `expire_snapshot` in this MR?

please refer to https://github.com/StarRocks/starrocks/blob/main/test/sql/test_iceberg/T/test_table_operation
author_association: CONTRIBUTOR
user: login: Youngwb
created_at: 2025-03-10T02:00:07Z
body: @Max-Cheng There is no such path 
![image](https://github.com/user-attachments/assets/d756188c-3ba9-4028-acea-ebe4fab64f43)
could you plz fix this issue?
author_association: CONTRIBUTOR
user: login: sonarqubecloud[bot]
created_at: 2025-03-11T02:32:08Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=StarRocks_starrocks&pullRequest=56385) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [3 New issues](https://sonarcloud.io/project/issues?id=StarRocks_starrocks&pullRequest=56385&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=StarRocks_starrocks&pullRequest=56385&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=StarRocks_starrocks&pullRequest=56385&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=StarRocks_starrocks&pullRequest=56385&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=StarRocks_starrocks&pullRequest=56385&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=StarRocks_starrocks&pullRequest=56385)


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-11T05:11:03Z
body: #### [Java-Extensions Incremental Coverage Report]


:white_check_mark: **pass** : 0 / 0 (0%)




author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-11T05:11:42Z
body: #### [FE Incremental Coverage Report]

:white_check_mark: **pass** : 65 / 80 (81.25%)




#### file detail

|   |path|covered_line|new_line|coverage|not_covered_line_detail|
|----|----|----|----|----|----|
|:large_blue_circle:|com/starrocks/connector/iceberg/IcebergAlterTableExecutor.java|62|77|80.52%|[538, 544, 545, 546, 568, 576, 577, 598, 614, 615, 623, 624, 635, 636, 637]|
|:large_blue_circle:|com/starrocks/connector/iceberg/IcebergUtil.java|1|1|100.00%|[]|
|:large_blue_circle:|com/starrocks/connector/iceberg/IcebergMetadata.java|1|1|100.00%|[]|
|:large_blue_circle:|com/starrocks/connector/iceberg/IcebergTableOperation.java|1|1|100.00%|[]|




author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-11T05:11:59Z
body: #### [BE Incremental Coverage Report]


:white_check_mark: **pass** : 0 / 0 (0%)




author_association: CONTRIBUTOR
user: login: Max-Cheng
created_at: 2025-03-11T05:52:29Z
body: @Youngwb @stephen-shelby @dirtysalt @DorianZheng Thanks for taking the time to review!
author_association: CONTRIBUTOR
user: login: Youngwb
created_at: 2025-03-11T05:52:35Z
body: @Max-Cheng Thx for your contribution.
author_association: CONTRIBUTOR