Code review comment: the contains and add operations of hashset is time-consuming. 

Code before review:
private void collectTableStatisticsAndCacheIcebergSplit(Table table, ScalarOperator predicate, long limit, Tracers tracers) {
IcebergTable icebergTable = (IcebergTable) table;
Optional<Snapshot> snapshot = icebergTable.getSnapshot();
if (!snapshot.isPresent()) {
return;
}
long snapshotId = snapshot.get().snapshotId();
String dbName = icebergTable.getRemoteDbName();
String tableName = icebergTable.getRemoteTableName();
IcebergFilter key = IcebergFilter.of(dbName, tableName, snapshotId, predicate);
org.apache.iceberg.Table nativeTbl = icebergTable.getNativeTable();
Types.StructType schema = nativeTbl.schema().asStruct();
Map<String, MetricsModes.MetricsMode> fieldToMetricsMode = getIcebergMetricsConfig(icebergTable);
if (!fieldToMetricsMode.isEmpty()) {
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".write_metrics_mode_default",
DEFAULT_WRITE_METRICS_MODE_DEFAULT);
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".non-default.size",
String.valueOf(fieldToMetricsMode.size()));
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".non-default.columns",
fieldToMetricsMode.toString());
}
List<ScalarOperator> scalarOperators = Utils.extractConjuncts(predicate);
ScalarOperatorToIcebergExpr.IcebergContext icebergContext = new ScalarOperatorToIcebergExpr.IcebergContext(schema);
Expression icebergPredicate = new ScalarOperatorToIcebergExpr().convert(scalarOperators, icebergContext);
TableScan scan = nativeTbl.newScan()
.useSnapshot(snapshotId)
.metricsReporter(metricsReporter)
.planWith(jobPlanningExecutor);
if (enableCollectColumnStatistics()) {
scan = scan.includeColumnStats();
}
if (icebergPredicate.op() != Expression.Operation.TRUE) {
scan = scan.filter(icebergPredicate);
}
CloseableIterable<FileScanTask> fileScanTaskIterable = TableScanUtil.splitFiles(
scan.planFiles(), scan.targetSplitSize());
CloseableIterator<FileScanTask> fileScanTaskIterator = fileScanTaskIterable.iterator();
Iterator<FileScanTask> fileScanTasks;
boolean canPruneManifests = limit != -1 && !icebergTable.isV2Format() && onlyHasPartitionPredicate(table, predicate)
&& limit < Integer.MAX_VALUE && nativeTbl.spec().specId() == 0 && enablePruneManifest();
if (canPruneManifests) {
fileScanTasks = Iterators.limit(fileScanTaskIterator, (int) limit);
} else {
fileScanTasks = fileScanTaskIterator;
}
List<FileScanTask> icebergScanTasks = Lists.newArrayList();
long totalReadCount = 0;
Set<String> filePaths = new HashSet<>();
while (fileScanTasks.hasNext()) {
FileScanTask scanTask = fileScanTasks.next();
FileScanTask icebergSplitScanTask = scanTask;
if (enableCollectColumnStatistics()) {
try (Timer ignored = Tracers.watchScope(EXTERNAL, "ICEBERG.buildSplitScanTask")) {
icebergSplitScanTask = buildIcebergSplitScanTask(scanTask, icebergPredicate, key);
}
List<Types.NestedField> fullColumns = nativeTbl.schema().columns();
Map<Integer, Type.PrimitiveType> idToTypeMapping = fullColumns.stream()
.filter(column -> column.type().isPrimitiveType())
.collect(Collectors.toMap(Types.NestedField::fieldId, column -> column.type().asPrimitiveType()));
Set<Integer> identityPartitionIds = nativeTbl.spec().fields().stream()
.filter(x -> x.transform().isIdentity())
.map(PartitionField::sourceId)
.collect(Collectors.toSet());
List<Types.NestedField> nonPartitionPrimitiveColumns = fullColumns.stream()
.filter(column -> !identityPartitionIds.contains(column.fieldId()) &&
column.type().isPrimitiveType())
.collect(toImmutableList());
try (Timer ignored = Tracers.watchScope(EXTERNAL, "ICEBERG.updateIcebergFileStats")) {
statisticProvider.updateIcebergFileStats(
icebergTable, scanTask, idToTypeMapping, nonPartitionPrimitiveColumns, key);
}
}
icebergScanTasks.add(icebergSplitScanTask);
if (canPruneManifests) {
String filePath = icebergSplitScanTask.file().path().toString();
if (!filePaths.contains(filePath)) {
filePaths.add(filePath);
totalReadCount += scanTask.file().recordCount();
}
if (totalReadCount >= limit) {
break;
}
}
}
try {
fileScanTaskIterable.close();
fileScanTaskIterator.close();
} catch (IOException e) {
}
Optional<ScanReport> metrics = metricsReporter.getReporter(catalogName, dbName, tableName, snapshotId, icebergPredicate);
if (metrics.isPresent()) {
Tracers.Module module = Tracers.Module.EXTERNAL;
String name = "ICEBERG.ScanMetrics." + metrics.get().tableName() + "["  + icebergPredicate + "]";
String value = metrics.get().scanMetrics().toString();
if (tracers == null) {
Tracers.record(module, name, value);
} else {
synchronized (this) {
Tracers.record(tracers, module, name, value);
}
}
}
splitTasks.put(key, icebergScanTasks);
scannedTables.add(key);
}

Line under review: String filePath = icebergSplitScanTask.file().path().toString();

Code after addressing the review:
private void collectTableStatisticsAndCacheIcebergSplit(Table table, ScalarOperator predicate, long limit, Tracers tracers) {
IcebergTable icebergTable = (IcebergTable) table;
Optional<Snapshot> snapshot = icebergTable.getSnapshot();
if (!snapshot.isPresent()) {
return;
}
long snapshotId = snapshot.get().snapshotId();
String dbName = icebergTable.getRemoteDbName();
String tableName = icebergTable.getRemoteTableName();
IcebergFilter key = IcebergFilter.of(dbName, tableName, snapshotId, predicate);
org.apache.iceberg.Table nativeTbl = icebergTable.getNativeTable();
Types.StructType schema = nativeTbl.schema().asStruct();
Map<String, MetricsModes.MetricsMode> fieldToMetricsMode = getIcebergMetricsConfig(icebergTable);
if (!fieldToMetricsMode.isEmpty()) {
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".write_metrics_mode_default",
DEFAULT_WRITE_METRICS_MODE_DEFAULT);
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".non-default.size",
String.valueOf(fieldToMetricsMode.size()));
Tracers.record(Tracers.Module.EXTERNAL, "ICEBERG.MetricsConfig." + nativeTbl + ".non-default.columns",
fieldToMetricsMode.toString());
}
List<ScalarOperator> scalarOperators = Utils.extractConjuncts(predicate);
ScalarOperatorToIcebergExpr.IcebergContext icebergContext = new ScalarOperatorToIcebergExpr.IcebergContext(schema);
Expression icebergPredicate = new ScalarOperatorToIcebergExpr().convert(scalarOperators, icebergContext);
TableScan scan = nativeTbl.newScan()
.useSnapshot(snapshotId)
.metricsReporter(metricsReporter)
.planWith(jobPlanningExecutor);
if (enableCollectColumnStatistics()) {
scan = scan.includeColumnStats();
}
if (icebergPredicate.op() != Expression.Operation.TRUE) {
scan = scan.filter(icebergPredicate);
}
CloseableIterable<FileScanTask> fileScanTaskIterable = TableScanUtil.splitFiles(
scan.planFiles(), scan.targetSplitSize());
CloseableIterator<FileScanTask> fileScanTaskIterator = fileScanTaskIterable.iterator();
Iterator<FileScanTask> fileScanTasks;
boolean canPruneManifests = limit != -1 && !icebergTable.isV2Format() && onlyHasPartitionPredicate(table, predicate)
&& limit < Integer.MAX_VALUE && nativeTbl.spec().specId() == 0 && enablePruneManifest();
if (canPruneManifests) {
fileScanTasks = Iterators.limit(fileScanTaskIterator, (int) limit);
} else {
fileScanTasks = fileScanTaskIterator;
}
List<FileScanTask> icebergScanTasks = Lists.newArrayList();
long totalReadCount = 0;
Set<String> filePaths = new HashSet<>();
while (fileScanTasks.hasNext()) {
FileScanTask scanTask = fileScanTasks.next();
FileScanTask icebergSplitScanTask = scanTask;
if (enableCollectColumnStatistics()) {
try (Timer ignored = Tracers.watchScope(EXTERNAL, "ICEBERG.buildSplitScanTask")) {
icebergSplitScanTask = buildIcebergSplitScanTask(scanTask, icebergPredicate, key);
}
List<Types.NestedField> fullColumns = nativeTbl.schema().columns();
Map<Integer, Type.PrimitiveType> idToTypeMapping = fullColumns.stream()
.filter(column -> column.type().isPrimitiveType())
.collect(Collectors.toMap(Types.NestedField::fieldId, column -> column.type().asPrimitiveType()));
Set<Integer> identityPartitionIds = nativeTbl.spec().fields().stream()
.filter(x -> x.transform().isIdentity())
.map(PartitionField::sourceId)
.collect(Collectors.toSet());
List<Types.NestedField> nonPartitionPrimitiveColumns = fullColumns.stream()
.filter(column -> !identityPartitionIds.contains(column.fieldId()) &&
column.type().isPrimitiveType())
.collect(toImmutableList());
try (Timer ignored = Tracers.watchScope(EXTERNAL, "ICEBERG.updateIcebergFileStats")) {
statisticProvider.updateIcebergFileStats(
icebergTable, scanTask, idToTypeMapping, nonPartitionPrimitiveColumns, key);
}
}
icebergScanTasks.add(icebergSplitScanTask);
if (canPruneManifests) {
String filePath = icebergSplitScanTask.file().path().toString();
if (!filePaths.contains(filePath)) {
filePaths.add(filePath);
totalReadCount += scanTask.file().recordCount();
}
if (totalReadCount >= limit) {
break;
}
}
}
try {
fileScanTaskIterable.close();
fileScanTaskIterator.close();
} catch (IOException e) {
}
Optional<ScanReport> metrics = metricsReporter.getReporter(catalogName, dbName, tableName, snapshotId, icebergPredicate);
if (metrics.isPresent()) {
Tracers.Module module = Tracers.Module.EXTERNAL;
String name = "ICEBERG.ScanMetrics." + metrics.get().tableName() + "["  + icebergPredicate + "]";
String value = metrics.get().scanMetrics().toString();
if (tracers == null) {
Tracers.record(module, name, value);
} else {
synchronized (this) {
Tracers.record(tracers, module, name, value);
}
}
}
splitTasks.put(key, icebergScanTasks);
scannedTables.add(key);
}