Repository: qdrant/qdrant
File: lib/shard/src/snapshots/snapshot_utils.rs (Rust)

Code before:

            // Delete local segment, if it's not present in partial snapshot
            let Some(segment_manifest) = snapshot_manifest.get(segment_id) else {
                delete_directories.push(segment_path);
                continue;
            };

            for (file, _local_version) in local_segment_manifest.file_versions() {
                let snapshot_version = segment_manifest.file_version(file);
                let is_removed = snapshot_version.is_none();

                if is_removed {
                    // If `file` is a regular file, delete it from disk, if it was
                    // *removed* from the snapshot
                    let path = segment_path.join(file);
                    delete_files.push(path);
                }
            }
        }

        let ShardDataFiles {
            wal_path: from_wal_path,
            segments_path: from_segments_path,
            newest_clocks_path: from_newest_clocks_path,
            oldest_clocks_path: from_oldest_clocks_path,
            applied_seq_path: _, // not part of the snapshot
        } = get_shard_data_files(snapshot_path);

        let ShardDataFiles {
            wal_path: to_wal_path,
            segments_path: to_segments_path,
            newest_clocks_path: to_newest_clocks_path,
            oldest_clocks_path: to_oldest_clocks_path,
            applied_seq_path: _, // not part of the snapshot
        } = get_shard_data_files(shard_path);

        merge_directories.push((from_segments_path, to_segments_path));
        replace_directories.push((from_wal_path, to_wal_path));

        if from_newest_clocks_path.exists() {
            move_files.push((from_newest_clocks_path, to_newest_clocks_path));
        }

        if from_oldest_clocks_path.exists() {
            move_files.push((from_oldest_clocks_path, to_oldest_clocks_path));
        }

        SnapshotMergePlan {
            move_files,
            replace_directories,
            merge_directories,

Review comment (suggestion):
Shouldn't it be part of the snapshot?

In most cases the segment files on disk should reflect the latest applied state.

But if we have a large pending queue of operations in the WAL, I guess it'd be nice to include all of it in the snapshot. Then we'd need the applied seq file too to not force replay everything on recovery, so that it'll behave the same as the source shard.

Code after:

            // Delete local segment, if it's not present in partial snapshot
            let Some(segment_manifest) = snapshot_manifest.get(segment_id) else {
                delete_directories.push(segment_path);
                continue;
            };

            for (file, _local_version) in local_segment_manifest.file_versions() {
                let snapshot_version = segment_manifest.file_version(file);
                let is_removed = snapshot_version.is_none();

                if is_removed {
                    // If `file` is a regular file, delete it from disk, if it was
                    // *removed* from the snapshot
                    let path = segment_path.join(file);
                    delete_files.push(path);
                }
            }
        }

        let ShardDataFiles {
            wal_path: from_wal_path,
            segments_path: from_segments_path,
            newest_clocks_path: from_newest_clocks_path,
            oldest_clocks_path: from_oldest_clocks_path,
            applied_seq_path: from_applied_seq_path,
        } = get_shard_data_files(snapshot_path);

        let ShardDataFiles {
            wal_path: to_wal_path,
            segments_path: to_segments_path,
            newest_clocks_path: to_newest_clocks_path,
            oldest_clocks_path: to_oldest_clocks_path,
            applied_seq_path: to_applied_seq_path,
        } = get_shard_data_files(shard_path);

        merge_directories.push((from_segments_path, to_segments_path));
        replace_directories.push((from_wal_path, to_wal_path));

        if from_newest_clocks_path.exists() {
            move_files.push((from_newest_clocks_path, to_newest_clocks_path));
        }

        if from_oldest_clocks_path.exists() {
            move_files.push((from_oldest_clocks_path, to_oldest_clocks_path));
        }

        if from_applied_seq_path.exists() {
            move_files.push((from_applied_seq_path, to_applied_seq_path));
        }