Repository: vercel/next.js
File: turbopack/crates/turbo-persistence/src/db.rs (Rust)

Code before:
                                                        path,
                                                        seq,
                                                        flags,
                                                    )?);

                                                    collector.entries.clear();
                                                }
                                            }

                                            collector.entries.push(current);

                                            // Early flush: once entries is past 50%
                                            // of any limit, the final file won't be
                                            // undersized, so flush last_entries to
                                            // reduce peak memory.
                                            if !collector.last_entries.is_empty()
                                                && (collector.total_key_size
                                                    + collector.total_value_size
                                                    > DATA_THRESHOLD_PER_COMPACTED_FILE / 2
                                                    || collector.entries.len()
                                                        >= MAX_ENTRIES_PER_COMPACTED_FILE / 2
                                                    || collector.value_block_tracker.is_half_full())
                                            {
                                                let seq = sequence_number
                                                    .fetch_add(1, Ordering::SeqCst)
                                                    + 1;
                                                keys_written += collector.last_entries.len() as u64;
                                                let mut flags = MetaEntryFlags::default();
                                                flags.set_cold(!is_used);
                                                collector.new_sst_files.push(create_sst_file(
                                                    &self.parallel_scheduler,
                                                    &collector.last_entries,
                                                    collector.last_entries_total_key_size,
                                                    path,
                                                    seq,
                                                    flags,
                                                )?);
                                                collector.last_entries.clear();
                                                collector.last_entries_total_key_size = 0;
                                            }
                                        } else {
                                            // Override value
                                            // TODO delete blob file
                                        }
                                    }
                                    current = Some(entry);
                                }
                                if let Some(entry) = current {
                                    let is_used = used_key_hashes
                                        .as_ref()
                                        .is_some_and(|amqf| amqf.contains_fingerprint(entry.hash));

Review comment (performance):
this is probably the least motivated improvement. the goal is to shave peak ram

if we are filling a collector with 256M of data, then at peak we have `last_entries` and `entries` both full which is 512M of data, with this the peak drops to 384M which is better but tl;dr; the `last_entries` optimization is kind of expensive

Code after:

                                                if !collector.entries.is_empty() {
                                                    let seq = sequence_number
                                                        .fetch_add(1, Ordering::SeqCst)
                                                        + 1;

                                                    keys_written += collector.entries.len() as u64;

                                                    let mut flags = MetaEntryFlags::default();
                                                    flags.set_cold(!is_used);
                                                    collector.new_sst_files.push(create_sst_file(
                                                        &self.parallel_scheduler,
                                                        &collector.entries,
                                                        selected_total_key_size,
                                                        path,
                                                        seq,
                                                        flags,
                                                    )?);

                                                    collector.entries.clear();
                                                }
                                            }

                                            collector.entries.push(current);

                                            // Early flush: once entries is past 50%
                                            // of any limit, the final file won't be
                                            // undersized, so flush last_entries to
                                            // reduce peak memory.
                                            if !collector.last_entries.is_empty()
                                                && collector.is_half_full()
                                            {
                                                let seq = sequence_number
                                                    .fetch_add(1, Ordering::SeqCst)
                                                    + 1;
                                                keys_written += collector.last_entries.len() as u64;
                                                let mut flags = MetaEntryFlags::default();
                                                flags.set_cold(!is_used);
                                                collector.new_sst_files.push(create_sst_file(
                                                    &self.parallel_scheduler,
                                                    &collector.last_entries,
                                                    collector.last_entries_total_key_size,
                                                    path,
                                                    seq,
                                                    flags,
                                                )?);
                                                collector.last_entries.clear();
                                                collector.last_entries_total_key_size = 0;
                                            }
                                        } else {
                                            // Override value