Repository: vercel/next.js
Pull request: [turbopack] Fix a crash under turbopackPersistentCaching

Address a crash found at head when testing caching while switching back and forth from `dev` and `build`.  

```
thread '<unnamed>' panicked at turbopack/crates/turbo-persistence/src/db.rs:798:17:
attempt to subtract with overflow
stack backtrace:
   0: __rustc::rust_begin_unwind
   1: core::panicking::panic_fmt
   2: core::panicking::panic_const::panic_const_sub_overflow
   3: <core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}> as core::iter::traits::iterator::Iterator>::fold::<(), core::iter::traits::iterator::Iterator::for_each::call<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>>>::extend_trusted<core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>::{closure#0}>::{closure#0}>
   4: <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>>::from_iter
   5: <turbo_persistence::db::TurboPersistence>::compact
```

Scenario:
 * build turbopack and next
 * cd bench/module-cost
 * enable persistent caching
 * pnpm i
 * pnpm dev-turbopack
 * load localhost:3000 and click all the buttons
 * pnpm build-turbopack
 * pnpm dev-turbopack
 * load localhost:3000 and see the crash

## What?

The `get_merge_segments` function and its caller had a different idea about what `max_merge_segment_count` means.  Unify how this is accounted to avoid underflow.



Closes PACK-5242

Changed files:
filename: turbopack/crates/turbo-persistence/src/compaction/selector.rs
patch: @@ -200,10 +200,13 @@ fn total_duplication_size(duplication: &IntervalMap<Option<DuplicationInfo>>) ->
 
 type MergeSegments = Vec<SmallVec<[usize; 1]>>;
 
+/// Computes the set of merge segments that should be run to compact the given compactables.
+///
+/// Returns both the mergeable segments and the actual number of merge segments that were created.
 pub fn get_merge_segments<T: Compactable>(
     compactables: &[T],
     config: &CompactConfig,
-) -> MergeSegments {
+) -> (MergeSegments, usize) {
     // Process all compactables in reverse order.
     // For each compactable, find the smallest set of compactables that overlaps with it and matches
     // the conditions.
@@ -259,7 +262,7 @@ pub fn get_merge_segments<T: Compactable>(
                     continue 'outer;
                 }
 
-                // If we are limited by size or count, we might also crate a merge segment if it's
+                // If we are limited by size or count, we might also create a merge segment if it's
                 // within the limits.
                 let valid_merge_job = current_set.len() >= config.min_merge_count
                     && duplication_size >= config.min_merge_duplication_bytes;
@@ -361,7 +364,7 @@ pub fn get_merge_segments<T: Compactable>(
         true
     });
 
-    merge_segments
+    (merge_segments, real_merge_segments)
 }
 
 #[cfg(test)]
@@ -398,7 +401,7 @@ mod tests {
             .into_iter()
             .map(|range| TestCompactable { range, size: 100 })
             .collect::<Vec<_>>();
-        let jobs = get_merge_segments(&compactables, config);
+        let (jobs, _) = get_merge_segments(&compactables, config);
         jobs.into_iter()
             .map(|job| job.into_iter().collect())
             .collect()
@@ -613,7 +616,7 @@ mod tests {
                 optimal_merge_duplication_bytes: 500,
                 max_merge_segment_count: 4,
             };
-            let jobs = get_merge_segments(&containers, &config);
+            let (jobs, _) = get_merge_segments(&containers, &config);
             if !jobs.is_empty() {
                 println!("{jobs:?}");
 

filename: turbopack/crates/turbo-persistence/src/db.rs
patch: @@ -794,8 +794,9 @@ impl TurboPersistence {
                 if compact_config.max_merge_segment_count == 0 {
                     return Vec::new();
                 }
-                let merge_jobs = get_merge_segments(ssts_with_ranges, &compact_config);
-                compact_config.max_merge_segment_count -= merge_jobs.len();
+                let (merge_jobs, real_merge_job_size) =
+                    get_merge_segments(ssts_with_ranges, &compact_config);
+                compact_config.max_merge_segment_count -= real_merge_job_size;
                 merge_jobs
             })
             .collect::<Vec<_>>();