Repository: vercel/next.js
Pull request: Turbopack: Improve errors when opening SST files

### What?

add more context to error messages for database failures

Closes PACK-4766

Changed files:
filename: turbopack/crates/turbo-persistence/src/db.rs
patch: @@ -652,7 +652,8 @@ impl TurboPersistence {
                 max_coverage,
                 max_merge_sequence,
                 max_merge_size,
-            )?;
+            )
+            .context("Failed to compact database")?;
         }
 
         if !new_meta_files.is_empty() {
@@ -664,7 +665,8 @@ impl TurboPersistence {
                 blob_seq_numbers_to_delete,
                 sequence_number: *sequence_number.get_mut(),
                 keys_written,
-            })?;
+            })
+            .context("Failed to commit the database compaction")?;
         }
 
         self.active_write_operation.store(false, Ordering::Release);
@@ -853,7 +855,7 @@ impl TurboPersistence {
                                 let index_in_meta = ssts_with_ranges[index].index_in_meta;
                                 let meta = &meta_files[meta_index];
                                 meta.entry(index_in_meta)
-                                    .sst(&self.path)?
+                                    .sst(meta)?
                                     .iter(key_block_cache, value_block_cache)
                             })
                             .collect::<Result<Vec<_>>>()?;
@@ -983,7 +985,10 @@ impl TurboPersistence {
                             keys_written,
                         })
                     })
-                    .collect::<Result<Vec<_>>>()?;
+                    .collect::<Result<Vec<_>>>()
+                    .with_context(|| {
+                        format!("Failed to merge database files for family {family}")
+                    })?;
 
                 let mut meta_file_builder = meta_file_builder.into_inner();
 
@@ -1017,8 +1022,7 @@ impl TurboPersistence {
 
                 let span = tracing::trace_span!("write meta file").entered();
                 let seq = sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
-                let meta_file =
-                    meta_file_builder.write(&self.path.join(format!("{seq:08}.meta")))?;
+                let meta_file = meta_file_builder.write(&self.path, seq)?;
                 drop(span);
 
                 let mut new_sst_files = Vec::with_capacity(

filename: turbopack/crates/turbo-persistence/src/meta_file.rs
patch: @@ -68,9 +68,15 @@ impl MetaEntry {
             .expect("AQMF data out of bounds")
     }
 
-    pub fn sst(&self, db_path: &Path) -> Result<&StaticSortedFile> {
-        self.sst
-            .get_or_try_init(|| StaticSortedFile::open(db_path, self.sst_data.clone()))
+    pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> {
+        self.sst.get_or_try_init(|| {
+            StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| {
+                format!(
+                    "Unable to open static sorted file referenced from {:08}.meta",
+                    meta.sequence_number()
+                )
+            })
+        })
     }
 
     /// Returns the key family and hash range of this file.
@@ -262,7 +268,14 @@ impl MetaFile {
                     GuardResult::Value(aqmf) => aqmf,
                     GuardResult::Guard(guard) => {
                         let aqmf = entry.aqmf(self.aqmf_data());
-                        let aqmf: Arc<qfilter::Filter> = Arc::new(pot::from_slice(aqmf)?);
+                        let aqmf: Arc<qfilter::Filter> =
+                            Arc::new(pot::from_slice(aqmf).with_context(|| {
+                                format!(
+                                    "Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
+                                    self.sequence_number,
+                                    entry.sequence_number()
+                                )
+                            })?);
                         let _ = guard.insert(aqmf.clone());
                         aqmf
                     }
@@ -275,19 +288,23 @@ impl MetaFile {
             } else {
                 let aqmf = entry.aqmf.get_or_try_init(|| {
                     let aqmf = entry.aqmf(self.aqmf_data());
-                    anyhow::Ok(pot::from_slice(aqmf)?)
+                    anyhow::Ok(pot::from_slice(aqmf).with_context(|| {
+                        format!(
+                            "Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
+                            self.sequence_number,
+                            entry.sequence_number()
+                        )
+                    })?)
                 })?;
                 if !aqmf.contains_fingerprint(key_hash) {
                     miss_result = MetaLookupResult::QuickFilterMiss;
                     continue;
                 }
             };
-            let result = entry.sst(&self.db_path)?.lookup(
-                key_hash,
-                key,
-                key_block_cache,
-                value_block_cache,
-            )?;
+            let result =
+                entry
+                    .sst(self)?
+                    .lookup(key_hash, key, key_block_cache, value_block_cache)?;
             if !matches!(result, SstLookupResult::NotFound) {
                 return Ok(MetaLookupResult::SstLookup(result));
             }

filename: turbopack/crates/turbo-persistence/src/meta_file_builder.rs
patch: @@ -4,6 +4,7 @@ use std::{
     path::Path,
 };
 
+use anyhow::{Context, Result};
 use byteorder::{BE, WriteBytesExt};
 
 use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta;
@@ -34,7 +35,13 @@ impl MetaFileBuilder {
     }
 
     #[tracing::instrument(level = "trace", skip_all)]
-    pub fn write(&self, file: &Path) -> io::Result<File> {
+    pub fn write(&self, db_path: &Path, seq: u32) -> Result<File> {
+        let file = db_path.join(format!("{seq:08}.meta"));
+        self.write_internal(&file)
+            .with_context(|| format!("Unable to write meta file {seq:08}.meta"))
+    }
+
+    fn write_internal(&self, file: &Path) -> io::Result<File> {
         let mut file = BufWriter::new(File::create(file)?);
         file.write_u32::<BE>(0xFE4ADA4A)?; // Magic number
         file.write_u32::<BE>(self.family)?;

filename: turbopack/crates/turbo-persistence/src/write_batch.rs
patch: @@ -397,10 +397,7 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
                 }
                 keys_written.fetch_add(entries, Ordering::Relaxed);
                 let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
-                let file = self.db_path.join(format!("{seq:08}.meta"));
-                let file = builder
-                    .write(&file)
-                    .with_context(|| format!("Unable to write meta file {seq:08}.meta"))?;
+                let file = builder.write(&self.db_path, seq)?;
                 Ok((seq, file))
             })
             .collect::<Result<Vec<_>>>()?;