Repository: vercel/next.js
Pull request: Turbopack: improve print_cache_item_size

### What?

* Convert it to a markdown table
* Show counts for all meta items
* Show size of output


Changed files:
filename: turbopack/crates/turbo-tasks-backend/src/backend/mod.rs
patch: @@ -1004,6 +1004,68 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
         let snapshot_time = Instant::now();
         drop(snapshot_request);
 
+        #[cfg(feature = "print_cache_item_size")]
+        #[derive(Default)]
+        struct TaskCacheStats {
+            data: usize,
+            data_compressed: usize,
+            data_count: usize,
+            meta: usize,
+            meta_compressed: usize,
+            meta_count: usize,
+            upper_count: usize,
+            collectibles_count: usize,
+            aggregated_collectibles_count: usize,
+            children_count: usize,
+            followers_count: usize,
+            collectibles_dependents_count: usize,
+            aggregated_dirty_containers_count: usize,
+            output_size: usize,
+        }
+        #[cfg(feature = "print_cache_item_size")]
+        impl TaskCacheStats {
+            fn compressed_size(data: &[u8]) -> Result<usize> {
+                Ok(lzzzz::lz4::Compressor::new()?.next_to_vec(
+                    data,
+                    &mut Vec::new(),
+                    lzzzz::lz4::ACC_LEVEL_DEFAULT,
+                )?)
+            }
+
+            fn add_data(&mut self, data: &[u8]) {
+                self.data += data.len();
+                self.data_compressed += Self::compressed_size(data).unwrap_or(0);
+                self.data_count += 1;
+            }
+
+            fn add_meta(&mut self, data: &[u8]) {
+                self.meta += data.len();
+                self.meta_compressed += Self::compressed_size(data).unwrap_or(0);
+                self.meta_count += 1;
+            }
+
+            fn add_counts(&mut self, storage: &TaskStorage) {
+                let counts = storage.meta_counts();
+                self.upper_count += counts.upper;
+                self.collectibles_count += counts.collectibles;
+                self.aggregated_collectibles_count += counts.aggregated_collectibles;
+                self.children_count += counts.children;
+                self.followers_count += counts.followers;
+                self.collectibles_dependents_count += counts.collectibles_dependents;
+                self.aggregated_dirty_containers_count += counts.aggregated_dirty_containers;
+                if let Some(output) = storage.get_output() {
+                    use turbo_bincode::turbo_bincode_encode;
+
+                    self.output_size += turbo_bincode_encode(&output)
+                        .map(|data| data.len())
+                        .unwrap_or(0);
+                }
+            }
+        }
+        #[cfg(feature = "print_cache_item_size")]
+        let task_cache_stats: Mutex<FxHashMap<_, TaskCacheStats>> =
+            Mutex::new(FxHashMap::default());
+
         let preprocess = |task_id: TaskId, inner: &TaskStorage| {
             if task_id.is_transient() {
                 return (None, None);
@@ -1021,6 +1083,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
         let process = |task_id: TaskId,
                        (meta, data): (Option<TaskStorage>, Option<TaskStorage>),
                        buffer: &mut TurboBincodeBuffer| {
+            #[cfg(feature = "print_cache_item_size")]
+            if let Some(ref m) = meta {
+                task_cache_stats
+                    .lock()
+                    .entry(self.debug_get_task_name(task_id))
+                    .or_default()
+                    .add_counts(m);
+            }
             (
                 task_id,
                 meta.map(|d| encode_task_data(task_id, &d, SpecificTaskDataCategory::Meta, buffer)),
@@ -1033,6 +1103,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                     return (task_id, None, None);
                 }
 
+                #[cfg(feature = "print_cache_item_size")]
+                if inner.flags.meta_modified() {
+                    task_cache_stats
+                        .lock()
+                        .entry(self.debug_get_task_name(task_id))
+                        .or_default()
+                        .add_counts(&inner);
+                }
+
                 // Encode meta/data directly from TaskStorage snapshot
                 (
                     task_id,
@@ -1049,42 +1128,6 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
             .storage
             .take_snapshot(&preprocess, &process, &process_snapshot);
 
-        #[cfg(feature = "print_cache_item_size")]
-        #[derive(Default)]
-        struct TaskCacheStats {
-            data: usize,
-            data_compressed: usize,
-            data_count: usize,
-            meta: usize,
-            meta_compressed: usize,
-            meta_count: usize,
-        }
-        #[cfg(feature = "print_cache_item_size")]
-        impl TaskCacheStats {
-            fn compressed_size(data: &[u8]) -> Result<usize> {
-                Ok(lzzzz::lz4::Compressor::new()?.next_to_vec(
-                    data,
-                    &mut Vec::new(),
-                    lzzzz::lz4::ACC_LEVEL_DEFAULT,
-                )?)
-            }
-
-            fn add_data(&mut self, data: &[u8]) {
-                self.data += data.len();
-                self.data_compressed += Self::compressed_size(data).unwrap_or(0);
-                self.data_count += 1;
-            }
-
-            fn add_meta(&mut self, data: &[u8]) {
-                self.meta += data.len();
-                self.meta_compressed += Self::compressed_size(data).unwrap_or(0);
-                self.meta_count += 1;
-            }
-        }
-        #[cfg(feature = "print_cache_item_size")]
-        let task_cache_stats: Mutex<FxHashMap<_, TaskCacheStats>> =
-            Mutex::new(FxHashMap::default());
-
         let task_snapshots = snapshot
             .into_iter()
             .filter_map(|iter| {
@@ -1100,7 +1143,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                                     #[cfg(feature = "print_cache_item_size")]
                                     task_cache_stats
                                         .lock()
-                                        .entry(self.debug_get_task_description(task_id))
+                                        .entry(self.debug_get_task_name(task_id))
                                         .or_default()
                                         .add_meta(&meta);
                                     Some(meta)
@@ -1120,7 +1163,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                                     #[cfg(feature = "print_cache_item_size")]
                                     task_cache_stats
                                         .lock()
-                                        .entry(self.debug_get_task_description(task_id))
+                                        .entry(self.debug_get_task_name(task_id))
                                         .or_default()
                                         .add_data(&data);
                                     Some(data)
@@ -1169,6 +1212,9 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                     .collect::<Vec<_>>();
                 if !task_cache_stats.is_empty() {
                     use turbo_tasks::util::FormatBytes;
+
+                    use crate::utils::markdown_table::print_markdown_table;
+
                     task_cache_stats.sort_unstable_by(|(key_a, stats_a), (key_b, stats_b)| {
                         (stats_b.data_compressed + stats_b.meta_compressed, key_b)
                             .cmp(&(stats_a.data_compressed + stats_a.meta_compressed, key_a))
@@ -1189,34 +1235,81 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                         )
                     );
 
-                    for (task_desc, stats) in task_cache_stats {
-                        println!(
-                            "  {} ({}) {task_desc} = {} ({}) meta {} x {} ({}), {} ({}) data {} x \
-                             {} ({})",
-                            FormatBytes(stats.data_compressed + stats.meta_compressed),
-                            FormatBytes(stats.data + stats.meta),
-                            FormatBytes(stats.meta_compressed),
-                            FormatBytes(stats.meta),
-                            stats.meta_count,
-                            FormatBytes(
-                                stats
-                                    .meta_compressed
-                                    .checked_div(stats.meta_count)
-                                    .unwrap_or(0)
-                            ),
-                            FormatBytes(stats.meta.checked_div(stats.meta_count).unwrap_or(0)),
-                            FormatBytes(stats.data_compressed),
-                            FormatBytes(stats.data),
-                            stats.data_count,
-                            FormatBytes(
-                                stats
-                                    .data_compressed
-                                    .checked_div(stats.data_count)
-                                    .unwrap_or(0)
-                            ),
-                            FormatBytes(stats.data.checked_div(stats.data_count).unwrap_or(0)),
-                        );
-                    }
+                    print_markdown_table(
+                        [
+                            "Task",
+                            " Total Size",
+                            " Data Size",
+                            " Data Count x Avg",
+                            " Data Count x Avg",
+                            " Meta Size",
+                            " Meta Count x Avg",
+                            " Meta Count x Avg",
+                            " Uppers",
+                            " Coll",
+                            " Agg Coll",
+                            " Children",
+                            " Followers",
+                            " Coll Deps",
+                            " Agg Dirty",
+                            " Output Size",
+                        ],
+                        task_cache_stats.iter(),
+                        |(task_desc, stats)| {
+                            [
+                                task_desc.to_string(),
+                                format!(
+                                    " {} ({})",
+                                    FormatBytes(stats.data_compressed + stats.meta_compressed),
+                                    FormatBytes(stats.data + stats.meta)
+                                ),
+                                format!(
+                                    " {} ({})",
+                                    FormatBytes(stats.data_compressed),
+                                    FormatBytes(stats.data)
+                                ),
+                                format!(" {} x", stats.data_count,),
+                                format!(
+                                    "{} ({})",
+                                    FormatBytes(
+                                        stats
+                                            .data_compressed
+                                            .checked_div(stats.data_count)
+                                            .unwrap_or(0)
+                                    ),
+                                    FormatBytes(
+                                        stats.data.checked_div(stats.data_count).unwrap_or(0)
+                                    ),
+                                ),
+                                format!(
+                                    " {} ({})",
+                                    FormatBytes(stats.meta_compressed),
+                                    FormatBytes(stats.meta)
+                                ),
+                                format!(" {} x", stats.meta_count,),
+                                format!(
+                                    "{} ({})",
+                                    FormatBytes(
+                                        stats
+                                            .meta_compressed
+                                            .checked_div(stats.meta_count)
+                                            .unwrap_or(0)
+                                    ),
+                                    FormatBytes(
+                                        stats.meta.checked_div(stats.meta_count).unwrap_or(0)
+                                    ),
+                                ),
+                                format!(" {}", stats.upper_count),
+                                format!(" {}", stats.collectibles_count),
+                                format!(" {}", stats.aggregated_collectibles_count),
+                                format!(" {}", stats.children_count),
+                                format!(" {}", stats.followers_count),
+                                format!(" {}", stats.collectibles_dependents_count),
+                                format!(" {}", stats.aggregated_dirty_containers_count),
+                                format!(" {}", FormatBytes(stats.output_size)),
+                            ]
+                        },
+                    );
                 }
             }
         }
@@ -1625,6 +1718,18 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
         }
     }
 
+    #[allow(dead_code)]
+    fn debug_get_task_name(&self, task_id: TaskId) -> String {
+        let task = self.storage.access_mut(task_id);
+        if let Some(value) = task.get_persistent_task_type() {
+            value.to_string()
+        } else if let Some(value) = task.get_transient_task_type() {
+            value.to_string()
+        } else {
+            "unknown".to_string()
+        }
+    }
+
     fn debug_get_cached_task_type(&self, task_id: TaskId) -> Option<Arc<CachedTaskType>> {
         let task = self.storage.access_mut(task_id);
         task.get_persistent_task_type().cloned()

filename: turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs
patch: @@ -80,7 +80,7 @@ struct TaskStorageSchema {
     /// The task's output value.
     /// Filtered during serialization to skip transient outputs (referencing transient tasks).
     #[field(storage = "direct", category = "meta", inline, filter_transient)]
-    output: Option<OutputValue>,
+    pub output: Option<OutputValue>,
 
     /// Upper nodes in the aggregation tree (reference counted).
     #[field(storage = "counter_map", category = "meta", inline, filter_transient)]
@@ -539,6 +539,32 @@ impl TaskStorage {
             reason: TaskExecutionReason::Initial,
         });
     }
+
+    /// Returns counts for aggregation tree and collectibles fields.
+    /// Used for cache size statistics.
+    pub fn meta_counts(&self) -> MetaCounts {
+        MetaCounts {
+            upper: self.upper().len(),
+            collectibles: self.collectibles().map_or(0, |c| c.len()),
+            aggregated_collectibles: self.aggregated_collectibles().map_or(0, |c| c.len()),
+            children: self.children().map_or(0, |c| c.len()),
+            followers: self.followers().map_or(0, |c| c.len()),
+            collectibles_dependents: self.collectibles_dependents().map_or(0, |c| c.len()),
+            aggregated_dirty_containers: self.aggregated_dirty_containers().map_or(0, |c| c.len()),
+        }
+    }
+}
+
+/// Counts for aggregation tree and collectibles fields.
+#[derive(Default)]
+pub struct MetaCounts {
+    pub upper: usize,
+    pub collectibles: usize,
+    pub aggregated_collectibles: usize,
+    pub children: usize,
+    pub followers: usize,
+    pub collectibles_dependents: usize,
+    pub aggregated_dirty_containers: usize,
 }
 
 // Support serialization filtering for CellDependents and CollectibleDependents

filename: turbopack/crates/turbo-tasks-backend/src/utils/markdown_table.rs
patch: @@ -0,0 +1,160 @@
+use std::fmt::Write;
+
+/// Prints a markdown table to stdout.
+/// See `write_markdown_table` for details.
+#[allow(dead_code)]
+pub fn print_markdown_table<T, const N: usize>(
+    headers: [&str; N],
+    data: impl IntoIterator<Item = T> + Clone,
+    get_fields: impl Fn(&T) -> [String; N],
+) {
+    write_markdown_table(&mut std::io::stdout(), headers, data, get_fields);
+}
+
+/// Writes a markdown table to the given writer.
+/// The headers and fields can specify alignment by starting or ending with a space:
+///   - " Text" - right aligned
+///   - " Text " - center aligned
+///   - "Text" - left aligned
+///
+/// Also, if multiple consecutive headers are identical, they will be form a merged header cell.
+#[allow(dead_code)]
+pub fn write_markdown_table<T, const N: usize>(
+    write: &mut impl std::io::Write,
+    headers: [&str; N],
+    data: impl IntoIterator<Item = T> + Clone,
+    get_fields: impl Fn(&T) -> [String; N],
+) {
+    let mut merged = headers.map(|_| false);
+    for i in 1..N {
+        if headers[i].trim() == headers[i - 1].trim() {
+            merged[i] = true;
+        }
+    }
+    // Measure max field size
+    let mut sizes = headers.map(|_| 1);
+    for item in data.clone() {
+        let fields = get_fields(&item);
+        for (i, field) in fields.iter().enumerate() {
+            let field_size = field.trim().len();
+            if field_size > sizes[i] {
+                sizes[i] = field_size;
+            }
+        }
+    }
+    // Add header size
+    let mut headers_sizes = sizes;
+    for (i, header) in headers.iter().enumerate() {
+        if merged[i] {
+            headers_sizes[i] = 0;
+            continue;
+        }
+        let header_size = header.trim().len();
+        let current_size = sizes[i]
+            + (i + 1..N)
+                .take_while(|&j| merged[j])
+                .map(|j| sizes[j] + 1)
+                .sum::<usize>();
+        if header_size > current_size {
+            sizes[i] += header_size - current_size;
+            headers_sizes[i] = header_size;
+        } else {
+            headers_sizes[i] = current_size;
+        }
+    }
+    // Print headers
+    {
+        let mut line = String::new();
+        for (i, header) in headers.iter().enumerate() {
+            let size = headers_sizes[i];
+            if size == 0 {
+                continue;
+            }
+            let right = header.starts_with(' ');
+            let center = header.ends_with(' ') && right;
+            let escaped_header = escape_markdown_cell(header.trim());
+            if center {
+                write!(line, "| {:^width$} ", escaped_header, width = size).unwrap();
+            } else if right {
+                write!(line, "| {:>width$} ", escaped_header, width = size).unwrap();
+            } else {
+                write!(line, "| {:<width$} ", escaped_header, width = size).unwrap();
+            }
+        }
+        writeln!(write, "{}|", line).unwrap();
+    }
+    // Print separator
+    {
+        let mut line = String::new();
+        for size in headers_sizes.iter() {
+            if *size == 0 {
+                continue;
+            }
+            write!(line, "| {:-<width$} ", "", width = *size).unwrap();
+        }
+        writeln!(write, "{}|", line).unwrap();
+    }
+    // Print rows
+    for item in data {
+        let row = get_fields(&item);
+        let mut line = String::new();
+        for (i, field) in row.iter().enumerate() {
+            let size = sizes[i];
+            let right = field.starts_with(' ');
+            let center = field.ends_with(' ') && right;
+            let escaped_field = escape_markdown_cell(field.trim());
+            let separator = if merged[i] { "" } else { "| " };
+            if center {
+                write!(line, "{separator}{:^width$} ", escaped_field, width = size).unwrap();
+            } else if right {
+                write!(line, "{separator}{:>width$} ", escaped_field, width = size).unwrap();
+            } else {
+                write!(line, "{separator}{:<width$} ", escaped_field, width = size).unwrap();
+            }
+        }
+        writeln!(write, "{}|", line).unwrap();
+    }
+}
+
+#[allow(dead_code)]
+fn escape_markdown_cell(content: &str) -> String {
+    content.replace('|', "\\|").replace('\n', " ")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::write_markdown_table;
+
+    #[test]
+    fn test_write_markdown_table() {
+        let headers = [" Name ", " Age", " Birthday (Age)", " Birthday (Age)"];
+        let data = vec![
+            (" Alice ", " 30", " 1990-01-01", "(30)"),
+            (" Bob ", " 25", " 2024", "(9)"),
+            (" Charlie ", " 35", " 1985-08-20", "(35)"),
+            (" N/A ", " ?", " N/A", "(?)"),
+        ];
+        let mut output = Vec::new();
+        write_markdown_table(&mut output, headers, data, |item| {
+            [
+                item.0.to_string(),
+                item.1.to_string(),
+                item.2.to_string(),
+                item.3.to_string(),
+            ]
+        });
+        let output = String::from_utf8(output).unwrap();
+        let expected = r#"
+|  Name   | Age |  Birthday (Age) |
+| ------- | --- | --------------- |
+|  Alice  |  30 | 1990-01-01 (30) |
+|   Bob   |  25 |       2024 (9)  |
+| Charlie |  35 | 1985-08-20 (35) |
+|   N/A   |   ? |        N/A (?)  |
+"#;
+        assert_eq!(
+            output.trim().lines().collect::<Vec<_>>(),
+            expected.trim().lines().collect::<Vec<_>>()
+        );
+    }
+}

filename: turbopack/crates/turbo-tasks-backend/src/utils/mod.rs
patch: @@ -3,6 +3,7 @@ pub mod chunked_vec;
 pub mod dash_map_drop_contents;
 pub mod dash_map_multi;
 pub mod dash_map_raw_entry;
+pub mod markdown_table;
 pub mod ptr_eq_arc;
 pub mod shard_amount;
 pub mod sharded;