Repository: davidlattimore/wild
Pull request #512: Build symbol name hashmap in parallel
Author: login: davidlattimore
Created: 2025-03-02T06:03:47Z | Merged: 2025-03-04T02:34:41Z



Diff:
diff --git a/libwild/src/resolution.rs b/libwild/src/resolution.rs
index 4e6561e1b..ba4d27df7 100644
--- a/libwild/src/resolution.rs
+++ b/libwild/src/resolution.rs
@@ -84,7 +84,6 @@ pub fn resolve_symbols_and_sections<'data>(
         canonicalise_undefined_symbols(undefined_symbols, &output_sections, &groups, symbol_db)?;
 
     crate::symbol_db::resolve_alternative_symbol_definitions(symbol_db, &groups)?;
-    crate::symbol_db::resolve_alternative_versioned_symbol_definitions(symbol_db, &groups)?;
 
     groups[PRELUDE_FILE_ID.group()].files[PRELUDE_FILE_ID.file()] =
         ResolvedFile::Prelude(ResolvedPrelude {
diff --git a/libwild/src/symbol_db.rs b/libwild/src/symbol_db.rs
index d74e6503c..d33442c8f 100644
--- a/libwild/src/symbol_db.rs
+++ b/libwild/src/symbol_db.rs
@@ -3,7 +3,6 @@
 
 use crate::args::Args;
 use crate::args::OutputKind;
-use crate::debug_assert_bail;
 use crate::error::Result;
 use crate::grouping::Group;
 use crate::hash::PassThroughHashMap;
@@ -32,6 +31,7 @@ use object::LittleEndian;
 use object::read::elf::Sym as _;
 use rayon::iter::IndexedParallelIterator;
 use rayon::iter::IntoParallelRefIterator;
+use rayon::iter::IntoParallelRefMutIterator;
 use rayon::iter::ParallelIterator;
 use std::collections::hash_map;
 use std::fmt::Display;
@@ -43,12 +43,7 @@ pub struct SymbolDb<'data> {
 
     pub(crate) groups: &'data [Group<'data>],
 
-    /// Mapping from global symbol names to a symbol ID with that name. If there are multiple
-    /// globals with the same name, then this will point to the one we encountered first, which may
-    /// not be the selected definition. In order to find the selected definition, you still need to
-    /// look a `symbol_definitions`.
-    name_to_id: PassThroughHashMap<UnversionedSymbolName<'data>, SymbolId>,
-    versioned_name_to_id: PassThroughHashMap<VersionedSymbolName<'data>, SymbolId>,
+    buckets: Vec<SymbolBucket<'data>>,
 
     /// Which file each symbol ID belongs to. Indexes past the end are assumed to be for custom
     /// section start/stop symbols.
@@ -61,24 +56,6 @@ pub struct SymbolDb<'data> {
 
     symbol_value_flags: Vec<ValueFlags>,
 
-    /// Global symbols that have multiple definitions. Indexed by symbol ID. It'd be nice if we
-    /// didn't need to store this and could just determine the canonical definition as we add
-    /// symbols. Unfortunately archive semantics make that impossible because we don't yet know
-    /// which archive entries will and won't be loaded. For the first symbol with a name, this
-    /// points to the last symbol with that name. That in turn then points to each previous
-    /// alternate definition with that name until the undefined symbol is reached.
-    alternative_definitions: Vec<SymbolId>,
-
-    /// Alternative definitions, but only for versioned symbols. This might be more efficient with a
-    /// proper multi-map that doesn't need a separate Vec for each value, however we don't expect
-    /// many entries here.
-    alternative_versioned_definitions: HashMap<SymbolId, Vec<SymbolId>>,
-
-    /// The symbol IDs of the first symbol with each name for which there are alternative
-    /// definitions. This can be used to find the head of a linked list in
-    /// `alternative_definitions`.
-    symbols_with_alternatives: Vec<SymbolId>,
-
     /// The number of symbols in each group, keyed by the index of the group.
     pub(crate) num_symbols_per_group: Vec<usize>,
 
@@ -89,17 +66,35 @@ pub struct SymbolDb<'data> {
     start_stop_symbol_names: Vec<UnversionedSymbolName<'data>>,
 }
 
+struct SymbolBucket<'data> {
+    /// Mapping from global symbol names to a symbol ID with that name. If there are multiple
+    /// globals with the same name, then this will point to the one we encountered first, which may
+    /// not be the selected definition. In order to find the selected definition, you still need to
+    /// look a `symbol_definitions`.
+    name_to_id: PassThroughHashMap<UnversionedSymbolName<'data>, SymbolId>,
+
+    versioned_name_to_id: PassThroughHashMap<VersionedSymbolName<'data>, SymbolId>,
+
+    /// Global symbols that have multiple definitions keyed by the first symbol with that name.
+    alternative_definitions: HashMap<SymbolId, Vec<SymbolId>>,
+
+    /// Alternative definitions, but only for versioned symbols. This might be more efficient with a
+    /// proper multi-map that doesn't need a separate Vec for each value, however we don't expect
+    /// many entries here.
+    alternative_versioned_definitions: HashMap<SymbolId, Vec<SymbolId>>,
+}
+
 /// A global symbol that hasn't been put into our database yet.
 #[derive(Clone, Copy)]
-pub(crate) struct PendingSymbol<'data> {
-    pub(crate) symbol_id: SymbolId,
-    pub(crate) name: PreHashed<UnversionedSymbolName<'data>>,
+struct PendingSymbol<'data> {
+    symbol_id: SymbolId,
+    name: PreHashed<UnversionedSymbolName<'data>>,
 }
 
 #[derive(Clone, Copy)]
-pub(crate) struct PendingVersionedSymbol<'data> {
-    pub(crate) symbol_id: SymbolId,
-    pub(crate) name: PreHashed<VersionedSymbolName<'data>>,
+struct PendingVersionedSymbol<'data> {
+    symbol_id: SymbolId,
+    name: PreHashed<VersionedSymbolName<'data>>,
 }
 
 /// An ID for a symbol. All symbols from all input files are allocated a unique symbol ID. The
@@ -227,10 +222,16 @@ impl Iterator for SymbolIdRangeIterator {
     }
 }
 
-#[derive(Default)]
 struct SymbolLoadOutputs<'data> {
-    pending_symbols: Vec<PendingSymbol<'data>>,
-    pending_versioned_symbols: Vec<PendingVersionedSymbol<'data>>,
+    /// Pending non-versioned symbols, grouped by hash bucket.
+    pending_symbols_by_bucket: Vec<PendingSymbolHashBucket<'data>>,
+}
+
+#[derive(Default, Clone)]
+struct PendingSymbolHashBucket<'data> {
+    symbols: Vec<PendingSymbol<'data>>,
+
+    versioned_symbols: Vec<PendingVersionedSymbol<'data>>,
 }
 
 impl<'data> SymbolDb<'data> {
@@ -275,7 +276,8 @@ impl<'data> SymbolDb<'data> {
             })
             .collect_vec();
 
-        let symbol_per_file = read_symbols(groups, &version_script, &mut per_group_writers, args)?;
+        let per_group_outputs =
+            read_symbols(groups, &version_script, &mut per_group_writers, args)?;
 
         for writer in per_group_writers {
             symbol_definitions_writer.return_shard(writer.resolutions);
@@ -284,13 +286,19 @@ impl<'data> SymbolDb<'data> {
         }
 
         let epilogue_file_id = groups.last().unwrap().files.last().unwrap().file_id();
-        let mut index = SymbolDb {
-            args,
+
+        let num_buckets = num_symbol_hash_buckets(args);
+        let mut buckets = Vec::new();
+        buckets.resize_with(num_buckets, || SymbolBucket {
             name_to_id: Default::default(),
             versioned_name_to_id: Default::default(),
-            alternative_definitions: vec![SymbolId::undefined(); num_symbols],
+            alternative_definitions: HashMap::with_hasher(RandomState::new()),
             alternative_versioned_definitions: HashMap::with_hasher(RandomState::new()),
-            symbols_with_alternatives: Vec::new(),
+        });
+
+        let mut index = SymbolDb {
+            args,
+            buckets,
             epilogue_file_id,
             symbol_file_ids,
             symbol_definitions,
@@ -299,94 +307,62 @@ impl<'data> SymbolDb<'data> {
             start_stop_symbol_names: Default::default(),
             symbol_value_flags,
         };
-        index.populate_symbol_db(symbol_per_file)?;
+
+        index.populate_symbol_db(&per_group_outputs)?;
+
         Ok(index)
     }
 
     #[tracing::instrument(skip_all, name = "Populate symbol map")]
-    fn populate_symbol_db(&mut self, symbol_per_file: Vec<SymbolLoadOutputs<'data>>) -> Result {
-        // The following approximation should be an upper bound on the number of global names we'll
-        // have. There will likely be at least a few global symbols with the same name, in which
-        // case the actual number will be slightly smaller.
-        let approx_num_symbols = symbol_per_file
-            .iter()
-            .map(|s| s.pending_symbols.len())
-            .sum();
-        self.name_to_id.reserve(approx_num_symbols);
-
-        for pending in symbol_per_file {
-            for symbol in pending.pending_symbols {
-                self.add_symbol(symbol);
-            }
+    fn populate_symbol_db(&mut self, per_group_outputs: &[SymbolLoadOutputs<'data>]) -> Result {
+        self.buckets
+            .par_iter_mut()
+            .enumerate()
+            .for_each(|(b, bucket)| {
+                // The following approximation should be an upper bound on the number of global
+                // names we'll have. There will likely be at least a few global symbols with the
+                // same name, in which case the actual number will be slightly smaller.
+                let approx_num_symbols = per_group_outputs
+                    .iter()
+                    .map(|s| s.pending_symbols_by_bucket[b].symbols.len())
+                    .sum();
+                bucket.name_to_id.reserve(approx_num_symbols);
+
+                for outputs in per_group_outputs {
+                    let pending = &outputs.pending_symbols_by_bucket[b];
+
+                    for symbol in &pending.symbols {
+                        bucket.add_symbol(symbol);
+                    }
 
-            for symbol in pending.pending_versioned_symbols {
-                self.add_versioned_symbol(symbol);
-            }
-        }
+                    for symbol in &pending.versioned_symbols {
+                        bucket.add_versioned_symbol(symbol);
+                    }
+                }
+            });
 
         Ok(())
     }
 
-    fn add_symbol(&mut self, pending: PendingSymbol<'data>) {
-        match self.name_to_id.entry(pending.name) {
-            hash_map::Entry::Occupied(entry) => {
-                let first_symbol_id = *entry.get();
-                self.add_extra_symbol_definition(first_symbol_id, pending.symbol_id);
-            }
-            hash_map::Entry::Vacant(entry) => {
-                entry.insert(pending.symbol_id);
-            }
-        }
-    }
-
-    fn add_versioned_symbol(&mut self, pending: PendingVersionedSymbol<'data>) {
-        match self.versioned_name_to_id.entry(pending.name) {
-            hash_map::Entry::Occupied(entry) => {
-                let first_symbol_id = *entry.get();
-                self.alternative_versioned_definitions
-                    .entry(first_symbol_id)
-                    .or_default()
-                    .push(pending.symbol_id);
-            }
-            hash_map::Entry::Vacant(entry) => {
-                entry.insert(pending.symbol_id);
-            }
-        }
-    }
-
-    fn add_extra_symbol_definition(&mut self, first_symbol_id: SymbolId, new_symbol_id: SymbolId) {
-        // Update the entry at `first_symbol_id` to point to the new last symbol (the
-        // pending symbol).
-        let previous_last = replace(
-            &mut self.alternative_definitions[first_symbol_id.as_usize()],
-            new_symbol_id,
-        );
-
-        // Our pending symbol is now last. Update its entry to point to the previous last
-        // symbol.
-        self.alternative_definitions[new_symbol_id.as_usize()] = previous_last;
-
-        if previous_last.is_undefined() {
-            // This is the first alternative definition for this name, note the first symbol
-            // ID for later when we resolve alternatives.
-            self.symbols_with_alternatives.push(first_symbol_id);
-        }
-    }
-
     pub(crate) fn add_start_stop_symbol(
         &mut self,
         symbol_name: PreHashed<UnversionedSymbolName<'data>>,
     ) -> SymbolId {
         let symbol_id = SymbolId::from_usize(self.symbol_definitions.len());
-        self.add_symbol(PendingSymbol {
+
+        let num_buckets = self.buckets.len();
+        self.buckets[symbol_name.hash() as usize % num_buckets].add_symbol(&PendingSymbol {
             symbol_id,
             name: symbol_name,
         });
+
         self.symbol_definitions.push(symbol_id);
         self.start_stop_symbol_names.push(*symbol_name);
         self.num_symbols_per_group[self.epilogue_file_id.group()] += 1;
+
         self.symbol_value_flags
             .push(ValueFlags::ADDRESS | ValueFlags::CAN_BYPASS_GOT);
+
         symbol_id
     }
 
@@ -489,20 +465,33 @@ impl<'data> SymbolDb<'data> {
         &self,
         prehashed: &PreHashed<UnversionedSymbolName>,
     ) -> Option<SymbolId> {
-        self.name_to_id.get(prehashed).copied()
+        let num_buckets = self.buckets.len();
+        self.buckets[prehashed.hash() as usize % num_buckets]
+            .name_to_id
+            .get(prehashed)
+            .copied()
     }
 
     pub(crate) fn get(&self, key: &PreHashedSymbolName) -> Option<SymbolId> {
+        let num_buckets = self.buckets.len();
+
         match key {
-            PreHashedSymbolName::Unversioned(key) => self.name_to_id.get(key).copied(),
-            PreHashedSymbolName::Versioned(key) => self.versioned_name_to_id.get(key).copied(),
+            PreHashedSymbolName::Unversioned(key) => self.buckets
+                [key.hash() as usize % num_buckets]
+                .name_to_id
+                .get(key)
+                .copied(),
+            PreHashedSymbolName::Versioned(key) => self.buckets[key.hash() as usize % num_buckets]
+                .versioned_name_to_id
+                .get(key)
+                .copied(),
         }
     }
 
     pub(crate) fn all_unversioned_symbols(
         &self,
     ) -> impl Iterator<Item = (&PreHashed<UnversionedSymbolName>, &SymbolId)> {
-        self.name_to_id.iter()
+        self.buckets.iter().flat_map(|b| b.name_to_id.iter())
     }
 
     fn symbol_strength(&self, symbol_id: SymbolId, resolved: &[ResolvedGroup]) -> SymbolStrength {
@@ -527,6 +516,42 @@ impl<'data> SymbolDb<'data> {
     }
 }
 
+impl<'data> SymbolBucket<'data> {
+    fn add_symbol(&mut self, pending: &PendingSymbol<'data>) {
+        match self.name_to_id.entry(pending.name) {
+            hash_map::Entry::Occupied(entry) => {
+                let first_symbol_id = *entry.get();
+                self.add_extra_symbol_definition(first_symbol_id, pending.symbol_id);
+            }
+            hash_map::Entry::Vacant(entry) => {
+                entry.insert(pending.symbol_id);
+            }
+        }
+    }
+
+    fn add_versioned_symbol(&mut self, pending: &PendingVersionedSymbol<'data>) {
+        match self.versioned_name_to_id.entry(pending.name) {
+            hash_map::Entry::Occupied(entry) => {
+                let first_symbol_id = *entry.get();
+                self.alternative_versioned_definitions
+                    .entry(first_symbol_id)
+                    .or_default()
+                    .push(pending.symbol_id);
+            }
+            hash_map::Entry::Vacant(entry) => {
+                entry.insert(pending.symbol_id);
+            }
+        }
+    }
+
+    fn add_extra_symbol_definition(&mut self, first_symbol_id: SymbolId, new_symbol_id: SymbolId) {
+        self.alternative_definitions
+            .entry(first_symbol_id)
+            .or_default()
+            .push(new_symbol_id);
+    }
+}
+
 /// For each symbol that has multiple definitions, some of which may be weak, some strong, some
 /// "common" symbols and some in archive entries that weren't loaded, resolve which version of the
 /// symbol we're using. The symbol we select will be the first strongly defined symbol in a loaded
@@ -537,61 +562,51 @@ pub(crate) fn resolve_alternative_symbol_definitions<'data>(
     symbol_db: &mut SymbolDb<'data>,
     resolved: &[ResolvedGroup],
 ) -> Result {
-    // For now, we do this from a single thread since we don't expect a lot of symbols will have
-    // multiple definitions. If it turns out that there are cases where it's actually taking
-    // significant time, then we could parallelise this without too much work.
-    let previous_definitions = take(&mut symbol_db.alternative_definitions);
-    let symbols_with_alternatives = take(&mut symbol_db.symbols_with_alternatives);
-    let mut alternatives = Vec::new();
-
-    for first in symbols_with_alternatives {
-        alternatives.clear();
-        let mut symbol_id = first;
-
-        loop {
-            symbol_id = previous_definitions[symbol_id.as_usize()];
-            if symbol_id.is_undefined() {
-                break;
-            }
+    let mut buckets = take(&mut symbol_db.buckets);
 
-            debug_assert_bail!(
-                !alternatives.contains(&symbol_id),
-                "Loop in symbol replacements for symbol {symbol_id}: {alternatives:?}"
-            );
+    let replacements: Vec<Vec<(SymbolId, SymbolId)>> = buckets
+        .par_iter_mut()
+        .map(|bucket| {
+            let mut replacements = Vec::new();
 
-            alternatives.push(symbol_id);
-        }
+            let mut add_replacement = |a, b| {
+                replacements.push((a, b));
+            };
 
-        let selected = select_symbol(symbol_db, first, &alternatives, resolved);
-        symbol_db.replace_definition(first, selected);
+            for (first, alternatives) in replace(
+                &mut bucket.alternative_definitions,
+                HashMap::with_hasher(RandomState::new()),
+            ) {
+                let selected = select_symbol(symbol_db, first, &alternatives, resolved);
+                add_replacement(first, selected);
 
-        for &alt in &alternatives {
-            symbol_db.replace_definition(alt, selected);
-        }
-    }
+                for alt in alternatives {
+                    add_replacement(alt, selected);
+                }
+            }
 
-    Ok(())
-}
+            for (first, alternatives) in replace(
+                &mut bucket.alternative_versioned_definitions,
+                HashMap::with_hasher(RandomState::new()),
+            ) {
+                let selected = select_symbol(symbol_db, first, &alternatives, resolved);
+                add_replacement(first, selected);
 
-#[tracing::instrument(skip_all, name = "Resolve alternative versioned symbol definitions")]
-pub(crate) fn resolve_alternative_versioned_symbol_definitions<'data>(
-    symbol_db: &mut SymbolDb<'data>,
-    resolved: &[ResolvedGroup],
-) -> Result {
-    let all_alternatives = replace(
-        &mut symbol_db.alternative_versioned_definitions,
-        HashMap::with_hasher(RandomState::new()),
-    );
+                for alt in alternatives {
+                    add_replacement(alt, selected);
+                }
+            }
 
-    for (first, alternatives) in all_alternatives {
-        let selected = select_symbol(symbol_db, first, &alternatives, resolved);
-        symbol_db.replace_definition(first, selected);
+            replacements
+        })
+        .collect();
 
-        for &alt in &alternatives {
-            symbol_db.replace_definition(alt, selected);
-        }
+    for (a, b) in replacements.iter().flat_map(|r| r.iter()) {
+        symbol_db.replace_definition(*a, *b);
     }
 
+    symbol_db.buckets = buckets;
+
     Ok(())
 }
 
@@ -673,11 +688,15 @@ fn read_symbols<'data, 'out>(
     symbols_out_by_file: &mut [SymbolInfoWriter],
     args: &Args,
 ) -> Result<Vec<SymbolLoadOutputs<'data>>> {
+    let num_buckets = num_symbol_hash_buckets(args);
+
     groups
         .par_iter()
         .zip(symbols_out_by_file)
         .map(|(group, symbols_out)| {
-            let mut outputs = SymbolLoadOutputs::default();
+            let mut outputs = SymbolLoadOutputs {
+                pending_symbols_by_bucket: vec![PendingSymbolHashBucket::default(); num_buckets],
+            };
 
             for file in &group.files {
                 let filename = file.filename();
@@ -831,12 +850,12 @@ trait SymbolLoader<'data> {
 
             if info.is_default {
                 let pending = PendingSymbol::from_prehashed(symbol_id, name);
-                outputs.pending_symbols.push(pending);
+                outputs.add_non_versioned(pending);
             }
 
             if let Some(version) = info.version_name {
                 let pending = PendingVersionedSymbol::from_prehashed(symbol_id, name, version);
-                outputs.pending_versioned_symbols.push(pending);
+                outputs.add_versioned(pending);
             }
 
             symbols_out.set_next(value_flags, resolution, file_id);
@@ -1124,9 +1143,6 @@ impl Prelude {
         outputs: &mut SymbolLoadOutputs,
         output_kind: OutputKind,
     ) {
-        outputs
-            .pending_symbols
-            .reserve(self.symbol_definitions.len());
         for definition in &self.symbol_definitions {
             let symbol_id = symbols_out.next;
             let value_flags = match definition {
@@ -1134,17 +1150,13 @@ impl Prelude {
                 InternalSymDefInfo::SectionStart(section_id) => {
                     let def = section_id.built_in_details();
                     let name = def.start_symbol_name(output_kind).unwrap().as_bytes();
-                    outputs
-                        .pending_symbols
-                        .push(PendingSymbol::new(symbol_id, name));
+                    outputs.add_non_versioned(PendingSymbol::new(symbol_id, name));
                     ValueFlags::ADDRESS | ValueFlags::CAN_BYPASS_GOT
                 }
                 InternalSymDefInfo::SectionEnd(section_id) => {
                     let def = section_id.built_in_details();
                     let name = def.end_symbol_name(output_kind).unwrap().as_bytes();
-                    outputs
-                        .pending_symbols
-                        .push(PendingSymbol::new(symbol_id, name));
+                    outputs.add_non_versioned(PendingSymbol::new(symbol_id, name));
                     ValueFlags::ADDRESS | ValueFlags::CAN_BYPASS_GOT
                 }
             };
@@ -1195,6 +1207,29 @@ impl<'data> PendingVersionedSymbol<'data> {
     }
 }
 
+/// Decides how many buckets we should use for symbol names.
+fn num_symbol_hash_buckets(args: &Args) -> usize {
+    args.num_threads.get()
+}
+
+impl<'data> SymbolLoadOutputs<'data> {
+    fn add_non_versioned(&mut self, pending: PendingSymbol<'data>) {
+        let num_buckets = self.pending_symbols_by_bucket.len();
+
+        self.pending_symbols_by_bucket[pending.name.hash() as usize % num_buckets]
+            .symbols
+            .push(pending);
+    }
+
+    fn add_versioned(&mut self, pending: PendingVersionedSymbol<'data>) {
+        let num_buckets = self.pending_symbols_by_bucket.len();
+
+        self.pending_symbols_by_bucket[pending.name.hash() as usize % num_buckets]
+            .versioned_symbols
+            .push(pending);
+    }
+}
+
 impl ShardKey for SymbolId {
     fn zero() -> Self {
         SymbolId(0)


Reviews:
user: login: mati865
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-02T20:21:32Z
user: login: davidlattimore
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T00:41:37Z
user: login: mati865
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-04T02:32:41Z

Review comments:
diff_hunk: @@ -48,6 +48,7 @@ uuid = { version = "1.13.1", features = ["v4"] }
 hex = "0.4.3"
 atomic-take = "1.1.0"
 normalize-path = "0.2.1"
+rustc-hash = "2.1.1"
path: libwild/Cargo.toml
user: login: mati865
body: We already have ahash which should be faster (at least with AES CPU feature enabled). Any reason to bring another hashing crate?
created_at: 2025-03-02T20:21:32Z
author_association: MEMBER
diff_hunk: @@ -48,6 +48,7 @@ uuid = { version = "1.13.1", features = ["v4"] }
 hex = "0.4.3"
 atomic-take = "1.1.0"
 normalize-path = "0.2.1"
+rustc-hash = "2.1.1"
path: libwild/Cargo.toml
user: login: davidlattimore
body: Good point. I used it because I had read somewhere that FxHash has better for primitive types (e.g. u32 in this case), whereas ahash was better for things like `&[u8]`. However, I just ran a benchmark (of the whole linker, not a microbenchmark) and if there's a difference, it's lost in the noise. So I've switch to ahash.
created_at: 2025-03-03T00:41:37Z
author_association: MEMBER

PR comments:
user: login: davidlattimore
created_at: 2025-03-03T00:46:33Z
body: This is a typical benchmark result for this change:

```
❯ OUT=/home/david/ttt throttle-count  poop '/home/david/tmp/clang-link-debug/0/run-with /home/david/tmp/wild-builds/2025-03-03 --no-fork --strip-debug' '/home/david/tmp/clang-link-debug/0/run-with target/release/wild --no-fork --strip-debug'
temp: +55.0°C
Benchmark 1 (20 runs): /home/david/tmp/clang-link-debug/0/run-with /home/david/tmp/wild-builds/2025-03-03 --no-fork --strip-debug
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           252ms ± 6.64ms     246ms …  276ms          1 ( 5%)        0%
  peak_rss           1.08GB ±  917KB    1.08GB … 1.08GB          0 ( 0%)        0%
  cpu_cycles         3.34G  ± 15.4M     3.31G  … 3.38G           0 ( 0%)        0%
  instructions       3.73G  ±  709K     3.73G  … 3.73G           0 ( 0%)        0%
  cache_references   66.5M  ±  223K     66.1M  … 66.8M           0 ( 0%)        0%
  cache_misses       32.8M  ± 95.1K     32.6M  … 33.0M           1 ( 5%)        0%
  branch_misses      7.14M  ±  117K     7.05M  … 7.39M           1 ( 5%)        0%
Benchmark 2 (21 runs): /home/david/tmp/clang-link-debug/0/run-with target/release/wild --no-fork --strip-debug
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           241ms ± 3.51ms     237ms …  251ms          1 ( 5%)        ⚡-  4.3% ±  1.3%
  peak_rss           1.08GB ± 1.89MB    1.08GB … 1.09GB          1 ( 5%)          +  0.3% ±  0.1%
  cpu_cycles         3.48G  ± 23.0M     3.41G  … 3.51G           1 ( 5%)        💩+  4.1% ±  0.4%
  instructions       3.75G  ± 1.01M     3.75G  … 3.75G           1 ( 5%)          +  0.6% ±  0.0%
  cache_references   68.9M  ±  219K     68.4M  … 69.2M           0 ( 0%)        💩+  3.5% ±  0.2%
  cache_misses       33.2M  ±  126K     33.0M  … 33.4M           0 ( 0%)        💩+  1.3% ±  0.2%
  branch_misses      7.12M  ± 30.6K     7.08M  … 7.20M           2 (10%)          -  0.4% ±  0.8%
Throttle pkg: 0 core: 0 ms: 0 temp: +67.0°C
```

I'm curious as to what others think as to whether this is a good idea or not. It does speed things up slightly, but it also increases CPU cycles and cache references. I guess the single-threaded wall-time doesn't seem to be hurt, so that's at least something:

```
❯ OUT=/home/david/ttt throttle-count  poop '/home/david/tmp/clang-link-debug/0/run-with /home/david/tmp/wild-builds/2025-03-03 --no-fork --strip-debug --threads=1' '/home/david/tmp/clang-link-debug/0/run-with target/release/wild --no-fork --strip-debug --threads=1'
temp: +47.0°C
Benchmark 1 (6 runs): /home/david/tmp/clang-link-debug/0/run-with /home/david/tmp/wild-builds/2025-03-03 --no-fork --strip-debug --threads=1
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           849ms ± 6.62ms     839ms …  858ms          0 ( 0%)        0%
  peak_rss           1.07GB ±  138KB    1.07GB … 1.07GB          0 ( 0%)        0%
  cpu_cycles         2.10G  ± 18.3M     2.07G  … 2.12G           0 ( 0%)        0%
  instructions       3.70G  ± 8.77K     3.70G  … 3.70G           0 ( 0%)        0%
  cache_references   54.8M  ±  472K     54.3M  … 55.5M           0 ( 0%)        0%
  cache_misses       29.6M  ±  128K     29.4M  … 29.8M           0 ( 0%)        0%
  branch_misses      6.67M  ±  255K     6.55M  … 7.19M           0 ( 0%)        0%
Benchmark 2 (6 runs): /home/david/tmp/clang-link-debug/0/run-with target/release/wild --no-fork --strip-debug --threads=1
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           851ms ± 13.9ms     837ms …  874ms          0 ( 0%)          +  0.2% ±  1.7%
  peak_rss           1.07GB ±  183KB    1.07GB … 1.07GB          0 ( 0%)          -  0.0% ±  0.0%
  cpu_cycles         2.13G  ± 30.2M     2.10G  … 2.17G           0 ( 0%)          +  1.5% ±  1.5%
  instructions       3.71G  ± 3.93K     3.71G  … 3.71G           1 (17%)          +  0.4% ±  0.0%
  cache_references   57.5M  ±  151K     57.4M  … 57.8M           0 ( 0%)        💩+  4.9% ±  0.8%
  cache_misses       29.8M  ± 75.3K     29.7M  … 29.9M           0 ( 0%)          +  0.7% ±  0.5%
  branch_misses      6.60M  ± 8.08K     6.59M  … 6.61M           0 ( 0%)          -  1.0% ±  3.5%
Throttle pkg: 0 core: 0 ms: 0 temp: +49.0°C
```
author_association: MEMBER
user: login: mati865
created_at: 2025-03-03T18:09:42Z
body: On Clang without debug info this doesn't seem to have any effect:
```
# Edited the script to output to /dev/null
❯ rg '\-o ' -A1 run-with
25:  -o \
26-  /dev/null \

❯ powerprofilesctl launch -p performance poop './run-with /tmp/clang/wild-e2ba8074 --no-fork' './run-with /tmp/clang/wild-eec936cb --no-fork'
Benchmark 1 (28 runs): ./run-with /tmp/clang/wild-e2ba8074 --no-fork
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           181ms ± 3.76ms     173ms …  189ms          0 ( 0%)        0%
  peak_rss            828MB ± 9.39MB     813MB …  846MB          0 ( 0%)        0%
  cpu_cycles         7.82G  ±  164M     7.52G  … 8.36G           1 ( 4%)        0%
  instructions       5.05G  ± 13.1M     5.02G  … 5.08G           0 ( 0%)        0%
  cache_references   86.1M  ±  537K     85.2M  … 87.2M           0 ( 0%)        0%
  cache_misses       23.6M  ±  204K     23.3M  … 24.1M           0 ( 0%)        0%
  branch_misses      8.32M  ± 29.6K     8.26M  … 8.37M           0 ( 0%)        0%
Benchmark 2 (28 runs): ./run-with /tmp/clang/wild-eec936cb --no-fork
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           180ms ± 3.75ms     172ms …  187ms          1 ( 4%)          -  0.5% ±  1.1%
  peak_rss            829MB ± 10.2MB     816MB …  854MB          0 ( 0%)          +  0.2% ±  0.6%
  cpu_cycles         8.20G  ±  153M     7.96G  … 8.58G           0 ( 0%)        💩+  4.9% ±  1.1%
  instructions       5.12G  ± 17.9M     5.09G  … 5.17G           2 ( 7%)        💩+  1.5% ±  0.2%
  cache_references   89.0M  ±  568K     87.9M  … 90.6M           1 ( 4%)        💩+  3.4% ±  0.3%
  cache_misses       25.2M  ±  168K     24.7M  … 25.7M           1 ( 4%)        💩+  6.4% ±  0.4%
  branch_misses      8.90M  ± 36.1K     8.84M  … 8.99M           2 ( 7%)        💩+  6.9% ±  0.2%
```
Unless I cap used threads:
```
❯ powerprofilesctl launch -p performance poop './run-with /tmp/clang/wild-e2ba8074 --no-fork --threads=16' './run-with /tmp/clang/wild-eec936cb --no-fork --threads=16'
Benchmark 1 (34 runs): ./run-with /tmp/clang/wild-e2ba8074 --no-fork --threads=16
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           149ms ± 2.25ms     145ms …  153ms          0 ( 0%)        0%
  peak_rss            806MB ± 4.54MB     801MB …  823MB          3 ( 9%)        0%
  cpu_cycles         4.11G  ± 70.1M     3.97G  … 4.26G           0 ( 0%)        0%
  instructions       4.83G  ± 15.2M     4.81G  … 4.89G           1 ( 3%)        0%
  cache_references   76.8M  ±  804K     75.5M  … 78.8M           0 ( 0%)        0%
  cache_misses       19.2M  ±  165K     18.8M  … 19.5M           0 ( 0%)        0%
  branch_misses      7.56M  ± 22.6K     7.52M  … 7.61M           0 ( 0%)        0%
Benchmark 2 (35 runs): ./run-with /tmp/clang/wild-eec936cb --no-fork --threads=16
  measurement          mean ± σ            min … max           outliers         delta
  wall_time           144ms ± 2.05ms     140ms …  148ms          0 ( 0%)        ⚡-  3.6% ±  0.7%
  peak_rss            807MB ± 2.98MB     803MB …  813MB          0 ( 0%)          +  0.1% ±  0.2%
  cpu_cycles         4.17G  ± 68.7M     4.02G  … 4.30G           0 ( 0%)          +  1.3% ±  0.8%
  instructions       4.86G  ± 11.3M     4.84G  … 4.88G           0 ( 0%)          +  0.6% ±  0.1%
  cache_references   78.4M  ±  583K     77.3M  … 79.9M           0 ( 0%)        💩+  2.1% ±  0.4%
  cache_misses       20.0M  ±  187K     19.6M  … 20.4M           0 ( 0%)        💩+  4.2% ±  0.4%
  branch_misses      7.89M  ± 65.5K     7.85M  … 8.25M           1 ( 3%)        💩+  4.3% ±  0.3%
```
My `/tmp` is on tmpfs.

I have tried with 4, 8, 16 and 32 threads on 16C/32T CPU and apparently for this benchmark caping used threads to the count of "real" cores is beneficial (16 threads were fastest by a small margin).
I'll look into the code later, but since there is 0-4% improvement, if the code doesn't get much worse this would be fine.
Personally I'm not concerned about the single thread, it often behaves more or less differently compared to more realistic scenarios (even CI these days starts at 2-4 vCPU).
author_association: MEMBER
user: login: davidlattimore
created_at: 2025-03-04T01:21:44Z
body: > Edited the script to output to /dev/null

Note, you can also run the script with `OUT=/dev/null`. Outputting to /dev/null will mean that we buffer the whole file in memory, then write it all to `/dev/null` at the end, so the performance might be different than writing to a tmpfs. My home directory is a tmpfs, which is why you'll often see in my commands that I set `OUT=/home/david/ttt`.

> I have tried with 4, 8, 16 and 32 threads on 16C/32T CPU and apparently for this benchmark caping used threads to the count of "real" cores is beneficial (16 threads were fastest by a small margin).

Wow, so it not only doesn't get faster going from 16 to 32 threads, but actually gets slower. That's really interesting. My laptop has 4 cores (8 threads) and I see faster runtimes when utilising all 8 threads.

> I'll look into the code later, but since there is 0-4% improvement, if the code doesn't get much worse this would be fine.
> Personally I'm not concerned about the single thread, it often behaves more or less differently compared to more realistic scenarios (even CI these days starts at 2-4 vCPU).

Thanks for looking into this. I'm leaning towards merging the change. If you're going to look at the code though, I'll wait for you to do that.

One other thing that occurred to me is that if we're building a hashmap per thread, prehashing doesn't really matter, so we could hash just prior to insertion into the hashmap, which might mean that we don't need to store the hashes.
author_association: MEMBER
user: login: mati865
created_at: 2025-03-04T02:32:23Z
body: > Note, you can also run the script with `OUT=/dev/null`.

Oh, neat. I thought it'd try to create /dev/null/bin, but should have tried.

> Outputting to /dev/null will mean that we buffer the whole file in memory, then write it all to `/dev/null` at the end, so the performance might be different than writing to a tmpfs.

I've found tmpfs and /dev/null to give comparable results, but /dev/null had less variation run to run. I can stick to tmpfs.

> Wow, so it not only doesn't get faster going from 16 to 32 threads, but actually gets slower. That's really interesting. My laptop has 4 cores (8 threads) and I see faster runtimes when utilising all 8 threads.

Yeah, with the insane number of symbols LLVM creates the buckets surely aren't too small for the job, so intuitively this should scale. Unless there is some bottleneck urlnrelated to CPU core count. That would require some profiling but is not a blocker.

> Thanks for looking into this. I'm leaning towards merging the change. If you're going to look at the code though, I'll wait for you to do that.

Even though I couldn't get to all the details here, I think this is good to go. The code is a bit more verbose which is expected here, but surely not "worse". Great job!

> One other thing that occurred to me is that if we're building a hashmap per thread, prehashing doesn't really matter, so we could hash just prior to insertion into the hashmap, which might mean that we don't need to store the hashes.

Hmm, that could help a bit with caches but I've found the hashmaps to be rather cache and predictor unfriendly.
author_association: MEMBER
user: login: marxin
created_at: 2025-03-04T06:35:35Z
body: I can measure an improvement if debug info is committed for Clang-21 binary:

```
Benchmark 1: Wild(after)
  Time (mean ± σ):     146.5 ms ±   3.2 ms    [User: 3.4 ms, System: 4.0 ms]
  Range (min … max):   137.2 ms … 151.8 ms    21 runs
 
Benchmark 2: Wild(before)
  Time (mean ± σ):     153.4 ms ±   4.0 ms    [User: 3.0 ms, System: 4.2 ms]
  Range (min … max):   142.9 ms … 160.4 ms    20 runs
 
Summary
  Wild(after) ran
    1.05 ± 0.04 times faster than Wild(before)
```

while the perf remains the same if I keep the debug info:

```
❯ hyperfine --warmup 3 './cmd -B ~/Programming/wild' './cmd -B /tmp/' -n "Wild(after)" -n "Wild(before)"
Benchmark 1: Wild(after)
  Time (mean ± σ):      3.904 s ±  0.080 s    [User: 0.003 s, System: 0.004 s]
  Range (min … max):    3.750 s …  4.058 s    10 runs
 
Benchmark 2: Wild(before)
  Time (mean ± σ):      3.919 s ±  0.068 s    [User: 0.002 s, System: 0.005 s]
  Range (min … max):    3.840 s …  4.071 s    10 runs
 
Summary
  Wild(after) ran
    1.00 ± 0.03 times faster than Wild(before)
```


author_association: COLLABORATOR
user: login: davidlattimore
created_at: 2025-03-04T06:46:08Z
body: That makes sense. With so much debug info, symbol processing becomes pretty much background noise. Thank you both for checking.

I realised that my thought of delaying computation of the hash can't work since we need the hash in order to know what bucket to put the symbol in. But I think there might still be some room for improvement with symbol processing. I particular the hashmap seems wasteful. I'm considering switching to hashbrown's HashTable, with hashes stored in an external Vec that's indexed by SymbolId and symbol names stored either in the same Vec, or a parallel one depending on which performs best. That way the HashTable only stores the SymbolIds.
author_association: MEMBER
user: login: mati865
created_at: 2025-03-15T15:47:53Z
body: > Wow, so it not only doesn't get faster going from 16 to 32 threads, but actually gets slower. That's really interesting. My laptop has 4 cores (8 threads) and I see faster runtimes when utilising all 8 threads.

FWIW, mold has the same behaviour. Maybe this example is just too small?
![Figure_1](https://github.com/user-attachments/assets/8fa54a08-a48f-46af-b937-ef02efb0b6ab)
![Figure_2](https://github.com/user-attachments/assets/848723f3-3b89-4d53-b308-034f15ade0db)
Either way, I think it's not worth any investigation at this point.
author_association: MEMBER