Repository: huggingface/tokenizers (language: rust)

Issue:
Unsound use of unsafe in `src/utils/parallelism.rs`
The static variable `USED_PARALLELISM` is accessed in `has_parallelism_been_used`, and modified in `MaybeParallelIterator::into_maybe_par_iter` and `MaybeParallelBridge::maybe_par_bridge`.
All these cases are unsafe because if they are done at the same time from different threads, they could cause a data race which has undefined behaviour.
The issue is that all of these are safe functions, and no checks are made to ensure that the unsafe operations are in fact safe.
This means that it would be possible to cause UB in safe rust by calling these functions from separate threads.
There maybe reasons to believe that this is unlikely or impossible given the rest of the library (I don't know the code base well enough to say), but that does not change the fact that this is unsound.

The easiest way to fix this would be to place the variable in a `Mutex`. I propose an implementation of this in #1492 .


Discussion / hints:

Fix patch:
diff --git a/tokenizers/src/utils/parallelism.rs b/tokenizers/src/utils/parallelism.rs
index 2ac2eaa80..b955731d1 100644
--- a/tokenizers/src/utils/parallelism.rs
+++ b/tokenizers/src/utils/parallelism.rs
@@ -5,14 +5,15 @@
 use rayon::iter::IterBridge;
 use rayon::prelude::*;
 use rayon_cond::CondIterator;
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering;
 
 // Re-export rayon current_num_threads
 pub use rayon::current_num_threads;
 
 pub const ENV_VARIABLE: &str = "TOKENIZERS_PARALLELISM";
 
-// Reading/Writing this variable should always happen on the main thread
-static mut USED_PARALLELISM: bool = false;
+static USED_PARALLELISM: AtomicBool = AtomicBool::new(false);
 
 /// Check if the TOKENIZERS_PARALLELISM env variable has been explicitly set
 pub fn is_parallelism_configured() -> bool {
@@ -21,7 +22,7 @@ pub fn is_parallelism_configured() -> bool {
 
 /// Check if at some point we used a parallel iterator
 pub fn has_parallelism_been_used() -> bool {
-    unsafe { USED_PARALLELISM }
+    USED_PARALLELISM.load(Ordering::SeqCst)
 }
 
 /// Get the currently set value for `TOKENIZERS_PARALLELISM` env variable
@@ -70,7 +71,7 @@ where
     fn into_maybe_par_iter(self) -> CondIterator<P, S> {
         let parallelism = get_parallelism();
         if parallelism {
-            unsafe { USED_PARALLELISM = true };
+            USED_PARALLELISM.store(true, Ordering::SeqCst);
         }
         CondIterator::new(self, parallelism)
     }
@@ -159,7 +160,7 @@ where
         let iter = CondIterator::from_serial(self);
 
         if get_parallelism() {
-            unsafe { USED_PARALLELISM = true };
+            USED_PARALLELISM.store(true, Ordering::SeqCst);
             CondIterator::from_parallel(iter.into_parallel().right().unwrap())
         } else {
             iter


Test patch: