Repository: python/cpython
Pull request: gh-111916: Make hashlib related modules thread-safe without the GIL
Still a work in progress but I'd be happy for feedback ;)

I'm not entirely sure what to do about `HASHLIB_GIL_MINSIZE`. IIUC the original code avoids releasing the GIL and getting a separate lock when the update buffer is short enough. I'm guessing that in the nogil version, we'll have to lock the mutex regardless of the length (i.e. essentially getting rid of the macro)? Is that correct? 

As a side note, I'm failing to build this PR with the GIL enabled. Running `./configure --with-pydebug` and `make -s -j2`, I'm getting this error:

```c
In file included from ./Modules/hashlib.h:7,
                 from ./Modules/md5module.c:29:
./Include/internal/pycore_lock.h: In function ‘PyMutex_LockFast’:
./Include/internal/pycore_lock.h:60:12: error: implicit declaration of function ‘_Py_atomic_compare_exchange_uint8’; did you mean ‘__atomic_compare_exchange_n’? [-Werror=implicit-function-declaration]
   60 |     return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED);
```

Even if I add `#include "pyatomic.h"` to `pycore_lock.h` I get the same error.. any idea why that is?



<!-- gh-issue-number: gh-111916 -->
* Issue: gh-111916
<!-- /gh-issue-number -->

Fixes #111916

Problem statement:
Make hashlib related modules thread-safe without the GIL
# Feature or enhancement

The hashlib based modules already have some locking to make some operations thread-safe (with the GIL), but the logic isn't sufficient if running with the GIL disabled.

Relevant files:
* Modules/_blake2/blake2b_impl.c
* Modules/_blake2/blake2s_impl.c
* Modules/_hashopenssl.c
* Modules/hashlib.h
* Modules/md5module.c
* Modules/sha1module.c
* Modules/sha2module.c
* Modules/sha3module.c

Basic idea:
1. Replace `PyThread_type_lock lock` with `PyMutex`. This should be both simpler and faster in general and avoid the need for dynamically assigning a lock, which can pose thread-safety issues without the GIL
2. Add a field `bool use_mutex` to indicate if the code should lock the mutex. This should always be set to `true` in `Py_NOGIL`. In the default build, we should dynamically set it to `true` in places where we previously allocated `self->lock`
3. Update `ENTER_HASHLIB` and `EXIT_HASHLIB` macros.



<!-- gh-linked-prs -->
### Linked PRs
* gh-111981
<!-- /gh-linked-prs -->

Hints from issue discussion:
Hey! I'd like to give this a try. I have a limited experience with cpython internals but I had a quick look at the code and it seems feasible to me 😅
Great, thanks! Let me know if you have any questions or there is anything I can help with.

Commit message: WIP: Make hashlib thread-safe without the GIL

Patch to review:
diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c
index c2cac98c7529eb..a6a3a21086bb3e 100644
--- a/Modules/_blake2/blake2b_impl.c
+++ b/Modules/_blake2/blake2b_impl.c
@@ -17,6 +17,7 @@
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_strhex.h"       // _Py_strhex()
 
@@ -42,7 +43,8 @@ typedef struct {
     PyObject_HEAD
     blake2b_param    param;
     blake2b_state    state;
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
 } BLAKE2bObject;
 
 #include "clinic/blake2b_impl.c.h"
@@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type)
 {
     BLAKE2bObject *self;
     self = (BLAKE2bObject *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        self->lock = NULL;
+    if (self == NULL) {
+        return NULL;
     }
+    INIT_MUTEX(self);
+
     return self;
 }
 
@@ -278,18 +282,17 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data)
 
     GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
-        self->lock = PyThread_allocate_lock();
-
-    if (self->lock != NULL) {
-       Py_BEGIN_ALLOW_THREADS
-       PyThread_acquire_lock(self->lock, 1);
-       blake2b_update(&self->state, buf.buf, buf.len);
-       PyThread_release_lock(self->lock);
-       Py_END_ALLOW_THREADS
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
+        Py_BEGIN_ALLOW_THREADS
+        PyMutex_Lock(&self->mutex);
+        blake2b_update(&self->state, buf.buf, buf.len);
+        PyMutex_Unlock(&self->mutex);
+        Py_END_ALLOW_THREADS
     } else {
         blake2b_update(&self->state, buf.buf, buf.len);
     }
+
     PyBuffer_Release(&buf);
 
     Py_RETURN_NONE;
@@ -389,10 +392,6 @@ py_blake2b_dealloc(PyObject *self)
     /* Try not to leave state in memory. */
     secure_zero_memory(&obj->param, sizeof(obj->param));
     secure_zero_memory(&obj->state, sizeof(obj->state));
-    if (obj->lock) {
-        PyThread_free_lock(obj->lock);
-        obj->lock = NULL;
-    }
 
     PyTypeObject *type = Py_TYPE(self);
     PyObject_Free(self);
diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c
index 1c47328ece13e8..018daff2c242ce 100644
--- a/Modules/_blake2/blake2s_impl.c
+++ b/Modules/_blake2/blake2s_impl.c
@@ -17,6 +17,7 @@
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_strhex.h"       // _Py_strhex()
 
@@ -42,7 +43,8 @@ typedef struct {
     PyObject_HEAD
     blake2s_param    param;
     blake2s_state    state;
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
 } BLAKE2sObject;
 
 #include "clinic/blake2s_impl.c.h"
@@ -51,7 +53,7 @@ typedef struct {
 module _blake2
 class _blake2.blake2s "BLAKE2sObject *" "&PyBlake2_BLAKE2sType"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b79d7ffe07286ce]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d47b0527b39c673f]*/
 
 
 static BLAKE2sObject *
@@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type)
 {
     BLAKE2sObject *self;
     self = (BLAKE2sObject *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        self->lock = NULL;
+    if (self == NULL) {
+        return NULL;
     }
+    INIT_MUTEX(self);
+
     return self;
 }
 
@@ -93,7 +97,7 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size,
                     int fanout, int depth, unsigned long leaf_size,
                     unsigned long long node_offset, int node_depth,
                     int inner_size, int last_node, int usedforsecurity)
-/*[clinic end generated code: output=556181f73905c686 input=4dda87723f23abb0]*/
+/*[clinic end generated code: output=32bfd8f043c6896f input=b947312abff46977]*/
 {
     BLAKE2sObject *self = NULL;
     Py_buffer buf;
@@ -247,7 +251,7 @@ Return a copy of the hash object.
 
 static PyObject *
 _blake2_blake2s_copy_impl(BLAKE2sObject *self)
-/*[clinic end generated code: output=5b90131c4eae275e input=0b9d44942f0fe4b2]*/
+/*[clinic end generated code: output=ff6acee5f93656ae input=e383c2d199fd8a2e]*/
 {
     BLAKE2sObject *cpy;
 
@@ -272,24 +276,23 @@ Update this hash object's state with the provided bytes-like object.
 
 static PyObject *
 _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data)
-/*[clinic end generated code: output=757dc087fec37815 input=97500db2f9de4aaa]*/
+/*[clinic end generated code: output=010dfcbe22654359 input=ffc4aa6a6a225d31]*/
 {
     Py_buffer buf;
 
     GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
-        self->lock = PyThread_allocate_lock();
-
-    if (self->lock != NULL) {
-       Py_BEGIN_ALLOW_THREADS
-       PyThread_acquire_lock(self->lock, 1);
-       blake2s_update(&self->state, buf.buf, buf.len);
-       PyThread_release_lock(self->lock);
-       Py_END_ALLOW_THREADS
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
+        Py_BEGIN_ALLOW_THREADS
+        PyMutex_Lock(&self->mutex);
+        blake2s_update(&self->state, buf.buf, buf.len);
+        PyMutex_Unlock(&self->mutex);
+        Py_END_ALLOW_THREADS
     } else {
         blake2s_update(&self->state, buf.buf, buf.len);
     }
+
     PyBuffer_Release(&buf);
 
     Py_RETURN_NONE;
@@ -303,7 +306,7 @@ Return the digest value as a bytes object.
 
 static PyObject *
 _blake2_blake2s_digest_impl(BLAKE2sObject *self)
-/*[clinic end generated code: output=40c566ca4bc6bc51 input=f41e0b8d6d937454]*/
+/*[clinic end generated code: output=a5864660f4bfc61a input=7d21659e9c5fff02]*/
 {
     uint8_t digest[BLAKE2S_OUTBYTES];
     blake2s_state state_cpy;
@@ -324,7 +327,7 @@ Return the digest value as a string of hexadecimal digits.
 
 static PyObject *
 _blake2_blake2s_hexdigest_impl(BLAKE2sObject *self)
-/*[clinic end generated code: output=15153eb5e59c52eb input=c77a1321567e8952]*/
+/*[clinic end generated code: output=b5598a87d8794a60 input=76930f6946351f56]*/
 {
     uint8_t digest[BLAKE2S_OUTBYTES];
     blake2s_state state_cpy;
@@ -389,10 +392,6 @@ py_blake2s_dealloc(PyObject *self)
     /* Try not to leave state in memory. */
     secure_zero_memory(&obj->param, sizeof(obj->param));
     secure_zero_memory(&obj->state, sizeof(obj->state));
-    if (obj->lock) {
-        PyThread_free_lock(obj->lock);
-        obj->lock = NULL;
-    }
 
     PyTypeObject *type = Py_TYPE(self);
     PyObject_Free(self);
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index ee6fb8b4b03643..d92ff97b3b9c85 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -22,6 +22,7 @@
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_hashtable.h"
 #include "pycore_pyhash.h"        // _Py_HashBytes()
@@ -227,16 +228,16 @@ typedef struct {
     PyObject_HEAD
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock   lock;  /* OpenSSL context lock */
+    bool use_mutex;
+    PyMutex mutex;
 } EVPobject;
 
 typedef struct {
     PyObject_HEAD
     HMAC_CTX *ctx;            /* OpenSSL hmac context */
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock lock;  /* HMAC context lock */
+    bool use_mutex;
+    PyMutex mutex;
 } HMACobject;
 
 #include "clinic/_hashopenssl.c.h"
@@ -414,8 +415,7 @@ newEVPobject(PyTypeObject *type)
     if (retval == NULL) {
         return NULL;
     }
-
-    retval->lock = NULL;
+    INIT_MUTEX(retval);
 
     retval->ctx = EVP_MD_CTX_new();
     if (retval->ctx == NULL) {
@@ -453,8 +453,6 @@ static void
 EVP_dealloc(EVPobject *self)
 {
     PyTypeObject *tp = Py_TYPE(self);
-    if (self->lock != NULL)
-        PyThread_free_lock(self->lock);
     EVP_MD_CTX_free(self->ctx);
     PyObject_Free(self);
     Py_DECREF(tp);
@@ -582,16 +580,11 @@ EVP_update(EVPobject *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
 
-    if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-        /* fail? lock = NULL and we fail over to non-threaded code. */
-    }
-
-    if (self->lock != NULL) {
+    if (self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         result = EVP_hash(self, view.buf, view.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         result = EVP_hash(self, view.buf, view.len);
@@ -1540,7 +1533,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
     }
 
     self->ctx = ctx;
-    self->lock = NULL;
+    self->mutex = (PyMutex){0};
 
     if ((msg_obj != NULL) && (msg_obj != Py_None)) {
         if (!_hmac_update(self, msg_obj))
@@ -1582,16 +1575,12 @@ _hmac_update(HMACobject *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);
 
-    if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-        /* fail? lock = NULL and we fail over to non-threaded code. */
-    }
-
-    if (self->lock != NULL) {
+    self->use_mutex = true;
+    if (view.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
@@ -1633,7 +1622,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self)
         return NULL;
     }
     retval->ctx = ctx;
-    retval->lock = NULL;
+    retval->mutex = (PyMutex){0};
 
     return (PyObject *)retval;
 }
@@ -1642,9 +1631,6 @@ static void
 _hmac_dealloc(HMACobject *self)
 {
     PyTypeObject *tp = Py_TYPE(self);
-    if (self->lock != NULL) {
-        PyThread_free_lock(self->lock);
-    }
     HMAC_CTX_free(self->ctx);
     PyObject_Free(self);
     Py_DECREF(tp);
diff --git a/Modules/hashlib.h b/Modules/hashlib.h
index a8bad9dd87a939..5c0c33b1e5b68f 100644
--- a/Modules/hashlib.h
+++ b/Modules/hashlib.h
@@ -1,5 +1,11 @@
 /* Common code for use by all hashlib related modules. */
 
+#ifndef Py_BUILD_CORE
+#define Py_BUILD_CORE
+#endif
+
+#include "pycore_lock.h"        // PyMutex
+
 /*
  * Given a PyObject* obj, fill in the Py_buffer* viewp with the result
  * of PyObject_GetBuffer.  Sets an exception and issues the erraction
@@ -48,18 +54,30 @@
 
 #include "pythread.h"
 #define ENTER_HASHLIB(obj) \
-    if ((obj)->lock) { \
-        if (!PyThread_acquire_lock((obj)->lock, 0)) { \
-            Py_BEGIN_ALLOW_THREADS \
-            PyThread_acquire_lock((obj)->lock, 1); \
-            Py_END_ALLOW_THREADS \
-        } \
+    if ((obj)->use_mutex) { \
+        Py_BEGIN_ALLOW_THREADS \
+        PyMutex_Lock(&(obj)->mutex); \
+        Py_END_ALLOW_THREADS \
     }
 #define LEAVE_HASHLIB(obj) \
-    if ((obj)->lock) { \
-        PyThread_release_lock((obj)->lock); \
+    if ((obj)->use_mutex) { \
+        PyMutex_Unlock(&(obj)->mutex); \
     }
 
+#ifdef Py_NOGIL
+#define INIT_MUTEX(obj) \
+    do { \
+        (obj)->mutex = (PyMutex){0}; \
+        (obj)->use_mutex = true; \
+    } while (0)
+#else
+#define INIT_MUTEX(obj) \
+    do { \
+        (obj)->mutex = (PyMutex){0}; \
+        (obj)->use_mutex = false; \
+    } while (0)
+#endif
+
 /* TODO(gpshead): We should make this a module or class attribute
  * to allow the user to optimize based on the platform they're using. */
 #define HASHLIB_GIL_MINSIZE 2048
diff --git a/Modules/md5module.c b/Modules/md5module.c
index 7883a8cbe6e246..cbb7b6b1fc8bab 100644
--- a/Modules/md5module.c
+++ b/Modules/md5module.c
@@ -54,8 +54,8 @@ typedef long long MD5_INT64; /* 64-bit integer */
 typedef struct {
     PyObject_HEAD
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
     Hacl_Streaming_MD5_state *hash_state;
 } MD5object;
 
@@ -78,7 +78,11 @@ static MD5object *
 newMD5object(MD5State * st)
 {
     MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type);
-    md5->lock = NULL;
+    if(!md5) {
+        return NULL;
+    }
+    INIT_MUTEX(md5);
+
     PyObject_GC_Track(md5);
     return md5;
 }
@@ -95,9 +99,6 @@ static void
 MD5_dealloc(MD5object *ptr)
 {
     Hacl_Streaming_MD5_legacy_free(ptr->hash_state);
-    if (ptr->lock != NULL) {
-        PyThread_free_lock(ptr->lock);
-    }
     PyTypeObject *tp = Py_TYPE((PyObject*)ptr);
     PyObject_GC_UnTrack(ptr);
     PyObject_GC_Del(ptr);
@@ -202,14 +203,12 @@ MD5Type_update(MD5object *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         update(self->hash_state, buf.buf, buf.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         update(self->hash_state, buf.buf, buf.len);
diff --git a/Modules/sha1module.c b/Modules/sha1module.c
index 3fd53123229ac4..28ac1f881111ff 100644
--- a/Modules/sha1module.c
+++ b/Modules/sha1module.c
@@ -49,7 +49,8 @@ typedef long long SHA1_INT64;        /* 64-bit integer */
 typedef struct {
     PyObject_HEAD
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
+    bool use_mutex;
+    PyMutex mutex;
     PyThread_type_lock lock;
     Hacl_Streaming_SHA1_state *hash_state;
 } SHA1object;
@@ -76,7 +77,8 @@ newSHA1object(SHA1State *st)
     if (sha == NULL) {
         return NULL;
     }
-    sha->lock = NULL;
+    INIT_MUTEX(sha);
+
     PyObject_GC_Track(sha);
     return sha;
 }
@@ -94,9 +96,6 @@ static void
 SHA1_dealloc(SHA1object *ptr)
 {
     Hacl_Streaming_SHA1_legacy_free(ptr->hash_state);
-    if (ptr->lock != NULL) {
-        PyThread_free_lock(ptr->lock);
-    }
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
     PyObject_GC_Del(ptr);
@@ -192,14 +191,12 @@ SHA1Type_update(SHA1object *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         update(self->hash_state, buf.buf, buf.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         update(self->hash_state, buf.buf, buf.len);
diff --git a/Modules/sha2module.c b/Modules/sha2module.c
index 6ad1ff2e05bfd8..9dc87203f24cbe 100644
--- a/Modules/sha2module.c
+++ b/Modules/sha2module.c
@@ -53,8 +53,8 @@ typedef struct {
     PyObject_HEAD
     int digestsize;
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
     Hacl_Streaming_SHA2_state_sha2_256 *state;
 } SHA256object;
 
@@ -62,8 +62,8 @@ typedef struct {
     PyObject_HEAD
     int digestsize;
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
     Hacl_Streaming_SHA2_state_sha2_512 *state;
 } SHA512object;
 
@@ -106,7 +106,8 @@ newSHA224object(sha2_state *state)
     if (!sha) {
         return NULL;
     }
-    sha->lock = NULL;
+    INIT_MUTEX(sha);
+
     PyObject_GC_Track(sha);
     return sha;
 }
@@ -119,7 +120,8 @@ newSHA256object(sha2_state *state)
     if (!sha) {
         return NULL;
     }
-    sha->lock = NULL;
+    INIT_MUTEX(sha);
+
     PyObject_GC_Track(sha);
     return sha;
 }
@@ -132,7 +134,8 @@ newSHA384object(sha2_state *state)
     if (!sha) {
         return NULL;
     }
-    sha->lock = NULL;
+    INIT_MUTEX(sha);
+
     PyObject_GC_Track(sha);
     return sha;
 }
@@ -145,7 +148,8 @@ newSHA512object(sha2_state *state)
     if (!sha) {
         return NULL;
     }
-    sha->lock = NULL;
+    INIT_MUTEX(sha);
+
     PyObject_GC_Track(sha);
     return sha;
 }
@@ -163,9 +167,6 @@ static void
 SHA256_dealloc(SHA256object *ptr)
 {
     Hacl_Streaming_SHA2_free_256(ptr->state);
-    if (ptr->lock != NULL) {
-        PyThread_free_lock(ptr->lock);
-    }
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
     PyObject_GC_Del(ptr);
@@ -176,9 +177,6 @@ static void
 SHA512_dealloc(SHA512object *ptr)
 {
     Hacl_Streaming_SHA2_free_512(ptr->state);
-    if (ptr->lock != NULL) {
-        PyThread_free_lock(ptr->lock);
-    }
     PyTypeObject *tp = Py_TYPE(ptr);
     PyObject_GC_UnTrack(ptr);
     PyObject_GC_Del(ptr);
@@ -376,14 +374,12 @@ SHA256Type_update(SHA256object *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         update_256(self->state, buf.buf, buf.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         update_256(self->state, buf.buf, buf.len);
@@ -410,14 +406,12 @@ SHA512Type_update(SHA512object *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         update_512(self->state, buf.buf, buf.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         update_512(self->state, buf.buf, buf.len);
diff --git a/Modules/sha3module.c b/Modules/sha3module.c
index 558d2005cff617..933e6fafd85b28 100644
--- a/Modules/sha3module.c
+++ b/Modules/sha3module.c
@@ -61,8 +61,8 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type"
 typedef struct {
     PyObject_HEAD
     // Prevents undefined behavior via multiple threads entering the C API.
-    // The lock will be NULL before threaded access has been enabled.
-    PyThread_type_lock lock;
+    bool use_mutex;
+    PyMutex mutex;
     Hacl_Streaming_Keccak_state *hash_state;
 } SHA3object;
 
@@ -76,7 +76,8 @@ newSHA3object(PyTypeObject *type)
     if (newobj == NULL) {
         return NULL;
     }
-    newobj->lock = NULL;
+    INIT_MUTEX(newobj);
+
     return newobj;
 }
 
@@ -169,9 +170,6 @@ static void
 SHA3_dealloc(SHA3object *self)
 {
     Hacl_Streaming_Keccak_free(self->hash_state);
-    if (self->lock != NULL) {
-        PyThread_free_lock(self->lock);
-    }
     PyTypeObject *tp = Py_TYPE(self);
     PyObject_Free(self);
     Py_DECREF(tp);
@@ -257,19 +255,20 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data)
 /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/
 {
     Py_buffer buf;
+
     GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+
+    self->use_mutex = true;
+    if (buf.len >= HASHLIB_GIL_MINSIZE) {
         Py_BEGIN_ALLOW_THREADS
-        PyThread_acquire_lock(self->lock, 1);
+        PyMutex_Lock(&self->mutex);
         sha3_update(self->hash_state, buf.buf, buf.len);
-        PyThread_release_lock(self->lock);
+        PyMutex_Unlock(&self->mutex);
         Py_END_ALLOW_THREADS
     } else {
         sha3_update(self->hash_state, buf.buf, buf.len);
     }
+
     PyBuffer_Release(&buf);
     Py_RETURN_NONE;
 }


Code review comments:
diff_hunk: @@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type)
 {
     BLAKE2bObject *self;
     self = (BLAKE2bObject *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        self->lock = NULL;
+    if (self == NULL) {
+        return NULL;
     }
+    INIT_MUTEX(self);
path: Modules/_blake2/blake2b_impl.c
text: @user1:
Maybe `HASHLIB_INIT_MUTEX` to make it a bit more clear where the macro is coming from.

diff_hunk: @@ -1,5 +1,11 @@
 /* Common code for use by all hashlib related modules. */
 
+#ifndef Py_BUILD_CORE
path: Modules/hashlib.h
text: @user1:
I don't think you want this here. You'll need to fix `md5module.c` instead.

diff_hunk: @@ -48,18 +54,30 @@
 
 #include "pythread.h"
 #define ENTER_HASHLIB(obj) \
-    if ((obj)->lock) { \
-        if (!PyThread_acquire_lock((obj)->lock, 0)) { \
-            Py_BEGIN_ALLOW_THREADS \
-            PyThread_acquire_lock((obj)->lock, 1); \
-            Py_END_ALLOW_THREADS \
-        } \
+    if ((obj)->use_mutex) { \
+        Py_BEGIN_ALLOW_THREADS \
path: Modules/hashlib.h
text: @user1:
Remove `Py_BEGIN_ALLOW_THREADS` and `Py_END_ALLOW_THREADS` from this macro.

It's not necessary here because `PyMutex_Lock` internally will release the GIL (or equivalent in nogil builds) if it blocks on acquiring the mutex.

diff_hunk: @@ -257,19 +255,20 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data)
 /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/
 {
     Py_buffer buf;
+
     GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-    }
-    if (self->lock != NULL) {
+
+    self->use_mutex = true;
path: Modules/sha3module.c
text: @user1:
Should follow the same pattern as `_blake2_blake2s_update`

diff_hunk: @@ -272,24 +276,23 @@ Update this hash object's state with the provided bytes-like object.
 
 static PyObject *
 _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data)
-/*[clinic end generated code: output=757dc087fec37815 input=97500db2f9de4aaa]*/
+/*[clinic end generated code: output=010dfcbe22654359 input=ffc4aa6a6a225d31]*/
 {
     Py_buffer buf;
 
     GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
 
-    if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
-        self->lock = PyThread_allocate_lock();
-
-    if (self->lock != NULL) {
-       Py_BEGIN_ALLOW_THREADS
-       PyThread_acquire_lock(self->lock, 1);
-       blake2s_update(&self->state, buf.buf, buf.len);
-       PyThread_release_lock(self->lock);
-       Py_END_ALLOW_THREADS
+    self->use_mutex = true;
path: Modules/_blake2/blake2s_impl.c
text: @user1:
This pattern should instead be:

```c
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
    self->use_mutex = true;
}
if (self->use_mutex) {
    Py_BEGIN_ALLOW_THREADS
    PyMutex_Lock(&self->mutex);
    blake2s_update(&self->state, buf.buf, buf.len);
    PyMutex_Unlock(&self->mutex);
    Py_END_ALLOW_THREADS
}
else {
    blake2s_update(&self->state, buf.buf, buf.len);
}
```

This keeps the default build behavior the same as before: the first time we see a big buffer, we use a mutex from then on. (And always use a mutex in nogil builds).

diff_hunk: @@ -582,16 +580,11 @@ EVP_update(EVPobject *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
 
-    if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-        /* fail? lock = NULL and we fail over to non-threaded code. */
-    }
-
-    if (self->lock != NULL) {
+    if (self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
path: Modules/_hashopenssl.c
text: @user1:
Should follow the same pattern as `_blake2_blake2s_update`

diff_hunk: @@ -1540,7 +1533,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
     }
 
     self->ctx = ctx;
-    self->lock = NULL;
+    self->mutex = (PyMutex){0};
path: Modules/_hashopenssl.c
text: @user1:
I think  `self->use_mutex` is not initialized here. Maybe use `INIT_MUTEX`?

diff_hunk: @@ -1582,16 +1575,12 @@ _hmac_update(HMACobject *self, PyObject *obj)
 
     GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);
 
-    if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
-        self->lock = PyThread_allocate_lock();
-        /* fail? lock = NULL and we fail over to non-threaded code. */
-    }
-
-    if (self->lock != NULL) {
+    self->use_mutex = true;
path: Modules/_hashopenssl.c
text: @user1:
Should follow the same pattern as `_blake2_blake2s_update`