Repository: pyca/cryptography

Issue:
Do we actually set-up threading properly for OpenSSL? TLDR; Nope.
OpenSSL needs you to call at least `CRYPTO_set_id_callback()` (or `CRYPTO_THREADID_set_callback()` depending on version.) The CPython built in `_ssl` module binds these its self using whatever version of OpenSSL Python is linked to.

I suspect that if `import _ssl` never happens, this initialization will never be done. Potentially CPython is not even using the same version of OpenSSL we are linked to and so we might not benefit from it anyway. (e.g. on Windows where OpenSSL is often statically linked to CPython.)

If we set our own callbacks and we are using the same OpenSSL as CPython when `_ssl` is imported our callbacks will be replaced. This is probably fine?

If they are using different versions of OpenSSL we might have some interesting dlopen binding issues to address too.



Discussion:
I built a test case that crashes on my Ubuntu machine unless `_ssl` is imported. https://gist.github.com/public/8114440

It is possible to implement the lock handler within Python.

https://gist.github.com/public/8132604

but it's pretty slow. The C lock handler in _ssl takes about ~5 seconds on my machine. The Python version takes ~20s on PyPy and ~40s on CPython.

The other option is to pretty much clone the C implementations of `_ssl_thread_locking_function` and `_setup_ssl_threads` code from CPythons Modules/_ssl.c and enable those ourselves when we initialize the backend.

However I'm not entirely sure how we handle platform compatibility there. PyPy has no C API for us to use and we can't use the CPython API from within cffi anyway afaik. (And that may not even let us avoid the GIL.) Wrapping up Windows and pthreads probably isn't a huge amount of work but it's pretty ugly :(


Resolving patch:
diff --git a/cryptography/hazmat/backends/openssl/backend.py b/cryptography/hazmat/backends/openssl/backend.py
--- a/cryptography/hazmat/backends/openssl/backend.py
+++ b/cryptography/hazmat/backends/openssl/backend.py
@@ -44,6 +44,8 @@ def __init__(self):
         self._ffi = self._binding.ffi
         self._lib = self._binding.lib
 
+        self._binding.init_static_locks()
+
         # adds all ciphers/digests for EVP
         self._lib.OpenSSL_add_all_algorithms()
         # registers available SSL/TLS ciphers and digests
diff --git a/cryptography/hazmat/bindings/openssl/binding.py b/cryptography/hazmat/bindings/openssl/binding.py
--- a/cryptography/hazmat/bindings/openssl/binding.py
+++ b/cryptography/hazmat/bindings/openssl/binding.py
@@ -14,6 +14,7 @@
 from __future__ import absolute_import, division, print_function
 
 import sys
+import threading
 
 from cryptography.hazmat.bindings.utils import build_ffi
 
@@ -70,6 +71,10 @@ class Binding(object):
         "x509v3",
     ]
 
+    _locks = None
+    _lock_cb_handle = None
+    _lock_init_lock = threading.Lock()
+
     ffi = None
     lib = None
 
@@ -95,3 +100,43 @@ def _ensure_ffi_initialized(cls):
     def is_available(cls):
         # OpenSSL is the only binding so for now it must always be available
         return True
+
+    @classmethod
+    def init_static_locks(cls):
+        with cls._lock_init_lock:
+            cls._ensure_ffi_initialized()
+
+            if not cls._lock_cb_handle:
+                cls._lock_cb_handle = cls.ffi.callback(
+                    "void(int, int, const char *, int)",
+                    cls._lock_cb
+                )
+
+            # use Python's implementation if available
+
+            __import__("_ssl")
+
+            if cls.lib.CRYPTO_get_locking_callback() != cls.ffi.NULL:
+                return
+
+            # otherwise setup our version
+
+            num_locks = cls.lib.CRYPTO_num_locks()
+            cls._locks = [threading.Lock() for n in range(num_locks)]
+
+            cls.lib.CRYPTO_set_locking_callback(cls._lock_cb_handle)
+
+    @classmethod
+    def _lock_cb(cls, mode, n, file, line):
+        lock = cls._locks[n]
+
+        if mode & cls.lib.CRYPTO_LOCK:
+            lock.acquire()
+        elif mode & cls.lib.CRYPTO_UNLOCK:
+            lock.release()
+        else:
+            raise RuntimeError(
+                "Unknown lock mode {0}: lock={1}, file={2}, line={3}".format(
+                    mode, n, file, line
+                )
+            )
diff --git a/cryptography/hazmat/bindings/openssl/crypto.py b/cryptography/hazmat/bindings/openssl/crypto.py
--- a/cryptography/hazmat/bindings/openssl/crypto.py
+++ b/cryptography/hazmat/bindings/openssl/crypto.py
@@ -27,6 +27,11 @@
 static const int CRYPTO_MEM_CHECK_OFF;
 static const int CRYPTO_MEM_CHECK_ENABLE;
 static const int CRYPTO_MEM_CHECK_DISABLE;
+static const int CRYPTO_LOCK;
+static const int CRYPTO_UNLOCK;
+static const int CRYPTO_READ;
+static const int CRYPTO_WRITE;
+static const int CRYPTO_LOCK_SSL;
 """
 
 FUNCTIONS = """
@@ -43,6 +48,7 @@
 void CRYPTO_set_id_callback(unsigned long (*)(void));
 unsigned long (*CRYPTO_get_id_callback(void))(void);
 void (*CRYPTO_get_locking_callback(void))(int, int, const char *, int);
+void CRYPTO_lock(int, int, const char *, int);
 
 void OPENSSL_free(void *);
 """
@@ -51,7 +57,6 @@
 void CRYPTO_add(int *, int, int);
 void CRYPTO_malloc_init(void);
 void CRYPTO_malloc_debug_init(void);
-
 """
 
 CUSTOMIZATIONS = """


Test patch:
diff --git a/tests/hazmat/bindings/test_openssl.py b/tests/hazmat/bindings/test_openssl.py
--- a/tests/hazmat/bindings/test_openssl.py
+++ b/tests/hazmat/bindings/test_openssl.py
@@ -11,6 +11,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+
 from cryptography.hazmat.bindings.openssl.binding import Binding
 
 
@@ -23,3 +25,74 @@ def test_binding_loads(self):
 
     def test_is_available(self):
         assert Binding.is_available() is True
+
+    def test_crypto_lock_init(self):
+        b = Binding()
+        b.init_static_locks()
+        lock_cb = b.lib.CRYPTO_get_locking_callback()
+        assert lock_cb != b.ffi.NULL
+
+    def _skip_if_not_fallback_lock(self, b):
+        # only run this test if we are using our locking cb
+        original_cb = b.lib.CRYPTO_get_locking_callback()
+        if original_cb != b._lock_cb_handle:
+            pytest.skip(
+                "Not using the fallback Python locking callback "
+                "implementation. Probably because import _ssl set one"
+            )
+
+    def test_fallback_crypto_lock_via_openssl_api(self):
+        b = Binding()
+        b.init_static_locks()
+
+        self._skip_if_not_fallback_lock(b)
+
+        # check that the lock state changes appropriately
+        lock = b._locks[b.lib.CRYPTO_LOCK_SSL]
+
+        # starts out unlocked
+        assert lock.acquire(False)
+        lock.release()
+
+        b.lib.CRYPTO_lock(
+            b.lib.CRYPTO_LOCK | b.lib.CRYPTO_READ,
+            b.lib.CRYPTO_LOCK_SSL, b.ffi.NULL, 0
+        )
+
+        # becomes locked
+        assert not lock.acquire(False)
+
+        b.lib.CRYPTO_lock(
+            b.lib.CRYPTO_UNLOCK | b.lib.CRYPTO_READ,
+            b.lib.CRYPTO_LOCK_SSL, b.ffi.NULL, 0
+        )
+
+        # then unlocked
+        assert lock.acquire(False)
+        lock.release()
+
+    def test_fallback_crypto_lock_via_binding_api(self):
+        b = Binding()
+        b.init_static_locks()
+
+        self._skip_if_not_fallback_lock(b)
+
+        lock = b._locks[b.lib.CRYPTO_LOCK_SSL]
+
+        with pytest.raises(RuntimeError):
+            b._lock_cb(0, b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
+
+        # errors shouldnt cause locking
+        assert lock.acquire(False)
+        lock.release()
+
+        b._lock_cb(b.lib.CRYPTO_LOCK | b.lib.CRYPTO_READ,
+                   b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
+        # locked
+        assert not lock.acquire(False)
+
+        b._lock_cb(b.lib.CRYPTO_UNLOCK | b.lib.CRYPTO_READ,
+                   b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
+        # unlocked
+        assert lock.acquire(False)
+        lock.release()