Repository: pyca/cryptography

Issue:
Yet another building on PyPy(5.8) issue
Hi,

I'm trying to build cryptography (tried both 1.9 and master) against PyPy 5.8beta0.
I'm running on osx sierra.

I've looked at several issues here, and have tried several suggested solutions, but none worked for me 😞 

I'm trying to link against brew openssl.
`brew list --versions openssl` -> `openssl 1.0.2j 1.0.2k 1.0.2l`

I've built pypy's openssl module without any special problem

```
env PYTHONPATH=/Users/omerba/Workspace/pypy \                                                                                                                               
       DYLD_LIBRARY_PATH="/Users/omerba/anaconda/lib" \
       CFLAGS="-I/usr/local/opt/openssl/include" \
       LDFLAGS="-L/usr/local/opt/openssl/lib" \
       /Users/omerba/Workspace/pypy/pypy3-c pypy/tool/build_cffi_imports.py
```
But when i try to build cryptography (in cryptography's dir):
```
 env PYTHONPATH=/Users/omerba/Workspace/pypy \                                                                                                                           
       DYLD_LIBRARY_PATH="/Users/omerba/anaconda/lib" \
       CFLAGS="-I/usr/local/opt/openssl/include" \
       LDFLAGS="-L/usr/local/opt/openssl/lib" \
       /Users/omerba/Workspace/pypy/pypy3-c setup.py install
```
I get:
```
running install
running bdist_egg
running egg_info
writing src/cryptography.egg-info/PKG-INFO
writing dependency_links to src/cryptography.egg-info/dependency_links.txt
writing entry points to src/cryptography.egg-info/entry_points.txt
writing requirements to src/cryptography.egg-info/requires.txt
writing top-level names to src/cryptography.egg-info/top_level.txt
reading manifest file 'src/cryptography.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
no previously-included directories found matching 'docs/_build'
warning: no previously-included files matching '*' found under directory 'vectors'
writing manifest file 'src/cryptography.egg-info/SOURCES.txt'
installing library code to build/bdist.macosx-10.12-x86_64/egg
running install_lib
running build_py
running build_ext
generating cffi module 'build/temp.macosx-10.12-x86_64-3.5/_padding.c'
already up-to-date
generating cffi module 'build/temp.macosx-10.12-x86_64-3.5/_constant_time.c'
already up-to-date
generating cffi module 'build/temp.macosx-10.12-x86_64-3.5/_openssl.c'
building '_openssl' extension
cc -pthread -DNDEBUG -O2 -I/usr/local/opt/openssl/include -fPIC -I/Users/omerba/Workspace/pypy/include -c build/temp.macosx-10.12-x86_64-3.5/_openssl.c -o build/temp.macosx-10.12-x86_64-3.5/build/temp.macosx-10.12-x86_64-3.5/_openssl.o
build/temp.macosx-10.12-x86_64-3.5/_openssl.c:2503:9: warning: comparison of function 'getentropy' not equal to a null pointer is always true [-Wtautological-pointer-compare]
    if (getentropy != NULL) {
        ^~~~~~~~~~    ~~~~
build/temp.macosx-10.12-x86_64-3.5/_openssl.c:2503:9: note: prefix with the address-of operator to silence this warning
    if (getentropy != NULL) {
        ^
        &
build/temp.macosx-10.12-x86_64-3.5/_openssl.c:3454:22: warning: comparison of constant 1152921504606846975 with expression of type 'unsigned int' is always false [-Wtautological-constant-out-of-range-compare]
        _ssl_locks = PyMem_New(PyThread_type_lock, _ssl_locks_count);
                     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/Users/omerba/Workspace/pypy/include/pymem.h:38:10: note: expanded from macro 'PyMem_New'
  ( ((n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \
     ~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
build/temp.macosx-10.12-x86_64-3.5/_openssl.c:74852:1: warning: control reaches end of non-void function [-Wreturn-type]
}
^
3 warnings generated.
cc -pthread -shared -L/usr/local/opt/openssl/lib -I/usr/local/opt/openssl/include build/temp.macosx-10.12-x86_64-3.5/build/temp.macosx-10.12-x86_64-3.5/_openssl.o -lssl -lcrypto -o build/lib.macosx-10.12-x86_64-3.5/cryptography/hazmat/bindings/_openssl.pypy3-58-x86_64-darwin.so
clang: warning: argument unused during compilation: '-pthread' [-Wunused-command-line-argument]
Undefined symbols for architecture x86_64:
  "_PyPyErr_NoMemory", referenced from:
      __setup_ssl_threads in _openssl.o
  "_PyPyMem_Free", referenced from:
      __setup_ssl_threads in _openssl.o
  "_PyPyMem_Malloc", referenced from:
      __setup_ssl_threads in _openssl.o
  "_PyPyThread_acquire_lock", referenced from:
      __ssl_thread_locking_function in _openssl.o
  "_PyPyThread_allocate_lock", referenced from:
      __setup_ssl_threads in _openssl.o
  "_PyPyThread_free_lock", referenced from:
      __setup_ssl_threads in _openssl.o
  "_PyPyThread_release_lock", referenced from:
      __ssl_thread_locking_function in _openssl.o
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
error: command 'cc' failed with exit status 1
```

Googling the issue only leads to [this](https://bitbucket.org/pypy/pypy/issues/2538/_ssl_buildpy-fails-on-macos-with-the-py35) exact same issue on the PyPy repo.

But the PyPy team seemed to have resolved it by making some changes to their cffi backend.
I'll admit that I tried to shamelessly copy these changes to cryptography's _cffi_src dir - which made the package install successfully, but then when I actually tried to use it:

`from cryptography.hazmat.backends import default_backend, openssl`

it blows up.. 

```
AttributeErrorTraceback (most recent call last)
<ipython-input-5-cc19c86b2edb> in <module>()
      1 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
----> 2 from cryptography.hazmat.backends import default_backend, openssl

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/backends/openssl/__init__.py in <module>()
      5 from __future__ import absolute_import, division, print_function
      6 
----> 7 from cryptography.hazmat.backends.openssl.backend import backend
      8 
      9 

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/backends/openssl/backend.py in <module>()
     47     _CertificateSigningRequest, _RevokedCertificate
     48 )
---> 49 from cryptography.hazmat.bindings.openssl import binding
     50 from cryptography.hazmat.primitives import hashes, serialization
     51 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/bindings/openssl/binding.py in <module>()
    154 # condition registering the OpenSSL locks. On Python 3.4+ the import lock
    155 # is per module so this approach will not work.
--> 156 Binding.init_static_locks()

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/bindings/openssl/binding.py in init_static_locks(cls)
    135     def init_static_locks(cls):
    136         with cls._lock_init_lock:
--> 137             cls._ensure_ffi_initialized()
    138             # Use Python's implementation if available, importing _ssl triggers
    139             # the setup for this.

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/bindings/openssl/binding.py in _ensure_ffi_initialized(cls)
    122         with cls._init_lock:
    123             if not cls._lib_loaded:
--> 124                 cls.lib = build_conditional_library(lib, CONDITIONAL_NAMES)
    125                 cls._lib_loaded = True
    126                 # initialize the SSL library

~/Workspace/pypy/site-packages/cryptography-1.9-py3.5-macosx-10.12-x86_64.egg/cryptography/hazmat/bindings/openssl/binding.py in build_conditional_library(lib, conditional_names)
     82     excluded_names = set()
     83     for condition, names in conditional_names.items():
---> 84         if not getattr(lib, condition):
     85             excluded_names |= set(names)
     86 

AttributeError: cffi library 'cryptography.hazmat.bindings._openssl' has no function, constant or global variable named 'Cryptography_HAS_DTLS'
```

Thanks for your work guys!


Discussion:
Okay, this is interesting. We don't currently test against the new pypy3 since it's still considered beta, but this appears to be (mostly) an artifact of our OpenSSL locking callbacks assuming CPython API. It looks like we need to integrate https://bitbucket.org/pypy/pypy/commits/198dc138680f96c391802fa1e77b8b6d2e0134e6?at=py3.5

I won't be able to take a look at this for a bit, but there's clearly a significant bug here.
Some more notes that might be useful:

PyPy3 doesn't have any official support for MacOS yet; ATM the only supported platform is x86-64 Linux.

I'm running CI against PyPy3 5.8.0-beta on x86-64 Linux, and cryptography 1.9 installs and runs fine there: https://travis-ci.org/python-trio/trio/jobs/242234740
(and that test suite doesn't touch cryptography directly, but it does pound on PyOpenSSL a fair amount). Maybe there's some wacky thing involving two-level namespaces or something that lets it work on Linux?

I'm not sure what to tell you about MacOS, but I would suggest adding PyPy3 to your Linux CI -- it's "beta" in the "think about whether you want to be the first person to run your business on this" sense, but it's [very solid](https://twitter.com/vorpalsmith/status/861703774720581632), and it needs testing to get out of beta :-).
Is PyPy3 still substantially slower than PyPy2? When we re-add PyPy3 we may need to turn off coverage on that config to avoid crazy long builds. We used to do this on PyPy2 < 4.0 but we've removed the config from our tox ini.
I haven't done any systematic comparison, but I've seen it be maybe 10% slower than the equivalent pypy2? There are probably optimization opportunities they aren't taking yet but it's definitely no longer the thing where it was using an ancient out of date version of the JIT.
I've added pypy3.5 to CI  in #3709. It was too slow with coverage turned on so I added a nocoverage target, but it works just fine on linux. This issue can stay open for now since we should still decide whether it's desirable to pull in pypy's patches to do locking without using the CPython API and test on macOS when pypy has official builds.
I can replicate this on macOS with homebrew pypy3 now.

Resolving patch:
diff --git a/src/_cffi_src/openssl/callbacks.py b/src/_cffi_src/openssl/callbacks.py
--- a/src/_cffi_src/openssl/callbacks.py
+++ b/src/_cffi_src/openssl/callbacks.py
@@ -10,7 +10,13 @@
 #include <openssl/x509_vfy.h>
 #include <openssl/crypto.h>
 
-#include <pythread.h>
+#ifdef _WIN32
+#include <Windows.h>
+#else
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#endif
 """
 
 TYPES = """
@@ -36,10 +42,47 @@
    locking callback for OpenSSL.
 
    Copyright 2001-2016 Python Software Foundation; All Rights Reserved.
+
+   It has been subsequently modified to use cross platform locking without
+   using CPython APIs by Armin Rigo of the PyPy project.
 */
 
+#ifdef _WIN32
+typedef CRITICAL_SECTION Cryptography_mutex;
+static __inline void cryptography_mutex_init(Cryptography_mutex *mutex) {
+    InitializeCriticalSection(mutex);
+}
+static __inline void cryptography_mutex_lock(Cryptography_mutex *mutex) {
+    EnterCriticalSection(mutex);
+}
+static __inline void cryptography_mutex_unlock(Cryptography_mutex *mutex) {
+    LeaveCriticalSection(mutex);
+}
+#else
+typedef pthread_mutex_t Cryptography_mutex;
+#define ASSERT_STATUS(call)                                             \
+    if ((call) != 0) {                                                  \
+        perror("Fatal error in callback initialization: " #call);       \
+        abort();                                                        \
+    }
+static inline void cryptography_mutex_init(Cryptography_mutex *mutex) {
+#if !defined(pthread_mutexattr_default)
+#  define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL)
+#endif
+    ASSERT_STATUS(pthread_mutex_init(mutex, pthread_mutexattr_default));
+}
+static inline void cryptography_mutex_lock(Cryptography_mutex *mutex) {
+    ASSERT_STATUS(pthread_mutex_lock(mutex));
+}
+static inline void cryptography_mutex_unlock(Cryptography_mutex *mutex) {
+    ASSERT_STATUS(pthread_mutex_unlock(mutex));
+}
+#endif
+
+
+
 static unsigned int _ssl_locks_count = 0;
-static PyThread_type_lock *_ssl_locks = NULL;
+static Cryptography_mutex *_ssl_locks = NULL;
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
                                          int line) {
@@ -63,35 +106,32 @@
     }
 
     if (mode & CRYPTO_LOCK) {
-        PyThread_acquire_lock(_ssl_locks[n], 1);
+        cryptography_mutex_lock(_ssl_locks + n);
     } else {
-        PyThread_release_lock(_ssl_locks[n]);
+        cryptography_mutex_unlock(_ssl_locks + n);
+    }
+}
+
+static void init_mutexes(void) {
+    int i;
+    for (i = 0; i < _ssl_locks_count; i++) {
+        cryptography_mutex_init(_ssl_locks + i);
     }
 }
 
-int _setup_ssl_threads(void) {
-    unsigned int i;
 
+int _setup_ssl_threads(void) {
     if (_ssl_locks == NULL) {
         _ssl_locks_count = CRYPTO_num_locks();
-        _ssl_locks = PyMem_New(PyThread_type_lock, _ssl_locks_count);
+        _ssl_locks = calloc(_ssl_locks_count, sizeof(Cryptography_mutex));
         if (_ssl_locks == NULL) {
-            PyErr_NoMemory();
             return 0;
         }
-        memset(_ssl_locks, 0, sizeof(PyThread_type_lock) * _ssl_locks_count);
-        for (i = 0;  i < _ssl_locks_count;  i++) {
-            _ssl_locks[i] = PyThread_allocate_lock();
-            if (_ssl_locks[i] == NULL) {
-                unsigned int j;
-                for (j = 0;  j < i;  j++) {
-                    PyThread_free_lock(_ssl_locks[j]);
-                }
-                PyMem_Free(_ssl_locks);
-                return 0;
-            }
-        }
+        init_mutexes();
         CRYPTO_set_locking_callback(_ssl_thread_locking_function);
+#ifndef _WIN32
+        pthread_atfork(NULL, NULL, &init_mutexes);
+#endif
     }
     return 1;
 }


Test patch: