Repository: numpy/numpy

Issue:
numpy >= 1.16 with pypy and ctypes errors
### Reproducing code example:
You can see an example of this failure on [Travis CI](https://travis-ci.org/ni/nimi-python/jobs/547906996#L1381)
```python
import numpy as np
return np.ctypeslib.as_ctypes(value)
```
where value is a numpy array with dtype='float64'

This works when restricting nimpy to '<1.16' on pypy and the latest numpy works with this on CPython.

While the examples I have put in this issue refer to pypy 2.7.13-7.1.1, this also applies to pypy 3.6.1-7.1.1

### Error message:

```
bin/nifake/nifake/session.py:72: in aux
    return f(*xs, **kws)
bin/nifake/nifake/session.py:1613: in write_waveform_numpy
    waveform_ctype = get_ctypes_pointer_for_buffer(value=waveform)  # case B510
bin/nifake/nifake/session.py:31: in get_ctypes_pointer_for_buffer
    return numpy.ctypeslib.as_ctypes(value)
.tox/64/pypy-test/site-packages/numpy/ctypeslib.py:539: in as_ctypes
    result = as_ctypes_type(dtype).from_address(addr)
.tox/64/pypy-test/site-packages/numpy/ctypeslib.py:502: in as_ctypes_type
    return _ctype_from_dtype(_dtype(dtype))
.tox/64/pypy-test/site-packages/numpy/ctypeslib.py:459: in _ctype_from_dtype
    return _ctype_from_dtype_subarray(dtype)
.tox/64/pypy-test/site-packages/numpy/ctypeslib.py:397: in _ctype_from_dtype_subarray
    ctype = _ctype_from_dtype(element_dtype)
.tox/64/pypy-test/site-packages/numpy/ctypeslib.py:461: in _ctype_from_dtype
    return _ctype_from_dtype_scalar(dtype)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dtype = dtype('float64')
    def _ctype_from_dtype_scalar(dtype):
        # swapping twice ensure that `=` is promoted to <, >, or |
        dtype_with_endian = dtype.newbyteorder('S').newbyteorder('S')
        dtype_native = dtype.newbyteorder('=')
        try:
            ctype = _scalar_type_map[dtype_native]
        except KeyError:
            raise NotImplementedError(
>               "Converting {!r} to a ctypes type".format(dtype)
            )
E           NotImplementedError: Converting dtype('float64') to a ctypes type```
```

### Numpy/Python version information:

```
OS:
    Name:      Linux
    Version:   #34~16.04.1-Ubuntu SMP Wed May 8 13:26:26 UTC 2019
    Bits:      64
Python:
    Version:   2.7.13 (8cdda8b8cdb8, Apr 14 2019, 14:06:44)
[PyPy 7.1.1 with GCC 6.2.0 20160901]
    Bits:      64
    Is_Venv:   True
    Installed Packages:
        nifake==1.1.3.dev0
        readline==6.2.4.1
        greenlet==0.4.13
        cffi==1.12.3
        wsgiref==0.1.2
        zipp==0.5.1
        wheel==0.33.1
        wcwidth==0.1.7
        six==1.12.0
        singledispatch==3.4.0.3
        setuptools==40.8.0
        scandir==1.10.0
        pytest==4.6.3
        pyparsing==2.4.0
        py==1.8.0
        pluggy==0.12.0
        pip==19.1.1
        pathlib2==2.3.3
        packaging==19.0
        numpy==1.16.4
        more-itertools==5.0.0
        mock==3.0.5
        markupsafe==1.1.1
        mako==1.0.12
        importlib-metadata==0.18
        funcsigs==1.0.2
        enum34==1.1.6
        coverage==4.5.3
        contextlib2==0.5.5
        configparser==3.7.4
        attrs==19.1.0
        atomicwrites==1.3.0
```




Discussion:
ctypes support improved after the release of PyPy 7.1.1. Numpy tests with a [nightly build](https://github.com/numpy/numpy/blob/master/tools/pypy-test.sh#L32). 
Can you print out dtype.char here?
I added `print('numpy #13807: {}'.format(expected_waveform.dtype.char))` to one of the test that fails. 

I get `numpy #13807: d` for all instances: CPython (passes), pypy/pypy3 with numpy<1.16 (passes), and pypy/pypy3 with numpy 1.16.4 (fails)
Thanks, I wanted to confirm the failure was not long double
@matti: this looks like a hashing/ dict lookup problem, not a ctypes problem
Can reproduce on latest PyPY with 1.16.4, but not with HEAD. The problem is that `np.dtype(ctypes.c_int)` produces `dtype('O')` on 1.16.4, `dtype('int32') on HEAD. Something was fixed and not backported, but I don't remember what.
`np.ctypeslib.as_ctypes_type(ctypes.c_int)` is a simpler test. Can you run that on both head and 1.16.4?
If we wish to fix this we should backport e67f8c7f to 1.16. It was part of PR #12594 to add PyPY3 to our CI, but 1.16.4 is the LTS python2 branch.

Resolving patch:
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py
--- a/numpy/core/_internal.py
+++ b/numpy/core/_internal.py
@@ -8,6 +8,7 @@
 
 import re
 import sys
+import platform
 
 from numpy.compat import unicode
 from numpy.core.overrides import set_module
@@ -17,6 +18,8 @@
 except ImportError:
     ctypes = None
 
+IS_PYPY = platform.python_implementation() == 'PyPy'
+
 if (sys.byteorder == 'little'):
     _nbo = b'<'
 else:
@@ -889,7 +892,12 @@ def npy_ctypes_check(cls):
     try:
         # ctypes class are new-style, so have an __mro__. This probably fails
         # for ctypes classes with multiple inheritance.
-        ctype_base = cls.__mro__[-2]
+        if IS_PYPY:
+            # (..., _ctypes.basics._CData, Bufferable, object)
+            ctype_base = cls.__mro__[-3]
+        else:
+            # # (..., _ctypes._CData, object)
+            ctype_base = cls.__mro__[-2]
         # right now, they're part of the _ctypes module
         return 'ctypes' in ctype_base.__module__
     except Exception:


Test patch:
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -36,7 +36,7 @@
     assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal,
     assert_array_equal, assert_raises_regex, assert_array_almost_equal,
     assert_allclose, IS_PYPY, HAS_REFCOUNT, assert_array_less, runstring,
-    temppath, suppress_warnings
+    temppath, suppress_warnings, break_cycles,
     )
 from numpy.core.tests._locales import CommaDecimalPointLocale
 
@@ -131,6 +131,7 @@ def test_writeable_from_buffer(self):
         assert_(vals.flags.writeable)
 
     @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies")
+    @pytest.mark.skipif(IS_PYPY, reason="PyPy always copies")
     def test_writeable_pickle(self):
         import pickle
         # Small arrays will be copied without setting base.
@@ -3816,7 +3817,7 @@ def test_roundtrip(self):
                         a, pickle.loads(pickle.dumps(a, protocol=proto)),
                         err_msg="%r" % a)
             del a, DATA, carray
-            gc.collect()
+            break_cycles()
             # check for reference leaks (gh-12793)
             for ref in refs:
                 assert ref() is None
@@ -7159,6 +7160,7 @@ class DummyArray2(object):
     assert_equal(arr1, arr2)
     assert_equal(arr1, arr3)
 
+@pytest.mark.skipif(IS_PYPY, reason='PyDict_GetItemString(.., "data") segfaults')
 def test_array_interface_offset():
     arr = np.array([1, 2, 3], dtype='int32')
     interface = dict(arr.__array_interface__)
@@ -7201,7 +7203,7 @@ def test_mem_seteventhook(self):
         # needs to be larger then limit of small memory cacher in ctors.c
         a = np.zeros(1000)
         del a
-        gc.collect()
+        break_cycles()
         _multiarray_tests.test_pydatamem_seteventhook_end()
 
 class TestMapIter(object):
@@ -7773,12 +7775,12 @@ def test_ctypes_data_as_holds_reference(self, arr):
 
         # `ctypes_ptr` should hold onto `arr`
         del arr
-        gc.collect()
+        break_cycles()
         assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference")
 
         # but when the `ctypes_ptr` object dies, so should `arr`
         del ctypes_ptr
-        gc.collect()
+        break_cycles()
         assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference")
 
 
@@ -7960,15 +7962,15 @@ class Dummy(object): pass
         assert_(isinstance(obj_subarray, RaisesInFinalize))
 
         # reference should still be held by obj_arr
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is not None, "object should not already be dead")
 
         del obj_arr
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is not None, "obj_arr should not hold the last reference")
 
         del obj_subarray
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is None, "no references should remain")
 
 
diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py
--- a/numpy/core/tests/test_numerictypes.py
+++ b/numpy/core/tests/test_numerictypes.py
@@ -5,7 +5,7 @@
 
 import pytest
 import numpy as np
-from numpy.testing import assert_, assert_equal, assert_raises
+from numpy.testing import assert_, assert_equal, assert_raises, IS_PYPY
 
 # This is the structure of the table used for plain objects:
 #
@@ -491,6 +491,7 @@ def test_issctype(rep, expected):
 
 @pytest.mark.skipif(sys.flags.optimize > 1,
                     reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1")
+@pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
 class TestDocStrings(object):
     def test_platform_dependent_aliases(self):
         if np.int64 is np.int_:
diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py
--- a/numpy/f2py/tests/test_block_docstring.py
+++ b/numpy/f2py/tests/test_block_docstring.py
@@ -4,7 +4,7 @@
 import pytest
 from . import util
 
-from numpy.testing import assert_equal
+from numpy.testing import assert_equal, IS_PYPY
 
 class TestBlockDocString(util.F2PyTest):
     code = """
@@ -18,6 +18,7 @@ class TestBlockDocString(util.F2PyTest):
 
     @pytest.mark.skipif(sys.platform=='win32',
                         reason='Fails with MinGW64 Gfortran (Issue #9673)')
+    @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
     def test_block_docstring(self):
         expected = "'i'-array(2,3)\n"
         assert_equal(self.module.block.__doc__, expected)
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -10,7 +10,7 @@
 from numpy import ma
 from numpy.testing import (
     assert_, assert_equal, assert_array_equal, assert_almost_equal,
-    assert_array_almost_equal, assert_raises, assert_allclose,
+    assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY,
     assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT,
     )
 import numpy.lib.function_base as nfb
@@ -3106,6 +3106,7 @@ def test_string_arg(self):
 class TestAdd_newdoc(object):
 
     @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
+    @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
     def test_add_doc(self):
         # test np.add_newdoc
         tgt = "Current flat index into the array."
diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py
--- a/numpy/testing/_private/utils.py
+++ b/numpy/testing/_private/utils.py
@@ -6,6 +6,7 @@
 
 import os
 import sys
+import platform
 import re
 import gc
 import operator
@@ -39,6 +40,7 @@
         'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY',
         'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare',
         '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles',
+        'break_cycles',
         ]
 
 
@@ -50,7 +52,7 @@ class KnownFailureException(Exception):
 KnownFailureTest = KnownFailureException  # backwards compat
 verbose = 0
 
-IS_PYPY = '__pypy__' in sys.modules
+IS_PYPY = platform.python_implementation() == 'PyPy'
 HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None
 
 
@@ -2255,6 +2257,7 @@ def _assert_no_gc_cycles_context(name=None):
 
     # not meaningful to test if there is no refcounting
     if not HAS_REFCOUNT:
+        yield
         return
 
     assert_(gc.isenabled())
@@ -2333,3 +2336,19 @@ def assert_no_gc_cycles(*args, **kwargs):
     args = args[1:]
     with _assert_no_gc_cycles_context(name=func.__name__):
         func(*args, **kwargs)
+
+def break_cycles():
+    """
+    Break reference cycles by calling gc.collect
+    Objects can call other objects' methods (for instance, another object's
+     __del__) inside their own __del__. On PyPy, the interpreter only runs
+    between calls to gc.collect, so multiple calls are needed to completely
+    release all cycles.
+    """
+
+    gc.collect()
+    if IS_PYPY:
+        # interpreter runs now, to call deleted objects' __del__ methods
+        gc.collect()
+        # one more, just to make sure
+        gc.collect()