Repository: numpy/numpy

Issue:
FutureWarning in recfunctions.join_by
Currently using numpy 1.12.0 on windows 7.

`joinedDataset = np.lib.recfunctions.join_by(['filename','x','y'], dataset, metricsDataset, jointype='inner', usemask=False)`

Yields the following warning:

> c:\Python35\lib\site-packages\numpy\ma\core.py:6589: FutureWarning: Assignment between structured arrays with different field names will change in numpy 1.13.
> 
> Previously fields in the dst would be set to the value of the identically-named field in the src. In numpy 1.13 fields will instead be assigned 'by position': The Nth field of the dst will be set to the Nth field of the src array.
> 
> See the release notes for details
>   d = np.concatenate([getdata(a) for a in arrays], axis)

I assume for numpy 1.13 recfunctions will be written to appropriately do the join.  Is this correct?


Discussion:
The PR for this behaviour change is #6053.

Added to the 1.13 milestone because in the likely event that that PR does not make it into 1.13, we need to update the warning messages
@ahaldane Can you answer the question?

I changed the warning message NumPy version in #9058.
Hmm OK, this is a legitimate warning.

It looks like I will need to change the `join_by` code a little bit. The problem presumably  occurs because `dataset.dtype` and `metricsDataset.dtype` have the same fields but in a different order.

Here is reproducible code:
```python
>>> a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u1')])
>>> b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
>>> numpy.lib.recfunctions.join_by(['a', 'b'], a, b, jointype='inner', usemask=False)
FutureWarning
```

The fix would/will be to change the line `ma.concatenate((r1k, r2k))` to `ma.concatenate((r1k[keys], r2k[keys]))`.

I'll have to look more closely for similar cases before we merge #6053.


Thanks @isaacgerg for the report, it is very helpful.
We probably want to silence this error or modify `join_by` in some way for 1.13. I'll take a look in a little while.

Resolving patch:
diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py
--- a/numpy/lib/recfunctions.py
+++ b/numpy/lib/recfunctions.py
@@ -495,7 +495,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False):
           dtype=[('a', '<i4')])
     """
     if _is_string_like(drop_names):
-        drop_names = [drop_names, ]
+        drop_names = [drop_names]
     else:
         drop_names = set(drop_names)
 
@@ -523,6 +523,31 @@ def _drop_descr(ndtype, drop_names):
     return _fix_output(output, usemask=usemask, asrecarray=asrecarray)
 
 
+def _keep_fields(base, keep_names, usemask=True, asrecarray=False):
+    """
+    Return a new array keeping only the fields in `keep_names`,
+    and preserving the order of those fields.
+
+    Parameters
+    ----------
+    base : array
+        Input array
+    keep_names : string or sequence
+        String or sequence of strings corresponding to the names of the
+        fields to keep. Order of the names will be preserved.
+    usemask : {False, True}, optional
+        Whether to return a masked array or not.
+    asrecarray : string or sequence, optional
+        Whether to return a recarray or a mrecarray (`asrecarray=True`) or
+        a plain ndarray or masked array with flexible dtype. The default
+        is False.
+    """
+    newdtype = [(n, base.dtype[n]) for n in keep_names]
+    output = np.empty(base.shape, dtype=newdtype)
+    output = recursive_fill_fields(base, output)
+    return _fix_output(output, usemask=usemask, asrecarray=asrecarray)
+
+
 def rec_drop_fields(base, drop_names):
     """
     Returns a new numpy.recarray with fields in `drop_names` dropped.
@@ -877,11 +902,14 @@ def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2',
         key = (key,)
 
     # Check the keys
+    if len(set(key)) != len(key):
+        dup = next(x for n,x in enumerate(key) if x in key[n+1:])
+        raise ValueError("duplicate join key %r" % dup)
     for name in key:
         if name not in r1.dtype.names:
-            raise ValueError('r1 does not have key field %s' % name)
+            raise ValueError('r1 does not have key field %r' % name)
         if name not in r2.dtype.names:
-            raise ValueError('r2 does not have key field %s' % name)
+            raise ValueError('r2 does not have key field %r' % name)
 
     # Make sure we work with ravelled arrays
     r1 = r1.ravel()
@@ -899,8 +927,10 @@ def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2',
         raise ValueError(msg)
 
     # Make temporary arrays of just the keys
-    r1k = drop_fields(r1, [n for n in r1names if n not in key])
-    r2k = drop_fields(r2, [n for n in r2names if n not in key])
+    #  (use order of keys in `r1` for back-compatibility)
+    key1 = [ n for n in r1names if n in key ]
+    r1k = _keep_fields(r1, key1)
+    r2k = _keep_fields(r2, key1)
 
     # Concatenate the two arrays for comparison
     aux = ma.concatenate((r1k, r2k))


Test patch:
diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py
--- a/numpy/lib/tests/test_recfunctions.py
+++ b/numpy/lib/tests/test_recfunctions.py
@@ -4,7 +4,7 @@
 import numpy.ma as ma
 from numpy.ma.mrecords import MaskedRecords
 from numpy.ma.testutils import assert_equal
-from numpy.testing import TestCase, run_module_suite, assert_
+from numpy.testing import TestCase, run_module_suite, assert_, assert_raises
 from numpy.lib.recfunctions import (
     drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields,
     find_duplicates, merge_arrays, append_fields, stack_arrays, join_by
@@ -633,6 +633,19 @@ def test_leftouter_join(self):
                            dtype=[('a', int), ('b', int), ('c', int), ('d', int)])
         assert_equal(test, control)
 
+    def test_different_field_order(self):
+        # gh-8940
+        a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u1')])
+        b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
+        # this should not give a FutureWarning:
+        j = join_by(['c', 'b'], a, b, jointype='inner', usemask=False)
+        assert_equal(j.dtype.names, ['b', 'c', 'a1', 'a2'])
+
+    def test_duplicate_keys(self):
+        a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u1')])
+        b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
+        assert_raises(ValueError, join_by, ['a', 'b', 'b'], a, b)
+
 
 class TestJoinBy2(TestCase):
     @classmethod