Repository: numba/numba

Issue:
A reference count handling bug
https://groups.google.com/a/continuum.io/forum/?utm_medium=email&utm_source=footer#!msg/numba-users/8yKajxuVJ10/CqaZXGA_AwAJ



Discussion:
The problem is yet another hidden alias problem I tried to fix in #1518.

The code provided by the user contains a basicblock with the following instructions:

``` llvm
  %.1328 = phi { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } [ %.1125, %switch.0.2 ], [ %.1126, %switch.1.2 ]
  %extracted.meminfo.36 = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 0
  %extracted.shape.36 = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 5
  %extracted.strides.36 = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 6
  call void @NRT_incref(i8* %extracted.meminfo.36)
  call void @NRT_decref(i8* null)
  %.1328.fca.1.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 1
  %.1328.fca.2.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 2
  %.1328.fca.3.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 3
  %.1328.fca.4.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 4
  %.1328.fca.5.0.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 5, 0
  %.1328.fca.6.0.extract = extractvalue { i8*, i8*, i64, i64, i64*, [1 x i64], [1 x i64] } %.1328, 6, 0
  call void @NRT_decref(i8* %extracted.meminfo.22)
  call void @NRT_decref(i8* %extracted.meminfo.23)
  %.1365 = insertvalue [1 x i64] undef, i64 %.1328.fca.5.0.extract, 0
  %.1366 = extractvalue [1 x i64] %.1365, 0
  %.1368.fca.0.insert = insertvalue [1 x i64] undef, i64 %.1328.fca.6.0.extract, 0
  %.1386 = load i64* %.1328.fca.4.extract
  call void @NRT_decref(i8* %extracted.meminfo.36)
```

The reference count pruning tries to remove the redundant pair of incref and decref `call void @NRT_incref(i8* %extracted.meminfo.36)` and `call void @NRT_decref(i8* %extracted.meminfo.36)`.
However, it does not know that `%extracted.meminfo.22` and/or `%extracted.meminfo.23` are potential alias to `%extracted.meminfo.36` and the following sequence may also access the underlying data:

``` llvm
  %.1365 = insertvalue [1 x i64] undef, i64 %.1328.fca.5.0.extract, 0
  %.1366 = extractvalue [1 x i64] %.1365, 0
  %.1368.fca.0.insert = insertvalue [1 x i64] undef, i64 %.1328.fca.6.0.extract, 0
  %.1386 = load i64* %.1328.fca.4.extract
```

After pruning, the above access is loading from data that is removed.

#1518 have only fixed problem due to hidden alias related to other incref.  This is hidden alias due to other decref.  A possible solution is to move all the decref to the last location of decref before pruning.  


Resolving patch:
diff --git a/numba/runtime/atomicops.py b/numba/runtime/atomicops.py
--- a/numba/runtime/atomicops.py
+++ b/numba/runtime/atomicops.py
@@ -245,7 +245,7 @@ def _extract_basic_blocks(func_lines):
         yield False, [func_lines[-1]]
 
     def _process_basic_block(bb_lines):
-        bb_lines = _move_decref_after_all_increfs(bb_lines)
+        bb_lines = _move_and_group_decref_after_all_increfs(bb_lines)
         bb_lines = _prune_redundant_refct_ops(bb_lines)
         return bb_lines
 
@@ -284,12 +284,20 @@ def _prune_redundant_refct_ops(bb_lines):
         return [ln for num, ln in enumerate(bb_lines)
                 if num not in to_remove]
 
-    def _move_decref_after_all_increfs(bb_lines):
+    def _move_and_group_decref_after_all_increfs(bb_lines):
         # find last incref
-        last_pos = 0
+        last_incref_pos = 0
         for pos, ln in enumerate(bb_lines):
             if _regex_incref.match(ln) is not None:
-                last_pos = pos + 1
+                last_incref_pos = pos + 1
+
+        # find last decref
+        last_decref_pos = 0
+        for pos, ln in enumerate(bb_lines):
+            if _regex_decref.match(ln) is not None:
+                last_decref_pos = pos + 1
+
+        last_pos = max(last_incref_pos, last_decref_pos)
 
         # find decrefs before last_pos
         decrefs = []
@@ -303,6 +311,7 @@ def _move_decref_after_all_increfs(bb_lines):
         # insert decrefs at last_pos
         return head + decrefs + bb_lines[last_pos:]
 
+
     # Early escape if NRT_incref is not used
     try:
         ll_module.get_function('NRT_incref')


Test patch:
diff --git a/numba/tests/test_nrt.py b/numba/tests/test_nrt.py
--- a/numba/tests/test_nrt.py
+++ b/numba/tests/test_nrt.py
@@ -306,6 +306,29 @@ def f():
         np.testing.assert_equal(a, b)
         np.testing.assert_equal(a, np.ones(10, dtype=np.float64))
 
+    def test_refct_pruning_issue_1526(self):
+        @njit
+        def udt(image, x, y):
+            next_loc = np.where(image == 1)
+
+            if len(next_loc[0]) == 0:
+                y_offset = 1
+                x_offset = 1
+            else:
+                y_offset = next_loc[0][0]
+                x_offset = next_loc[1][0]
+
+            next_loc_x = (x - 1) + x_offset
+            next_loc_y = (y - 1) + y_offset
+
+            return next_loc_x, next_loc_y
+
+        a = np.array([[1, 0, 1, 0, 1, 0, 0, 1, 0, 0]])
+        expect = udt.py_func(a, 1, 6)
+        got = udt(a, 1, 6)
+
+        self.assertEqual(expect, got)
+
 
 if __name__ == '__main__':
     unittest.main()