Repository: scipy/scipy

Issue:
Massive performance regression in cKDTree.query with L_inf distance in scipy 0.17
Hi,

While running some of my own code, I noticed a major slowdown in scipy 0.17 which I tracked to the cKDtree.query call with L_inf distance.
I did a git bisect and tracked the commit that caused the issue -- the commit is :
58a10c2688398ab440de8d6d7b5fba29671df182
Here is the example program that finishes in ~ 10 seconds before this commit and pretty much never finishes after it

```
import scipy.spatial.ckdtree
import numpy as np
arr1 = np.random.uniform(size=(6000000, 3))
tree2 = scipy.spatial.ckdtree.cKDTree(arr1)
distances, ind = tree2.query(arr1, 1, p=np.inf)
```

I also checked that if I change the distance from L_inf to L_2, then program finishes pretty quickly. I also tried to stop the program with gdb a few times to see where the code is mostly spending the time on and it seems to be here scipy/spatial/ckdtree/src/query.cxx:311. I also tried to dig into the problematic commit but I don't know enough about ckdtree internals to suggest a patch. 

Thanks, 
 Sergey

If relevant, I'm seeing this issue on a both systems that I tried  64bit RHEL6.7 with gcc 4.4 and Debian 7.9 with gcc-4.7. In all the cases it is python 2.7



Discussion:

Resolving patch:
diff --git a/benchmarks/benchmarks/spatial.py b/benchmarks/benchmarks/spatial.py
--- a/benchmarks/benchmarks/spatial.py
+++ b/benchmarks/benchmarks/spatial.py
@@ -106,27 +106,74 @@ class Neighbors(Benchmark):
         [1, 2, np.inf],
         [0.2, 0.5],
         BOX_SIZES, LEAF_SIZES,
+        ['cKDTree', 'cKDTree_weighted'],
     ]
-    param_names = ['(m, n1, n2)', 'p', 'probe radius', 'boxsize', 'leafsize']
+    param_names = ['(m, n1, n2)', 'p', 'probe radius', 'boxsize', 'leafsize', 'cls']
 
-    def setup(self, mn1n2, p, probe_radius, boxsize, leafsize):
+    def setup(self, mn1n2, p, probe_radius, boxsize, leafsize, cls):
         m, n1, n2 = mn1n2
 
         self.data1 = np.random.uniform(size=(n1, m))
         self.data2 = np.random.uniform(size=(n2, m))
 
+        self.w1 = np.ones(n1)
+        self.w2 = np.ones(n2)
+
         self.T1 = cKDTree(self.data1, boxsize=boxsize, leafsize=leafsize)
         self.T2 = cKDTree(self.data2, boxsize=boxsize, leafsize=leafsize)
 
-    def time_sparse_distance_matrix(self, mn1n2, p, probe_radius, boxsize, leafsize):
+    def time_sparse_distance_matrix(self, mn1n2, p, probe_radius, boxsize, leafsize, cls):
         self.T1.sparse_distance_matrix(self.T2, probe_radius, p=p)
 
-    def time_count_neighbors(self, mn1n2, p, probe_radius, boxsize, leafsize):
+    def time_count_neighbors(self, mn1n2, p, probe_radius, boxsize, leafsize, cls):
         """
         Count neighbors kd-tree
-        dim | # points T1 | # points T2 | p | probe radius |  BoxSize | LeafSize
+        dim | # points T1 | # points T2 | p | probe radius |  BoxSize | LeafSize | cls
+        """
+
+        if cls != 'cKDTree_weighted':
+            self.T1.count_neighbors(self.T2, probe_radius, p=p)
+        else:
+            self.T1.count_neighbors(self.T2, probe_radius, self_weights=self.w1, other_weights=self.w2, p=p)
+
+class CNeighbors(Benchmark):
+    params = [
+        [
+          (2,1000,1000),
+          (8,1000,1000),
+          (16,1000,1000)
+        ],
+        [2, 10, 100, 400, 1000],
+    ]
+    param_names = ['(m, n1, n2)', 'Nr']
+
+    def setup(self, mn1n2, Nr):
+        m, n1, n2 = mn1n2
+
+        data1 = np.random.uniform(size=(n1, m))
+        data2 = np.random.uniform(size=(n2, m))
+        self.w1 = np.ones(len(data1))
+        self.w2 = np.ones(len(data2))
+ 
+        self.T1d = cKDTree(data1, leafsize=1)
+        self.T2d = cKDTree(data2, leafsize=1)
+        self.T1s = cKDTree(data1, leafsize=8)
+        self.T2s = cKDTree(data2, leafsize=8)
+        self.r = np.linspace(0, 0.5, Nr)
+
+    def time_count_neighbors_deep(self, mn1n2, Nr):
+        """
+        Count neighbors for a very deep kd-tree
+        dim | # points T1 | # points T2 | Nr
+        """
+        self.T1d.count_neighbors(self.T2d, self.r)
+
+    def time_count_neighbors_shallow(self, mn1n2, Nr):
+        """
+        Count neighbors for a shallow kd-tree
+        dim | # points T1 | # points T2 | Nr
         """
-        self.T1.count_neighbors(self.T2, probe_radius, p=p)
+        self.T1s.count_neighbors(self.T2s, self.r)
 
 def generate_spherical_points(num_points):
         # generate uniform points on sphere (see:


Test patch:
diff --git a/scipy/spatial/tests/test_kdtree.py b/scipy/spatial/tests/test_kdtree.py
--- a/scipy/spatial/tests/test_kdtree.py
+++ b/scipy/spatial/tests/test_kdtree.py
@@ -4,13 +4,17 @@
 from __future__ import division, print_function, absolute_import
 
 from numpy.testing import (assert_equal, assert_array_equal,
-    assert_almost_equal, assert_array_almost_equal, assert_, run_module_suite)
+    assert_almost_equal, assert_array_almost_equal, assert_,
+    assert_raises,
+    run_module_suite)
 
 import numpy as np
 from scipy.spatial import KDTree, Rectangle, distance_matrix, cKDTree
 from scipy.spatial.ckdtree import cKDTreeNode
 from scipy.spatial import minkowski_distance
 
+import itertools
+
 def distance_box(a, b, p, boxsize):
     diff = a - b
     diff[diff > 0.5 * boxsize] -= boxsize
@@ -1065,19 +1069,11 @@ def test_ckdtree_box():
 
 def test_ckdtree_box_upper_bounds():
     data = np.linspace(0, 2, 10).reshape(-1, 1)
-    try:
-        cKDTree(data, leafsize=1, boxsize=1.0)
-    except ValueError:
-        return
-    raise AssertionError("ValueError is not raised")
+    assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=1.0)
 
 def test_ckdtree_box_lower_bounds():
     data = np.linspace(-1, 1, 10)
-    try:
-        cKDTree(data, leafsize=1, boxsize=1.0)
-    except ValueError:
-        return
-    raise AssertionError("ValueError is not raised")
+    assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=1.0)
 
 def simulate_periodic_box(kdtree, data, k, boxsize):
     dd = []
@@ -1105,6 +1101,10 @@ def simulate_periodic_box(kdtree, data, k, boxsize):
     
 def test_ckdtree_memuse():
     # unit test adaptation of gh-5630
+
+    # NOTE: this will fail when run via valgrind,
+    # because rss is no longer a reliable memory usage indicator.
+
     try:
         import resource
     except ImportError:
@@ -1139,6 +1139,71 @@ def test_ckdtree_memuse():
     # ideally zero leaks, but errors might accidentally happen
     # outside cKDTree
     assert_(num_leaks < 10)
+
+def test_ckdtree_weights():
+
+    data = np.linspace(0, 1, 4).reshape(-1, 1)
+    tree1 = cKDTree(data, leafsize=1)
+    weights = np.ones(len(data), dtype='f4')
+
+    nw = tree1._build_weights(weights)
+    assert_array_equal(nw, [4, 2, 1, 1, 2, 1, 1])
+
+    assert_raises(ValueError, tree1._build_weights, weights[:-1])
+
+    for i in range(10):
+        # since weights are uniform, these shall agree:
+        c1 = tree1.count_neighbors(tree1, np.linspace(0, 10, i))
+        c2 = tree1.count_neighbors(tree1, np.linspace(0, 10, i), 
+                weights=(weights, weights))
+        c3 = tree1.count_neighbors(tree1, np.linspace(0, 10, i), 
+                weights=(weights, None))
+        c4 = tree1.count_neighbors(tree1, np.linspace(0, 10, i), 
+                weights=(None, weights))
+        c5 = tree1.count_neighbors(tree1, np.linspace(0, 10, i), 
+                weights=weights)
+
+        assert_array_equal(c1, c2)
+        assert_array_equal(c1, c3)
+        assert_array_equal(c1, c4)
+
+    for i in range(len(data)):
+        # this tests removal of one data point by setting weight to 0
+        w1 = weights.copy()
+        w1[i] = 0
+        data2 = data[w1 != 0]
+        w2 = weights[w1 != 0]
+        tree2 = cKDTree(data2)
+
+        c1 = tree1.count_neighbors(tree1, np.linspace(0, 10, 100),
+                weights=(w1, w1))
+        # "c2 is correct"
+        c2 = tree2.count_neighbors(tree2, np.linspace(0, 10, 100))
+
+        assert_array_equal(c1, c2)
+
+        #this asserts for two different trees, singular weights
+        # crashes
+        assert_raises(ValueError, tree1.count_neighbors,
+            tree2, np.linspace(0, 10, 100), weights=w1)
+
+def test_ckdtree_count_neighbous_multiple_r():
+    n = 2000
+    m = 2
+    np.random.seed(1234)
+    data = np.random.normal(size=(n, m))
+    kdtree = cKDTree(data, leafsize=1)
+    r0 = [0, 0.01, 0.01, 0.02, 0.05]
+    i0 = np.arange(len(r0))
+    n0 = kdtree.count_neighbors(kdtree, r0)
+    nnc = kdtree.count_neighbors(kdtree, r0, cumulative=False)
+    assert_equal(n0, nnc.cumsum())
+
+    for i, r in zip(itertools.permutations(i0), 
+                    itertools.permutations(r0)):
+        # permute n0 by i and it shall agree 
+        n = kdtree.count_neighbors(kdtree, r)
+        assert_array_equal(n, n0[list(i)])
     
 def test_len0_arrays():
     # make sure len-0 arrays are handled correctly
@@ -1186,6 +1251,38 @@ def test_len0_arrays():
     z = np.empty(shape=(0,2), dtype=np.intp)
     assert_array_equal(y, z)
 
+def test_ckdtree_duplicated_inputs():
+    # check ckdtree with duplicated inputs
+    n = 1024 
+    for m in range(1, 8):
+        data = np.concatenate([
+            np.ones((n // 2, m)) * 1,
+            np.ones((n // 2, m)) * 2], axis=0)
+
+        # it shall not divide more than 3 nodes.
+        # root left (1), and right (2)
+        kdtree = cKDTree(data, leafsize=1)
+        assert_equal(kdtree.size, 3)
+
+        kdtree = cKDTree(data)
+        assert_equal(kdtree.size, 3)
+
+        # if compact_nodes are disabled, the number
+        # of nodes is n (per leaf) + (m - 1)* 2 (splits per dimension) + 1
+        # and the root
+        kdtree = cKDTree(data, compact_nodes=False, leafsize=1)
+        assert_equal(kdtree.size, n + m * 2 - 1)
+
+def test_ckdtree_noncumulative_nondecreasing():
+    # check ckdtree with duplicated inputs
+
+    # it shall not divide more than 3 nodes.
+    # root left (1), and right (2)
+    kdtree = cKDTree([[0]], leafsize=1)
+
+    assert_raises(ValueError, kdtree.count_neighbors,
+        kdtree, [0.1, 0], cumulative=False)
+
 def test_short_knn():
 
     # The test case is based on github: #6425 by @SteveDoyle2