Repository: scipy/scipy

Issue:
scipy.stats.mode extremely slow (Trac #905)
_Original ticket http://projects.scipy.org/scipy/ticket/905 on 2009-03-31 by trac user parejkoj, assigned to unknown._

I'm trying to run scipy.stats.mode (in scipy.stats.stats) on a 2-d ndarray, but it takes an extremely long time.  The data is an image (astronomical data), with:

data.shape = (227,3104)
data.dtype = ('float32')

A quick-and-dirty mode function like this:

```
def mode(data):
    counts = {}
    for x in data.flatten():
    counts[x] = counts.get(x,0) + 1
    maxcount = max(counts.values())
    modelist = []
    for x in counts:
    if counts[x] == maxcount:
        modelist.append(x)
    return modelist,maxcount
#...
```

takes only a second or so to run, while stats.mode(data) or stats.mode(data.flatten()) takes ~20 minutes, no matter how I set the axis parameter.  Am I missing something, or shouldn't it be significantly faster?

I'm using scipy 0.6.0 from Fink on Mac OS X Leopard with python2.5, but there appear to be no changes to the mode function between 0.6.0 and the current svn.



Discussion:
_@josef-pkt wrote on 2009-03-31_

scipy.stats.mode, and some other functions, loop over the array for each unique element (with numpy.where or similar) which is _very_ slow if the number of unique elements is large. 

For the general case with arbitrary types of unique elements, I haven't found anything faster than the loop with a python dict.
I expect numpy.argwhere to be faster for the second loop to find the indices of the 
max. 

```
>>> np.argwhere(np.array([1,2,3,3,3]) == 3)
array([[2],
       [3],
       [4]])
```

For integers there is numpy.bincount, and for floats a histogram might be more appropriate either numpy.histogram or scipy.ndimage.measurements.histogram.

_@rgommers wrote on 2010-07-17_

How about including this explanation in the docstring then? Right now it's not clear that (a) this has to be so slow and (b) that for ints/floats this is not the best function to use.

Milestone changed to `0.9.0` by @rgommers on 2010-07-17

gh-3035 is a duplicate of this issue in which another faster implementation was proposed.


Resolving patch:
diff --git a/benchmarks/benchmarks/stats.py b/benchmarks/benchmarks/stats.py
--- a/benchmarks/benchmarks/stats.py
+++ b/benchmarks/benchmarks/stats.py
@@ -92,5 +92,18 @@ def time_distribution(self, distribution, properties):
                 stats.beta.rvs(size=1000, a=5, b=3, loc=4, scale=10)
             elif properties == 'fit':
                 stats.beta.fit(self.x, a=5, b=3, loc=4, scale=10)
-        
+
+
+class DescriptiveStats(Benchmark):
+    param_names = ['n_levels']
+    params = [
+        [10, 1000]
+    ]
+
+    def setup(self, n_levels):
+        np.random.seed(12345678)
+        self.levels = np.random.randint(n_levels, size=(1000, 10))
+
+    def time_mode(self, n_levels):
+        stats.mode(self.levels, axis=0)
 
diff --git a/scipy/stats/mstats_basic.py b/scipy/stats/mstats_basic.py
--- a/scipy/stats/mstats_basic.py
+++ b/scipy/stats/mstats_basic.py
@@ -312,9 +312,7 @@ def _mode1D(a):
         elif cnt.size:
             return (rep[cnt.argmax()], cnt.max())
         else:
-            not_masked_indices = ma.flatnotmasked_edges(a)
-            first_not_masked_index = not_masked_indices[0]
-            return (a[first_not_masked_index], 1)
+            return (a.min(), 1)
 
     if axis is None:
         output = _mode1D(ma.ravel(a))
diff --git a/scipy/stats/stats.py b/scipy/stats/stats.py
--- a/scipy/stats/stats.py
+++ b/scipy/stats/stats.py
@@ -438,19 +438,42 @@ def mode(a, axis=0, nan_policy='propagate'):
         a = ma.masked_invalid(a)
         return mstats_basic.mode(a, axis)
 
-    scores = np.unique(np.ravel(a))       # get ALL unique values
-    testshape = list(a.shape)
-    testshape[axis] = 1
-    oldmostfreq = np.zeros(testshape, dtype=a.dtype)
-    oldcounts = np.zeros(testshape, dtype=int)
-    for score in scores:
-        template = (a == score)
-        counts = np.expand_dims(np.sum(template, axis), axis)
-        mostfrequent = np.where(counts > oldcounts, score, oldmostfreq)
-        oldcounts = np.maximum(counts, oldcounts)
-        oldmostfreq = mostfrequent
-
-    return ModeResult(mostfrequent, oldcounts)
+    if (NumpyVersion(np.__version__) < '1.9.0') or (a.dtype == object and np.nan in set(a)):
+        # Fall back to a slower method since np.unique does not work with NaN
+        # or for older numpy which does not support return_counts
+        scores = set(np.ravel(a))  # get ALL unique values
+        testshape = list(a.shape)
+        testshape[axis] = 1
+        oldmostfreq = np.zeros(testshape, dtype=a.dtype)
+        oldcounts = np.zeros(testshape, dtype=int)
+
+        for score in scores:
+            template = (a == score)
+            counts = np.expand_dims(np.sum(template, axis), axis)
+            mostfrequent = np.where(counts > oldcounts, score, oldmostfreq)
+            oldcounts = np.maximum(counts, oldcounts)
+            oldmostfreq = mostfrequent
+
+        return ModeResult(mostfrequent, oldcounts)
+
+    def _mode1D(a):
+        vals, cnts = np.unique(a, return_counts=True)
+        return vals[cnts.argmax()], cnts.max()
+
+    # np.apply_along_axis will convert the _mode1D tuples to a numpy array, casting types in the process
+    # This recreates the results without that issue
+    # View of a, rotated so the requested axis is last
+    in_dims = list(range(a.ndim))
+    a_view = np.transpose(a, in_dims[:axis] + in_dims[axis+1:] + [axis])
+
+    inds = np.ndindex(a_view.shape[:-1])
+    modes = np.empty(a_view.shape[:-1], dtype=a.dtype)
+    counts = np.zeros(a_view.shape[:-1], dtype=np.int)
+    for ind in inds:
+        modes[ind], counts[ind] = _mode1D(a_view[ind])
+    newshape = list(a.shape)
+    newshape[axis] = 1
+    return ModeResult(modes.reshape(newshape), counts.reshape(newshape))
 
 
 def _mask_to_limits(a, limits, inclusive):


Test patch:
diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py
--- a/scipy/stats/tests/test_stats.py
+++ b/scipy/stats/tests/test_stats.py
@@ -1359,7 +1359,6 @@ def test_strings(self):
         assert_equal(vals[0][0], 'showers')
         assert_equal(vals[1][0], 2)
 
-    @pytest.mark.xfail(sys.version_info > (3,), reason='numpy github issue 641')
     def test_mixed_objects(self):
         objects = [10, True, np.nan, 'hello', 10]
         arr = np.empty((5,), dtype=object)
@@ -1422,6 +1421,16 @@ def test_mode_nan(self):
         assert_raises(ValueError, stats.mode, data1, nan_policy='raise')
         assert_raises(ValueError, stats.mode, data1, nan_policy='foobar')
 
+    @pytest.mark.parametrize("data", [
+        [3, 5, 1, 1, 3],
+        [3, np.nan, 5, 1, 1, 3],
+        [3, 5, 1],
+        [3, np.nan, 5, 1],
+    ])
+    def test_smallest_equal(self, data):
+        result = stats.mode(data, nan_policy='omit')
+        assert_equal(result[0][0], 1)
+
 
 class TestVariability(object):