Repository: pandas-dev/pandas

Issue:
Mean overflows for integer dtypes
Say there is a array of type int64
for convenience, that me just put is some large number

test1 = pd.Series(20150515061816532, index=list(range(500)), dtype='int64')
test1.describe()
Out[152]: 
count    5.000000e+02
**mean    -1.674297e+16**
std      0.000000e+00
min      2.015052e+16
25%      2.015052e+16
50%      2.015052e+16
75%      2.015052e+16
max      2.015052e+16

Look at the mean, it overflow, and become negative. Obviously the mean should be 20150515061816532

In [153]: test1.sum()
**Out[153]: -8371486542801285616** This is wrong.

The computation should have been sum them up as **float**, and devided by total count.

I think we need to examine other parts of code that involve similar situation.

Mean overflows for integer dtypes
Say there is a array of type int64
for convenience, that me just put is some large number

test1 = pd.Series(20150515061816532, index=list(range(500)), dtype='int64')
test1.describe()
Out[152]: 
count    5.000000e+02
**mean    -1.674297e+16**
std      0.000000e+00
min      2.015052e+16
25%      2.015052e+16
50%      2.015052e+16
75%      2.015052e+16
max      2.015052e+16

Look at the mean, it overflow, and become negative. Obviously the mean should be 20150515061816532

In [153]: test1.sum()
**Out[153]: -8371486542801285616** This is wrong.

The computation should have been sum them up as **float**, and devided by total count.

I think we need to examine other parts of code that involve similar situation.



Discussion:
How do I label it as Bug?

How do I label it as Bug?


Resolving patch:
diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py
--- a/pandas/core/nanops.py
+++ b/pandas/core/nanops.py
@@ -20,7 +20,7 @@
                                 is_complex_dtype, is_integer_dtype,
                                 is_bool_dtype, is_object_dtype,
                                 is_datetime64_dtype, is_timedelta64_dtype,
-                                is_datetime_or_timedelta_dtype,
+                                is_datetime_or_timedelta_dtype, _get_dtype,
                                 is_int_or_datetime_dtype, is_any_int_dtype)
 
 
@@ -254,8 +254,16 @@ def nansum(values, axis=None, skipna=True):
 @bottleneck_switch()
 def nanmean(values, axis=None, skipna=True):
     values, mask, dtype, dtype_max = _get_values(values, skipna, 0)
-    the_sum = _ensure_numeric(values.sum(axis, dtype=dtype_max))
-    count = _get_counts(mask, axis)
+
+    dtype_sum = dtype_max
+    dtype_count = np.float64
+    if is_integer_dtype(dtype):
+        dtype_sum = np.float64
+    elif is_float_dtype(dtype):
+        dtype_sum = dtype
+        dtype_count = dtype
+    count = _get_counts(mask, axis, dtype=dtype_count)
+    the_sum = _ensure_numeric(values.sum(axis, dtype=dtype_sum))
 
     if axis is not None and getattr(the_sum, 'ndim', False):
         the_mean = the_sum / count
@@ -557,15 +565,16 @@ def _maybe_arg_null_out(result, axis, mask, skipna):
     return result
 
 
-def _get_counts(mask, axis):
+def _get_counts(mask, axis, dtype=float):
+    dtype = _get_dtype(dtype)
     if axis is None:
-        return float(mask.size - mask.sum())
+        return dtype.type(mask.size - mask.sum())
 
     count = mask.shape[axis] - mask.sum(axis)
     try:
-        return count.astype(float)
+        return count.astype(dtype)
     except AttributeError:
-        return np.array(count, dtype=float)
+        return np.array(count, dtype=dtype)
 
 
 def _maybe_null_out(result, axis, mask):


Test patch:
diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py
--- a/pandas/tests/test_nanops.py
+++ b/pandas/tests/test_nanops.py
@@ -5,7 +5,7 @@
 
 import numpy as np
 
-from pandas.core.common import isnull
+from pandas.core.common import isnull, is_integer_dtype
 import pandas.core.nanops as nanops
 import pandas.util.testing as tm
 
@@ -323,6 +323,32 @@ def test_nanmean(self):
                         allow_complex=False, allow_obj=False,
                         allow_str=False, allow_date=False, allow_tdelta=True)
 
+    def test_nanmean_overflow(self):
+        # GH 10155
+        # In the previous implementation mean can overflow for int dtypes, it
+        # is now consistent with numpy
+        from pandas import Series
+
+        # numpy < 1.9.0 is not computing this correctly
+        from distutils.version import LooseVersion
+        if LooseVersion(np.__version__) >= '1.9.0':
+            for a in [2 ** 55, -2 ** 55, 20150515061816532]:
+                s = Series(a, index=range(500), dtype=np.int64)
+                result = s.mean()
+                np_result = s.values.mean()
+                self.assertEqual(result, a)
+                self.assertEqual(result, np_result)
+                self.assertTrue(result.dtype == np.float64)
+
+        # check returned dtype
+        for dtype in [np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]:
+            s = Series(range(10), dtype=dtype)
+            result = s.mean()
+            if is_integer_dtype(dtype):
+                self.assertTrue(result.dtype == np.float64)
+            else:
+                self.assertTrue(result.dtype == dtype)
+
     def test_nanmedian(self):
         self.check_funs(nanops.nanmedian, np.median,
                         allow_complex=False, allow_str=False, allow_date=False,