Repository: pandas-dev/pandas

Issue:
BUG: allow .std() of timedelta types
This currently raises `TypeError` as its disabled in `core/nanops.py`, see here: https://github.com/pydata/pandas/commit/d29d4c6623a29db36a4eb8d062bdcfe603e2b490

The issue is that `.var()` can generate a value that is too big for the timedeltas to hold propery, and std CALLS var.

solns:
- reduce the value of the timedeltas going into var to make them work (and inflate on exit), e.g. divide by 1e9 or something. lose some small amount of precision.
- allow std to return a valid value by passing a flag thru to var (and don't wrap the results). a bit hacky

```
In [7]: max_int = np.iinfo(np.int64).max

In [8]: max_int
Out[8]: 9223372036854775807

In [9]: pd.Timedelta(max_int)
Out[9]: Timedelta('106751 days 23:47:16.854775')

In [10]: big_float = 3e19

In [12]: pd.Timedelta(big_float)
---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
OverflowError: Python int too large to convert to C long

In [13]: pd.Timedelta(np.sqrt(big_float))
Out[13]: Timedelta('0 days 00:00:05.477225')
```



Discussion:

Resolving patch:
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -3950,60 +3950,42 @@ def mad(self,  axis=None, skipna=None, level=None, **kwargs):
             return np.abs(demeaned).mean(axis=axis, skipna=skipna)
         cls.mad = mad
 
-        @Substitution(outname='variance',
-                      desc="Return unbiased variance over requested "
-                           "axis.\n\nNormalized by N-1 by default. "
-                           "This can be changed using the ddof argument")
-        @Appender(_num_doc)
-        def var(self, axis=None, skipna=None, level=None, ddof=1, **kwargs):
-            if skipna is None:
-                skipna = True
-            if axis is None:
-                axis = self._stat_axis_number
-            if level is not None:
-                return self._agg_by_level('var', axis=axis, level=level,
-                                          skipna=skipna, ddof=ddof)
+        def _make_stat_function_ddof(name, desc, f):
 
-            return self._reduce(nanops.nanvar, axis=axis, skipna=skipna,
-                                ddof=ddof)
-        cls.var = var
-
-        @Substitution(outname='stdev',
-                      desc="Return unbiased standard deviation over requested "
-                           "axis.\n\nNormalized by N-1 by default. "
-                           "This can be changed using the ddof argument")
-        @Appender(_num_doc)
-        def std(self, axis=None, skipna=None, level=None, ddof=1, **kwargs):
-            if skipna is None:
-                skipna = True
-            if axis is None:
-                axis = self._stat_axis_number
-            if level is not None:
-                return self._agg_by_level('std', axis=axis, level=level,
-                                          skipna=skipna, ddof=ddof)
-            result = self.var(axis=axis, skipna=skipna, ddof=ddof)
-            if getattr(result, 'ndim', 0) > 0:
-                return result.apply(np.sqrt)
-            return np.sqrt(result)
-        cls.std = std
-
-        @Substitution(outname='standarderror',
-                      desc="Return unbiased standard error of the mean over "
-                           "requested axis.\n\nNormalized by N-1 by default. "
-                           "This can be changed using the ddof argument")
-        @Appender(_num_doc)
-        def sem(self, axis=None, skipna=None, level=None, ddof=1, **kwargs):
-            if skipna is None:
-                skipna = True
-            if axis is None:
-                axis = self._stat_axis_number
-            if level is not None:
-                return self._agg_by_level('sem', axis=axis, level=level,
-                                          skipna=skipna, ddof=ddof)
+            @Substitution(outname=name, desc=desc)
+            @Appender(_num_doc)
+            def stat_func(self, axis=None, skipna=None, level=None, ddof=1,
+                          **kwargs):
+                if skipna is None:
+                    skipna = True
+                if axis is None:
+                    axis = self._stat_axis_number
+                if level is not None:
+                    return self._agg_by_level(name, axis=axis, level=level,
+                                              skipna=skipna, ddof=ddof)
+                return self._reduce(f, axis=axis,
+                                    skipna=skipna, ddof=ddof)
+            stat_func.__name__ = name
+            return stat_func
 
-            return self._reduce(nanops.nansem, axis=axis, skipna=skipna,
-                                ddof=ddof)
-        cls.sem = sem
+        cls.sem = _make_stat_function_ddof(
+            'sem',
+            "Return unbiased standard error of the mean over "
+            "requested axis.\n\nNormalized by N-1 by default. "
+            "This can be changed using the ddof argument",
+            nanops.nansem)
+        cls.var = _make_stat_function_ddof(
+            'var',
+            "Return unbiased variance over requested "
+            "axis.\n\nNormalized by N-1 by default. "
+            "This can be changed using the ddof argument",
+            nanops.nanvar)
+        cls.std = _make_stat_function_ddof(
+            'std',
+            "Return unbiased standard deviation over requested "
+            "axis.\n\nNormalized by N-1 by default. "
+            "This can be changed using the ddof argument",
+            nanops.nanstd)
 
         @Substitution(outname='compounded',
                       desc="Return the compound percentage of the values for "
diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py
--- a/pandas/core/nanops.py
+++ b/pandas/core/nanops.py
@@ -228,7 +228,7 @@ def _wrap_results(result, dtype):
         if not isinstance(result, np.ndarray):
             result = lib.Timedelta(result)
         else:
-            result = result.view(dtype)
+            result = result.astype('i8').view(dtype)
 
     return result
 
@@ -295,7 +295,7 @@ def get_median(x):
     if values.ndim > 1:
         # there's a non-empty array to apply over otherwise numpy raises
         if notempty:
-            return np.apply_along_axis(get_median, axis, values)
+            return _wrap_results(np.apply_along_axis(get_median, axis, values), dtype)
 
         # must return the correct shape, but median is not defined for the
         # empty set so return nans of shape "everything but the passed axis"
@@ -305,7 +305,7 @@ def get_median(x):
         dims = np.arange(values.ndim)
         ret = np.empty(shp[dims != axis])
         ret.fill(np.nan)
-        return ret
+        return _wrap_results(ret, dtype)
 
     # otherwise return a scalar value
     return _wrap_results(get_median(values) if notempty else np.nan, dtype)
@@ -329,15 +329,8 @@ def _get_counts_nanvar(mask, axis, ddof):
     return count, d
 
 
-@disallow('M8','m8')
-@bottleneck_switch(ddof=1)
-def nanvar(values, axis=None, skipna=True, ddof=1):
-
-    # we are going to allow timedelta64[ns] here
-    # but NOT going to coerce them to the Timedelta type
-    # as this could cause overflow
-    # so var cannot be computed (but std can!)
-
+def _nanvar(values, axis=None, skipna=True, ddof=1):
+    # private nanvar calculator
     mask = isnull(values)
     if not _is_floating_dtype(values):
         values = values.astype('f8')
@@ -352,6 +345,23 @@ def nanvar(values, axis=None, skipna=True, ddof=1):
     XX = _ensure_numeric((values ** 2).sum(axis))
     return np.fabs((XX - X ** 2 / count) / d)
 
+@disallow('M8')
+@bottleneck_switch(ddof=1)
+def nanstd(values, axis=None, skipna=True, ddof=1):
+
+    result = np.sqrt(_nanvar(values, axis=axis, skipna=skipna, ddof=ddof))
+    return _wrap_results(result, values.dtype)
+
+@disallow('M8','m8')
+@bottleneck_switch(ddof=1)
+def nanvar(values, axis=None, skipna=True, ddof=1):
+
+    # we are going to allow timedelta64[ns] here
+    # but NOT going to coerce them to the Timedelta type
+    # as this could cause overflow
+    # so var cannot be computed (but std can!)
+    return _nanvar(values, axis=axis, skipna=skipna, ddof=ddof)
+
 @disallow('M8','m8')
 def nansem(values, axis=None, skipna=True, ddof=1):
     var = nanvar(values, axis, skipna, ddof=ddof)
@@ -517,7 +527,7 @@ def nankurt(values, axis=None, skipna=True):
         return result
 
 
-@disallow('M8')
+@disallow('M8','m8')
 def nanprod(values, axis=None, skipna=True):
     mask = isnull(values)
     if skipna and not _is_any_int_dtype(values):


Test patch:
diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py
--- a/pandas/tests/test_nanops.py
+++ b/pandas/tests/test_nanops.py
@@ -332,6 +332,10 @@ def test_nanvar(self):
         self.check_funs_ddof(nanops.nanvar, np.var,
                              allow_complex=False, allow_date=False, allow_tdelta=False)
 
+    def test_nanstd(self):
+        self.check_funs_ddof(nanops.nanstd, np.std,
+                             allow_complex=False, allow_date=False, allow_tdelta=True)
+
     def test_nansem(self):
         tm.skip_if_no_package('scipy.stats')
         self.check_funs_ddof(nanops.nansem, np.var,
diff --git a/pandas/tseries/tests/test_timedeltas.py b/pandas/tseries/tests/test_timedeltas.py
--- a/pandas/tseries/tests/test_timedeltas.py
+++ b/pandas/tseries/tests/test_timedeltas.py
@@ -479,6 +479,9 @@ def test_timedelta_ops(self):
         expected = to_timedelta(timedelta(seconds=9))
         self.assertEqual(result, expected)
 
+        result = td.to_frame().mean()
+        self.assertEqual(result[0], expected)
+
         result = td.quantile(.1)
         expected = Timedelta(np.timedelta64(2600,'ms'))
         self.assertEqual(result, expected)
@@ -487,18 +490,28 @@ def test_timedelta_ops(self):
         expected = to_timedelta('00:00:08')
         self.assertEqual(result, expected)
 
+        result = td.to_frame().median()
+        self.assertEqual(result[0], expected)
+
         # GH 6462
         # consistency in returned values for sum
         result = td.sum()
         expected = to_timedelta('00:01:21')
         self.assertEqual(result, expected)
 
-        # you can technically do a std, but var overflows
-        # so this is tricky
-        self.assertRaises(TypeError, lambda : td.std())
+        result = td.to_frame().sum()
+        self.assertEqual(result[0], expected)
+
+        # std
+        result = td.std()
+        expected = to_timedelta(Series(td.dropna().values).std())
+        self.assertEqual(result, expected)
+
+        result = td.to_frame().std()
+        self.assertEqual(result[0], expected)
 
         # invalid ops
-        for op in ['skew','kurt','sem','var']:
+        for op in ['skew','kurt','sem','var','prod']:
             self.assertRaises(TypeError, lambda : getattr(td,op)())
 
     def test_timedelta_ops_scalar(self):