Repository: pandas-dev/pandas

Issue:
BUG: resample/groupby with NaT in the grouper
related to this: https://github.com/pydata/pandas/issues/6992

These are broken

```
In [70]: df.set_index('A').resample('M',how='count')
ValueError: month must be in 1..12

In [73]: df.groupby(pd.Grouper(freq='MS',key='A')).count()
ValueError: month must be in 1..12
```

Separately, this should not also return the A column

```
In [74]: df.dropna(how='any',subset=['A']).groupby(pd.Grouper(key='A',freq='M')).count()
Out[74]: 
            A  B
A               
2014-10-31  3  3
2014-11-30  2  2
2014-12-31  1  1
```

```
In [57]: data
Out[57]: 
0   2014-10-01
1   2014-10-01
2   2014-10-31
3   2014-11-15
4   2014-11-30
5          NaT
6   2014-12-01
dtype: datetime64[ns]

In [58]: df = DataFrame(dict(A = data, B = np.arange(len(data))))
```

This is a work-around

```
In [59]: df.dropna(how='any',subset=['A']).set_index('A').resample('M',how='count')
Out[59]: 
            B
A            
2014-10-31  3
2014-11-30  2
2014-12-31  1
```



Discussion:
Is this only with count?

I don't think so; I think its an issue with the `NaT` in the grouping itself


Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -28,6 +28,7 @@
 from pandas import _np_version_under1p7
 import pandas.lib as lib
 from pandas.lib import Timestamp
+import pandas.tslib as tslib
 import pandas.algos as _algos
 import pandas.hashtable as _hash
 
@@ -1581,7 +1582,11 @@ def groups(self):
 
         # this is mainly for compat
         # GH 3881
-        return dict(zip(self.binlabels,self.bins))
+        result = {}
+        for key, value in zip(self.binlabels, self.bins):
+            if key is not tslib.NaT:
+                result[key] = value
+        return result
 
     @property
     def nkeys(self):
@@ -1605,7 +1610,8 @@ def get_iterator(self, data, axis=0):
 
         start = 0
         for edge, label in zip(self.bins, self.binlabels):
-            yield label, slicer(start,edge)
+            if label is not tslib.NaT:
+                yield label, slicer(start,edge)
             start = edge
 
         if start < length:
@@ -1636,7 +1642,7 @@ def indices(self):
 
         i = 0
         for label, bin in zip(self.binlabels, self.bins):
-            if i < bin:
+            if label is not tslib.NaT and i < bin:
                 indices[label] = list(range(i, bin))
                 i = bin
         return indices
@@ -1647,7 +1653,8 @@ def ngroups(self):
 
     @cache_readonly
     def result_index(self):
-        return self.binlabels
+        mask = self.binlabels.asi8 == tslib.iNaT
+        return self.binlabels[~mask]
 
     @property
     def levels(self):
diff --git a/pandas/tseries/resample.py b/pandas/tseries/resample.py
--- a/pandas/tseries/resample.py
+++ b/pandas/tseries/resample.py
@@ -13,6 +13,7 @@
 
 from pandas.lib import Timestamp
 import pandas.lib as lib
+import pandas.tslib as tslib
 
 
 _DEFAULT_METHOD = 'mean'
@@ -186,6 +187,10 @@ def _get_time_bins(self, ax):
             elif not trimmed:
                 labels = labels[:-1]
 
+        if (ax_values == tslib.iNaT).any():
+            binner = binner.insert(0, tslib.NaT)
+            labels = labels.insert(0, tslib.NaT)
+
         # if we end up with more labels than bins
         # adjust the labels
         # GH4076
@@ -352,14 +357,14 @@ def _get_range_edges(axis, offset, closed='left', base=0):
     if isinstance(offset, compat.string_types):
         offset = to_offset(offset)
 
+    first, last = axis.min(), axis.max()
     if isinstance(offset, Tick):
         day_nanos = _delta_to_nanoseconds(timedelta(1))
         # #1165
         if (day_nanos % offset.nanos) == 0:
-            return _adjust_dates_anchored(axis[0], axis[-1], offset,
+            return _adjust_dates_anchored(first, last, offset,
                                           closed=closed, base=base)
 
-    first, last = axis.min(), axis.max()
     if not isinstance(offset, Tick):  # and first.time() != last.time():
         # hack!
         first = tools.normalize_date(first)


Test patch:
diff --git a/pandas/tseries/tests/test_resample.py b/pandas/tseries/tests/test_resample.py
--- a/pandas/tseries/tests/test_resample.py
+++ b/pandas/tseries/tests/test_resample.py
@@ -744,28 +744,32 @@ def test_resample_consistency(self):
 
     def test_resample_timegrouper(self):
         # GH 7227
-        dates = [datetime(2014, 10, 1), datetime(2014, 9, 3),
+        dates1 = [datetime(2014, 10, 1), datetime(2014, 9, 3),
                   datetime(2014, 11, 5), datetime(2014, 9, 5),
                   datetime(2014, 10, 8), datetime(2014, 7, 15)]
 
-        df = DataFrame(dict(A=dates, B=np.arange(len(dates))))
-        result = df.set_index('A').resample('M', how='count')
-        exp_idx = pd.DatetimeIndex(['2014-07-31', '2014-08-31', '2014-09-30',
-                                    '2014-10-31', '2014-11-30'], freq='M', name='A')
-        expected = DataFrame({'B': [1, 0, 2, 2, 1]}, index=exp_idx)
-        assert_frame_equal(result, expected)
+        dates2 = dates1[:2] + [pd.NaT] + dates1[2:4] + [pd.NaT] + dates1[4:]
+        dates3 = [pd.NaT] + dates1 + [pd.NaT]
 
-        result = df.groupby(pd.Grouper(freq='M', key='A')).count()
-        assert_frame_equal(result, expected)
+        for dates in [dates1, dates2, dates3]:
+            df = DataFrame(dict(A=dates, B=np.arange(len(dates))))
+            result = df.set_index('A').resample('M', how='count')
+            exp_idx = pd.DatetimeIndex(['2014-07-31', '2014-08-31', '2014-09-30',
+                                        '2014-10-31', '2014-11-30'], freq='M', name='A')
+            expected = DataFrame({'B': [1, 0, 2, 2, 1]}, index=exp_idx)
+            assert_frame_equal(result, expected)
 
-        df = DataFrame(dict(A=dates, B=np.arange(len(dates)), C=np.arange(len(dates))))
-        result = df.set_index('A').resample('M', how='count')
-        expected = DataFrame({'B': [1, 0, 2, 2, 1], 'C': [1, 0, 2, 2, 1]},
-                             index=exp_idx, columns=['B', 'C'])
-        assert_frame_equal(result, expected)
+            result = df.groupby(pd.Grouper(freq='M', key='A')).count()
+            assert_frame_equal(result, expected)
 
-        result = df.groupby(pd.Grouper(freq='M', key='A')).count()
-        assert_frame_equal(result, expected)
+            df = DataFrame(dict(A=dates, B=np.arange(len(dates)), C=np.arange(len(dates))))
+            result = df.set_index('A').resample('M', how='count')
+            expected = DataFrame({'B': [1, 0, 2, 2, 1], 'C': [1, 0, 2, 2, 1]},
+                                 index=exp_idx, columns=['B', 'C'])
+            assert_frame_equal(result, expected)
+
+            result = df.groupby(pd.Grouper(freq='M', key='A')).count()
+            assert_frame_equal(result, expected)
 
 
 def _simple_ts(start, end, freq='D'):
@@ -1302,6 +1306,84 @@ def test_fails_on_no_datetime_index(self):
                                        "but got an instance of %r" % name):
                 df.groupby(TimeGrouper('D'))
 
+    def test_aggregate_normal(self):
+        # check TimeGrouper's aggregation is identical as normal groupby
+
+        n = 20
+        data = np.random.randn(n, 4)
+        normal_df = DataFrame(data, columns=['A', 'B', 'C', 'D'])
+        normal_df['key'] = [1, 2, 3, 4, 5] * 4
+
+        dt_df = DataFrame(data, columns=['A', 'B', 'C', 'D'])
+        dt_df['key'] = [datetime(2013, 1, 1), datetime(2013, 1, 2), datetime(2013, 1, 3),
+                        datetime(2013, 1, 4), datetime(2013, 1, 5)] * 4
+
+        normal_grouped = normal_df.groupby('key')
+        dt_grouped = dt_df.groupby(TimeGrouper(key='key', freq='D'))
+
+        for func in ['min', 'max', 'prod', 'var', 'std', 'mean']:
+            expected = getattr(normal_grouped, func)()
+            dt_result = getattr(dt_grouped, func)()
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            assert_frame_equal(expected, dt_result)
+
+        for func in ['count', 'sum']:
+            expected = getattr(normal_grouped, func)()
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            dt_result = getattr(dt_grouped, func)()
+            assert_frame_equal(expected, dt_result)
+
+        """
+        for func in ['first', 'last']:
+            expected = getattr(normal_grouped, func)()
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            dt_result = getattr(dt_grouped, func)()
+            assert_frame_equal(expected, dt_result)
+
+        for func in ['nth']:
+            expected = getattr(normal_grouped, func)(3)
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            dt_result = getattr(dt_grouped, func)(3)
+            assert_frame_equal(expected, dt_result)
+        """
+        # if TimeGrouper is used included, 'size' 'first','last' and 'nth' doesn't work yet
+
+    def test_aggregate_with_nat(self):
+        # check TimeGrouper's aggregation is identical as normal groupby
+
+        n = 20
+        data = np.random.randn(n, 4)
+        normal_df = DataFrame(data, columns=['A', 'B', 'C', 'D'])
+        normal_df['key'] = [1, 2, np.nan, 4, 5] * 4
+
+        dt_df = DataFrame(data, columns=['A', 'B', 'C', 'D'])
+        dt_df['key'] = [datetime(2013, 1, 1), datetime(2013, 1, 2), pd.NaT,
+                        datetime(2013, 1, 4), datetime(2013, 1, 5)] * 4
+
+        normal_grouped = normal_df.groupby('key')
+        dt_grouped = dt_df.groupby(TimeGrouper(key='key', freq='D'))
+
+        for func in ['min', 'max', 'prod']:
+            normal_result = getattr(normal_grouped, func)()
+            dt_result = getattr(dt_grouped, func)()
+            pad = DataFrame([[np.nan, np.nan, np.nan, np.nan]],
+                            index=[3], columns=['A', 'B', 'C', 'D'])
+            expected = normal_result.append(pad)
+            expected = expected.sort_index()
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            assert_frame_equal(expected, dt_result)
+
+        for func in ['count', 'sum']:
+            normal_result = getattr(normal_grouped, func)()
+            pad = DataFrame([[0, 0, 0, 0]], index=[3], columns=['A', 'B', 'C', 'D'])
+            expected = normal_result.append(pad)
+            expected = expected.sort_index()
+            expected.index = date_range(start='2013-01-01', freq='D', periods=5, name='key')
+            dt_result = getattr(dt_grouped, func)()
+            assert_frame_equal(expected, dt_result)
+
+        # if NaT is included, 'var', 'std', 'mean', 'size', 'first','last' and 'nth' doesn't work yet
+
 
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],