Repository: statsmodels/statsmodels

Issue:
BUG: TSA models if date index without known frequency
If given a Pandas `endog`, TSA models require the index to contain dates. If the date index does not have a frequency, an attempt is made to infer it, and it appears that the intent is to raise an error if this inference fails (`statsmodels/tsa/base/tsa_model.py:56`):

``` python
if not freq:
    try:
        freq = datetools._infer_freq(dates)
    except:
        raise ValueError("Frequency inference failed. Use `freq` "
                "keyword.")
```

However, `_infer_freq` doesn't usually raise an exception (from a brief look, it only ever raises an exception if the length of the series is less than 3), but it will often return `None` if the frequency can't be inferred.

I'm guessing we want to raise an exception here if the inferred frequency is None. I'll make a PR to add that behavior, unless I've misunderstood.



Discussion:
Although I guess this would be backwards incompatible.


Resolving patch:
diff --git a/statsmodels/tsa/base/tsa_model.py b/statsmodels/tsa/base/tsa_model.py
--- a/statsmodels/tsa/base/tsa_model.py
+++ b/statsmodels/tsa/base/tsa_model.py
@@ -3,7 +3,7 @@
 
 import datetime
 
-from pandas import to_datetime, DatetimeIndex, Period, PeriodIndex
+from pandas import to_datetime, DatetimeIndex, Period, PeriodIndex, Timestamp
 
 from statsmodels.base import data
 import statsmodels.base.model as base
@@ -71,6 +71,15 @@ def _init_dates(self, dates, freq):
         self.data.dates = dates
         self.data.freq = freq
 
+        # Test for nanoseconds in early pandas versions
+        if freq is not None and _freq_to_pandas[freq].freqstr == 'N':
+            from distutils.version import LooseVersion
+            from pandas import __version__ as pd_version
+            if LooseVersion(pd_version) < '0.14':
+                raise NotImplementedError('Nanosecond index not available in'
+                                          ' Pandas < 0.14')
+
+
     def _get_exog_names(self):
         return self.data.xnames
 
@@ -235,6 +244,18 @@ def _make_predict_dates(self):
             dates = self.data.dates.__class__(start=dtstart,
                                               end=dtend,
                                               freq=pandas_freq)
+
+            if pandas_freq.freqstr == 'N':
+                _dtend = dtend
+                if isinstance(dates[-1], Period):
+                    _dtend = pd.to_datetime(_dtend).to_period(dates.freq)
+                if not dates[-1] == _dtend:
+                    # TODO: this is a hack because a DatetimeIndex with
+                    # nanosecond frequency does not include "end"
+                    dtend = Timestamp(dtend.value + 1)
+                    dates = self.data.dates.__class__(start=dtstart,
+                                                      end=dtend,
+                                                      freq=pandas_freq)
         # handle
         elif freq is None and (isinstance(dtstart, (int, long)) and
                                isinstance(dtend, (int, long))):


Test patch:
diff --git a/statsmodels/tsa/base/tests/test_base.py b/statsmodels/tsa/base/tests/test_base.py
--- a/statsmodels/tsa/base/tests/test_base.py
+++ b/statsmodels/tsa/base/tests/test_base.py
@@ -1,9 +1,9 @@
 import numpy as np
 import numpy.testing as npt
 import pandas as pd
-
+from statsmodels.tsa.base.datetools import _freq_to_pandas
 from statsmodels.tsa.base.tsa_model import TimeSeriesModel
-from statsmodels.tools.testing import assert_equal
+from statsmodels.tools.testing import assert_equal, assert_raises
 
 
 def test_pandas_nodates_index():
@@ -14,6 +14,38 @@ def test_pandas_nodates_index():
 
     npt.assert_raises(ValueError, TimeSeriesModel, s)
 
+    # Test with a non-date index that doesn't raise an exception because it
+    # can be coerced into a nanosecond DatetimeIndex
+    # (This test doesn't make sense for Numpy < 1.7 since they don't have
+    # nanosecond support)
+    # (This test also doesn't make sense for Pandas < 0.14 since we don't
+    # support nanosecond index in Pandas < 0.14)
+    try:
+        # Check for Numpy < 1.7
+        _freq_to_pandas['N']
+    except:
+        pass
+    else:
+        data = [988, 819, 964]
+        # index=pd.date_range('1970-01-01', periods=3, freq='QS')
+        index = pd.to_datetime([100, 101, 102])
+        s = pd.Series(data, index=index)
+
+        # Alternate test for Pandas < 0.14
+        from distutils.version import LooseVersion
+        from pandas import __version__ as pd_version
+        if LooseVersion(pd_version) < '0.14':
+            assert_raises(NotImplementedError, TimeSeriesModel, s)
+        else:
+            actual_str = (index[0].strftime('%Y-%m-%d %H:%M:%S.%f') +
+                          str(index[0].value))
+            assert_equal(actual_str, '1970-01-01 00:00:00.000000100')
+            mod = TimeSeriesModel(s)
+            start = mod._get_predict_start(0)
+            end, out_of_sample = mod._get_predict_end(4)
+            mod._make_predict_dates()
+            assert_equal(len(mod.data.predict_dates), 5)
+
 def test_predict_freq():
     # test that predicted dates have same frequency
     x = np.arange(1,36.)