Repository: statsmodels/statsmodels

Issue:
test failure tsa.filters  precision too high
most of the filter tests use `assert_equal` for floating point results.
Very often repeated calls to a function will **not** always give the exactly same answer.

I'm replacing `assert_equal(cycle.values, ndcycle)` in the tests in cfitz_pandas, but not the other ones. They might also need adjustments, but I haven't seen any failures yet in those.

```

======================================================================
FAIL: statsmodels.tsa.filters.tests.test_filters.test_cfitz_pandas
----------------------------------------------------------------------
Traceback (most recent call last):
  File "E:\Josef\testing\tox\py33\lib\site-packages\nose-1.3.0-py3.3.egg\nose\case.py", line 198, in runTest
    self.test(*self.arg)
  File "E:\Josef\testing\tox\py33\Lib\site-packages\statsmodels\tsa\filters\tests\test_filters.py", line 575,
in test_cfitz_pandas
    assert_equal(cycle.values, ndcycle)
  File "E:\Josef\testing\tox\py33\Lib\site-packages\numpy\testing\utils.py", line 257, in assert_equal
    return assert_array_equal(actual, desired, err_msg, verbose)
  File "E:\Josef\testing\tox\py33\Lib\site-packages\numpy\testing\utils.py", line 719, in assert_array_equal
    verbose=verbose, header='Arrays are not equal')
  File "E:\Josef\testing\tox\py33\Lib\site-packages\numpy\testing\utils.py", line 645, in assert_array_compare
    raise AssertionError(msg)
AssertionError:
Arrays are not equal

(mismatch 0.9852216748768541%)
 x: array([ 0.23792689,  0.77000717,  1.17773562,  1.25675402,  0.97212805,
        0.49188918,  0.07018866, -0.13043176, -0.13415474, -0.09207268,
       -0.12983069, -0.2417271 , -0.31990194, -0.27873995, -0.14891048,...
 y: array([ 0.23792689,  0.77000717,  1.17773562,  1.25675402,  0.97212805,
        0.49188918,  0.07018866, -0.13043176, -0.13415474, -0.09207268,
       -0.12983069, -0.2417271 , -0.31990194, -0.27873995, -0.14891048,...

----------------------------------------------------------------------
```



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -495,7 +495,7 @@ def get_data_files():
     package_data.update({"statsmodels.tsa.vector_ar.data" : ["*.dat"]})
     package_data.update({"statsmodels.tsa.vector_ar.data" : ["*.dat"]})
     # Why are we installing this stuff?
-    package_data.update({"statsmodels.libqstrung" : ["*.r, *.txt"]})
+    package_data.update({"statsmodels.libqstrung" : ["*.r, *.txt", "*.dat"]})
 
     #TODO: deal with this. Not sure if it ever worked for bdists
     #('docs/build/htmlhelp/statsmodelsdoc.chm',
diff --git a/statsmodels/tsa/arima_model.py b/statsmodels/tsa/arima_model.py
--- a/statsmodels/tsa/arima_model.py
+++ b/statsmodels/tsa/arima_model.py
@@ -866,8 +866,9 @@ def __new__(cls, endog, order, exog=None, dates=None, freq=None,
         if d == 0: # then we just use an ARMA model
             return ARMA(endog, (p,q), exog, dates, freq, missing)
         else:
-            return super(ARIMA, cls).__new__(cls, endog, order, exog, dates,
-                                             freq, missing)
+            mod = super(ARIMA, cls).__new__(cls)
+            mod.__init__(endog, order, exog, dates, freq, missing)
+            return mod
 
     def __init__(self, endog, order, exog=None, dates=None, freq=None,
                        missing='none'):


Test patch:
diff --git a/statsmodels/graphics/tests/test_mosaicplot.py b/statsmodels/graphics/tests/test_mosaicplot.py
--- a/statsmodels/graphics/tests/test_mosaicplot.py
+++ b/statsmodels/graphics/tests/test_mosaicplot.py
@@ -15,6 +15,9 @@
 except:
     have_matplotlib = False
 
+import pandas
+pandas_old = int(pandas.__version__.split('.')[1]) < 9
+
 # the main drawing function
 from statsmodels.graphics.mosaicplot import mosaic
 # other functions to be tested for accuracy
@@ -25,7 +28,7 @@
 from statsmodels.graphics.mosaicplot import _split_rect
 
 
-@dec.skipif(not have_matplotlib)
+@dec.skipif(not have_matplotlib or pandas_old)
 def test_data_conversion():
     # It will not reorder the elements
     # so the dictionary will look odd
@@ -96,7 +99,7 @@ def test_mosaic_simple():
     #pylab.show()
 
 
-@dec.skipif(not have_matplotlib)
+@dec.skipif(not have_matplotlib or pandas_old)
 def test_mosaic():
     # make the same analysis on a known dataset
 
diff --git a/statsmodels/iolib/tests/test_foreign.py b/statsmodels/iolib/tests/test_foreign.py
--- a/statsmodels/iolib/tests/test_foreign.py
+++ b/statsmodels/iolib/tests/test_foreign.py
@@ -17,6 +17,10 @@
 from statsmodels.datasets import macrodata
 
 
+import pandas
+pandas_old = int(pandas.__version__.split('.')[1]) < 9
+
+
 # Test precisions
 DECIMAL_4 = 4
 DECIMAL_3 = 3
@@ -93,17 +97,29 @@ def test_stata_writer_pandas():
     #as of 0.9.0 pandas only supports i8 and f8
     dta = dta.astype(np.dtype([('year', 'i8'),
                                ('quarter', 'i8')] + dtype.descr[2:]))
+    dta4 = dta.astype(np.dtype([('year', 'i4'),
+                               ('quarter', 'i4')] + dtype.descr[2:]))
     dta = DataFrame.from_records(dta)
+    dta4 = DataFrame.from_records(dta4)
+    # dta is int64 'i8'  given to Stata writer
     writer = StataWriter(buf, dta)
     writer.write_file()
     buf.seek(0)
     dta2 = genfromdta(buf)
-    ptesting.assert_frame_equal(dta.reset_index(), DataFrame.from_records(dta2))
+    dta5 = DataFrame.from_records(dta2)
+    # dta2 is int32 'i4'  returned from Stata reader
+
+    if dta5.dtypes[1] is np.dtype('int64'):
+        ptesting.assert_frame_equal(dta.reset_index(), dta5)
+    else:
+        # don't check index because it has different size, int32 versus int64
+        ptesting.assert_frame_equal(dta4, dta5[dta5.columns[1:]])
 
 def test_stata_writer_unicode():
     # make sure to test with characters outside the latin-1 encoding
     pass
 
+@dec.skipif(pandas_old)
 def test_genfromdta_datetime():
     results = [(datetime(2006, 11, 19, 23, 13, 20), 1479596223000,
             datetime(2010, 1, 20), datetime(2010, 1, 8), datetime(2010, 1, 1),
@@ -155,6 +171,7 @@ def test_date_converters():
         assert_equal(_datetime_to_stata_elapsed(
                      _stata_elapsed_date_to_datetime(i, "ty"), "ty"), i)
 
+@dec.skipif(pandas_old)
 def test_datetime_roundtrip():
     dta = np.array([(1, datetime(2010, 1, 1), 2),
                     (2, datetime(2010, 2, 1), 3),
diff --git a/statsmodels/stats/tests/test_power.py b/statsmodels/stats/tests/test_power.py
--- a/statsmodels/stats/tests/test_power.py
+++ b/statsmodels/stats/tests/test_power.py
@@ -715,6 +715,9 @@ def test_power_solver():
 
 def test_power_solver_warn():
     # messing up the solver to trigger warning
+    # I wrote this with scipy 0.9,
+    # convergence behavior of scipy 0.11 is different,
+    # fails at a different case, but is successful where it failed before
 
     pow_ = 0.69219411243824214 # from previous function
     nip = smp.NormalIndPower()
@@ -722,25 +725,29 @@ def test_power_solver_warn():
     nip.start_bqexp['nobs1'] = {'upp': 50, 'low': -20}
     val = nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1,
                           alternative='larger')
-    assert_almost_equal(val, 1600, decimal=4)
-    assert_equal(nip.cache_fit_res[0], 1)
-    assert_equal(len(nip.cache_fit_res), 3)
-
-    # case that has convergence failure, and should warn
-    nip.start_ttp['nobs1'] = np.nan
-
-    from statsmodels.tools.sm_exceptions import ConvergenceWarning
-    assert_warns(ConvergenceWarning, nip.solve_power, 0.1, nobs1=None,
-                  alpha=0.01, power=pow_, ratio=1, alternative='larger')
-
-    import warnings
-    with warnings.catch_warnings():  # python >= 2.6
-        warnings.simplefilter("ignore")
-        val = nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1,
-                              alternative='larger')
-        assert_equal(nip.cache_fit_res[0], 0)
+    import scipy
+    if scipy.__version__ < '0.10':
+        assert_almost_equal(val, 1600, decimal=4)
+        assert_equal(nip.cache_fit_res[0], 1)
         assert_equal(len(nip.cache_fit_res), 3)
 
+        # case that has convergence failure, and should warn
+        nip.start_ttp['nobs1'] = np.nan
+
+        from statsmodels.tools.sm_exceptions import ConvergenceWarning
+        assert_warns(ConvergenceWarning, nip.solve_power, 0.1, nobs1=None,
+                      alpha=0.01, power=pow_, ratio=1, alternative='larger')
+        # this converges with scipy 0.11  ???
+        # nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1, alternative='larger')
+
+        import warnings
+        with warnings.catch_warnings():  # python >= 2.6
+            warnings.simplefilter("ignore")
+            val = nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1,
+                                  alternative='larger')
+            assert_equal(nip.cache_fit_res[0], 0)
+            assert_equal(len(nip.cache_fit_res), 3)
+
 
 
 if __name__ == '__main__':
diff --git a/statsmodels/tsa/filters/tests/test_filters.py b/statsmodels/tsa/filters/tests/test_filters.py
--- a/statsmodels/tsa/filters/tests/test_filters.py
+++ b/statsmodels/tsa/filters/tests/test_filters.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from numpy.testing import assert_almost_equal, assert_equal
+from numpy.testing import assert_almost_equal, assert_equal, assert_allclose
 from numpy import array, column_stack
 from statsmodels.datasets import macrodata
 from statsmodels.tsa.base.datetools import dates_from_range
@@ -572,7 +572,7 @@ def test_cfitz_pandas():
     dta.index = index
     cycle, trend = cffilter(dta["infl"])
     ndcycle, ndtrend = cffilter(dta['infl'].values)
-    assert_equal(cycle.values, ndcycle)
+    assert_allclose(cycle.values, ndcycle, rtol=1e-14)
     assert_equal(cycle.index[0], datetime(1959, 3, 31))
     assert_equal(cycle.index[-1], datetime(2009, 9, 30))
     assert_equal(cycle.name, "infl")
@@ -580,7 +580,7 @@ def test_cfitz_pandas():
     #2d
     cycle, trend = cffilter(dta[["infl","unemp"]])
     ndcycle, ndtrend = cffilter(dta[['infl', 'unemp']].values)
-    assert_equal(cycle.values, ndcycle)
+    assert_allclose(cycle.values, ndcycle, rtol=1e-14)
     assert_equal(cycle.index[0], datetime(1959, 3, 31))
     assert_equal(cycle.index[-1], datetime(2009, 9, 30))
     assert_equal(cycle.columns.values, ["infl", "unemp"])