Repository: statsmodels/statsmodels

Issue:
.conf_int() fails on linear regression without intercept
Below is the problem. When I undertake a regression without an intercept I cannot retrieve the confidence interval report (calling .conf_int()). However, if I include the intercept in the regression the confidence interval is reported as expected. I also suspect the R^2 is incorrectly reported (statsmodels shows same value for both with and without intercept).

``` python
model = ols('LREO_recovery ~ 0 + HREO_recovery', df_product)
results = model.fit()
print results.summary()

                            OLS Regression Results                            
==============================================================================
Dep. Variable:          LREO_recovery   R-squared:                       0.999
Model:                            OLS   Adj. R-squared:                  0.999
Method:                 Least Squares   F-statistic:                       inf
Date:                Sat, 16 Mar 2013   Prob (F-statistic):                nan
Time:                        20:35:35   Log-Likelihood:                 165.81
No. Observations:                  54   AIC:                            -329.6
Df Residuals:                      53   BIC:                            -327.6
Df Model:                           0                                         
=================================================================================
                    coef    std err          t      P>|t|      [95.0% Conf. Int.]
---------------------------------------------------------------------------------
HREO_recovery     1.0029      0.003    373.607      0.000         0.998     1.008
==============================================================================
Omnibus:                       21.160   Durbin-Watson:                   1.978
Prob(Omnibus):                  0.000   Jarque-Bera (JB):               67.387
Skew:                           0.891   Prob(JB):                     2.33e-15
Kurtosis:                       8.175   Cond. No.                         1.00
==============================================================================

results.conf_int()
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-45-6d86a730d4c3> in <module>()
----> 1 results.conf_int()

/usr/local/lib/python2.7/dist-packages/statsmodels/base/wrapper.pyc in wrapper(self, *args, **kwargs)
     87         results = object.__getattribute__(self, '_results')
     88         data = results.model._data
---> 89         return data.wrap_output(func(results, *args, **kwargs), how)
     90 
     91     argspec = inspect.getargspec(func)

/usr/local/lib/python2.7/dist-packages/statsmodels/base/data.pyc in wrap_output(self, obj, how)
    104     def wrap_output(self, obj, how='columns'):
    105         if how == 'columns':
--> 106             return self.attach_columns(obj)
    107         elif how == 'rows':
    108             return self.attach_rows(obj)

/usr/local/lib/python2.7/dist-packages/statsmodels/base/data.pyc in attach_columns(self, result)
    166     def attach_columns(self, result):
    167         if result.squeeze().ndim <= 1:
--> 168             return Series(result, index=self.xnames)
    169         else: # for e.g., confidence intervals
    170             return DataFrame(result, index=self.xnames)

/usr/local/lib/python2.7/dist-packages/pandas/core/series.pyc in __new__(cls, data, index, dtype, name, copy)
    370 
    371         subarr = _sanitize_array(data, index, dtype, copy,
--> 372                                  raise_cast_failure=True)
    373 
    374         if not isinstance(subarr, np.ndarray):

/usr/local/lib/python2.7/dist-packages/pandas/core/series.pyc in _sanitize_array(data, index, dtype, copy, raise_cast_failure)
   3127     elif subarr.ndim > 1:
   3128         if isinstance(data, np.ndarray):
-> 3129             raise Exception('Data must be 1-dimensional')
   3130         else:
   3131             subarr = _asarray_tuplesafe(data, dtype=dtype)

Exception: Data must be 1-dimensional

model = ols('LREO_recovery ~ HREO_recovery', df_product)
results = model.fit()
print results.summary()

                            OLS Regression Results                            
==============================================================================
Dep. Variable:          LREO_recovery   R-squared:                       0.999
Model:                            OLS   Adj. R-squared:                  0.999
Method:                 Least Squares   F-statistic:                 3.925e+04
Date:                Sat, 16 Mar 2013   Prob (F-statistic):           1.60e-76
Time:                        20:35:51   Log-Likelihood:                 166.13
No. Observations:                  54   AIC:                            -328.3
Df Residuals:                      52   BIC:                            -324.3
Df Model:                           1                                         
=================================================================================
                    coef    std err          t      P>|t|      [95.0% Conf. Int.]
---------------------------------------------------------------------------------
Intercept         0.0023      0.003      0.789      0.434        -0.004     0.008
HREO_recovery     0.9995      0.005    198.115      0.000         0.989     1.010
==============================================================================
Omnibus:                       26.677   Durbin-Watson:                   1.990
Prob(Omnibus):                  0.000   Jarque-Bera (JB):               93.924
Skew:                           1.169   Prob(JB):                     4.02e-21
Kurtosis:                       9.023   Cond. No.                         4.09
==============================================================================

results.conf_int()
Out[47]: 
                      0         1
Intercept     -0.003528  0.008103
HREO_recovery  0.989421  1.009669

```



Discussion:
This works for me on master with the most recent versions of patsy and pandas. 

```
In [16]: import statsmodels.formula.api as smf
In [17]: import pandas as pd
In [18]: url = "http://vincentarelbundock.github.com/Rdatasets/csv/HistData/Guerry.csv"
In [19]: dat = pd.read_csv(url)
In [20]: dat = dat[['Lottery', 'Literacy', 'Wealth']].dropna()
In [21]: smf.ols('Lottery ~ Literacy + Wealth - 1', dat).fit().conf_int()
Out[21]: 
                 0         1
Literacy  0.018886  0.377035
Wealth    0.593651  0.900307
```

Edit: It looks like 0 should work too. Can you share your data so I can look at it? I can't replicate the error here...

I also get different R2 with and without intercept

I can provide the set of data that produces the problem. Is there a facility to upload the dataframe (as a pickle)? 

Do you have an example with a single variable such as I have used? 

From the traceback it looks like all we need is an additional squeeze int

```
 166     def attach_columns(self, result):
    167         if result.squeeze().ndim <= 1:
--> 168             return Series(result, index=self.xnames)
```

or maybe we shouldn't squeeze in the if condition.

Earlier (before pandas) we had the problem with one column regressors, for example regressing on a constant only. My guess is that should raise a similar exception.

(I haven't tried yet an example.)

On a completely different point: I'm not sure why we use the pandas objects in this part, there should be a proper api for the access to the different elements of the results, and my version of summary was written with arrays in mind.

The R^2 behavior has been changed a while ago in master to be correct also when there is no constant.
Skipper added a more consistent handling of whether we have a constant included or not. Previously several of the result statistics, r_squared and similar, assumed that a constant is present.

Ok, I misunderstood: summary works fine because it uses the underlying ndarrays

results.conf_int hits a bug in the wrapper code for pandas data

```
>>> y = pa.Series(np.random.randn(10))
>>> x = pa.Series(np.ones(10))
>>> res = sm.OLS(y, x).fit()

>>> res.conf_int()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "e:\josef\eclipsegworkspace\statsmodels-git\statsmodels-josef_new\statsmodels\base\wrapper.py", line 90, in wrapper
    return data.wrap_output(func(results, *args, **kwargs), how)
  File "e:\josef\eclipsegworkspace\statsmodels-git\statsmodels-josef_new\statsmodels\base\data.py", line 251, in wrap_output
    return self.attach_columns(obj)
  File "e:\josef\eclipsegworkspace\statsmodels-git\statsmodels-josef_new\statsmodels\base\data.py", line 325, in attach_columns
    return Series(result, index=self.xnames)
  File "c:\python26\lib\site-packages\pandas-0.7.3-py2.6-win32.egg\pandas\core\series.py", line 273, in __new__
    raise_cast_failure=True)
  File "c:\python26\lib\site-packages\pandas-0.7.3-py2.6-win32.egg\pandas\core\series.py", line 2464, in _sanitize_array
    raise Exception('Data must be 1-dimensional')
Exception: Data must be 1-dimensional

>>> res._results.conf_int()
array([[-0.29378162,  0.91694844]])
>>> res._results.conf_int().ndim
2
```

I don't know if we have enough test cases that use pandas to change this easily. We also might not have enough unit tests to check the output array dimension across various corner cases.

My impression is now that we shouldn't have the squeeze in the if statement, but I don't know which cases/attributes this is supposed to catch. 
(But in any case it looks inconsistent to check the dimension of the squeezed array and then give an array with ndim>1 to pandas.Series, unless the behavior of pandas.Series has changed in this respect.)

Simply using `Series(results.squeeze())` won't work as it now complains about something related to index length or something like that (looked at it yesterday, can't recall details). 

And Series still required a squeezed array. This won't work for example: `Series(np.ones((1,3)))`

I suspected something like this. I think creating a Series is the wrong thing, because it's the wrong axis.

I think what we need for conf_int is a DataFrame with dimension (2,1) or (1,2) ( I don't know which axis it goes when we have more than one confidence interval).

dropping the squeeze from the `if` condition, would send conf_int to the DataFrame path instead of the Series path.


Resolving patch:
diff --git a/statsmodels/base/data.py b/statsmodels/base/data.py
--- a/statsmodels/base/data.py
+++ b/statsmodels/base/data.py
@@ -321,7 +321,10 @@ def _get_row_labels(self, arr):
             return self.orig_endog.index
 
     def attach_columns(self, result):
-        if result.squeeze().ndim <= 1:
+        # this can either be a 1d array or a scalar
+        # don't squeeze because it might be a 2d row array
+        # if it needs a squeeze, the bug is elsewhere
+        if result.ndim <= 1:
             return Series(result, index=self.xnames)
         else: # for e.g., confidence intervals
             return DataFrame(result, index=self.xnames)


Test patch:
diff --git a/statsmodels/regression/tests/test_regression.py b/statsmodels/regression/tests/test_regression.py
--- a/statsmodels/regression/tests/test_regression.py
+++ b/statsmodels/regression/tests/test_regression.py
@@ -1,6 +1,7 @@
 """
 Test functions for models.regression
 """
+import pandas
 import numpy as np
 from numpy.testing import (assert_almost_equal, assert_, assert_approx_equal,
                             assert_raises, assert_equal)
@@ -635,6 +636,15 @@ def test_const_indicator():
     mod = OLS(y, X, hasconst=True).fit()
     assert_almost_equal(modc.rsquared, mod.rsquared, 12)
 
+def test_706():
+    # make sure one regressor pandas Series gets passed to DataFrame
+    # for conf_int.
+    y = pandas.Series(np.random.randn(10))
+    x = pandas.Series(np.ones(10))
+    res = OLS(y,x).fit()
+    conf_int = res.conf_int()
+    np.testing.assert_equal(conf_int.shape, (1, 2))
+    np.testing.assert_(isinstance(conf_int, pandas.DataFrame))
 
 if __name__=="__main__":