Repository: statsmodels/statsmodels

Issue:
summary_col always includes mislabeled R-squared statistics
#### Describe the bug
summary_col always appends two extra rows to the table, with the adjusted R-squared (incorrectly labelled R-squared) and the regular R-squared (unlabeled) whether you want them or not. In earlier versions (0.10 and below) this didn't happen. 

#### Code Sample, a copy-pastable example if possible


```python
import statsmodels.api as sm
from statsmodels.iolib.summary2 import summary_col

y, X =  [1, 1, 4, 2], sm.add_constant([1,2,3,4]), 
mod = sm.OLS(endog=y, exog=X).fit()
table = summary_col(results=mod , info_dict=None)
print(mod.summary())
print(table)
```
Model summary:
```
                           OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       0.300
Model:                            OLS   Adj. R-squared:                 -0.050
Method:                 Least Squares   F-statistic:                    0.8571
Date:                Wed, 11 Mar 2020   Prob (F-statistic):              0.452
Time:                        15:05:45   Log-Likelihood:                -5.7733
No. Observations:                   4   AIC:                             15.55
Df Residuals:                       2   BIC:                             14.32
Df Model:                           1                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [0.025      0.975]
------------------------------------------------------------------------------
const          0.5000      1.775      0.282      0.805      -7.136       8.136
x1             0.6000      0.648      0.926      0.452      -2.188       3.388
==============================================================================
Omnibus:                          nan   Durbin-Watson:                   3.067
Prob(Omnibus):                    nan   Jarque-Bera (JB):                0.667
Skew:                           0.892   Prob(JB):                        0.717
Kurtosis:                       2.097   Cond. No.                         7.47
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
```
summary_col table:
```
==================
             y    
------------------
const     0.5000  
          (1.7748)
x1        0.6000  
          (0.6481)
R-squared -0.0500 
          0.3000  
==================
Standard errors in
parentheses.
```

#### Output of ``import statsmodels.api as sm; sm.show_versions()``

<details>

INSTALLED VERSIONS
------------------
Python: 3.7.2.final.0

statsmodels
===========

Installed: 0.11.1 (C:\Users\Axel\Anaconda3\lib\site-packages\statsmodels)

Required Dependencies
=====================

cython: 0.29.15 (C:\Users\Axel\Anaconda3\lib\site-packages\Cython)
numpy: 1.18.1 (C:\Users\Axel\Anaconda3\lib\site-packages\numpy)
scipy: 1.2.1 (C:\Users\Axel\Anaconda3\lib\site-packages\scipy)
pandas: 1.0.0 (C:\Users\Axel\Anaconda3\lib\site-packages\pandas)
    dateutil: 2.8.1 (C:\Users\Axel\Anaconda3\lib\site-packages\dateutil)
patsy: 0.5.1 (C:\Users\Axel\Anaconda3\lib\site-packages\patsy)

Optional Dependencies
=====================

matplotlib: 3.0.3 (C:\Users\Axel\Anaconda3\lib\site-packages\matplotlib)
    backend: module://ipykernel.pylab.backend_inline 
cvxopt: Not installed
joblib: 0.14.1 (C:\Users\Axel\Anaconda3\lib\site-packages\joblib)

Developer Tools
================

IPython: 7.12.0 (C:\Users\Axel\Anaconda3\lib\site-packages\IPython)
    jinja2: 2.11.1 (C:\Users\Axel\Anaconda3\lib\site-packages\jinja2)
sphinx: 2.4.0 (C:\Users\Axel\Anaconda3\lib\site-packages\sphinx)
    pygments: 2.5.2 (C:\Users\Axel\Anaconda3\lib\site-packages\pygments)
pytest: 5.3.5 (C:\Users\Axel\Anaconda3\lib\site-packages\pytest)
virtualenv: Not installed


</details>


Discussion:

Resolving patch:
diff --git a/statsmodels/iolib/summary2.py b/statsmodels/iolib/summary2.py
--- a/statsmodels/iolib/summary2.py
+++ b/statsmodels/iolib/summary2.py
@@ -1,5 +1,4 @@
-from statsmodels.compat.python import (lrange, iterkeys, iteritems, lzip,
-                                       itervalues)
+from statsmodels.compat.python import iterkeys, iteritems, lzip, itervalues
 
 from collections import OrderedDict
 import datetime
@@ -29,12 +28,13 @@ def __repr__(self):
         return str(type(self)) + '\n"""\n' + self.__str__() + '\n"""'
 
     def _repr_html_(self):
-        '''Display as HTML in IPython notebook.'''
+        """Display as HTML in IPython notebook."""
         return self.as_html()
 
     def add_df(self, df, index=True, header=True, float_format='%.4f',
                align='r'):
-        '''Add the contents of a DataFrame to summary table
+        """
+        Add the contents of a DataFrame to summary table
 
         Parameters
         ----------
@@ -47,7 +47,7 @@ def add_df(self, df, index=True, header=True, float_format='%.4f',
             Formatting to float data columns
         align : str
             Data alignment (l/c/r)
-        '''
+        """
 
         settings = {'index': index, 'header': header,
                     'float_format': float_format, 'align': align}
@@ -55,7 +55,7 @@ def add_df(self, df, index=True, header=True, float_format='%.4f',
         self.settings.append(settings)
 
     def add_array(self, array, align='r', float_format="%.4f"):
-        '''Add the contents of a Numpy array to summary table
+        """Add the contents of a Numpy array to summary table
 
         Parameters
         ----------
@@ -64,14 +64,14 @@ def add_array(self, array, align='r', float_format="%.4f"):
             Formatting to array if type is float
         align : str
             Data alignment (l/c/r)
-        '''
+        """
 
         table = pd.DataFrame(array)
         self.add_df(table, index=False, header=False,
                     float_format=float_format, align=align)
 
     def add_dict(self, d, ncols=2, align='l', float_format="%.4f"):
-        '''Add the contents of a Dict to summary table
+        """Add the contents of a Dict to summary table
 
         Parameters
         ----------
@@ -82,7 +82,9 @@ def add_dict(self, d, ncols=2, align='l', float_format="%.4f"):
             Number of columns of the output table
         align : str
             Data alignment (l/c/r)
-        '''
+        float_format : str
+            Formatting to float data columns
+        """
 
         keys = [_formatter(x, float_format) for x in iterkeys(d)]
         vals = [_formatter(x, float_format) for x in itervalues(d)]
@@ -97,17 +99,17 @@ def add_dict(self, d, ncols=2, align='l', float_format="%.4f"):
         self.add_array(data, align=align)
 
     def add_text(self, string):
-        '''Append a note to the bottom of the summary table. In ASCII tables,
+        """Append a note to the bottom of the summary table. In ASCII tables,
         the note will be wrapped to table width. Notes are not indendented.
-        '''
+        """
         self.extra_txt.append(string)
 
     def add_title(self, title=None, results=None):
-        '''Insert a title on top of the summary table. If a string is provided
+        """Insert a title on top of the summary table. If a string is provided
         in the title argument, that string is printed. If no title string is
         provided but a results instance is provided, statsmodels attempts
         to construct a useful title automatically.
-        '''
+        """
         if isinstance(title, str):
             self.title = title
         else:
@@ -121,14 +123,14 @@ def add_title(self, title=None, results=None):
 
     def add_base(self, results, alpha=0.05, float_format="%.4f", title=None,
                  xname=None, yname=None):
-        '''Try to construct a basic summary instance.
+        """Try to construct a basic summary instance.
 
         Parameters
         ----------
         results : Model results instance
         alpha : float
             significance level for the confidence intervals (optional)
-        float_formatting: str
+        float_format: str
             Float formatting for summary of parameters (optional)
         title : str
             Title of the summary table (optional)
@@ -136,7 +138,7 @@ def add_base(self, results, alpha=0.05, float_format="%.4f", title=None,
             Names of the independent variables (optional)
         yname : str
             Name of the dependent variable (optional)
-        '''
+        """
 
         param = summary_params(results, alpha=alpha, use_t=results.use_t)
         info = summary_model(results)
@@ -149,8 +151,8 @@ def add_base(self, results, alpha=0.05, float_format="%.4f", title=None,
         self.add_title(title=title, results=results)
 
     def as_text(self):
-        '''Generate ASCII Summary Table
-        '''
+        """Generate ASCII Summary Table
+        """
 
         tables = self.tables
         settings = self.settings
@@ -173,7 +175,7 @@ def as_text(self):
         if title is not None:
             title = title
             if len(title) < widest:
-                title = ' ' * int(widest/2 - len(title)/2) + title
+                title = ' ' * int(widest / 2 - len(title) / 2) + title
         else:
             title = ''
 
@@ -186,8 +188,8 @@ def as_text(self):
         return out
 
     def as_html(self):
-        '''Generate HTML Summary Table
-        '''
+        """Generate HTML Summary Table
+        """
 
         tables = self.tables
         settings = self.settings
@@ -199,8 +201,8 @@ def as_html(self):
         return tab
 
     def as_latex(self):
-        '''Generate LaTeX Summary Table
-        '''
+        """Generate LaTeX Summary Table
+        """
         tables = self.tables
         settings = self.settings
         title = self.title
@@ -227,11 +229,11 @@ def as_latex(self):
 
 
 def _measure_tables(tables, settings):
-    '''Compare width of ascii tables in a list and calculate padding values.
+    """Compare width of ascii tables in a list and calculate padding values.
     We add space to each col_sep to get us as close as possible to the
     width of the largest table. Then, we add a few spaces to the first
     column to pad the rest.
-    '''
+    """
 
     simple_tables = _simple_tables(tables, settings)
     tab = [x.as_text() for x in simple_tables]
@@ -263,8 +265,9 @@ def _measure_tables(tables, settings):
 
 
 def summary_model(results):
-    '''Create a dict with information about the model
-    '''
+    """
+    Create a dict with information about the model
+    """
 
     def time_now(*args, **kwds):
         now = datetime.datetime.now()
@@ -313,7 +316,7 @@ def time_now(*args, **kwds):
 
 def summary_params(results, yname=None, xname=None, alpha=.05, use_t=True,
                    skip_header=False, float_format="%.4f"):
-    '''create a summary table of parameters from results instance
+    """create a summary table of parameters from results instance
 
     Parameters
     ----------
@@ -329,7 +332,7 @@ def summary_params(results, yname=None, xname=None, alpha=.05, use_t=True,
     use_t : bool
         indicator whether the p-values are based on the Student-t
         distribution (if True) or on the normal distribution (if False)
-    skip_headers : bool
+    skip_header : bool
         If false (default), then the header row is added. If true, then no
         header row is added.
     float_format : str
@@ -338,7 +341,7 @@ def summary_params(results, yname=None, xname=None, alpha=.05, use_t=True,
     Returns
     -------
     params_table : SimpleTable instance
-    '''
+    """
 
     if isinstance(results, tuple):
         results, params, bse, tvalues, pvalues, conf_int = results
@@ -355,10 +358,10 @@ def summary_params(results, yname=None, xname=None, alpha=.05, use_t=True,
 
     if use_t:
         data.columns = ['Coef.', 'Std.Err.', 't', 'P>|t|',
-                        '[' + str(alpha/2), str(1-alpha/2) + ']']
+                        '[' + str(alpha / 2), str(1 - alpha / 2) + ']']
     else:
         data.columns = ['Coef.', 'Std.Err.', 'z', 'P>|z|',
-                        '[' + str(alpha/2), str(1-alpha/2) + ']']
+                        '[' + str(alpha / 2), str(1 - alpha / 2) + ']']
 
     if not xname:
         try:
@@ -373,8 +376,8 @@ def summary_params(results, yname=None, xname=None, alpha=.05, use_t=True,
 
 # Vertical summary instance for multiple models
 def _col_params(result, float_format='%.4f', stars=True):
-    '''Stack coefficients and standard errors in single column
-    '''
+    """Stack coefficients and standard errors in single column
+    """
 
     # Extract parameters
     res = summary_params(result)
@@ -393,31 +396,24 @@ def _col_params(result, float_format='%.4f', stars=True):
         res.loc[idx, res.columns[0]] = res.loc[idx, res.columns[0]] + '*'
     # Stack Coefs and Std.Errors
     res = res.iloc[:, :2]
-    res = res.iloc[:, :2]
-    rsquared = rsquared_adj = np.nan
-    if hasattr(result, 'rsquared'):
-        rsquared = result.rsquared
-    if hasattr(result, 'rsquared_adj'):
-        rsquared_adj = result.rsquared_adj
-    r_result = pd.DataFrame({'Basic': [rsquared], 'Adj.': [rsquared_adj]},
-                            index=['R-squared'])
-    if not np.all(np.isnan(np.asarray(r_result))):
-        for col in r_result:
-            r_result[col] = r_result[col].apply(lambda x: float_format % x)
-        try:
-            res = pd.DataFrame(res).append(r_result, sort=True)
-        except TypeError:
-            # TODO: Remove when min pandas >= 0.23
-            res = pd.DataFrame(res).append(r_result)
     res = res.stack()
+
+    rsquared = getattr(result, 'rsquared', np.nan)
+    rsquared_adj = getattr(result, 'rsquared_adj', np.nan)
+    r2 = pd.Series({('R-squared', ""): rsquared,
+                    ('R-squared Adj.', ""): rsquared_adj})
+
+    if r2.notnull().any():
+        r2 = r2.apply(lambda x: float_format % x)
+        res = pd.concat([res, r2], axis=0)
     res = pd.DataFrame(res)
     res.columns = [str(result.model.endog_names)]
     return res
 
 
 def _col_info(result, info_dict=None):
-    '''Stack model info in a column
-    '''
+    """Stack model info in a column
+    """
 
     if info_dict is None:
         info_dict = {}
@@ -445,7 +441,7 @@ def _make_unique(list_of_names):
     header = []
     for _name in list_of_names:
         name_counter[_name] += "I"
-        header.append(_name+" " + name_counter[_name])
+        header.append(_name + " " + name_counter[_name])
     return header
 
 
@@ -465,7 +461,7 @@ def summary_col(results, float_format='%.4f', model_names=(), stars=False,
         unique, a roman number will be appended to all model names
     stars : bool
         print significance stars
-    info_dict : dict
+    info_dict : dict, default None
         dict of functions to be applied to results instances to retrieve
         model info. To use specific information for different models, add a
         (nested) info_dict with model name as the key.
@@ -505,21 +501,28 @@ def merg(x, y):
 
     if regressor_order:
         varnames = summ.index.get_level_values(0).tolist()
+        vc = pd.Series(varnames).value_counts()
+        varnames = vc.loc[vc == 2].index.tolist()
         ordered = [x for x in regressor_order if x in varnames]
-        unordered = [x for x in varnames if x not in regressor_order + ['']]
-        order = ordered + list(np.unique(unordered))
-
-        def f(idx):
-            return sum([[x + 'coef', x + 'stde'] for x in idx], [])
-
-        summ.index = f(pd.unique(varnames))
-        summ = summ.reindex(f(order))
-        summ.index = [x[:-4] for x in summ.index]
+        unordered = [x for x in varnames if x not in regressor_order]
+        new_order = ordered + unordered
+        other = [x for x in summ.index.get_level_values(0)
+                 if x not in new_order]
+        new_order += other
         if drop_omitted:
-            summ = summ.loc[regressor_order]
-
-    idx = pd.Series(lrange(summ.shape[0])) % 2 == 1
-    summ.index = np.where(idx, '', summ.index.get_level_values(0))
+            for uo in unordered:
+                new_order.remove(uo)
+        summ = summ.loc[new_order]
+
+    idx = []
+    index = summ.index.get_level_values(0)
+    for i in range(0, index.shape[0], 2):
+        idx.append(index[i])
+        if (i + 1) < index.shape[0] and (index[i] == index[i + 1]):
+            idx.append("")
+        else:
+            idx.append(index[i + 1])
+    summ.index = idx
 
     # add infos about the models.
     if info_dict:


Test patch:
diff --git a/statsmodels/iolib/tests/test_summary2.py b/statsmodels/iolib/tests/test_summary2.py
--- a/statsmodels/iolib/tests/test_summary2.py
+++ b/statsmodels/iolib/tests/test_summary2.py
@@ -9,6 +9,7 @@
 from statsmodels.tools.tools import add_constant
 from statsmodels.regression.linear_model import OLS
 
+
 class TestSummaryLatex(object):
 
     def test_summarycol(self):
@@ -19,42 +20,42 @@ def test_summarycol(self):
 \begin{center}
 \begin{tabular}{lcc}
 \hline
-          &   y I    &   y II    \\
+               &   y I    &   y II    \\
 \midrule
-const     & 7.7500   & 12.4231   \\
-          & (1.1058) & (3.1872)  \\
-x1        & -0.7500  & -1.5769   \\
-          & (0.2368) & (0.6826)  \\
-R-squared & 0.6930   & 0.5202    \\
-          & 0.7697   & 0.6401    \\
+const          & 7.7500   & 12.4231   \\
+               & (1.1058) & (3.1872)  \\
+x1             & -0.7500  & -1.5769   \\
+               & (0.2368) & (0.6826)  \\
+R-squared      & 0.7697   & 0.6401    \\
+R-squared Adj. & 0.6930   & 0.5202    \\
 \hline
 \end{tabular}
 \end{center}
 \end{table}
 '''
-        x = [1,5,7,3,5]
+        x = [1, 5, 7, 3, 5]
         x = add_constant(x)
-        y1 = [6,4,2,7,4]
-        y2 = [8,5,0,12,4]
-        reg1 = OLS(y1,x).fit()
-        reg2 = OLS(y2,x).fit()
-        actual = summary_col([reg1,reg2]).as_latex()
+        y1 = [6, 4, 2, 7, 4]
+        y2 = [8, 5, 0, 12, 4]
+        reg1 = OLS(y1, x).fit()
+        reg2 = OLS(y2, x).fit()
+        actual = summary_col([reg1, reg2]).as_latex()
         actual = '\n%s\n' % actual
         assert_equal(desired, actual)
 
     def test_summarycol_float_format(self):
         # Test for latex output of summary_col object
         desired = r"""
-=====================
-           y I   y II
----------------------
-const     7.7   12.4 
-          (1.1) (3.2)
-x1        -0.7  -1.6 
-          (0.2) (0.7)
-R-squared 0.7   0.5  
-          0.8   0.6  
-=====================
+==========================
+                y I   y II
+--------------------------
+const          7.7   12.4 
+               (1.1) (3.2)
+x1             -0.7  -1.6 
+               (0.2) (0.7)
+R-squared      0.8   0.6  
+R-squared Adj. 0.7   0.5  
+==========================
 Standard errors in
 parentheses.
 """  # noqa:W291
@@ -68,6 +69,11 @@ def test_summarycol_float_format(self):
         actual = '%s\n' % actual
         assert_equal(actual, desired)
 
+        starred = summary_col([reg1, reg2], stars=True, float_format='%0.1f')
+        assert "7.7***" in str(starred)
+        assert "12.4**" in str(starred)
+        assert "12.4***" not in str(starred)
+
     def test_summarycol_drop_omitted(self):
         # gh-3702
         x = [1, 5, 7, 3, 5]
@@ -112,17 +118,17 @@ def test_OLSsummary(self):
         # Test that latex output of regular OLS output still contains
         # multiple tables
 
-        x = [1,5,7,3,5]
+        x = [1, 5, 7, 3, 5]
         x = add_constant(x)
-        y1 = [6,4,2,7,4]
-        reg1 = OLS(y1,x).fit()
+        y1 = [6, 4, 2, 7, 4]
+        reg1 = OLS(y1, x).fit()
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
             actual = reg1.summary().as_latex()
         string_to_find = r'''\end{tabular}
 \begin{tabular}'''
         result = string_to_find in actual
-        assert(result is True)
+        assert (result is True)
 
 
 def test_ols_summary_rsquared_label():
@@ -142,3 +148,13 @@ def test_ols_summary_rsquared_label():
         assert r2_str in str(reg_without_constant.summary2())
     with pytest.warns(UserWarning):
         assert r2_str in str(reg_without_constant.summary())
+
+
+def test_summary_col_r2():
+    # GH 6578
+    y = [1, 1, 4, 2] * 4
+    x = add_constant([1, 2, 3, 4] * 4)
+    mod = OLS(endog=y, exog=x).fit()
+    table = summary_col(results=mod)
+    assert "R-squared  " in str(table)
+    assert "R-squared Adj." in str(table)