Repository: pandas-dev/pandas

Issue:
wide_to_long does not convert integer suffixes to int
#### Code Sample, a copy-pastable example if possible

```python
>>> df = pd.DataFrame({
                'A_1': [1.0, 2.0],
                'A_2': [3.0, 4.0],
                'X': ['X1', 'X2']})

   A_1  A_2   X
0  1.0  3.0  X1
1  2.0  4.0  X2

>>> df2 = pd.wide_to_long(df, stubnames='A', i='X', j='num', sep='_')
          A
X  num     
X1 1    1.0
X2 1    2.0
X1 2    3.0
X2 2    4.0

>>> df2.reset_index().dtypes
X       object
num     object
A      float64
dtype: object
```
#### Problem description

`wide_to_long` should attempt to coerce integer strings to ints. I have a pull request ready for this.

Additionally, I'd think it might be good to deprecate `wide_to_long` and `lreshape` in favor of expanded `melt` functionality.

#### Expected Output
```
>>> df2.reset_index().dtypes
X       object
num     int64
A      float64
dtype: object
```
#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.6.1.final.0
python-bits: 64
OS: Darwin
OS-release: 15.6.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: en_US.UTF-8

pandas: 0.20.3
pytest: 3.0.7
pip: 9.0.1
setuptools: 35.0.2
Cython: 0.25.2
numpy: 1.13.1
scipy: 0.19.0
xarray: None
IPython: 6.0.0
sphinx: 1.5.5
patsy: 0.4.1
dateutil: 2.6.1
pytz: 2017.2
blosc: None
bottleneck: 1.2.0
tables: 3.4.2
numexpr: 2.6.2
feather: None
matplotlib: 2.0.2
openpyxl: 2.4.7
xlrd: 1.0.0
xlwt: 1.2.0
xlsxwriter: 0.9.6
lxml: 3.7.3
bs4: 4.6.0
html5lib: 0.9999999
sqlalchemy: 1.1.9
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
pandas_gbq: None
pandas_datareader: 0.3.0.post
</details>



Discussion:

Resolving patch:
diff --git a/pandas/core/reshape/melt.py b/pandas/core/reshape/melt.py
--- a/pandas/core/reshape/melt.py
+++ b/pandas/core/reshape/melt.py
@@ -13,6 +13,7 @@
 
 import re
 from pandas.core.dtypes.missing import notna
+from pandas.core.tools.numeric import to_numeric
 
 
 @Appender(_shared_docs['melt'] %
@@ -199,6 +200,9 @@ def wide_to_long(df, stubnames, i, j, sep="", suffix=r'\d+'):
 
         .. versionadded:: 0.20.0
 
+        .. versionchanged:: 0.22.0
+            When all suffixes are numeric, they are cast to int64/float64.
+
     Returns
     -------
     DataFrame
@@ -278,8 +282,8 @@ def wide_to_long(df, stubnames, i, j, sep="", suffix=r'\d+'):
 
     Going from long back to wide just takes some creative use of `unstack`
 
-    >>> w = l.reset_index().set_index(['famid', 'birth', 'age']).unstack()
-    >>> w.columns = pd.Index(w.columns).str.join('')
+    >>> w = l.unstack()
+    >>> w.columns = w.columns.map('{0[0]}{0[1]}'.format)
     >>> w.reset_index()
        famid  birth  ht1  ht2
     0      1      1  2.8  3.4
@@ -333,16 +337,63 @@ def wide_to_long(df, stubnames, i, j, sep="", suffix=r'\d+'):
     >>> list(stubnames)
     ['A(quarterly)', 'B(quarterly)']
 
+    All of the above examples have integers as suffixes. It is possible to
+    have non-integers as suffixes.
+
+    >>> df = pd.DataFrame({
+    ...     'famid': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+    ...     'birth': [1, 2, 3, 1, 2, 3, 1, 2, 3],
+    ...     'ht_one': [2.8, 2.9, 2.2, 2, 1.8, 1.9, 2.2, 2.3, 2.1],
+    ...     'ht_two': [3.4, 3.8, 2.9, 3.2, 2.8, 2.4, 3.3, 3.4, 2.9]
+    ... })
+    >>> df
+       birth  famid  ht_one  ht_two
+    0      1      1     2.8     3.4
+    1      2      1     2.9     3.8
+    2      3      1     2.2     2.9
+    3      1      2     2.0     3.2
+    4      2      2     1.8     2.8
+    5      3      2     1.9     2.4
+    6      1      3     2.2     3.3
+    7      2      3     2.3     3.4
+    8      3      3     2.1     2.9
+
+    >>> l = pd.wide_to_long(df, stubnames='ht', i=['famid', 'birth'], j='age',
+                            sep='_', suffix='\w')
+    >>> l
+    ... # doctest: +NORMALIZE_WHITESPACE
+                      ht
+    famid birth age
+    1     1     one  2.8
+                two  3.4
+          2     one  2.9
+                two  3.8
+          3     one  2.2
+                two  2.9
+    2     1     one  2.0
+                two  3.2
+          2     one  1.8
+                two  2.8
+          3     one  1.9
+                two  2.4
+    3     1     one  2.2
+                two  3.3
+          2     one  2.3
+                two  3.4
+          3     one  2.1
+                two  2.9
+
     Notes
     -----
     All extra variables are left untouched. This simply uses
     `pandas.melt` under the hood, but is hard-coded to "do the right thing"
-    in a typicaly case.
+    in a typical case.
     """
     def get_var_names(df, stub, sep, suffix):
-        regex = "^{stub}{sep}{suffix}".format(
+        regex = r'^{stub}{sep}{suffix}$'.format(
             stub=re.escape(stub), sep=re.escape(sep), suffix=suffix)
-        return df.filter(regex=regex).columns.tolist()
+        pattern = re.compile(regex)
+        return [col for col in df.columns if pattern.match(col)]
 
     def melt_stub(df, stub, i, j, value_vars, sep):
         newdf = melt(df, id_vars=i, value_vars=value_vars,
@@ -350,9 +401,12 @@ def melt_stub(df, stub, i, j, value_vars, sep):
         newdf[j] = Categorical(newdf[j])
         newdf[j] = newdf[j].str.replace(re.escape(stub + sep), "")
 
+        # GH17627 Cast numerics suffixes to int/float
+        newdf[j] = to_numeric(newdf[j], errors='ignore')
+
         return newdf.set_index(i + [j])
 
-    if any(map(lambda s: s in df.columns.tolist(), stubnames)):
+    if any([col in stubnames for col in df.columns]):
         raise ValueError("stubname can't be identical to a column name")
 
     if not is_list_like(stubnames):
@@ -368,8 +422,7 @@ def melt_stub(df, stub, i, j, value_vars, sep):
     if df[i].duplicated().any():
         raise ValueError("the id variables need to uniquely identify each row")
 
-    value_vars = list(map(lambda stub:
-                          get_var_names(df, stub, sep, suffix), stubnames))
+    value_vars = [get_var_names(df, stub, sep, suffix) for stub in stubnames]
 
     value_vars_flattened = [e for sublist in value_vars for e in sublist]
     id_vars = list(set(df.columns.tolist()).difference(value_vars_flattened))


Test patch:
diff --git a/pandas/tests/reshape/test_melt.py b/pandas/tests/reshape/test_melt.py
--- a/pandas/tests/reshape/test_melt.py
+++ b/pandas/tests/reshape/test_melt.py
@@ -308,12 +308,12 @@ def test_simple(self):
         exp_data = {"X": x.tolist() + x.tolist(),
                     "A": ['a', 'b', 'c', 'd', 'e', 'f'],
                     "B": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
-                    "year": ['1970', '1970', '1970', '1980', '1980', '1980'],
+                    "year": [1970, 1970, 1970, 1980, 1980, 1980],
                     "id": [0, 1, 2, 0, 1, 2]}
-        exp_frame = DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(['id', 'year'])[["X", "A", "B"]]
-        long_frame = wide_to_long(df, ["A", "B"], i="id", j="year")
-        tm.assert_frame_equal(long_frame, exp_frame)
+        expected = DataFrame(exp_data)
+        expected = expected.set_index(['id', 'year'])[["X", "A", "B"]]
+        result = wide_to_long(df, ["A", "B"], i="id", j="year")
+        tm.assert_frame_equal(result, expected)
 
     def test_stubs(self):
         # GH9204
@@ -348,12 +348,12 @@ def test_separating_character(self):
         exp_data = {"X": x.tolist() + x.tolist(),
                     "A": ['a', 'b', 'c', 'd', 'e', 'f'],
                     "B": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
-                    "year": ['1970', '1970', '1970', '1980', '1980', '1980'],
+                    "year": [1970, 1970, 1970, 1980, 1980, 1980],
                     "id": [0, 1, 2, 0, 1, 2]}
-        exp_frame = DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(['id', 'year'])[["X", "A", "B"]]
-        long_frame = wide_to_long(df, ["A", "B"], i="id", j="year", sep=".")
-        tm.assert_frame_equal(long_frame, exp_frame)
+        expected = DataFrame(exp_data)
+        expected = expected.set_index(['id', 'year'])[["X", "A", "B"]]
+        result = wide_to_long(df, ["A", "B"], i="id", j="year", sep=".")
+        tm.assert_frame_equal(result, expected)
 
     def test_escapable_characters(self):
         np.random.seed(123)
@@ -376,14 +376,14 @@ def test_escapable_characters(self):
         exp_data = {"X": x.tolist() + x.tolist(),
                     "A(quarterly)": ['a', 'b', 'c', 'd', 'e', 'f'],
                     "B(quarterly)": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
-                    "year": ['1970', '1970', '1970', '1980', '1980', '1980'],
+                    "year": [1970, 1970, 1970, 1980, 1980, 1980],
                     "id": [0, 1, 2, 0, 1, 2]}
-        exp_frame = DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(
+        expected = DataFrame(exp_data)
+        expected = expected.set_index(
             ['id', 'year'])[["X", "A(quarterly)", "B(quarterly)"]]
-        long_frame = wide_to_long(df, ["A(quarterly)", "B(quarterly)"],
-                                  i="id", j="year")
-        tm.assert_frame_equal(long_frame, exp_frame)
+        result = wide_to_long(df, ["A(quarterly)", "B(quarterly)"],
+                              i="id", j="year")
+        tm.assert_frame_equal(result, expected)
 
     def test_unbalanced(self):
         # test that we can have a varying amount of time variables
@@ -396,11 +396,11 @@ def test_unbalanced(self):
                     'A': [1.0, 3.0, 2.0, 4.0],
                     'B': [5.0, np.nan, 6.0, np.nan],
                     'id': [0, 0, 1, 1],
-                    'year': ['2010', '2011', '2010', '2011']}
-        exp_frame = pd.DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(['id', 'year'])[["X", "A", "B"]]
-        long_frame = wide_to_long(df, ['A', 'B'], i='id', j='year')
-        tm.assert_frame_equal(long_frame, exp_frame)
+                    'year': [2010, 2011, 2010, 2011]}
+        expected = pd.DataFrame(exp_data)
+        expected = expected.set_index(['id', 'year'])[["X", "A", "B"]]
+        result = wide_to_long(df, ['A', 'B'], i='id', j='year')
+        tm.assert_frame_equal(result, expected)
 
     def test_character_overlap(self):
         # Test we handle overlapping characters in both id_vars and value_vars
@@ -415,19 +415,19 @@ def test_character_overlap(self):
             'BBBZ': [91, 92, 93]
         })
         df['id'] = df.index
-        exp_frame = pd.DataFrame({
+        expected = pd.DataFrame({
             'BBBX': [91, 92, 93, 91, 92, 93],
             'BBBZ': [91, 92, 93, 91, 92, 93],
             'A': ['a11', 'a22', 'a33', 'a21', 'a22', 'a23'],
             'B': ['b11', 'b12', 'b13', 'b21', 'b22', 'b23'],
             'BB': [1, 2, 3, 4, 5, 6],
             'id': [0, 1, 2, 0, 1, 2],
-            'year': ['11', '11', '11', '12', '12', '12']})
-        exp_frame = exp_frame.set_index(['id', 'year'])[
+            'year': [11, 11, 11, 12, 12, 12]})
+        expected = expected.set_index(['id', 'year'])[
             ['BBBX', 'BBBZ', 'A', 'B', 'BB']]
-        long_frame = wide_to_long(df, ['A', 'B', 'BB'], i='id', j='year')
-        tm.assert_frame_equal(long_frame.sort_index(axis=1),
-                              exp_frame.sort_index(axis=1))
+        result = wide_to_long(df, ['A', 'B', 'BB'], i='id', j='year')
+        tm.assert_frame_equal(result.sort_index(axis=1),
+                              expected.sort_index(axis=1))
 
     def test_invalid_separator(self):
         # if an invalid separator is supplied a empty data frame is returned
@@ -445,13 +445,13 @@ def test_invalid_separator(self):
                     'year': [],
                     'A': [],
                     'B': []}
-        exp_frame = pd.DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(['id', 'year'])[[
+        expected = pd.DataFrame(exp_data).astype({'year': 'int'})
+        expected = expected.set_index(['id', 'year'])[[
             'X', 'A2010', 'A2011', 'B2010', 'A', 'B']]
-        exp_frame.index.set_levels([[0, 1], []], inplace=True)
-        long_frame = wide_to_long(df, ['A', 'B'], i='id', j='year', sep=sep)
-        tm.assert_frame_equal(long_frame.sort_index(axis=1),
-                              exp_frame.sort_index(axis=1))
+        expected.index.set_levels([0, 1], level=0, inplace=True)
+        result = wide_to_long(df, ['A', 'B'], i='id', j='year', sep=sep)
+        tm.assert_frame_equal(result.sort_index(axis=1),
+                              expected.sort_index(axis=1))
 
     def test_num_string_disambiguation(self):
         # Test that we can disambiguate number value_vars from
@@ -467,19 +467,19 @@ def test_num_string_disambiguation(self):
             'Arating_old': [91, 92, 93]
         })
         df['id'] = df.index
-        exp_frame = pd.DataFrame({
+        expected = pd.DataFrame({
             'Arating': [91, 92, 93, 91, 92, 93],
             'Arating_old': [91, 92, 93, 91, 92, 93],
             'A': ['a11', 'a22', 'a33', 'a21', 'a22', 'a23'],
             'B': ['b11', 'b12', 'b13', 'b21', 'b22', 'b23'],
             'BB': [1, 2, 3, 4, 5, 6],
             'id': [0, 1, 2, 0, 1, 2],
-            'year': ['11', '11', '11', '12', '12', '12']})
-        exp_frame = exp_frame.set_index(['id', 'year'])[
+            'year': [11, 11, 11, 12, 12, 12]})
+        expected = expected.set_index(['id', 'year'])[
             ['Arating', 'Arating_old', 'A', 'B', 'BB']]
-        long_frame = wide_to_long(df, ['A', 'B', 'BB'], i='id', j='year')
-        tm.assert_frame_equal(long_frame.sort_index(axis=1),
-                              exp_frame.sort_index(axis=1))
+        result = wide_to_long(df, ['A', 'B', 'BB'], i='id', j='year')
+        tm.assert_frame_equal(result.sort_index(axis=1),
+                              expected.sort_index(axis=1))
 
     def test_invalid_suffixtype(self):
         # If all stubs names end with a string, but a numeric suffix is
@@ -497,13 +497,13 @@ def test_invalid_suffixtype(self):
                     'year': [],
                     'A': [],
                     'B': []}
-        exp_frame = pd.DataFrame(exp_data)
-        exp_frame = exp_frame.set_index(['id', 'year'])[[
-            'X', 'Aone', 'Atwo', 'Bone', 'A', 'B']]
-        exp_frame.index.set_levels([[0, 1], []], inplace=True)
-        long_frame = wide_to_long(df, ['A', 'B'], i='id', j='year')
-        tm.assert_frame_equal(long_frame.sort_index(axis=1),
-                              exp_frame.sort_index(axis=1))
+        expected = pd.DataFrame(exp_data).astype({'year': 'int'})
+
+        expected = expected.set_index(['id', 'year'])
+        expected.index.set_levels([0, 1], level=0, inplace=True)
+        result = wide_to_long(df, ['A', 'B'], i='id', j='year')
+        tm.assert_frame_equal(result.sort_index(axis=1),
+                              expected.sort_index(axis=1))
 
     def test_multiple_id_columns(self):
         # Taken from http://www.ats.ucla.edu/stat/stata/modules/reshapel.htm
@@ -513,17 +513,17 @@ def test_multiple_id_columns(self):
             'ht1': [2.8, 2.9, 2.2, 2, 1.8, 1.9, 2.2, 2.3, 2.1],
             'ht2': [3.4, 3.8, 2.9, 3.2, 2.8, 2.4, 3.3, 3.4, 2.9]
         })
-        exp_frame = pd.DataFrame({
+        expected = pd.DataFrame({
             'ht': [2.8, 3.4, 2.9, 3.8, 2.2, 2.9, 2.0, 3.2, 1.8,
                    2.8, 1.9, 2.4, 2.2, 3.3, 2.3, 3.4, 2.1, 2.9],
             'famid': [1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3],
             'birth': [1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3],
-            'age': ['1', '2', '1', '2', '1', '2', '1', '2', '1',
-                    '2', '1', '2', '1', '2', '1', '2', '1', '2']
+            'age': [1, 2, 1, 2, 1, 2, 1, 2, 1,
+                    2, 1, 2, 1, 2, 1, 2, 1, 2]
         })
-        exp_frame = exp_frame.set_index(['famid', 'birth', 'age'])[['ht']]
-        long_frame = wide_to_long(df, 'ht', i=['famid', 'birth'], j='age')
-        tm.assert_frame_equal(long_frame, exp_frame)
+        expected = expected.set_index(['famid', 'birth', 'age'])[['ht']]
+        result = wide_to_long(df, 'ht', i=['famid', 'birth'], j='age')
+        tm.assert_frame_equal(result, expected)
 
     def test_non_unique_idvars(self):
         # GH16382
@@ -535,3 +535,87 @@ def test_non_unique_idvars(self):
         })
         with pytest.raises(ValueError):
             wide_to_long(df, ['A_A', 'B_B'], i='x', j='colname')
+
+    def test_cast_j_int(self):
+        df = pd.DataFrame({
+            'actor_1': ['CCH Pounder', 'Johnny Depp', 'Christoph Waltz'],
+            'actor_2': ['Joel David Moore', 'Orlando Bloom', 'Rory Kinnear'],
+            'actor_fb_likes_1': [1000.0, 40000.0, 11000.0],
+            'actor_fb_likes_2': [936.0, 5000.0, 393.0],
+            'title': ['Avatar', "Pirates of the Caribbean", 'Spectre']})
+
+        expected = pd.DataFrame({
+            'actor': ['CCH Pounder',
+                      'Johnny Depp',
+                      'Christoph Waltz',
+                      'Joel David Moore',
+                      'Orlando Bloom',
+                      'Rory Kinnear'],
+            'actor_fb_likes': [1000.0, 40000.0, 11000.0, 936.0, 5000.0, 393.0],
+            'num': [1, 1, 1, 2, 2, 2],
+            'title': ['Avatar',
+                      'Pirates of the Caribbean',
+                      'Spectre',
+                      'Avatar',
+                      'Pirates of the Caribbean',
+                      'Spectre']}).set_index(['title', 'num'])
+        result = wide_to_long(df, ['actor', 'actor_fb_likes'],
+                              i='title', j='num', sep='_')
+
+        tm.assert_frame_equal(result, expected)
+
+    def test_identical_stubnames(self):
+        df = pd.DataFrame({'A2010': [1.0, 2.0],
+                           'A2011': [3.0, 4.0],
+                           'B2010': [5.0, 6.0],
+                           'A': ['X1', 'X2']})
+        with pytest.raises(ValueError):
+            wide_to_long(df, ['A', 'B'], i='A', j='colname')
+
+    def test_nonnumeric_suffix(self):
+        df = pd.DataFrame({'treatment_placebo': [1.0, 2.0],
+                           'treatment_test': [3.0, 4.0],
+                           'result_placebo': [5.0, 6.0],
+                           'A': ['X1', 'X2']})
+        expected = pd.DataFrame({
+            'A': ['X1', 'X1', 'X2', 'X2'],
+            'colname': ['placebo', 'test', 'placebo', 'test'],
+            'result': [5.0, np.nan, 6.0, np.nan],
+            'treatment': [1.0, 3.0, 2.0, 4.0]})
+        expected = expected.set_index(['A', 'colname'])
+        result = wide_to_long(df, ['result', 'treatment'],
+                              i='A', j='colname', suffix='[a-z]+', sep='_')
+        tm.assert_frame_equal(result, expected)
+
+    def test_mixed_type_suffix(self):
+        df = pd.DataFrame({
+            'treatment_1': [1.0, 2.0],
+            'treatment_foo': [3.0, 4.0],
+            'result_foo': [5.0, 6.0],
+            'result_1': [0, 9],
+            'A': ['X1', 'X2']})
+        expected = pd.DataFrame({
+            'A': ['X1', 'X2', 'X1', 'X2'],
+            'colname': ['1', '1', 'foo', 'foo'],
+            'result': [0.0, 9.0, 5.0, 6.0],
+            'treatment': [1.0, 2.0, 3.0, 4.0]}).set_index(['A', 'colname'])
+        result = wide_to_long(df, ['result', 'treatment'],
+                              i='A', j='colname', suffix='.+', sep='_')
+        tm.assert_frame_equal(result, expected)
+
+    def test_float_suffix(self):
+        df = pd.DataFrame({
+            'treatment_1.1': [1.0, 2.0],
+            'treatment_2.1': [3.0, 4.0],
+            'result_1.2': [5.0, 6.0],
+            'result_1': [0, 9],
+            'A': ['X1', 'X2']})
+        expected = pd.DataFrame({
+            'A': ['X1', 'X1', 'X1', 'X1', 'X2', 'X2', 'X2', 'X2'],
+            'colname': [1, 1.1, 1.2, 2.1, 1, 1.1, 1.2, 2.1],
+            'result': [0.0, np.nan, 5.0, np.nan, 9.0, np.nan, 6.0, np.nan],
+            'treatment': [np.nan, 1.0, np.nan, 3.0, np.nan, 2.0, np.nan, 4.0]})
+        expected = expected.set_index(['A', 'colname'])
+        result = wide_to_long(df, ['result', 'treatment'],
+                              i='A', j='colname', suffix='[0-9.]+', sep='_')
+        tm.assert_frame_equal(result, expected)