Repository: pandas-dev/pandas

Issue:
merge fails to add suffixes on multiindex columns
#### Code Sample, a copy-pastable example if possible

```python
import pandas as pd
index_tuples=[]

for word_group in ["a", "b", "c", "d"]:
    for correctness in ["1", "2", "3"]:
        index_tuples.append([word_group, correctness])

index = pd.MultiIndex.from_tuples(index_tuples, names=["outer", "inner"])

frame_x = pd.DataFrame(columns = index)
frame_x["id"]=""

frame_y = pd.DataFrame(columns = index)
frame_y["id"]=""

print(frame_x.merge(frame_y, on="id").columns)

```
#### Problem description

I'm trying to merge to dataframes. Both have a multiindex and an "id" column. The merge happens on "id", the outer layer of the multiindex should receive suffixes. Depending on the number of indices in the multiindex this doesn't work. Only some of the multiindex columns receive suffixes, other's don't. For the codesample, I've set up an empty dataframe, the behaviour is the same when it is filled with data.

The issue seems non-deterministic. Sometimes it happens, sometimes it doesn't. Here is a video:
https://imgur.com/a/rbSvuSl

I'm using pandas version 0.25.1
This is a conda environment, here is its yml file: https://gist.github.com/lhk/ab3cf1f95be37a23789792fd75beef93

#### Expected Output
All columns in the multiindex should receive either _x or _y as suffix.

#### Output of ``pd.show_versions()``

<details>
INSTALLED VERSIONS
------------------
commit           : None
python           : 3.6.9.final.0
python-bits      : 64
OS               : Linux
OS-release       : 5.0.0-21-generic
machine          : x86_64
processor        : x86_64
byteorder        : little
LC_ALL           : None
LANG             : en_US.UTF-8
LOCALE           : en_US.UTF-8

pandas           : 0.25.1
numpy            : 1.16.5
pytz             : 2019.2
dateutil         : 2.8.0
pip              : 19.2.2
setuptools       : 41.0.1
Cython           : None
pytest           : None
hypothesis       : None
sphinx           : None
blosc            : None
feather          : None
xlsxwriter       : None
lxml.etree       : None
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : 2.10.1
IPython          : 7.8.0
pandas_datareader: None
bs4              : None
bottleneck       : None
fastparquet      : None
gcsfs            : None
lxml.etree       : None
matplotlib       : 3.1.1
numexpr          : None
odfpy            : None
openpyxl         : None
pandas_gbq       : None
pyarrow          : None
pytables         : None
s3fs             : None
scipy            : 1.3.1
sqlalchemy       : None
tables           : None
xarray           : None
xlrd             : None
xlwt             : None
xlsxwriter       : None

</details>



Discussion:

Resolving patch:
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -365,6 +365,15 @@ def _verify_integrity(self, codes=None, levels=None):
                     "Level values must be unique: {values} on "
                     "level {level}".format(values=[value for value in level], level=i)
                 )
+        if self.sortorder is not None:
+            if self.sortorder > self._lexsort_depth():
+                raise ValueError(
+                    "Value for sortorder must be inferior or equal "
+                    "to actual lexsort_depth: "
+                    "sortorder {sortorder} with lexsort_depth {lexsort_depth}".format(
+                        sortorder=self.sortorder, lexsort_depth=self._lexsort_depth()
+                    )
+                )
 
         codes = [
             self._validate_codes(level, code) for level, code in zip(levels, codes)
@@ -1783,16 +1792,23 @@ def is_lexsorted(self):
     @cache_readonly
     def lexsort_depth(self):
         if self.sortorder is not None:
-            if self.sortorder == 0:
-                return self.nlevels
-            else:
-                return 0
+            return self.sortorder
+
+        return self._lexsort_depth()
 
+    def _lexsort_depth(self) -> int:
+        """
+        Compute and return the lexsort_depth, the number of levels of the
+        MultiIndex that are sorted lexically
+
+        Returns
+        ------
+        int
+        """
         int64_codes = [ensure_int64(level_codes) for level_codes in self.codes]
         for k in range(self.nlevels, 0, -1):
             if libalgos.is_lexsorted(int64_codes[:k]):
                 return k
-
         return 0
 
     def _sort_levels_monotonic(self):


Test patch:
diff --git a/pandas/tests/indexes/multi/test_indexing.py b/pandas/tests/indexes/multi/test_indexing.py
--- a/pandas/tests/indexes/multi/test_indexing.py
+++ b/pandas/tests/indexes/multi/test_indexing.py
@@ -96,7 +96,6 @@ def test_slice_locs_not_contained():
     index = MultiIndex(
         levels=[[0, 2, 4, 6], [0, 2, 4]],
         codes=[[0, 0, 0, 1, 1, 2, 3, 3, 3], [0, 1, 2, 1, 2, 2, 0, 1, 2]],
-        sortorder=0,
     )
 
     result = index.slice_locs((1, 0), (5, 2))
diff --git a/pandas/tests/reshape/merge/test_merge.py b/pandas/tests/reshape/merge/test_merge.py
--- a/pandas/tests/reshape/merge/test_merge.py
+++ b/pandas/tests/reshape/merge/test_merge.py
@@ -2094,3 +2094,33 @@ def test_merge_equal_cat_dtypes2():
 
     # Categorical is unordered, so don't check ordering.
     tm.assert_frame_equal(result, expected, check_categorical=False)
+
+
+def test_merge_multiindex_columns():
+    # Issue #28518
+    # Verify that merging two dataframes give the expected labels
+    # The original cause of this issue come from a bug lexsort_depth and is tested in
+    # test_lexsort_depth
+
+    letters = ["a", "b", "c", "d"]
+    numbers = ["1", "2", "3"]
+    index = pd.MultiIndex.from_product((letters, numbers), names=["outer", "inner"])
+
+    frame_x = pd.DataFrame(columns=index)
+    frame_x["id"] = ""
+    frame_y = pd.DataFrame(columns=index)
+    frame_y["id"] = ""
+
+    l_suf = "_x"
+    r_suf = "_y"
+    result = frame_x.merge(frame_y, on="id", suffixes=((l_suf, r_suf)))
+
+    # Constructing the expected results
+    expected_labels = [l + l_suf for l in letters] + [l + r_suf for l in letters]
+    expected_index = pd.MultiIndex.from_product(
+        [expected_labels, numbers], names=["outer", "inner"]
+    )
+    expected = pd.DataFrame(columns=expected_index)
+    expected["id"] = ""
+
+    tm.assert_frame_equal(result, expected)
diff --git a/pandas/tests/test_multilevel.py b/pandas/tests/test_multilevel.py
--- a/pandas/tests/test_multilevel.py
+++ b/pandas/tests/test_multilevel.py
@@ -2056,6 +2056,53 @@ def test_is_lexsorted(self):
         assert not index.is_lexsorted()
         assert index.lexsort_depth == 0
 
+    def test_raise_invalid_sortorder(self):
+        # Test that the MultiIndex constructor raise when a incorrect sortorder is given
+        # Issue #28518
+
+        levels = [[0, 1], [0, 1, 2]]
+
+        # Correct sortorder
+        MultiIndex(
+            levels=levels, codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]], sortorder=2
+        )
+
+        with pytest.raises(ValueError, match=r".* sortorder 2 with lexsort_depth 1.*"):
+            MultiIndex(
+                levels=levels,
+                codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 2, 1]],
+                sortorder=2,
+            )
+
+        with pytest.raises(ValueError, match=r".* sortorder 1 with lexsort_depth 0.*"):
+            MultiIndex(
+                levels=levels,
+                codes=[[0, 0, 1, 0, 1, 1], [0, 1, 0, 2, 2, 1]],
+                sortorder=1,
+            )
+
+    def test_lexsort_depth(self):
+        # Test that lexsort_depth return the  correct sortorder
+        # when it was given to the MultiIndex const.
+        # Issue #28518
+
+        levels = [[0, 1], [0, 1, 2]]
+
+        index = MultiIndex(
+            levels=levels, codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]], sortorder=2
+        )
+        assert index.lexsort_depth == 2
+
+        index = MultiIndex(
+            levels=levels, codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 2, 1]], sortorder=1
+        )
+        assert index.lexsort_depth == 1
+
+        index = MultiIndex(
+            levels=levels, codes=[[0, 0, 1, 0, 1, 1], [0, 1, 0, 2, 2, 1]], sortorder=0
+        )
+        assert index.lexsort_depth == 0
+
     def test_sort_index_and_reconstruction(self):
 
         # 15622