Repository: databricks/koalas

Issue:
ops on diff Series works not properly when it has MultiIndex.
Assuming that there are two `Series` with `MultiIndex` like below.

```python
>>> kser1
lama    speed      45.0
koalas  length    200.0
cow     speed       1.2
        power      30.0
        length    250.0
falcon  speed       1.5
        weight    320.0
        power       1.0
Name: 0, dtype: float64

>>> kser2
lama    speed     -45.0
        weight    200.0
        length     -1.2
cow     speed      30.0
        weight   -250.0
        length      1.5
falcon  speed     320.0
        weight      1.0
        length     -0.3
Name: 0, dtype: float64
```

now `kser1 + kser2`  works like below.

```python
>>> kser1 + kser2
cow     length    None
falcon  weight    None
cow     speed     None
falcon  speed     None
koalas  length    None
lama    speed     None
cow     power     None
falcon  power     None
        falcon    None
        falcon    None
        falcon    None
lama    lama      None
        lama      None
        lama      None
cow     cow       None
        cow       None
        cow       None
Name: 0, dtype: object
```

but i think it's not correct, since pandas works like below.

```python
>>> kser1.to_pandas() + kser2.to_pandas()
cow     length    251.5
        power       NaN
        speed      31.2
        weight      NaN
falcon  length      NaN
        power       NaN
        speed     321.5
        weight    321.0
koalas  length      NaN
lama    length      NaN
        speed       0.0
        weight      NaN
Name: 0, dtype: float64
```


Discussion:
I'm working on this.

Resolving patch:
diff --git a/databricks/koalas/utils.py b/databricks/koalas/utils.py
--- a/databricks/koalas/utils.py
+++ b/databricks/koalas/utils.py
@@ -72,21 +72,23 @@ def combine_frames(this, *args, how="full"):
         join_scols = []
         merged_index_scols = []
 
+        # Note that the order of each element in index_map is guaranteed according to the index
+        # level.
+        this_and_that_index_map = zip(this_index_map, that_index_map)
+
         # If the same named index is found, that's used.
-        for this_column, this_name in this_index_map:
-            for that_col, that_name in that_index_map:
-                if this_name == that_name:
-                    # We should merge the Spark columns into one
-                    # to mimic pandas' behavior.
-                    this_scol = this._internal.scol_for(this_column)
-                    that_scol = that._internal.scol_for(that_col)
-                    join_scol = this_scol == that_scol
-                    join_scols.append(join_scol)
-                    merged_index_scols.append(
-                        F.when(
-                            this_scol.isNotNull(), this_scol
-                        ).otherwise(that_scol).alias(this_column))
-                    break
+        for (this_column, this_name), (that_column, that_name) in this_and_that_index_map:
+            if this_name == that_name:
+                # We should merge the Spark columns into one
+                # to mimic pandas' behavior.
+                this_scol = this._internal.scol_for(this_column)
+                that_scol = that._internal.scol_for(that_column)
+                join_scol = this_scol == that_scol
+                join_scols.append(join_scol)
+                merged_index_scols.append(
+                    F.when(
+                        this_scol.isNotNull(), this_scol
+                    ).otherwise(that_scol).alias(this_column))
             else:
                 raise ValueError("Index names must be exactly matched currently.")
 


Test patch:
diff --git a/databricks/koalas/tests/test_ops_on_diff_frames.py b/databricks/koalas/tests/test_ops_on_diff_frames.py
--- a/databricks/koalas/tests/test_ops_on_diff_frames.py
+++ b/databricks/koalas/tests/test_ops_on_diff_frames.py
@@ -78,6 +78,30 @@ def pdf6(self):
             'e': [4, 5, 6, 3, 2, 1, 0, 0, 0],
         }, index=list(range(9))).set_index(['a', 'b'])
 
+    @property
+    def pser1(self):
+        midx = pd.MultiIndex([['lama', 'cow', 'falcon', 'koala'],
+                              ['speed', 'weight', 'length', 'power']],
+                             [[0, 3, 1, 1, 1, 2, 2, 2],
+                              [0, 2, 0, 3, 2, 0, 1, 3]])
+        return pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1], index=midx)
+
+    @property
+    def pser2(self):
+        midx = pd.MultiIndex([['lama', 'cow', 'falcon'],
+                              ['speed', 'weight', 'length']],
+                             [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+                              [0, 1, 2, 0, 1, 2, 0, 1, 2]])
+        return pd.Series([-45, 200, -1.2, 30, -250, 1.5, 320, 1, -0.3], index=midx)
+
+    @property
+    def pser3(self):
+        midx = pd.MultiIndex([['koalas', 'cow', 'falcon'],
+                              ['speed', 'weight', 'length']],
+                             [[0, 0, 0, 1, 1, 1, 2, 2, 2],
+                              [1, 1, 2, 0, 0, 2, 2, 2, 1]])
+        return pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
+
     @property
     def kdf1(self):
         return ks.from_pandas(self.pdf1)
@@ -102,6 +126,18 @@ def kdf5(self):
     def kdf6(self):
         return ks.from_pandas(self.pdf6)
 
+    @property
+    def kser1(self):
+        return ks.from_pandas(self.pser1)
+
+    @property
+    def kser2(self):
+        return ks.from_pandas(self.pser2)
+
+    @property
+    def kser3(self):
+        return ks.from_pandas(self.pser3)
+
     def test_ranges(self):
         self.assert_eq(
             (ks.range(10) + ks.range(10)).sort_index(),
@@ -118,6 +154,10 @@ def test_arithmetic(self):
         kdf2 = self.kdf2
         pdf1 = self.pdf1
         pdf2 = self.pdf2
+        kser1 = self.kser1
+        pser1 = self.pser1
+        kser2 = self.kser2
+        pser2 = self.pser2
 
         # Series
         self.assert_eq(
@@ -162,6 +202,23 @@ def test_arithmetic(self):
             (kdf1 + kdf2).sort_index(),
             (pdf1 + pdf2).sort_index(), almost=True)
 
+        # MultiIndex Series
+        self.assert_eq(
+            (kser1 + kser2).sort_index(),
+            (pser1 + pser2).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 - kser2).sort_index(),
+            (pser1 - pser2).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 * kser2).sort_index(),
+            (pser1 * pser2).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 / kser2).sort_index(),
+            (pser1 / pser2).sort_index(), almost=True)
+
     def test_arithmetic_chain(self):
         kdf1 = self.kdf1
         kdf2 = self.kdf2
@@ -169,6 +226,12 @@ def test_arithmetic_chain(self):
         pdf1 = self.pdf1
         pdf2 = self.pdf2
         pdf3 = self.pdf3
+        kser1 = self.kser1
+        pser1 = self.pser1
+        kser2 = self.kser2
+        pser2 = self.pser2
+        kser3 = self.kser3
+        pser3 = self.pser3
 
         # Series
         self.assert_eq(
@@ -217,6 +280,23 @@ def test_arithmetic_chain(self):
             (kdf1 + kdf2 - kdf3).sort_index(),
             (pdf1 + pdf2 - pdf3).sort_index(), almost=True)
 
+        # MultiIndex Series
+        self.assert_eq(
+            (kser1 + kser2 - kser3).sort_index(),
+            (pser1 + pser2 - pser3).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 * kser2 * kser3).sort_index(),
+            (pser1 * pser2 * pser3).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 - kser2 / kser3).sort_index(),
+            (pser1 - pser2 / pser3).sort_index(), almost=True)
+
+        self.assert_eq(
+            (kser1 + kser2 * kser3).sort_index(),
+            (pser1 + pser2 * pser3).sort_index(), almost=True)
+
     def test_getitem_boolean_series(self):
         pdf1 = pd.DataFrame({'A': [0, 1, 2, 3, 4], 'B': [100, 200, 300, 400, 500]})
         pdf2 = pd.DataFrame({'A': [0, -1, -2, -3, -4], 'B': [-100, -200, -300, -400, -500]})