Repository: IAMconsortium/pyam
Pull request #269 (2019-09-24 02:37:59)

Fix bug in `append`
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

## Adding to RELEASE_NOTES.md (remove section after adding to RELEASE_NOTES.md)

Please add a single line in the release notes similar to the following:

```
- (#XX)[http://link-to-pr.com] Added feature which does something
```

# Description of PR

At the moment, `append` sometimes swaps the `extra_cols`. This PR fixes that behaviour.


coveralls: 
[![Coverage Status](https://coveralls.io/builds/25885527/badge)](https://coveralls.io/builds/25885527)

Coverage decreased (-0.3%) to 87.835% when pulling **e9f1e3e3641938ce41c2763450f0393e84e34f2c on znicholls:append-bug** into **127680a8f412659c440655c4238c33cb45b528da on IAMconsortium:master**.


Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 919b0a2..144fe77 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 
 # Next Release
 
+- [#269](https://github.com/IAMconsortium/pyam/pull/269) Ensure append doesn't accidentally swap indexes
 - [#268](https://github.com/IAMconsortium/pyam/pull/268) Update `aggregate_region` so it can find variables below sub-cateogories too
 - [#261](https://github.com/IAMconsortium/pyam/pull/261) Add a check that `keep` in `filter()` is a boolean
 - [#254](https://github.com/IAMconsortium/pyam/pull/254) Hotfix for aggregating missing regions and filtering empty dataframes
diff --git a/pyam/core.py b/pyam/core.py
index 5cbf1ff..f527ceb 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -232,8 +232,9 @@ class IamDataFrame(object):
             ret.meta = ret.meta.append(other.meta.loc[diff, :], **sort_kwarg)
 
         # append other.data (verify integrity for no duplicates)
-        _data = ret.data.set_index(ret._LONG_IDX).append(
-            other.data.set_index(other._LONG_IDX), verify_integrity=True)
+        _data = ret.data.set_index(sorted(ret._LONG_IDX)).append(
+            other.data.set_index(sorted(other._LONG_IDX)),
+            verify_integrity=True)
 
         # merge extra columns in `data` and set `LONG_IDX`
         ret.extra_cols += [i for i in other.extra_cols


Test patch:
diff --git a/tests/test_feature_append_rename_convert.py b/tests/test_feature_append_rename_convert.py
index ea1ad1f..b34d47c 100644
--- a/tests/test_feature_append_rename_convert.py
+++ b/tests/test_feature_append_rename_convert.py
@@ -91,6 +91,37 @@ def test_append_same_scenario(meta_df):
     npt.assert_array_equal(ts.iloc[2], ts.iloc[3])
 
 
+@pytest.mark.parametrize("shuffle_cols", [True, False])
+def test_append_extra_col(test_df, shuffle_cols):
+    base_data = test_df.data.copy()
+
+    base_data["col_1"] = "hi"
+    base_data["col_2"] = "bye"
+    base_df = IamDataFrame(base_data)
+
+    other_data = base_data[base_data["variable"] == "Primary Energy"].copy()
+    other_data["variable"] = "Primary Energy|Gas"
+    other_df = IamDataFrame(other_data)
+
+    if shuffle_cols:
+        c1_idx = other_df._LONG_IDX.index("col_1")
+        c2_idx = other_df._LONG_IDX.index("col_2")
+        other_df._LONG_IDX[c1_idx] = "col_2"
+        other_df._LONG_IDX[c2_idx] = "col_1"
+
+    res = base_df.append(other_df)
+
+    def check_meta_is(iamdf, meta_col, val):
+        for checker in [iamdf.timeseries().reset_index(), iamdf.data]:
+            meta_vals = checker[meta_col].unique()
+            assert len(meta_vals) == 1, meta_vals
+            assert meta_vals[0] == val, meta_vals
+
+    # ensure meta merged correctly
+    check_meta_is(res, "col_1", "hi")
+    check_meta_is(res, "col_2", "bye")
+
+
 def test_append_duplicates(test_df_year):
     other = copy.deepcopy(test_df_year)
     pytest.raises(ValueError, test_df_year.append, other=other)