Repository: IAMconsortium/pyam
Pull request #113 (2018-10-18 07:58:34)

fix issue with region mismatch
This fixes an issue in the region mapping where duplicate values would cause double counting.


Patch:
diff --git a/pyam/core.py b/pyam/core.py
index ce44784..29be323 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -921,7 +921,7 @@ class IamDataFrame(object):
         return ax
 
     def map_regions(self, map_col, agg=None, copy_col=None, fname=None,
-                    region_col=None, inplace=False):
+                    region_col=None, remove_duplicates=False, inplace=False):
         """Plot regional data for a single model, scenario, variable, and year
 
         see pyam.plotting.region_plot() for all available options
@@ -939,6 +939,12 @@ class IamDataFrame(object):
             Use a non-default region mapping file
         region_col: string, optional
             Use a non-default column name for regions to map from.
+        remove_duplicates: bool, optional, default: False
+            If there are duplicates in the mapping from one regional level to
+            another, then remove these duplicates by counting the most common
+            mapped value.
+            This option is most useful when mapping from high resolution
+            (e.g., model regions) to low resolution (e.g., 5_region).
         inplace : bool, default False
             if True, do operation inplace and return None
         """
@@ -958,7 +964,28 @@ class IamDataFrame(object):
             _col = region_col or '{}.REGION'.format(model)
             _map = mapping.rename(columns={_col.lower(): 'region'})
             _map = _map[['region', map_col]].dropna().drop_duplicates()
-
+            _map = _map[_map['region'].isin(_df['region'])]
+            if remove_duplicates and _map['region'].duplicated().any():
+                # find duplicates
+                where_dup = _map['region'].duplicated(keep=False)
+                dups = _map[where_dup]
+                logger().warning("""
+                Duplicate entries found for the following regions.
+                Mapping will occur only for the most common instance.
+                {}""".format(dups['region'].unique()))
+                # get non duplicates
+                _map = _map[~where_dup]
+                # order duplicates by the count frequency
+                dups = (dups
+                        .groupby(['region', map_col])
+                        .size()
+                        .reset_index(name='count')
+                        .sort_values(by='count', ascending=False)
+                        .drop('count', axis=1))
+                # take top occurance
+                dups = dups[~dups['region'].duplicated(keep='first')]
+                # combine them back
+                _map = pd.concat([_map, dups])
             if copy_col is not None:
                 df[copy_col] = df['region']
 


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index a839a97..28c2674 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -763,6 +763,27 @@ def test_48b():
     pd.testing.assert_frame_equal(obs, exp, check_index_type=False)
 
 
+def test_48c():
+    # tests fix for #48 mapping few->many, dropping duplicates
+
+    exp = IamDataFrame(pd.DataFrame([
+        ['model', 'scen', 'AGO', 'var', 'unit', 1, 6],
+        ['model', 'scen1', 'AGO', 'var', 'unit', 2, 7],
+    ], columns=['model', 'scenario', 'region',
+                'variable', 'unit', 2005, 2010],
+    )).data.reset_index(drop=True)
+
+    df = IamDataFrame(pd.DataFrame([
+        ['model', 'scen', 'R5MAF', 'var', 'unit', 1, 6],
+        ['model', 'scen1', 'R5MAF', 'var', 'unit', 2, 7],
+    ], columns=['model', 'scenario', 'region',
+                'variable', 'unit', 2005, 2010],
+    ))
+    obs = df.map_regions('iso', region_col='r5_region',
+                         remove_duplicates=True).data
+    pd.testing.assert_frame_equal(obs, exp, check_index_type=False)
+
+
 def test_rename_variable():
     df = IamDataFrame(pd.DataFrame([
         ['model', 'scen', 'SST', 'test_1', 'unit', 1, 5],