Repository: IAMconsortium/pyam

Issue:
Allow undefined units?
Currently, pyam raises an error when initializing from a pandas.DataFrame with `nan` or when reading from xlsx/csv with empty cells in any index/coordinate dimension. This makes sense for model, scenario, variable and region, but there could be timeseries without a natural unit (e.g., population or number power plants  if not given in an order of magnitude, shares, probabilities if given on the range [0, 1]).

The reason why this was introduced in the first place is that pandas has issues when filtering and other operations when `nan` are present.

One solution could be to allow variables without units by replacing `nan` with `''` (an empty string). That way, filtering and other operations should behave as expected, and writing to csv/xlsx should again generate empty cells (same as the original when reading from a file).

Discussion:

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 8226694..8324afc 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -2,6 +2,7 @@
 
 ## Individual updates
 
+- [#454](https://github.com/IAMconsortium/pyam/pull/454) Enable dimensionless units and fix `info()` if IamDataFrame is empty
 - [#451](https://github.com/IAMconsortium/pyam/pull/451) Fix unit conversions from C to CO2eq
 - [#450](https://github.com/IAMconsortium/pyam/pull/450) Defer logging set-up to when the first logging message is generated
 - [#445](https://github.com/IAMconsortium/pyam/pull/445) Prevent conflicts between attributes and data/meta columns
diff --git a/doc/source/data.rst b/doc/source/data.rst
index ea9bcf1..cf06255 100644
--- a/doc/source/data.rst
+++ b/doc/source/data.rst
@@ -82,9 +82,18 @@ in wide format.
 
 .. note::
 
-    When using custom columns for annotations:
-    **pyam** drops any data rows where the 'value' column is 'nan',
-    and it raises an error for 'nan' in any other column.
+    If there are :code:`numpy.nan` in a **pandas.DataFrame**
+    or empty cells in xlsx/csv files when initializing an **IamDataFrame**,
+    it will behave as follows:
+
+    ========= =============================================
+    column    behaviour
+    ========= =============================================
+    'value'   ignore/drop 'nan'
+    'unit'    replace 'nan' by an empty string (:code:`''`)
+    all other raise an error
+    ========= =============================================
+
     Hence, if you are adding variable/region-specific meta information to
     'data', you need to make sure that you **add a value to every single row**.
 
diff --git a/pyam/core.py b/pyam/core.py
index aac267b..e886bf2 100755
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -228,7 +228,7 @@ class IamDataFrame(object):
                              self.meta.dtypes[0:meta_rows])])
         # print `...` if more than `meta_rows` columns
         if len(self.meta.columns) > meta_rows:
-            info += '\n * ...'
+            info += '\n   ...'
 
         # add info on size (optional)
         if memory_usage:
diff --git a/pyam/utils.py b/pyam/utils.py
index 0ad2576..cb98067 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -261,6 +261,9 @@ def format_data(df, **kwargs):
     df['value'] = df['value'].astype('float64')
     df.dropna(inplace=True, subset=['value'])
 
+    # replace missing units by an empty string for user-friendly filtering
+    df.loc[df.unit.isnull(), 'unit'] = ''
+
     # verify that there are no nan's left (in columns)
     null_rows = df.isnull().values
     if null_rows.any():
@@ -518,10 +521,12 @@ def datetime_match(data, dts):
 
 def print_list(x, n):
     """Return a printable string of a list shortened to n characters"""
-    # subtract count added at end from line width
-    x = list(map(str, x))
+    # if list is empty, only write count
+    if len(x) == 0:
+        return '(0)'
 
-    # write number of elements
+    # write number of elements, subtract count added at end from line width
+    x = [i if i != '' else "''" for i in map(str, x)]
     count = f' ({len(x)})'
     n -= len(count)
 


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 1f27562..ca48099 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -28,14 +28,6 @@ df_filter_by_meta_nonmatching_idx = pd.DataFrame([
 ], columns=['model', 'scenario', 'region', 2010, 2020]
 ).set_index(['model', 'region'])
 
-df_with_na_columns = pd.DataFrame([
-    ['model_a', 'scen_a', 'World', 'Primary Energy', np.nan, 1, 6.],
-    ['model_a', 'scen_a', 'World', 'Primary Energy|Coal', 'EJ/yr', 0.5, 3],
-    ['model_a', 'scen_b', 'World', 'Primary Energy', 'EJ/yr', 2, 7],
-],
-    columns=IAMC_IDX + [2005, 2010],
-)
-
 df_empty = pd.DataFrame([], columns=IAMC_IDX + [2005, 2010])
 
 
@@ -79,8 +71,10 @@ def test_init_df_with_duplicates_raises(test_df):
         IamDataFrame(_df)
 
 
-def test_init_df_with_na_unit(test_df):
-    pytest.raises(ValueError, IamDataFrame, data=df_with_na_columns)
+def test_init_df_with_na_scenario(test_pd_df):
+    # missing values in an index dimension raises an error
+    test_pd_df.loc[1, 'scenario'] = np.nan
+    pytest.raises(ValueError, IamDataFrame, data=test_pd_df)
 
 
 def test_init_df_with_float_cols(test_pd_df):
@@ -162,6 +156,26 @@ def test_print(test_df_year):
     assert obs == exp
 
 
+def test_print_empty(test_df_year):
+    """Assert that `print(IamDataFrame)` (and `info()`) returns as expected"""
+    exp = '\n'.join([
+        "<class 'pyam.core.IamDataFrame'>",
+        'Index dimensions:',
+        ' * model    : (0)',
+        ' * scenario : (0)',
+        'Timeseries data coordinates:',
+        '   region   : (0)',
+        '   variable : (0)',
+        '   unit     : (0)',
+        '   year     : (0)',
+        'Meta indicators:',
+        '   exclude (bool) (0)',
+        '   number (int64) (0)',
+        '   string (object) (0)'])
+    obs = test_df_year.filter(model='foo').info()
+    assert obs == exp
+
+
 def test_as_pandas(test_df):
     # test that `as_pandas()` returns the right columns
     df = test_df.copy()
diff --git a/tests/test_io.py b/tests/test_io.py
index 4c51da7..c3453a9 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -47,6 +47,28 @@ def test_io_xlsx(test_df, meta_args):
         os.remove(file)
 
 
+def test_init_df_with_na_unit(test_pd_df):
+    # missing values in the unit column are replaced by an empty string
+    test_pd_df.loc[1, 'unit'] = np.nan
+    df = IamDataFrame(test_pd_df)
+    assert df.unit == ['', 'EJ/yr']
+
+    # writing to file and importing as pandas returns `nan`, not empty string
+    file = 'na_unit.csv'
+    df.to_csv(file)
+    df_csv = pd.read_csv(file)
+    assert np.isnan(df_csv.loc[1, 'Unit'])
+    IamDataFrame('na_unit.csv')  # reading from file as IamDataFrame works
+    os.remove(file)
+
+    file = 'na_unit.xlsx'
+    df.to_excel(file)
+    df_excel = pd.read_excel(file)
+    assert np.isnan(df_excel.loc[1, 'Unit'])
+    IamDataFrame('na_unit.xlsx')  # reading from file as IamDataFrame works
+    os.remove(file)
+
+
 @pytest.mark.parametrize("args", [{}, dict(sheet_name='meta')])
 def test_load_meta(test_df, args):
     file = os.path.join(TEST_DATA_DIR, 'testing_metadata.xlsx')