Repository: IAMconsortium/pyam

Issue:
Unexpected failures with incorrect permissions in the IIASA database system
When a user has access to a Scenario Explorer database instance but no permission to view any models (or no scenarios exist), querying data from the API fails with an error similar to

```
  if nmissing == len(indexer):
      if use_interval_msg:
          key = list(key)
              raise KeyError(f"None of [{key}] are in the [{axis_name}]")
E KeyError: "None of [Index(['model', 'scenario', 'version', 'is_default', 'run_id'], dtype='object')] are in the [columns]"
```

The pyam.iiasa module should check that a response exists before casting to an **IamDataFrame**.

Discussion:

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 1f98daf..757e2f8 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -2,6 +2,7 @@
 
 ## Individual updates
 
+- [#697](https://github.com/IAMconsortium/pyam/pull/697) Add warning if IIASA API returns empty result
 - [#695](https://github.com/IAMconsortium/pyam/pull/695) Remove unused meta levels during initialization
 - [#688](https://github.com/IAMconsortium/pyam/pull/688) Remove ixmp as optional dependency
 - [#684](https://github.com/IAMconsortium/pyam/pull/684) Use new IIASA-manager API with token refresh 
diff --git a/pyam/iiasa.py b/pyam/iiasa.py
index d26ea75..f8d83af 100644
--- a/pyam/iiasa.py
+++ b/pyam/iiasa.py
@@ -2,6 +2,7 @@ from pathlib import Path
 import json
 import logging
 import requests
+from fnmatch import fnmatch
 
 import httpx
 import jwt
@@ -251,21 +252,15 @@ class Connection(object):
         """Currently connected resource (database API connection)"""
         return self._connected
 
-    def index(self, default=True):
-        """Return the index of models and scenarios in the connected resource
-
-        Parameters
-        ----------
-        default : bool, optional
-            If `True`, return *only* the default version of a model/scenario.
-            Any model/scenario without a default version is omitted.
-            If `False`, returns all versions.
-        """
-        cols = ["version"] if default else ["version", "is_default"]
-        return self._query_index(default)[META_IDX + cols].set_index(META_IDX)
+    @property
+    def meta_columns(self):
+        """Return the list of meta indicators in the connected resource"""
+        url = "/".join([self._base_url, "metadata/types"])
+        r = requests.get(url, headers=self.auth())
+        _check_response(r)
+        return pd.read_json(r.text, orient="records")["name"]
 
-    @lru_cache()
-    def _query_index(self, default=True, meta=False):
+    def _query_index(self, default=True, meta=False, cols=[], **kwargs):
         # TODO: at present this reads in all data for all scenarios,
         #  it could be sped up in the future to try to query a subset
         _default = "true" if default else "false"
@@ -275,18 +270,39 @@ class Connection(object):
         r = requests.get(url, headers=self.auth())
         _check_response(r)
 
-        # cast response to dataframe and return
-        return pd.read_json(r.text, orient="records")
+        # cast response to dataframe, apply filter by kwargs, and return
+        runs = pd.read_json(r.text, orient="records")
+        if runs.empty:
+            logger.warning("No permission to view model(s) or no scenarios exist.")
+            return pd.DataFrame([], columns=META_IDX + ["version", "run_id"] + cols)
 
-    @property
-    def meta_columns(self):
-        """Return the list of meta indicators in the connected resource"""
-        url = "/".join([self._base_url, "metadata/types"])
-        r = requests.get(url, headers=self.auth())
-        _check_response(r)
-        return pd.read_json(r.text, orient="records")["name"]
+        if kwargs:
+            keep = np.ones(len(runs), dtype=bool)
+            for key, values in kwargs.items():
+                if key not in META_IDX + ["version"]:
+                    raise ValueError(f"Invalid filter: '{key}'")
+                keep_col = pd.Series([fnmatch(v, values) for v in runs[key].values])
+                keep = np.logical_and(keep, keep_col)
+            return runs[keep]
+        else:
+            return runs
 
-    def meta(self, default=True, **kwargs):
+    def index(self, default=True, **kwargs):
+        """Return the index of models and scenarios
+
+        Parameters
+        ----------
+        default : bool, optional
+            If `True`, return *only* the default version of a model/scenario.
+            Any model/scenario without a default version is omitted.
+            If `False`, returns all versions.
+         kwargs
+            Arguments to filer by *model* and *scenario*, `*` can be used as wildcard
+        """
+        cols = ["version"] if default else ["version", "is_default"]
+        return self._query_index(default, **kwargs)[META_IDX + cols].set_index(META_IDX)
+
+    def meta(self, default=True, run_id=False, **kwargs):
         """Return categories and indicators (meta) of scenarios
 
         Parameters
@@ -295,27 +311,25 @@ class Connection(object):
             Return *only* the default version of each scenario.
             Any (`model`, `scenario`) without a default version is omitted.
             If `False`, return all versions.
+        run_id : bool, optional
+            Include "run id" column
+        kwargs
+            Arguments to filer by *model* and *scenario*, `*` can be used as wildcard
         """
-        df = self._query_index(default, meta=True)
+        df = self._query_index(default, meta=True, **kwargs)
 
         cols = ["version"] if default else ["version", "is_default"]
-        if kwargs:
-            if kwargs.get("run_id", False):
-                cols.append("run_id")
+        if run_id:
+            cols.append("run_id")
 
-        # catching an issue where the query above does not yield any scenarios
-        if df.empty:
-            logger.warning("No permission to view model(s) or they do not exist.")
-            meta = pd.DataFrame([], columns=META_IDX + cols)
-        else:
-            meta = df[META_IDX + cols]
-            if df.metadata.any():
-                extra_meta = pd.DataFrame.from_records(df.metadata)
-                meta = pd.concat([meta, extra_meta], axis=1)
+        meta = df[META_IDX + cols]
+        if not meta.empty and df.metadata.any():
+            extra_meta = pd.DataFrame.from_records(df.metadata)
+            meta = pd.concat([meta, extra_meta], axis=1)
 
         return meta.set_index(META_IDX + ([] if default else ["version"]))
 
-    def properties(self, default=True):
+    def properties(self, default=True, **kwargs):
         """Return the audit properties of scenarios
 
         Parameters
@@ -324,17 +338,17 @@ class Connection(object):
             Return *only* the default version of each scenario.
             Any (`model`, `scenario`) without a default version is omitted.
             If :obj:`False`, return all versions.
+        kwargs
+            Arguments to filer by *model* and *scenario*, `*` can be used as wildcard
         """
-        _df = self._query_index(default, meta=True)
         audit_cols = ["cre_user", "cre_date", "upd_user", "upd_date"]
-        audit_mapping = dict([(i, i.replace("_", "ate_")) for i in audit_cols])
         other_cols = ["version"] if default else ["version", "is_default"]
+        cols = audit_cols + other_cols
 
-        return (
-            _df[META_IDX + other_cols + audit_cols]
-            .set_index(META_IDX)
-            .rename(columns=audit_mapping)
-        )
+        _df = self._query_index(default, meta=True, cols=cols, **kwargs)
+        audit_mapping = dict([(i, i.replace("_", "ate_")) for i in audit_cols])
+
+        return _df.set_index(META_IDX).rename(columns=audit_mapping)
 
     def models(self):
         """List all models in the connected resource"""
@@ -504,6 +518,10 @@ class Connection(object):
         else:
             _meta = self._query_index(default=default).set_index(META_IDX)
 
+        # return nothing if no data exists at all
+        if _meta.empty:
+            return
+
         # retrieve data
         _args = json.dumps(self._query_post(_meta, default=default, **kwargs))
         url = "/".join([self._base_url, "runs/bulk/ts"])


Test patch:
diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py
index 8478e09..727e643 100644
--- a/tests/test_iiasa.py
+++ b/tests/test_iiasa.py
@@ -167,39 +167,71 @@ def test_meta_columns(conn):
     npt.assert_array_equal(conn.meta_columns, META_COLS)
 
 
+@pytest.mark.parametrize("kwargs", [{}, dict(model="model_a")])
 @pytest.mark.parametrize("default", [True, False])
-def test_index(conn, default):
+def test_index(conn, kwargs, default):
     # test that connection returns the correct index
+    obs = conn.index(default=default, **kwargs)
+
     if default:
         exp = META_DF.loc[META_DF.is_default, ["version"]]
+        if kwargs:
+            exp = exp.iloc[0:2]
     else:
         exp = META_DF[VERSION_COLS]
+        if kwargs:
+            exp = exp.iloc[0:3]
+
+    pdt.assert_frame_equal(obs, exp, check_dtype=False)
+
+
+def test_index_empty(conn):
+    # test that an empty filter does not yield an error
+    # solves https://github.com/IAMconsortium/pyam/issues/676
+    conn.index(model="foo").empty
+
 
-    pdt.assert_frame_equal(conn.index(default=default), exp, check_dtype=False)
+def test_index_illegal_column(conn):
+    # test that filtering by an illegal column raises an error
+    with pytest.raises(ValueError, match="Invalid filter: 'foo'"):
+        conn.index(foo="bar")
 
 
+@pytest.mark.parametrize("kwargs", [{}, dict(model="model_a")])
 @pytest.mark.parametrize("default", [True, False])
-def test_meta(conn, default):
+def test_meta(conn, kwargs, default):
     # test that connection returns the correct meta dataframe
+    obs = conn.meta(default=default, **kwargs)
+
     v = "version"
     if default:
         exp = META_DF.loc[META_DF.is_default, [v] + META_COLS]
+        if kwargs:
+            exp = exp.iloc[0:2]
     else:
         exp = META_DF[VERSION_COLS + META_COLS].set_index(v, append=True)
+        if kwargs:
+            exp = exp.iloc[0:3]
 
-    pdt.assert_frame_equal(conn.meta(default=default), exp, check_dtype=False)
+    pdt.assert_frame_equal(obs, exp, check_dtype=False)
 
 
+@pytest.mark.parametrize("kwargs", [{}, dict(model="model_a")])
 @pytest.mark.parametrize("default", [True, False])
-def test_properties(conn, default):
+def test_properties(conn, kwargs, default):
     # test that connection returns the correct properties dataframe
-    obs = conn.properties(default=default)
+    obs = conn.properties(default, **kwargs)
+
     if default:
         exp_cols = ["version"]
         exp = META_DF.loc[META_DF.is_default, exp_cols]
+        if kwargs:
+            exp = exp.iloc[0:2]
     else:
         exp_cols = VERSION_COLS
         exp = META_DF[exp_cols]
+        if kwargs:
+            exp = exp.iloc[0:3]
 
     # assert that the expected audit columns are included
     for col in ["create_user", "create_date", "update_user", "update_date"]: