Repository: fsspec/filesystem_spec

Issue:
KeyError when dict values and templates are used in ReferenceFileSystem
#1562 added support for `ReferenceFileSystem` to hold dicts. However, when templates are also present in the input JSON, initializing the file system results in a `KeyError`:

```python
    def _process_references1(self, references, template_overrides=None):
        if not self.simple_templates or self.templates:
            import jinja2
        self.references = {}
        self._process_templates(references.get("templates", {}))
    
        @lru_cache(1000)
        def _render_jinja(u):
            return jinja2.Template(u).render(**self.templates)
    
        for k, v in references.get("refs", {}).items():
            if isinstance(v, str):
                if v.startswith("base64:"):
                    self.references[k] = base64.b64decode(v[7:])
                self.references[k] = v
            elif self.templates:
>               u = v[0]
E               KeyError: 0

fsspec/implementations/reference.py:962: KeyError
```

This can be resolved by moving the dict check to above the `elif self.templates:`

I'll submit a PR to fix this. (cc @bendichter)

Discussion:

Patch:
diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py
index 06254fe..6e3de91 100644
--- a/fsspec/implementations/reference.py
+++ b/fsspec/implementations/reference.py
@@ -956,6 +956,8 @@ class ReferenceFileSystem(AsyncFileSystem):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             elif self.templates:
                 u = v[0]
                 if "{{" in u:
@@ -968,8 +970,6 @@ class ReferenceFileSystem(AsyncFileSystem):
                     else:
                         u = _render_jinja(u)
                 self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]]
-            elif isinstance(v, dict):
-                self.references[k] = json.dumps(v)
             else:
                 self.references[k] = v
         self.references.update(self._process_gen(references.get("gen", [])))


Test patch:
diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py
index 3c27119..762d831 100644
--- a/fsspec/implementations/tests/test_reference.py
+++ b/fsspec/implementations/tests/test_reference.py
@@ -229,6 +229,8 @@ jdata = """{
 
 def test_spec1_expand():
     pytest.importorskip("jinja2")
+    from fsspec.implementations.reference import json as json_impl
+
     in_data = {
         "version": 1,
         "templates": {"u": "server.domain/path", "f": "{{c}}"},
@@ -252,6 +254,7 @@ def test_spec1_expand():
             "key2": ["http://{{u}}", 10000, 100],
             "key3": ["http://{{f(c='text')}}", 10000, 100],
             "key4": ["http://target_url"],
+            "key5": {"key": "value"},
         },
     }
     fs = fsspec.filesystem(
@@ -263,6 +266,7 @@ def test_spec1_expand():
         "key2": ["http://server.domain/path", 10000, 100],
         "key3": ["http://text", 10000, 100],
         "key4": ["http://target_url"],
+        "key5": json_impl.dumps(in_data["refs"]["key5"]),
         "gen_key0": ["http://server.domain/path_0", 1000, 1000],
         "gen_key1": ["http://server.domain/path_1", 2000, 1000],
         "gen_key2": ["http://server.domain/path_2", 3000, 1000],
@@ -275,6 +279,8 @@ def test_spec1_expand():
 
 def test_spec1_expand_simple():
     pytest.importorskip("jinja2")
+    from fsspec.implementations.reference import json as json_impl
+
     in_data = {
         "version": 1,
         "templates": {"u": "server.domain/path"},
@@ -282,6 +288,7 @@ def test_spec1_expand_simple():
             "key0": "base64:ZGF0YQ==",
             "key2": ["http://{{u}}", 10000, 100],
             "key4": ["http://target_url"],
+            "key5": {"key": "value"},
         },
     }
     fs = fsspec.filesystem("reference", fo=in_data, target_protocol="http")
@@ -294,6 +301,7 @@ def test_spec1_expand_simple():
     )
     assert fs.references["key2"] == ["http://not.org/p", 10000, 100]
     assert fs.cat("key0") == b"data"
+    assert fs.cat("key5") == json_impl.dumps(in_data["refs"]["key5"]).encode("utf-8")
 
 
 def test_spec1_gen_variants():