Repository: intake/intake
Pull request #644 (2022-01-21 20:38:04)

BUG: allow None as json compression
If you have a json without compression, you would use the default of None.

However, this will currently raise a `ValueError` as None is not in `VALID_COMPRESSIONS`.
https://github.com/intake/intake/blob/master/intake/source/jsonfiles.py#L60

This PR fixes this by also checking for a None value.

cc. @blackary 

Don't think CI failures are related.


Patch:
diff --git a/intake/source/jsonfiles.py b/intake/source/jsonfiles.py
index ed5f665..e7168c2 100644
--- a/intake/source/jsonfiles.py
+++ b/intake/source/jsonfiles.py
@@ -56,11 +56,11 @@ class JSONFileSource(DataSource):
         self._dataframe = None
         self._file = None
         self.compression = compression
-
-        if compression not in VALID_COMPRESSIONS:
-            raise ValueError(
-                f"Compression value {compression} must be one of {VALID_COMPRESSIONS}"
-            )
+        if compression is not None:
+            if compression not in VALID_COMPRESSIONS:
+                raise ValueError(
+                    f"Compression value {compression} must be one of {VALID_COMPRESSIONS}"
+                )
         self.mode = "rt" if text_mode else "rb"
         self.encoding = text_encoding
         self._read = read
@@ -130,10 +130,11 @@ class JSONLinesFileSource(DataSource):
         self._dataframe = None
         self._file = None
         self.compression = compression
-        if compression not in VALID_COMPRESSIONS:
-            raise ValueError(
-                f"Compression value {compression} must be one of {VALID_COMPRESSIONS}"
-            )
+        if compression is not None:
+            if compression not in VALID_COMPRESSIONS:
+                raise ValueError(
+                    f"Compression value {compression} must be one of {VALID_COMPRESSIONS}"
+                )
         self.mode = "rt" if text_mode else "rb"
         self.encoding = text_encoding
         self._read = read


Test patch:
diff --git a/intake/source/tests/test_json.py b/intake/source/tests/test_json.py
index da641a9..5ac7f18 100644
--- a/intake/source/tests/test_json.py
+++ b/intake/source/tests/test_json.py
@@ -50,6 +50,20 @@ def test_jsonfile(json_file: str):
     assert out["hello"] == "world"
 
 
+def test_jsonfile_none(json_file: str):
+    try:
+        j = JSONFileSource(json_file, text_mode=True, compression=None)
+        out = j.read()
+        # compression=None should not raise exception for uncompressed files
+        assert json_file.endswith(".json")
+    except UnicodeDecodeError:
+        # compression=None should raise exception for compressed files
+        assert not json_file.endswith(".json")
+        return
+    assert isinstance(out, dict)
+    assert out["hello"] == "world"
+
+
 def test_jsonlfile(jsonl_file: str):
     j = JSONLinesFileSource(jsonl_file, compression="infer")
     out = j.read()
@@ -62,6 +76,25 @@ def test_jsonlfile(jsonl_file: str):
     assert out[1] == [1, 2, 3]
 
 
+def test_jsonfilel_none(jsonl_file: str):
+    try:
+        j = JSONLinesFileSource(jsonl_file, compression=None)
+        out = j.read()
+        # compression=None should not raise exception for uncompressed files
+        assert jsonl_file.endswith(".jsonl")
+    except UnicodeDecodeError:
+        # compression=None should raise exception for compressed files
+        assert not jsonl_file.endswith(".jsonl")
+        return
+    assert isinstance(out, list)
+
+    assert isinstance(out[0], dict)
+    assert out[0]["hello"] == "world"
+
+    assert isinstance(out[1], list)
+    assert out[1] == [1, 2, 3]
+
+
 def test_jsonl_head(jsonl_file: str):
     j = JSONLinesFileSource(jsonl_file, compression="infer")
     out = j.head(1)