Repository: fsspec/filesystem_spec
Pull request #973 (2022-05-28 03:32:14)

fix httpfile serialization bug
This implements the suggestion in https://github.com/pangeo-forge/pangeo-forge-recipes/pull/370#issuecomment-1140028052, which fixes a bug in how `HTTPFile` is serialized.

It's a bit worrying to me that serialization of `HTTPFile` is not covered by tests sufficiently to have already surfaced this bug. Therefore, I would like to use this PR to contribute some improvements to testing as well.

martindurant: I think this PR is fine, and solves one arm of he possibilities in https://github.com/fsspec/filesystem_spec/issues/579 .

Another possible thing to consider, is making OpenFile file-like, calling into `.open()` when required. Seems like a rather large change to the core API.

Patch:
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
index 126b410..8d65c27 100644
--- a/fsspec/implementations/http.py
+++ b/fsspec/implementations/http.py
@@ -642,7 +642,7 @@ class HTTPFile(AbstractBufferedFile):
                 self.url,
                 self.mode,
                 self.blocksize,
-                self.cache.name,
+                self.cache.name if self.cache else "none",
                 self.size,
             ),
         )
diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py
index de4a41d..d106285 100644
--- a/fsspec/implementations/local.py
+++ b/fsspec/implementations/local.py
@@ -354,7 +354,7 @@ class LocalFileOpener(io.IOBase):
 
     def __enter__(self):
         self._incontext = True
-        return self.f.__enter__()
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         self._incontext = False


Test patch:
diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
index a71838c..6bb5145 100644
--- a/fsspec/implementations/tests/test_http.py
+++ b/fsspec/implementations/tests/test_http.py
@@ -163,6 +163,11 @@ def test_file_pickle(server):
     # via HTTPFile
     h = fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
     out = server + "/index/realfile"
+
+    with fsspec.open(out, headers={"give_length": "true", "head_ok": "true"}) as f:
+        pic = pickle.loads(pickle.dumps(f))
+        assert pic.read() == data
+
     with h.open(out, "rb") as f:
         pic = pickle.dumps(f)
         assert f.read() == data
diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py
index 9c41392..b32ad34 100644
--- a/fsspec/implementations/tests/test_local.py
+++ b/fsspec/implementations/tests/test_local.py
@@ -587,6 +587,20 @@ def test_pickle(tmpdir):
     with pytest.raises(ValueError):
         pickle.dumps(f)
 
+    # with context
+    with fs.open(fn0, "rb") as f:
+        f.seek(1)
+        f2 = pickle.loads(pickle.dumps(f))
+        assert f2.tell() == 1
+        assert f2.read() == f.read()
+
+    # with fsspec.open https://github.com/fsspec/filesystem_spec/issues/579
+    with fsspec.open(fn0, "rb") as f:
+        f.seek(1)
+        f2 = pickle.loads(pickle.dumps(f))
+        assert f2.tell() == 1
+        assert f2.read() == f.read()
+
 
 def test_strip_protocol_expanduser():
     path = "file://~\\foo\\bar" if WIN else "file://~/foo/bar"