nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: fsspec/filesystem_spec
Pull request #1145 (2023-01-03 17:46:15)

Fix type returned by calling .ls on memory file
`MemoryFileSystem.ls()` would always return a list with a single detailed dict in it, even when `detail=False` (the default). It should return a list with a single str as per the docs.

martindurant: Thanks for this. Can we also change the default to `detail=True`, which is what AbstractFileSystem and the others have?
FHTMitchell: OK done, I suppose it's ok for me to fix this test?
martindurant: Yes please

Patch:
diff --git a/fsspec/implementations/memory.py b/fsspec/implementations/memory.py
index cf0b2c5..120f5aa 100644
--- a/fsspec/implementations/memory.py
+++ b/fsspec/implementations/memory.py
@@ -31,10 +31,12 @@ class MemoryFileSystem(AbstractFileSystem):
         path = path.lstrip("/").rstrip("/")
         return "/" + path if path else ""
 
-    def ls(self, path, detail=False, **kwargs):
+    def ls(self, path, detail=True, **kwargs):
         path = self._strip_protocol(path)
         if path in self.store:
             # there is a key with this exact name
+            if not detail:
+                return [path]
             return [
                 {
                     "name": path,


Test patch:
diff --git a/fsspec/implementations/tests/test_dask.py b/fsspec/implementations/tests/test_dask.py
index d40f267..c4c05e1 100644
--- a/fsspec/implementations/tests/test_dask.py
+++ b/fsspec/implementations/tests/test_dask.py
@@ -26,5 +26,5 @@ def cli(tmpdir):
 def test_basic(cli):
 
     fs = fsspec.filesystem("dask", target_protocol="memory")
-    assert fs.ls("") == ["/afile"]
+    assert fs.ls("", detail=False) == ["/afile"]
     assert fs.cat("/afile") == b"data"
diff --git a/fsspec/implementations/tests/test_memory.py b/fsspec/implementations/tests/test_memory.py
index 7cf29e4..1239701 100644
--- a/fsspec/implementations/tests/test_memory.py
+++ b/fsspec/implementations/tests/test_memory.py
@@ -44,8 +44,11 @@ def test_ls(m):
     assert m.ls("/dir", False) == ["/dir/afile", "/dir/dir1"]
     assert m.ls("/dir", True)[0]["type"] == "file"
     assert m.ls("/dir", True)[1]["type"] == "directory"
+    assert m.ls("/dir/afile", False) == ["/dir/afile"]
+    assert m.ls("/dir/afile", True)[0]["type"] == "file"
 
     assert len(m.ls("/dir/dir1")) == 2
+    assert len(m.ls("/dir/afile")) == 1
 
 
 def test_directories(m):
Repository: fsspec/filesystem_spec
Pull request #1222 (2023-03-27 09:59:48)

Handle local file URI without authority
The `fsspec` Local implementation currently doesn't handle file URIs that skip the authority properly, as per https://www.rfc-editor.org/rfc/rfc8089#appendix-A - ref: https://github.com/fsspec/universal_pathlib/issues/81

(The pytest CI seems to hang on my fork, not certain why)

martindurant: Seems like CI is having problems generally... we will investigate.
martindurant: Now github itself is having issues, so we wait :)
https://www.githubstatus.com/
SaravananSathyanandhaQC: The green tick of joy doesn't show here for some reason, but it's showing on my fork

Patch:
diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py
index 23d6dac..034d06f 100644
--- a/fsspec/implementations/local.py
+++ b/fsspec/implementations/local.py
@@ -214,6 +214,8 @@ class LocalFileSystem(AbstractFileSystem):
         path = stringify_path(path)
         if path.startswith("file://"):
             path = path[7:]
+        elif path.startswith("file:"):
+            path = path[5:]
         return make_path_posix(path).rstrip("/") or cls.root_marker
 
     def _isfilestore(self):


Test patch:
diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py
index 8e7053f..72a2d6e 100644
--- a/fsspec/implementations/tests/test_local.py
+++ b/fsspec/implementations/tests/test_local.py
@@ -628,6 +628,13 @@ def test_strip_protocol_expanduser():
     assert not LocalFileSystem._strip_protocol("./").endswith("/")
 
 
+def test_strip_protocol_no_authority():
+    path = "file:\\foo\\bar" if WIN else "file:/foo/bar"
+    stripped = LocalFileSystem._strip_protocol(path)
+    assert "file:" not in stripped
+    assert stripped.endswith("/foo/bar")
+
+
 def test_mkdir_twice_faile(tmpdir):
     fn = os.path.join(tmpdir, "test")
     fs = fsspec.filesystem("file")
Repository: fsspec/filesystem_spec
Pull request #1585 (2024-04-20 06:52:57)

Remove mv_file(): not in the specs, while mv() is


Patch:
diff --git a/fsspec/core.py b/fsspec/core.py
index 1e07b5e..bd4f98d 100644
--- a/fsspec/core.py
+++ b/fsspec/core.py
@@ -379,6 +379,7 @@ def url_to_fs(url, **kwargs):
     urlpath : str
         The file-systems-specific URL for ``url``.
     """
+    url = stringify_path(url)
     # non-FS arguments that appear in fsspec.open()
     # inspect could keep this in sync with open()'s signature
     known_kwargs = {
diff --git a/fsspec/implementations/arrow.py b/fsspec/implementations/arrow.py
index e3dd17f..f9fea70 100644
--- a/fsspec/implementations/arrow.py
+++ b/fsspec/implementations/arrow.py
@@ -139,8 +139,6 @@ class ArrowFSWrapper(AbstractFileSystem):
         path2 = self._strip_protocol(path2).rstrip("/")
         self.fs.move(path1, path2)
 
-    mv_file = mv
-
     @wrap_exceptions
     def rm_file(self, path):
         path = self._strip_protocol(path)
diff --git a/fsspec/implementations/dirfs.py b/fsspec/implementations/dirfs.py
index b1b3f6d..1b311ab 100644
--- a/fsspec/implementations/dirfs.py
+++ b/fsspec/implementations/dirfs.py
@@ -329,8 +329,8 @@ class DirFileSystem(AsyncFileSystem):
     def rmdir(self, path):
         return self.fs.rmdir(self._join(path))
 
-    def mv_file(self, path1, path2, **kwargs):
-        return self.fs.mv_file(
+    def mv(self, path1, path2, **kwargs):
+        return self.fs.mv(
             self._join(path1),
             self._join(path2),
             **kwargs,
diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py
index 9d2b335..7938a91 100644
--- a/fsspec/implementations/local.py
+++ b/fsspec/implementations/local.py
@@ -150,7 +150,7 @@ class LocalFileSystem(AbstractFileSystem):
     def put_file(self, path1, path2, callback=None, **kwargs):
         return self.cp_file(path1, path2, **kwargs)
 
-    def mv_file(self, path1, path2, **kwargs):
+    def mv(self, path1, path2, **kwargs):
         path1 = self._strip_protocol(path1, remove_trailing_slash=True)
         path2 = self._strip_protocol(path2, remove_trailing_slash=True)
         shutil.move(path1, path2)


Test patch:
diff --git a/fsspec/implementations/tests/test_dirfs.py b/fsspec/implementations/tests/test_dirfs.py
index e42b9b6..645ce9a 100644
--- a/fsspec/implementations/tests/test_dirfs.py
+++ b/fsspec/implementations/tests/test_dirfs.py
@@ -542,10 +542,10 @@ def test_rmdir(mocker, dirfs):
     dirfs.fs.rmdir.assert_called_once_with(f"{PATH}/dir")
 
 
-def test_mv_file(mocker, dirfs):
-    dirfs.fs.mv_file = mocker.Mock()
-    dirfs.mv_file("one", "two", **KWARGS)
-    dirfs.fs.mv_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)
+def test_mv(mocker, dirfs):
+    dirfs.fs.mv = mocker.Mock()
+    dirfs.mv("one", "two", **KWARGS)
+    dirfs.fs.mv.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)
 
 
 def test_touch(mocker, dirfs):
Repository: fsspec/filesystem_spec
Pull request #1627 (2024-06-15 06:11:31)

Fix `AbstractFileSystem.from_dict` not being idempotent
The existing implementation mutates the input dictionary which causes the function to fail if called multiple times in succession. This PR fixes this issue.

martindurant: Good spot.

Patch:
diff --git a/fsspec/spec.py b/fsspec/spec.py
index f55a536..951e9a2 100644
--- a/fsspec/spec.py
+++ b/fsspec/spec.py
@@ -1470,6 +1470,8 @@ class AbstractFileSystem(metaclass=_Cached):
         """
         from .json import FilesystemJSONDecoder
 
+        dct = dict(dct)  # Defensive copy
+
         cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
         if cls is None:
             raise ValueError("Not a serialized AbstractFileSystem")


Test patch:
diff --git a/fsspec/tests/test_spec.py b/fsspec/tests/test_spec.py
index 27ea0f4..32db78c 100644
--- a/fsspec/tests/test_spec.py
+++ b/fsspec/tests/test_spec.py
@@ -888,6 +888,30 @@ def test_json_fs_attr():
     assert DummyTestFS.from_json(outb) is b
 
 
+def test_dict():
+    a = DummyTestFS(1)
+    b = DummyTestFS(2, bar=1)
+
+    outa = a.to_dict()
+    outb = b.to_dict()
+
+    assert isinstance(outa, dict)
+    assert a != b
+    assert "bar" in outb
+
+    assert DummyTestFS.from_dict(outa) is a
+    assert DummyTestFS.from_dict(outb) is b
+
+
+def test_dict_idempotent():
+    a = DummyTestFS(1)
+
+    outa = a.to_dict()
+
+    assert DummyTestFS.from_dict(outa) is a
+    assert DummyTestFS.from_dict(outa) is a
+
+
 def test_from_dict_valid():
     fs = DummyTestFS.from_dict({"cls": "fsspec.tests.test_spec.DummyTestFS"})
     assert isinstance(fs, DummyTestFS)
Repository: fsspec/filesystem_spec
Pull request #1628 (2024-06-17 12:12:04)

DirFS: Fix `pipe(dict[str, bytes])`
`.pipe()` can be called with a dictionary as input. This is currently not supported by `DirFS` as `._join()` silently maps the dictionary input to a list which causes a cryptic error message error when trying to use `.pipe()` with dictionary input.


Patch:
diff --git a/fsspec/implementations/dirfs.py b/fsspec/implementations/dirfs.py
index 1b311ab..04f7479 100644
--- a/fsspec/implementations/dirfs.py
+++ b/fsspec/implementations/dirfs.py
@@ -56,6 +56,8 @@ class DirFileSystem(AsyncFileSystem):
             if not path:
                 return self.path
             return self.fs.sep.join((self.path, self._strip_protocol(path)))
+        if isinstance(path, dict):
+            return {self._join(_path): value for _path, value in path.items()}
         return [self._join(_path) for _path in path]
 
     def _relpath(self, path):
diff --git a/fsspec/spec.py b/fsspec/spec.py
index f55a536..951e9a2 100644
--- a/fsspec/spec.py
+++ b/fsspec/spec.py
@@ -1470,6 +1470,8 @@ class AbstractFileSystem(metaclass=_Cached):
         """
         from .json import FilesystemJSONDecoder
 
+        dct = dict(dct)  # Defensive copy
+
         cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
         if cls is None:
             raise ValueError("Not a serialized AbstractFileSystem")


Test patch:
diff --git a/fsspec/implementations/tests/test_dirfs.py b/fsspec/implementations/tests/test_dirfs.py
index 300bdd8..c04ba66 100644
--- a/fsspec/implementations/tests/test_dirfs.py
+++ b/fsspec/implementations/tests/test_dirfs.py
@@ -162,6 +162,11 @@ def test_pipe(dirfs):
     dirfs.fs.pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
 
 
+def test_pipe_dict(dirfs):
+    dirfs.pipe({"file": b"foo"}, *ARGS, **KWARGS)
+    dirfs.fs.pipe.assert_called_once_with({f"{PATH}/file": b"foo"}, *ARGS, **KWARGS)
+
+
 @pytest.mark.asyncio
 async def test_async_pipe_file(adirfs):
     await adirfs._pipe_file("file", *ARGS, **KWARGS)
diff --git a/fsspec/tests/test_spec.py b/fsspec/tests/test_spec.py
index 27ea0f4..32db78c 100644
--- a/fsspec/tests/test_spec.py
+++ b/fsspec/tests/test_spec.py
@@ -888,6 +888,30 @@ def test_json_fs_attr():
     assert DummyTestFS.from_json(outb) is b
 
 
+def test_dict():
+    a = DummyTestFS(1)
+    b = DummyTestFS(2, bar=1)
+
+    outa = a.to_dict()
+    outb = b.to_dict()
+
+    assert isinstance(outa, dict)
+    assert a != b
+    assert "bar" in outb
+
+    assert DummyTestFS.from_dict(outa) is a
+    assert DummyTestFS.from_dict(outb) is b
+
+
+def test_dict_idempotent():
+    a = DummyTestFS(1)
+
+    outa = a.to_dict()
+
+    assert DummyTestFS.from_dict(outa) is a
+    assert DummyTestFS.from_dict(outa) is a
+
+
 def test_from_dict_valid():
     fs = DummyTestFS.from_dict({"cls": "fsspec.tests.test_spec.DummyTestFS"})
     assert isinstance(fs, DummyTestFS)
Repository: fsspec/filesystem_spec
Pull request #1639 (2024-06-27 21:22:37)

DirFS: Handle paths with no leading `/` (#1638)
If you wrap an `S3FileSystem` with `DirFileSystem`, `ls()` breaks, since `DirFileSystem` assumes that the underlying FS returns paths with a leading `/`, but `S3FileSystem` does not (see #1638).

This change to _relpath() removes the leading `/` from the `prefix` used in constructing the relative path when the incoming `path` has no leading `/`, but the `DirFileSystem` path does.

martindurant: Thanks for sorting this out.
It sort of feels untidy, but I can't think of a better way to do this :)

Patch:
diff --git a/fsspec/implementations/dirfs.py b/fsspec/implementations/dirfs.py
index 04f7479..6a28909 100644
--- a/fsspec/implementations/dirfs.py
+++ b/fsspec/implementations/dirfs.py
@@ -64,9 +64,15 @@ class DirFileSystem(AsyncFileSystem):
         if isinstance(path, str):
             if not self.path:
                 return path
-            if path == self.path:
+            # We need to account for S3FileSystem returning paths that do not
+            # start with a '/'
+            if path == self.path or (
+                self.path.startswith(self.fs.sep) and path == self.path[1:]
+            ):
                 return ""
             prefix = self.path + self.fs.sep
+            if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep):
+                prefix = prefix[1:]
             assert path.startswith(prefix)
             return path[len(prefix) :]
         return [self._relpath(_path) for _path in path]


Test patch:
diff --git a/fsspec/implementations/tests/test_dirfs.py b/fsspec/implementations/tests/test_dirfs.py
index c04ba66..990ab11 100644
--- a/fsspec/implementations/tests/test_dirfs.py
+++ b/fsspec/implementations/tests/test_dirfs.py
@@ -82,6 +82,8 @@ def test_dirfs(fs, asyncfs):
         ("", "foo", "foo"),
         ("root", "", "root"),
         ("root", "foo", "root/foo"),
+        ("/root", "", "/root"),
+        ("/root", "foo", "/root/foo"),
     ],
 )
 def test_path(fs, root, rel, full):
@@ -90,6 +92,18 @@ def test_path(fs, root, rel, full):
     assert dirfs._relpath(full) == rel
 
 
+@pytest.mark.parametrize(
+    "root, rel, full",
+    [
+        ("/root", "foo", "root/foo"),
+        ("/root", "", "root"),
+    ],
+)
+def test_path_no_leading_slash(fs, root, rel, full):
+    dirfs = DirFileSystem(root, fs)
+    assert dirfs._relpath(full) == rel
+
+
 def test_sep(mocker, dirfs):
     sep = mocker.Mock()
     dirfs.fs.sep = sep
Repository: fsspec/filesystem_spec
Pull request #1798 (2025-02-27 13:48:57)

dirfs: Ensures consistency of details returned by the “ls” and “info” methods.
Before this changes, the name field into the information returned by the 'info' method was an absolute path while the name for the same entry returned by the 'ls' method was a relative path. As a result, the name in the info of the 'info' method was unusable for api calls.

martindurant: Thanks for fixing this oversight

Patch:
diff --git a/fsspec/implementations/dirfs.py b/fsspec/implementations/dirfs.py
index a944452..c0623b8 100644
--- a/fsspec/implementations/dirfs.py
+++ b/fsspec/implementations/dirfs.py
@@ -231,10 +231,16 @@ class DirFileSystem(AsyncFileSystem):
         return self.fs.exists(self._join(path))
 
     async def _info(self, path, **kwargs):
-        return await self.fs._info(self._join(path), **kwargs)
+        info = await self.fs._info(self._join(path), **kwargs)
+        info = info.copy()
+        info["name"] = self._relpath(info["name"])
+        return info
 
     def info(self, path, **kwargs):
-        return self.fs.info(self._join(path), **kwargs)
+        info = self.fs.info(self._join(path), **kwargs)
+        info = info.copy()
+        info["name"] = self._relpath(info["name"])
+        return info
 
     async def _ls(self, path, detail=True, **kwargs):
         ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy()


Test patch:
diff --git a/fsspec/implementations/tests/test_dirfs.py b/fsspec/implementations/tests/test_dirfs.py
index 46ddd0c..f589694 100644
--- a/fsspec/implementations/tests/test_dirfs.py
+++ b/fsspec/implementations/tests/test_dirfs.py
@@ -327,12 +327,14 @@ def test_exists(dirfs):
 
 @pytest.mark.asyncio
 async def test_async_info(adirfs):
-    assert await adirfs._info("file", **KWARGS) == adirfs.fs._info.return_value
+    adirfs.fs._info.return_value = {"name": f"{PATH}/file", "foo": "bar"}
+    assert await adirfs._info("file", **KWARGS) == {"name": "file", "foo": "bar"}
     adirfs.fs._info.assert_called_once_with(f"{PATH}/file", **KWARGS)
 
 
 def test_info(dirfs):
-    assert dirfs.info("file", **KWARGS) == dirfs.fs.info.return_value
+    dirfs.fs.info.return_value = {"name": f"{PATH}/file", "foo": "bar"}
+    assert dirfs.info("file", **KWARGS) == {"name": "file", "foo": "bar"}
     dirfs.fs.info.assert_called_once_with(f"{PATH}/file", **KWARGS)