Repository: fsspec/universal_pathlib (python)

Issue:
Dependencies: introduce minimum version requirement for fsspec
We should introduce a minimum version requirement for fsspec and run tests against the smallest supported version to avoid issues like #138 

Pull request:
require fsspec>=2022.1.0
This PR closes #141 by introducing a minimum version requirement for `fsspec>=2022.1.0`. I went as far back as reasonable before larger parts of the test suite started failing.

Some notes:
- I'm xfailing the few tests that require slightly newer versions of fsspec for specific implementations. (I.e. some mkdir stuff for `gcsfs<2022.7.1`, or a webdav test that needs `fsspec>=2022.5.0`, etc...)

TODO:
- [x] add CI job testing against `fsspec==2022.1.0`
- [x] decide if we should:
  - fix the fs specific behavior for these xfailed tests :no_entry_sign: 
  - add min version requirements per implementation :no_entry_sign: 
  - just document them (my preferred option right now) :heavy_check_mark: 

Patch:
diff --git a/README.md b/README.md
index 160588e..0ab6c5f 100644
--- a/README.md
+++ b/README.md
@@ -181,6 +181,15 @@ universal_pathlib.implementations =
     myproto = my_module.submodule:MyPath
 ```
 
+### Known issues solvable by installing newer upstream dependencies
+
+Some issues in UPath's behavior with specific filesystems can be fixed by installing newer versions of
+the dependencies. The following list will be kept up to date whenever we encounter more:
+
+- **UPath().glob()** fsspec fixed its glob behavior when handling `**` patterns in versions `fsspec>=2023.9.0`
+- **GCSPath().mkdir()** a few mkdir quirks are solved by installing `gcsfs>=2022.7.1`
+- **fsspec.filesystem(WebdavPath().protocol)** the webdav protocol was added to fsspec in version `fsspec>=2022.5.0`
+
 ## Contributing
 
 Contributions are very welcome.
diff --git a/noxfile.py b/noxfile.py
index 1ef5b9d..71d86e2 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -23,6 +23,20 @@ def tests(session: nox.Session) -> None:
     )
 
 
+@nox.session(python="3.8", name="tests-minversion")
+def tests_minversion(session: nox.Session) -> None:
+    session.install("fsspec==2022.1.0", ".[dev]")
+    session.run(
+        "pytest",
+        "-m",
+        "not hdfs",
+        "--cov",
+        "--cov-config=pyproject.toml",
+        *session.posargs,
+        env={"COVERAGE_FILE": f".coverage.{session.python}"},
+    )
+
+
 @nox.session
 def lint(session: nox.Session) -> None:
     session.install("pre-commit")
diff --git a/setup.cfg b/setup.cfg
index 90586e6..161eee8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,7 +23,7 @@ python_requires = >=3.8
 zip_safe = False
 packages = find:
 install_requires=
-    fsspec
+    fsspec>=2022.1.0
 
 [options.extras_require]
 tests =
diff --git a/upath/core.py b/upath/core.py
index 5ba049a..63abf4c 100644
--- a/upath/core.py
+++ b/upath/core.py
@@ -276,7 +276,10 @@ class UPath(Path):
     def _make_child_relpath(self: PT, part: str) -> PT:
         # This is an optimization used for dir walking.  `part` must be
         # a single part relative to this path.
-        parts = self._parts + [part]
+        if self._parts[-1:] == [""] and part:
+            parts = self._parts[:-1] + [part]
+        else:
+            parts = self._parts + [part]
         return self._from_parsed_parts(
             self._drv, self._root, parts, url=self._url, **self._kwargs
         )
@@ -415,8 +418,8 @@ class UPath(Path):
 
     def _sub_path(self, name):
         # only want the path name with iterdir
-        sp = self._path
-        return re.sub(f"^({sp}|{sp[1:]})/", "", name)
+        sp = re.escape(self._path)
+        return re.sub(f"^({sp}|{sp[1:]})/?", "", name)
 
     def absolute(self: PT) -> PT:
         # fsspec paths are always absolute
diff --git a/upath/implementations/hdfs.py b/upath/implementations/hdfs.py
index c00f893..19e5a57 100644
--- a/upath/implementations/hdfs.py
+++ b/upath/implementations/hdfs.py
@@ -19,8 +19,19 @@ class _HDFSAccessor(upath.core._FSSpecAccessor):
         else:
             if not kwargs.get("exist_ok", False) and self._fs.exists(pth):
                 raise FileExistsError(pth)
+            print(kwargs, self._fs.exists(pth), pth)
             return self._fs.mkdir(pth, create_parents=create_parents, **kwargs)
 
+    def listdir(self, path, **kwargs):
+        try:
+            yield from super().listdir(path, **kwargs)
+        except OSError as err:
+            if err.args and err.args[0].startswith(
+                "GetFileInfo expects base_dir of selector to be a directory"
+            ):
+                raise NotADirectoryError(path)
+            raise
+
 
 class HDFSPath(upath.core.UPath):
     _default_accessor = _HDFSAccessor


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 41e6eba..dff0f69 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -22,6 +22,12 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ['3.8', '3.9', '3.10', '3.11']
+        fsspec: ['']
+
+        include:
+          - os: ubuntu-20.04
+            pyv: '3.8'
+            fsspec: 'minversion'
 
     steps:
     - name: Check out the repository
@@ -41,7 +47,7 @@ jobs:
         nox --version
 
     - name: Run tests
-      run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml
+      run: nox -s tests-${{ matrix.fsspec || matrix.pyv }} -- --cov-report=xml
 
   lint:
     runs-on: ubuntu-latest
diff --git a/upath/tests/cases.py b/upath/tests/cases.py
index 8a276ab..f91b4c2 100644
--- a/upath/tests/cases.py
+++ b/upath/tests/cases.py
@@ -131,6 +131,11 @@ class BaseTests:
         assert {p.name for p in pl_iter} == {u.name for u in up_iter}
         assert next(self.path.parent.iterdir()).exists()
 
+    def test_iterdir_trailing_slash(self):
+        files_noslash = list(self.path.joinpath("folder1").iterdir())
+        files_slash = list(self.path.joinpath("folder1/").iterdir())
+        assert files_noslash == files_slash
+
     def test_parents(self):
         p = self.path.joinpath("folder1", "file1.txt")
         assert p.is_file()
diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py
index f72eeae..3c892c1 100644
--- a/upath/tests/implementations/test_gcs.py
+++ b/upath/tests/implementations/test_gcs.py
@@ -5,6 +5,7 @@ from upath.implementations.cloud import GCSPath
 
 from ..cases import BaseTests
 from ..utils import skip_on_windows
+from ..utils import xfail_if_version
 
 
 @skip_on_windows
@@ -34,3 +35,15 @@ class TestGCSPath(BaseTests):
     @pytest.mark.skip
     def test_makedirs_exist_ok_false(self):
         pass
+
+    @xfail_if_version("gcsfs", lt="2022.7.1", reason="requires gcsfs>=2022.7.1")
+    def test_mkdir(self):
+        super().test_mkdir()
+
+    @xfail_if_version("gcsfs", lt="2022.7.1", reason="requires gcsfs>=2022.7.1")
+    def test_mkdir_exists_ok_false(self):
+        super().test_mkdir_exists_ok_false()
+
+    @xfail_if_version("gcsfs", lt="2022.7.1", reason="requires gcsfs>=2022.7.1")
+    def test_mkdir_exists_ok_true(self):
+        super().test_mkdir_exists_ok_true()
diff --git a/upath/tests/implementations/test_hdfs.py b/upath/tests/implementations/test_hdfs.py
index 9ba8eba..c2b75cf 100644
--- a/upath/tests/implementations/test_hdfs.py
+++ b/upath/tests/implementations/test_hdfs.py
@@ -19,9 +19,6 @@ class TestUPathHDFS(BaseTests):
     def test_is_HDFSPath(self):
         assert isinstance(self.path, HDFSPath)
 
-    def test_chmod(self):
-        # todo
-        pass
-
-    def test_fsspec_compat(self):
+    @pytest.mark.skip
+    def test_makedirs_exist_ok_false(self):
         pass
diff --git a/upath/tests/implementations/test_webdav.py b/upath/tests/implementations/test_webdav.py
index e9a9678..756d456 100644
--- a/upath/tests/implementations/test_webdav.py
+++ b/upath/tests/implementations/test_webdav.py
@@ -3,6 +3,7 @@ import pytest  # noqa: F401
 from upath import UPath
 
 from ..cases import BaseTests
+from ..utils import xfail_if_version
 
 
 class TestUPathWebdav(BaseTests):
@@ -20,3 +21,7 @@ class TestUPathWebdav(BaseTests):
         base_url = storage_options.pop("base_url")
         assert storage_options == self.path.fs.storage_options
         assert base_url == self.path.fs.client.base_url
+
+    @xfail_if_version("fsspec", lt="2022.5.0", reason="requires fsspec>=2022.5.0")
+    def test_read_with_fsspec(self):
+        super().test_read_with_fsspec()