Repository: fsspec/universal_pathlib
Pull request #58 (2022-04-18 10:41:19)

Fix iterdir for http-path and refactorings
When calling `iterdir` on a `HTTPPath`, it uses the base implementation from `UPath`. Since `fsspec` returns not only the path but the complete address (i.e. `http://example.com/path/to/a`) and the `_sub_path` implementation from `UPath` expects only `/path/to/a`, it fails.

To fix this, this PR introduces a `_sub_path` implementation to `HTTPPath` which takes the `scheme` and `netloc` into account.

Additional changes:
- Reimplements `is_file` and `is_dir` within `HTTPPath`. [`fsspec` only distinguishes between both based on the existence of a trailing slash](https://github.com/fsspec/filesystem_spec/blob/45dcfa99e2dc320bf072c28bef767b71f8299a4a/fsspec/implementations/http.py#L183) which is not really great for real world deployments.
- Re-add S3 tests in CI by fixing `moto`
- S3, GCS, Http tests now reuse most of `BaseTests`
- `UPath` got a dedicated `relative_to` method, because the `pathlib.Path` version removed the kwargs
- `UPath` now has stubs for not-implemented methods instead of `__getattribute__` magic
- Some fixes for Windows (e.g. remove use of `os.path.join` in `GCSPath` and `S3Path`)
- Add `tests/utils` module
- Refactor `_Registry` to use a dedicated dict for implementations instead of class attributes (eliminates module-attribute naming conflicts, e.g. `http = http.HTTPPath`)
- `is_file`, `is_dir`, `is_symlink` no longer raise `FileNotFoundError`, but return `False` if path is non-existent
- Strips `_url` from `_kwargs` to reduce redundancy + refactorings in `_from_parts` and `_from_parsed_parts`


Patch:
diff --git a/noxfile.py b/noxfile.py
index 24e1234..5e23486 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -27,7 +27,9 @@ def install(session):
 
 @nox.session(python=False)
 def smoke(session):
-    session.install(*"pytest aiohttp requests gcsfs".split())
+    session.install(
+        *"pytest aiohttp requests gcsfs s3fs moto[s3,server]".split()
+    )
     session.run(*"pytest --skiphdfs -vv upath".split())
 
 
diff --git a/upath/core.py b/upath/core.py
index 95d1e7f..e2cbc61 100644
--- a/upath/core.py
+++ b/upath/core.py
@@ -1,5 +1,6 @@
 import pathlib
 import re
+import sys
 from typing import Union
 import urllib
 
@@ -88,21 +89,14 @@ class _FSSpecAccessor:
 
 class UPath(pathlib.Path):
 
-    __slots__ = ("_url", "_kwargs", "_closed", "_accessor")
+    __slots__ = (
+        "_url",
+        "_kwargs",
+        "_accessor",  # overwritten because of default in Python 3.10
+    )
     _flavour = pathlib._posix_flavour
     _default_accessor = _FSSpecAccessor
 
-    not_implemented = [
-        "cwd",
-        "home",
-        "expanduser",
-        "group",
-        "lchmod",
-        "lstat",
-        "owner",
-        "readlink",
-    ]
-
     def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]:
         args_list = list(args)
         first = args_list.pop(0)
@@ -116,7 +110,6 @@ class UPath(pathlib.Path):
             )
 
             new_kwargs = getattr(other, "_kwargs", {}).copy()
-            new_kwargs.pop("_url", None)
             new_kwargs.update(kwargs)
 
             return other.__class__(
@@ -136,9 +129,8 @@ class UPath(pathlib.Path):
             import upath.registry
 
             cls = upath.registry._registry[parsed_url.scheme]
-            kwargs["_url"] = parsed_url
             args_list.insert(0, parsed_url.path)
-            return cls._from_parts(tuple(args_list), **kwargs)
+            return cls._from_parts(tuple(args_list), url=parsed_url, **kwargs)
 
         # treat as local filesystem, return PosixPath or WindowsPath
         return pathlib.Path(*args, **kwargs)
@@ -146,34 +138,29 @@ class UPath(pathlib.Path):
     def __getattr__(self, item):
         if item == "_accessor":
             # cache the _accessor attribute on first access
-            kw = self._kwargs.copy()
-            kw.pop("_url", None)
-            self._accessor = _accessor = self._default_accessor(self._url, **kw)
+            kwargs = self._kwargs.copy()
+            self._accessor = _accessor = self._default_accessor(
+                self._url, **kwargs
+            )
             return _accessor
         else:
             raise AttributeError(item)
 
-    def __getattribute__(self, item):
-        if item == "__class__":
-            return super().__getattribute__("__class__")
-        if item in getattr(self.__class__, "not_implemented"):
-            raise NotImplementedError(f"UPath has no attribute {item}")
-        else:
-            return super().__getattribute__(item)
-
     def _make_child(self, args):
-        drv, root, parts = self._parse_args(args, **self._kwargs)
+        drv, root, parts = self._parse_args(args)
         drv, root, parts = self._flavour.join_parsed_parts(
             self._drv, self._root, self._parts, drv, root, parts
         )
-        return self._from_parsed_parts(drv, root, parts, **self._kwargs)
+        return self._from_parsed_parts(
+            drv, root, parts, url=self._url, **self._kwargs
+        )
 
     def _make_child_relpath(self, part):
         # This is an optimization used for dir walking.  `part` must be
         # a single part relative to this path.
         parts = self._parts + [part]
         return self._from_parsed_parts(
-            self._drv, self._root, parts, **self._kwargs
+            self._drv, self._root, parts, url=self._url, **self._kwargs
         )
 
     def _format_parsed_parts(self, drv, root, parts):
@@ -213,7 +200,9 @@ class UPath(pathlib.Path):
         parts = self._parts
         if len(parts) == 1 and (drv or root):
             return self
-        return self._from_parsed_parts(drv, root, parts[:-1], **self._kwargs)
+        return self._from_parsed_parts(
+            drv, root, parts[:-1], url=self._url, **self._kwargs
+        )
 
     def stat(self):
         return self._accessor.stat(self)
@@ -222,8 +211,6 @@ class UPath(pathlib.Path):
         """Iterate over the files in this directory.  Does not yield any
         result for the special paths '.' and '..'.
         """
-        if self._closed:
-            self._raise_closed()
         for name in self._accessor.listdir(self):
             # fsspec returns dictionaries
             if isinstance(name, dict):
@@ -234,8 +221,29 @@ class UPath(pathlib.Path):
             # only want the path name with iterdir
             name = self._sub_path(name)
             yield self._make_child_relpath(name)
-            if self._closed:
-                self._raise_closed()
+
+    def relative_to(self, *other):
+        for other_item in other:
+            if not isinstance(other_item, self.__class__) and not isinstance(
+                other_item, str
+            ):
+                raise ValueError(
+                    f"{repr(self)} and {repr(other_item)} are "
+                    "not of compatible classes."
+                )
+            if not isinstance(other_item, str) and (
+                other_item._url.scheme != self._url.scheme
+                or other_item._url.netloc != self._url.netloc
+                or other_item._kwargs != self._kwargs
+            ):
+                raise ValueError(
+                    f"{self} and {other_item} do not share the same "
+                    "base URL and storage options."
+                )
+        output = super().relative_to(*other)
+        output._url = self._url
+        output._kwargs = self._kwargs
+        return output
 
     def glob(self, pattern):
         path = self.joinpath(pattern)
@@ -263,24 +271,33 @@ class UPath(pathlib.Path):
             return self._accessor.exists(self)
 
     def is_dir(self):
-        info = self._accessor.info(self)
-        if info["type"] == "directory":
-            return True
+        try:
+            info = self._accessor.info(self)
+            if info["type"] == "directory":
+                return True
+        except FileNotFoundError:
+            return False
         return False
 
     def is_file(self):
-        info = self._accessor.info(self)
-        if info["type"] == "file":
-            return True
+        try:
+            info = self._accessor.info(self)
+            if info["type"] == "file":
+                return True
+        except FileNotFoundError:
+            return False
         return False
 
     def is_mount(self):
         return False
 
     def is_symlink(self):
-        info = self._accessor.info(self)
-        if "islink" in info:
-            return info["islink"]
+        try:
+            info = self._accessor.info(self)
+            if "islink" in info:
+                return info["islink"]
+        except FileNotFoundError:
+            return False
         return False
 
     def is_socket(self):
@@ -295,16 +312,6 @@ class UPath(pathlib.Path):
     def is_char_device(self):
         return False
 
-    def chmod(self, mod):
-        raise NotImplementedError
-
-    def rename(self, target):
-        # can be implemented, but may be tricky
-        raise NotImplementedError
-
-    def touch(self, trunicate=True, **kwargs):
-        self._accessor.touch(self, trunicate=trunicate, **kwargs)
-
     def unlink(self, missing_ok=False):
         if not self.exists():
             if not missing_ok:
@@ -323,45 +330,80 @@ class UPath(pathlib.Path):
             raise NotDirectoryError
         self._accessor.rm(self, recursive=recursive)
 
-    @classmethod
-    def _parse_args(cls, args, **kwargs):
-        return super(UPath, cls)._parse_args(args)
+    def chmod(self, mod):
+        raise NotImplementedError
+
+    def rename(self, target):
+        # can be implemented, but may be tricky
+        raise NotImplementedError
+
+    def cwd(self):
+        raise NotImplementedError
+
+    def home(self):
+        raise NotImplementedError
+
+    def expanduser(self):
+        raise NotImplementedError
+
+    def group(self):
+        raise NotImplementedError
+
+    def lchmod(self, mode):
+        raise NotImplementedError
+
+    def lstat(self):
+        raise NotImplementedError
+
+    def owner(self):
+        raise NotImplementedError
+
+    def readlink(self):
+        raise NotImplementedError
+
+    def touch(self, trunicate=True, **kwargs):
+        self._accessor.touch(self, trunicate=trunicate, **kwargs)
 
     @classmethod
-    def _from_parts(cls, args, **kwargs):
+    def _from_parts(cls, args, url=None, **kwargs):
         obj = object.__new__(cls)
-        drv, root, parts = obj._parse_args(args, **kwargs)
+        drv, root, parts = obj._parse_args(args)
         obj._drv = drv
-        obj._parts = parts
-        obj._closed = False
+        if sys.version_info < (3, 9):
+            obj._closed = False
+        obj._url = url
         obj._kwargs = kwargs.copy()
-        obj._url = kwargs.pop("_url", None) or None
 
         if not root:
             if not parts:
                 root = "/"
+                parts = ["/"]
             elif parts[0] == "/":
-                root = parts.pop(0)
+                root = parts[1:]
         obj._root = root
+        obj._parts = parts
 
         return obj
 
     @classmethod
-    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
+    def _from_parsed_parts(cls, drv, root, parts, url=None, **kwargs):
         obj = object.__new__(cls)
         obj._drv = drv
         obj._parts = parts
-        obj._closed = False
+        if sys.version_info < (3, 9):
+            obj._closed = False
+        obj._url = url
         obj._kwargs = kwargs.copy()
-        obj._url = kwargs.pop("_url", None) or None
 
         if not root:
             if not parts:
                 root = "/"
             elif parts[0] == "/":
                 root = parts.pop(0)
-        obj._root = root
+        if len(obj._parts) == 0 or obj._parts[0] != root:
+            obj._parts.insert(0, root)
 
+        obj._root = root
         return obj
 
     @property
@@ -380,7 +422,6 @@ class UPath(pathlib.Path):
         )
 
         kwargs = self._kwargs.copy()
-        kwargs.pop("_url")
 
         # Create a new object
         out = self.__class__(
@@ -390,16 +431,11 @@ class UPath(pathlib.Path):
         return out
 
     def __setstate__(self, state):
-        kwargs = state["_kwargs"].copy()
-        kwargs["_url"] = self._url
-        self._kwargs = kwargs
+        self._kwargs = state["_kwargs"].copy()
 
     def __reduce__(self):
-        kwargs = self._kwargs.copy()
-        kwargs.pop("_url", None)
-
         return (
             self.__class__,
             (self._format_parsed_parts(self._drv, self._root, self._parts),),
-            {"_kwargs": kwargs},
+            {"_kwargs": self._kwargs.copy()},
         )
diff --git a/upath/implementations/gcs.py b/upath/implementations/gcs.py
index 497069c..2613c57 100644
--- a/upath/implementations/gcs.py
+++ b/upath/implementations/gcs.py
@@ -1,5 +1,4 @@
 import upath.core
-import os
 import re
 
 
@@ -11,7 +10,7 @@ class _GCSAccessor(upath.core._FSSpecAccessor):
         """
         netloc has already been set to project via `GCSPath._from_parts`
         """
-        s = os.path.join(self._url.netloc, s.lstrip("/"))
+        s = f"{self._url.netloc}/{s.lstrip('/')}"
         return s
 
 
@@ -20,25 +19,26 @@ class GCSPath(upath.core.UPath):
     _default_accessor = _GCSAccessor
 
     @classmethod
-    def _from_parts(cls, args, **kwargs):
-        obj = super()._from_parts(args, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parts(cls, args, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+        obj = super()._from_parts(args, url, **kwargs)
         return obj
 
     @classmethod
-    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
-        obj = super()._from_parsed_parts(drv, root, parts, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parsed_parts(cls, drv, root, parts, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+        obj = super()._from_parsed_parts(drv, root, parts, url, **kwargs)
         return obj
 
     def _sub_path(self, name):
-        """gcs returns path as `{bucket}/<path>` with listdir
-        and glob, so here we can add the netloc to the sub string
-        so it gets subbed out as well
+        """
+        `gcsfs` returns the full path as `<bucket>/<path>` with `listdir` and
+        `glob`. However, in `iterdir` and `glob` we only want the relative path
+        to `self`.
         """
         sp = self.path
         subed = re.sub(f"^({self._url.netloc})?/?({sp}|{sp[1:]})/?", "", name)
diff --git a/upath/implementations/http.py b/upath/implementations/http.py
index 901e7b9..408d46f 100644
--- a/upath/implementations/http.py
+++ b/upath/implementations/http.py
@@ -23,8 +23,8 @@ class _HTTPAccessor(upath.core._FSSpecAccessor):
                         args.insert(0, first_arg)
                     args = tuple(args)
                 else:
-                    new_url = self._url.replace(path=kwargs["path"])
-                    unparsed = urllib.urlunparse(new_url)
+                    new_url = self._url._replace(path=kwargs["path"])
+                    unparsed = urllib.parse.urlunparse(new_url)
                     kwargs["path"] = unparsed
             return func(*args, **kwargs)
 
@@ -33,3 +33,38 @@ class _HTTPAccessor(upath.core._FSSpecAccessor):
 
 class HTTPPath(upath.core.UPath):
     _default_accessor = _HTTPAccessor
+
+    def is_dir(self):
+        try:
+            return self._path_type() == "directory"
+        except FileNotFoundError:
+            return False
+
+    def is_file(self):
+        try:
+            return self._path_type() == "file"
+        except FileNotFoundError:
+            return False
+
+    def _path_type(self):
+        info = self._accessor.info(self)
+        if (
+            info["type"] == "directory"
+            or next(self.iterdir(), None) is not None
+        ):
+            return "directory"
+        return "file"
+
+    def _sub_path(self, name):
+        """
+        `fsspec` returns the full path as `scheme://netloc/<path>` with
+        `listdir` and `glob`. However, in `iterdir` and `glob` we only want the
+        relative path to `self`.
+        """
+        complete_address = self._format_parsed_parts(None, None, [self.path])
+
+        if name.startswith(complete_address):
+            name = name[len(complete_address) :]  # noqa: E203
+        name = name.strip("/")
+
+        return name
diff --git a/upath/implementations/memory.py b/upath/implementations/memory.py
index 2a3fe51..994cd9b 100644
--- a/upath/implementations/memory.py
+++ b/upath/implementations/memory.py
@@ -6,12 +6,6 @@ class _MemoryAccessor(upath.core._FSSpecAccessor):
         super().__init__(*args, **kwargs)
         self._fs.root_marker = ""
 
-    def _format_path(self, s):
-        """If the filesystem backend doesn't have a root_marker, strip the
-        leading slash of a path
-        """
-        return s
-
 
 class MemoryPath(upath.core.UPath):
     _default_accessor = _MemoryAccessor
@@ -20,8 +14,6 @@ class MemoryPath(upath.core.UPath):
         """Iterate over the files in this directory.  Does not yield any
         result for the special paths '.' and '..'.
         """
-        if self._closed:
-            self._raise_closed()
         for name in self._accessor.listdir(self):
             # fsspec returns dictionaries
             if isinstance(name, dict):
@@ -33,5 +25,3 @@ class MemoryPath(upath.core.UPath):
             name = name.rstrip("/")
             name = self._sub_path(name)
             yield self._make_child_relpath(name)
-            if self._closed:
-                self._raise_closed()
diff --git a/upath/implementations/s3.py b/upath/implementations/s3.py
index 69d553b..b3f229f 100644
--- a/upath/implementations/s3.py
+++ b/upath/implementations/s3.py
@@ -1,4 +1,3 @@
-import os
 import re
 
 import upath.core
@@ -12,7 +11,7 @@ class _S3Accessor(upath.core._FSSpecAccessor):
         """If the filesystem backend doesn't have a root_marker, strip the
         leading slash of a path and add the bucket
         """
-        s = os.path.join(self._url.netloc, s.lstrip("/"))
+        s = f"{self._url.netloc}/{s.lstrip('/')}"
         return s
 
 
@@ -20,25 +19,27 @@ class S3Path(upath.core.UPath):
     _default_accessor = _S3Accessor
 
     @classmethod
-    def _from_parts(cls, args, **kwargs):
-        obj = super()._from_parts(args, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parts(cls, args, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+        obj = super()._from_parts(args, url, **kwargs)
         return obj
 
     @classmethod
-    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
-        obj = super()._from_parsed_parts(drv, root, parts, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parsed_parts(cls, drv, root, parts, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+
+        obj = super()._from_parsed_parts(drv, root, parts, url, **kwargs)
         return obj
 
     def _sub_path(self, name):
-        """s3fs returns path as `{bucket}/<path>` with listdir
-        and glob, so here we can add the netloc to the sub string
-        so it gets subbed out as well
+        """
+        `s3fs` returns the full path as `<bucket>/<path>` with `listdir` and
+        `glob`. However, in `iterdir` and `glob` we only want the relative path
+        to `self`.
         """
         sp = self.path
         subed = re.sub(f"^({self._url.netloc})?/?({sp}|{sp[1:]})/?", "", name)
diff --git a/upath/registry.py b/upath/registry.py
index 6b84708..98f80bf 100644
--- a/upath/registry.py
+++ b/upath/registry.py
@@ -1,22 +1,27 @@
+from typing import Dict, Type
 import warnings
 
 import upath
+from upath.core import UPath
 
 
 class _Registry:
     from upath.implementations import hdfs, http, memory, s3, gcs
 
-    http = http.HTTPPath
-    hdfs = hdfs.HDFSPath
-    s3a = s3.S3Path
-    s3 = s3.S3Path
-    memory = memory.MemoryPath
-    gs = gcs.GCSPath
-    gcs = gcs.GCSPath
+    known_implementations: Dict[str, Type[UPath]] = {
+        "https": http.HTTPPath,
+        "http": http.HTTPPath,
+        "hdfs": hdfs.HDFSPath,
+        "s3a": s3.S3Path,
+        "s3": s3.S3Path,
+        "memory": memory.MemoryPath,
+        "gs": gcs.GCSPath,
+        "gcs": gcs.GCSPath,
+    }
 
     def __getitem__(self, item):
-        implemented_path = getattr(self, item, None)
-        if not implemented_path:
+        implementation = self.known_implementations.get(item, None)
+        if not implementation:
             warning_str = (
                 f"{item} filesystem path not explicitly implemented. "
                 "falling back to default implementation. "
@@ -24,7 +29,7 @@ class _Registry:
             )
             warnings.warn(warning_str, UserWarning)
             return upath.UPath
-        return implemented_path
+        return implementation
 
 
 _registry = _Registry()


Test patch:
diff --git a/upath/tests/cases.py b/upath/tests/cases.py
index 856d02c..c96a055 100644
--- a/upath/tests/cases.py
+++ b/upath/tests/cases.py
@@ -1,9 +1,11 @@
 import pickle
-import sys
 from pathlib import Path
+import re
+import sys
 
 import pytest
 from upath import UPath
+from .utils import posixify
 
 
 class BaseTests:
@@ -38,10 +40,11 @@ class BaseTests:
         mock_glob = list(self.path.glob("**.txt"))
         path_glob = list(pathlib_base.glob("**/*.txt"))
 
-        root = "/" if sys.platform.startswith("win") else ""
-        mock_glob_normalized = sorted([a.path for a in mock_glob])
+        mock_glob_normalized = sorted(
+            [a.relative_to(self.path).path for a in mock_glob]
+        )
         path_glob_normalized = sorted(
-            [f"{root}{a}".replace("\\", "/") for a in path_glob]
+            [f"/{posixify(a.relative_to(pathlib_base))}" for a in path_glob]
         )
 
         assert mock_glob_normalized == path_glob_normalized
@@ -53,14 +56,17 @@ class BaseTests:
     def test_is_dir(self):
         assert self.path.is_dir()
 
-        path = self.path.joinpath("file1.txt")
+        path = self.path / "file1.txt"
         assert not path.is_dir()
+        assert not (self.path / "not-existing-dir").is_dir()
 
     def test_is_file(self):
-        path = self.path.joinpath("file1.txt")
+        path = self.path / "file1.txt"
         assert path.is_file()
         assert not self.path.is_file()
 
+        assert not (self.path / "not-existing-file.txt").is_file()
+
     def test_is_mount(self):
         assert self.path.is_mount() is False
 
@@ -85,12 +91,25 @@ class BaseTests:
         up_iter = list(self.path.iterdir())
         pl_iter = list(pl_path.iterdir())
 
+        for x in up_iter:
+            assert x.name != ""
+            assert x.exists()
+
+        assert len(up_iter) == len(pl_iter)
+        assert set(p.name for p in pl_iter) == set(u.name for u in up_iter)
+        assert next(self.path.parent.iterdir()).exists()
+
+    def test_iterdir2(self, local_testdir):
+        pl_path = Path(local_testdir) / "folder1"
+
+        up_iter = list((self.path / "folder1").iterdir())
+        pl_iter = list(pl_path.iterdir())
+
         for x in up_iter:
             assert x.exists()
 
         assert len(up_iter) == len(pl_iter)
-        pnames = [p.name for p in pl_iter]
-        assert all(map(lambda x: x.name in pnames, up_iter))
+        assert set(p.name for p in pl_iter) == set(u.name for u in up_iter)
         assert next(self.path.parent.iterdir()).exists()
 
     def test_lchmod(self):
@@ -205,21 +224,26 @@ class BaseTests:
 
     def test_fsspec_compat(self):
         fs = self.path.fs
-        scheme = self.path._url.scheme
         content = b"a,b,c\n1,2,3\n4,5,6"
 
-        p1 = f"{scheme}:///tmp/output1.csv"
-        upath1 = UPath(p1)
+        def strip_scheme(path):
+            root = "" if sys.platform.startswith("win") else "/"
+            return root + re.sub("^[a-z0-9]+:/*", "", str(path))
+
+        upath1 = self.path / "output1.csv"
+        p1 = strip_scheme(upath1)
         upath1.write_bytes(content)
+        assert fs is upath1.fs
         with fs.open(p1) as f:
             assert f.read() == content
         upath1.unlink()
 
         # write with fsspec, read with upath
-        p2 = f"{scheme}:///tmp/output2.csv"
+        upath2 = self.path / "output2.csv"
+        p2 = strip_scheme(upath2)
+        assert fs is upath2.fs
         with fs.open(p2, "wb") as f:
             f.write(content)
-        upath2 = UPath(p2)
         assert upath2.read_bytes() == content
         upath2.unlink()
 
@@ -245,7 +269,8 @@ class BaseTests:
         assert path.fs.storage_options == recovered_path.fs.storage_options
 
     def test_child_path(self):
-        path_a = UPath(f"{self.path}/folder")
+        path_str = str(self.path).rstrip("/")
+        path_a = UPath(f"{path_str}/folder")
         path_b = self.path / "folder"
 
         assert str(path_a) == str(path_b)
diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py
index 7d00932..059f762 100644
--- a/upath/tests/conftest.py
+++ b/upath/tests/conftest.py
@@ -13,6 +13,8 @@ from fsspec.registry import register_implementation, _registry
 
 import fsspec
 
+from .utils import posixify
+
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -91,7 +93,7 @@ def htcluster():
             stdout=subprocess.DEVNULL,
         )
     except FileNotFoundError as err:
-        if err.errno == 2 and 'htcluster' == err.filename:
+        if err.errno == 2 and "htcluster" == err.filename:
             pytest.skip("htcluster not installed")
         else:
             raise
@@ -135,56 +137,66 @@ def s3_server():
     if "BOTO_CONFIG" not in os.environ:  # pragma: no cover
         os.environ["BOTO_CONFIG"] = "/dev/null"
     if "AWS_ACCESS_KEY_ID" not in os.environ:  # pragma: no cover
-        os.environ["AWS_ACCESS_KEY_ID"] = "foo"
+        os.environ["AWS_ACCESS_KEY_ID"] = "testing"
     if "AWS_SECRET_ACCESS_KEY" not in os.environ:  # pragma: no cover
-        os.environ["AWS_SECRET_ACCESS_KEY"] = "bar"
+        os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    if "AWS_SECURITY_TOKEN" not in os.environ:  # pragma: no cover
+        os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    if "AWS_SESSION_TOKEN" not in os.environ:  # pragma: no cover
+        os.environ["AWS_SESSION_TOKEN"] = "testing"
+    if "AWS_DEFAULT_REGION" not in os.environ:  # pragma: no cover
+        os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
     requests = pytest.importorskip("requests")
 
     pytest.importorskip("moto")
 
     port = 5555
-    endpoint_uri = "http://127.0.0.1:%s/" % port
+    endpoint_uri = f"http://127.0.0.1:{port}/"
     proc = subprocess.Popen(
-        shlex.split("moto_server s3 -p %s" % port),
+        shlex.split(f"moto_server s3 -p {port}"),
         stderr=subprocess.DEVNULL,
         stdout=subprocess.DEVNULL,
     )
-
-    timeout = 5
-    while timeout > 0:
-        try:
-            r = requests.get(endpoint_uri)
-            if r.ok:
-                break
-        except Exception:  # pragma: no cover
-            pass
-        timeout -= 0.1  # pragma: no cover
-        time.sleep(0.1)  # pragma: no cover
-    anon = False
-    s3so = dict(
-        client_kwargs={"endpoint_url": endpoint_uri}, use_listings_cache=True
-    )
-    yield anon, s3so
-    proc.terminate()
-    proc.wait()
+    try:
+        timeout = 5
+        while timeout > 0:
+            try:
+                r = requests.get(endpoint_uri, timeout=10)
+                if r.ok:
+                    break
+            except Exception:  # pragma: no cover
+                pass
+            timeout -= 0.1  # pragma: no cover
+            time.sleep(0.1)  # pragma: no cover
+        anon = False
+        s3so = dict(
+            client_kwargs={"endpoint_url": endpoint_uri},
+            use_listings_cache=True,
+        )
+        yield anon, s3so
+    finally:
+        proc.terminate()
+        proc.wait()
 
 
 @pytest.fixture
-def s3(s3_server, tempdir, local_testdir):
-    s3fs = pytest.importorskip("s3fs")
+def s3_fixture(s3_server, local_testdir):
+    pytest.importorskip("s3fs")
     anon, s3so = s3_server
-    s3 = s3fs.S3FileSystem(anon=False, **s3so)
-    s3.mkdir(tempdir, create_parents=True)
+    s3 = fsspec.filesystem("s3", anon=False, **s3so)
+    bucket_name = "test_bucket"
+    if s3.exists(bucket_name):
+        for dir, _, keys in s3.walk(bucket_name):
+            for key in keys:
+                s3.rm(f"{dir}/{key}")
+    else:
+        s3.mkdir(bucket_name)
     for x in Path(local_testdir).glob("**/*"):
+        target_path = f"{bucket_name}/{posixify(x.relative_to(local_testdir))}"
         if x.is_file():
-            text = x.read_text().encode("utf8")
-            if not s3.exists(str(x.parent)):
-                s3.mkdir(str(x.parent), create_parents=True)
-            with s3.open(str(x), "wb") as f:
-                f.write(text)
-        else:
-            s3.mkdir(str(x))
-    yield anon, s3so
+            s3.upload(str(x), target_path)
+    s3.invalidate_cache()
+    yield f"s3://{bucket_name}", anon, s3so
 
 
 def stop_docker(container):
@@ -194,10 +206,7 @@ def stop_docker(container):
         subprocess.call(["docker", "rm", "-f", "-v", cid])
 
 
-TEST_PROJECT = os.environ.get("GCSFS_TEST_PROJECT", "test_project")
-
-
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 def docker_gcs():
     if "STORAGE_EMULATOR_HOST" in os.environ:
         # assume using real API or otherwise have a server already set up
@@ -219,9 +228,8 @@ def docker_gcs():
     timeout = 10
     while True:
         try:
-            r = requests.get(url + "/storage/v1/b")
+            r = requests.get(url + "/storage/v1/b", timeout=10)
             if r.ok:
-                print("url: ", url)
                 yield url
                 break
         except Exception as e:  # noqa: E722
@@ -233,36 +241,56 @@ def docker_gcs():
 
 
 @pytest.fixture
-def gcs(docker_gcs, tempdir, local_testdir, populate=True):
-    try:
-        from gcsfs.core import GCSFileSystem
-    except ImportError:
-        pytest.skip("gcsfs not installed")
-
-    # from gcsfs.credentials import GoogleCredentials
-    GCSFileSystem.clear_instance_cache()
+def gcs_fixture(docker_gcs, local_testdir):
+    pytest.importorskip("gcsfs")
     gcs = fsspec.filesystem("gcs", endpoint_url=docker_gcs)
-    try:
-        # ensure we're empty.
-        try:
-            gcs.rm("tmp", recursive=True)
-        except FileNotFoundError:
-            pass
-        try:
-            gcs.mkdir("tmp")
-            print("made tmp dir")
-        except Exception:
-            pass
-        if populate:
-            for x in Path(local_testdir).glob("**/*"):
-                if x.is_file():
-                    gcs.upload(str(x), str(x))
-                else:
-                    gcs.mkdir(str(x))
-        gcs.invalidate_cache()
-        yield docker_gcs
-    finally:
+    bucket_name = "test_bucket"
+    if gcs.exists(bucket_name):
+        for dir, _, keys in gcs.walk(bucket_name):
+            for key in keys:
+                gcs.rm(f"{dir}/{key}")
+    else:
+        gcs.mkdir(bucket_name)
+    for x in Path(local_testdir).glob("**/*"):
+        target_path = f"{bucket_name}/{posixify(x.relative_to(local_testdir))}"
+        if x.is_file():
+            gcs.upload(str(x), target_path)
+    gcs.invalidate_cache()
+    yield f"gs://{bucket_name}", docker_gcs
+
+
+@pytest.fixture(scope="session")
+def http_server():
+    with tempfile.TemporaryDirectory() as tempdir:
+        requests = pytest.importorskip("requests")
+        pytest.importorskip("http.server")
+        proc = subprocess.Popen(
+            shlex.split(f"python -m http.server --directory {tempdir} 8080")
+        )
         try:
-            gcs.rm(gcs.find("tmp"))
-        except:  # noqa: E722
-            pass
+            url = "http://127.0.0.1:8080/folder"
+            path = Path(tempdir) / "folder"
+            path.mkdir()
+            timeout = 10
+            while True:
+                try:
+                    r = requests.get(url, timeout=10)
+                    if r.ok:
+                        yield path, url
+                        break
+                except Exception as e:  # noqa: E722
+                    timeout -= 1
+                    if timeout < 0:
+                        raise SystemError from e
+                    time.sleep(1)
+        finally:
+            proc.terminate()
+            proc.wait()
+
+
+@pytest.fixture
+def http_fixture(local_testdir, http_server):
+    http_path, http_url = http_server
+    shutil.rmtree(http_path)
+    shutil.copytree(local_testdir, http_path)
+    yield http_url
diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py
index 0e86865..e488e94 100644
--- a/upath/tests/implementations/test_gcs.py
+++ b/upath/tests/implementations/test_gcs.py
@@ -1,20 +1,20 @@
 import pytest
-import sys
 
 from upath import UPath
 from upath.implementations.gcs import GCSPath
 from upath.errors import NotDirectoryError
-from upath.tests.cases import BaseTests
+from ..cases import BaseTests
+from ..utils import skip_on_windows
 
 
-@pytest.mark.skipif(sys.platform.startswith("win"), reason="Windows bad")
+@skip_on_windows
 @pytest.mark.usefixtures("path")
 class TestGCSPath(BaseTests):
     @pytest.fixture(autouse=True, scope="function")
-    def path(self, local_testdir, gcs):
-        scheme = "gs:/"
-        self.path = UPath(f"{scheme}{local_testdir}", endpoint_url=gcs)
-        self.endpoint_url = gcs
+    def path(self, gcs_fixture):
+        path, endpoint_url = gcs_fixture
+        self.path = UPath(path, endpoint_url=endpoint_url)
+        self.endpoint_url = endpoint_url
 
     def test_is_GCSPath(self):
         assert isinstance(self.path, GCSPath)
@@ -24,16 +24,7 @@ class TestGCSPath(BaseTests):
         new_dir.joinpath("test.txt").touch()
         assert new_dir.exists()
 
-    def test_glob(self, pathlib_base):
-        mock_glob = list(self.path.glob("**.txt"))
-        path_glob = list(pathlib_base.glob("**/*.txt"))
-
-        assert len(mock_glob) == len(path_glob)
-        assert all(
-            map(lambda m: m.path in [str(p)[4:] for p in path_glob], mock_glob)
-        )
-
-    def test_rmdir(self, local_testdir):
+    def test_rmdir(self):
         dirname = "rmdir_test"
         mock_dir = self.path.joinpath(dirname)
         mock_dir.joinpath("test.txt").write_text("hello")
@@ -42,23 +33,3 @@ class TestGCSPath(BaseTests):
         assert not mock_dir.exists()
         with pytest.raises(NotDirectoryError):
             self.path.joinpath("file1.txt").rmdir()
-
-    def test_fsspec_compat(self):
-        fs = self.path.fs
-        scheme = self.path._url.scheme
-        content = b"a,b,c\n1,2,3\n4,5,6"
-
-        p1 = f"{scheme}:///tmp/output1.csv"
-        upath1 = UPath(p1, endpoint_url=self.endpoint_url)
-        upath1.write_bytes(content)
-        with fs.open(p1) as f:
-            assert f.read() == content
-        upath1.unlink()
-
-        # write with fsspec, read with upath
-        p2 = f"{scheme}:///tmp/output2.csv"
-        with fs.open(p2, "wb") as f:
-            f.write(content)
-        upath2 = UPath(p2, endpoint_url=self.endpoint_url)
-        assert upath2.read_bytes() == content
-        upath2.unlink()
diff --git a/upath/tests/implementations/test_hdfs.py b/upath/tests/implementations/test_hdfs.py
index 82dab23..9bcc309 100644
--- a/upath/tests/implementations/test_hdfs.py
+++ b/upath/tests/implementations/test_hdfs.py
@@ -4,7 +4,7 @@ import pytest  # noqa: F401
 
 from upath import UPath
 from upath.implementations.hdfs import HDFSPath
-from upath.tests.cases import BaseTests
+from ..cases import BaseTests
 
 
 @pytest.mark.hdfs
diff --git a/upath/tests/implementations/test_http.py b/upath/tests/implementations/test_http.py
index 31e2c69..8c7a217 100644
--- a/upath/tests/implementations/test_http.py
+++ b/upath/tests/implementations/test_http.py
@@ -4,6 +4,8 @@ from fsspec import get_filesystem_class
 
 from upath import UPath
 from upath.implementations.http import HTTPPath
+from ..cases import BaseTests
+from ..utils import skip_on_windows
 
 try:
     get_filesystem_class("http")
@@ -15,3 +17,34 @@ def test_httppath():
     path = UPath("http://example.com")
     assert isinstance(path, HTTPPath)
     assert path.exists()
+
+
+def test_httpspath():
+    path = UPath("https://example.com")
+    assert isinstance(path, HTTPPath)
+    assert path.exists()
+
+
+@skip_on_windows
+class TestUPathHttp(BaseTests):
+    @pytest.fixture(autouse=True, scope="function")
+    def path(self, http_fixture):
+        self.path = UPath(http_fixture)
+
+    def test_work_at_root(self):
+        assert "folder" in (f.name for f in self.path.parent.iterdir())
+
+    def test_mkdir(self):
+        pass
+
+    def test_touch_unlink(self):
+        pass
+
+    def test_write_bytes(self, pathlib_base):
+        pass
+
+    def test_write_text(self, pathlib_base):
+        pass
+
+    def test_fsspec_compat(self):
+        pass
diff --git a/upath/tests/implementations/test_memory.py b/upath/tests/implementations/test_memory.py
index c4730b9..eeea03c 100644
--- a/upath/tests/implementations/test_memory.py
+++ b/upath/tests/implementations/test_memory.py
@@ -1,30 +1,18 @@
-import sys
 import pytest
 
 from upath import UPath
 from upath.implementations.memory import MemoryPath
-from upath.tests.cases import BaseTests
+from ..cases import BaseTests
 
 
 class TestMemoryPath(BaseTests):
     @pytest.fixture(autouse=True)
     def path(self, local_testdir):
-        path = f"memory:/{local_testdir}"
+        if not local_testdir.startswith("/"):
+            local_testdir = "/" + local_testdir
+        path = f"memory:{local_testdir}"
         self.path = UPath(path)
         self.prepare_file_system()
 
     def test_is_MemoryPath(self):
         assert isinstance(self.path, MemoryPath)
-
-    def test_glob(self, pathlib_base):
-        mock_glob = list(self.path.glob("**.txt"))
-        path_glob = list(pathlib_base.glob("**/*.txt"))
-
-        assert len(mock_glob) == len(path_glob)
-        if not sys.platform.startswith("win"):  # need to fix windows tests here
-            assert all(
-                map(
-                    lambda m: m.path in [str(p)[4:] for p in path_glob],
-                    mock_glob,
-                )
-            )
diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py
index 9d2f78a..017d80d 100644
--- a/upath/tests/implementations/test_s3.py
+++ b/upath/tests/implementations/test_s3.py
@@ -5,14 +5,13 @@ import pytest  # noqa: F401
 from upath import UPath
 from upath.errors import NotDirectoryError
 from upath.implementations.s3 import S3Path
-from upath.tests.cases import BaseTests
+from ..cases import BaseTests
 
 
 class TestUPathS3(BaseTests):
     @pytest.fixture(autouse=True)
-    def path(self, local_testdir, s3):
-        anon, s3so = s3
-        path = f"s3:/{local_testdir}"
+    def path(self, s3_fixture):
+        path, anon, s3so = s3_fixture
         self.path = UPath(path, anon=anon, **s3so)
         self.anon = anon
         self.s3so = s3so
@@ -32,7 +31,7 @@ class TestUPathS3(BaseTests):
         new_dir.joinpath("test.txt").touch()
         assert new_dir.exists()
 
-    def test_rmdir(self, local_testdir):
+    def test_rmdir(self):
         dirname = "rmdir_test"
         mock_dir = self.path.joinpath(dirname)
         mock_dir.joinpath("test.txt").touch()
@@ -41,6 +40,27 @@ class TestUPathS3(BaseTests):
         with pytest.raises(NotDirectoryError):
             self.path.joinpath("file1.txt").rmdir()
 
+    def test_relative_to(self):
+        assert "s3://test_bucket/file.txt" == str(
+            UPath("s3://test_bucket/file.txt").relative_to(
+                UPath("s3://test_bucket")
+            )
+        )
+
+    def test_iterdir_root(self):
+        client_kwargs = self.path._kwargs["client_kwargs"]
+        bucket_path = UPath(
+            "s3://other_test_bucket", client_kwargs=client_kwargs
+        )
+        bucket_path.mkdir(mode="private")
+
+        (bucket_path / "test1.txt").touch()
+        (bucket_path / "test2.txt").touch()
+
+        for x in bucket_path.iterdir():
+            assert x.name != ""
+            assert x.exists()
+
     def test_touch_unlink(self):
         path = self.path.joinpath("test_touch.txt")
         path.touch()
@@ -55,35 +75,6 @@ class TestUPathS3(BaseTests):
         # file doesn't exists, but missing_ok is True
         path.unlink(missing_ok=True)
 
-    def test_glob(self, pathlib_base):
-        mock_glob = list(self.path.glob("**.txt"))
-        path_glob = list(pathlib_base.glob("**/*.txt"))
-
-        assert len(mock_glob) == len(path_glob)
-        assert all(
-            map(lambda m: m.path in [str(p)[4:] for p in path_glob], mock_glob)
-        )
-
-    def test_fsspec_compat(self):
-        fs = self.path.fs
-        scheme = self.path._url.scheme
-        content = b"a,b,c\n1,2,3\n4,5,6"
-
-        p1 = f"{scheme}:///tmp/output1.csv"
-        upath1 = UPath(p1, anon=self.anon, **self.s3so)
-        upath1.write_bytes(content)
-        with fs.open(p1) as f:
-            assert f.read() == content
-        upath1.unlink()
-
-        # write with fsspec, read with upath
-        p2 = f"{scheme}:///tmp/output2.csv"
-        with fs.open(p2, "wb") as f:
-            f.write(content)
-        upath2 = UPath(p2, anon=self.anon, **self.s3so)
-        assert upath2.read_bytes() == content
-        upath2.unlink()
-
     @pytest.mark.parametrize(
         "joiner", [["bucket", "path", "file"], "bucket/path/file"]
     )
diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py
index 3c9296d..27a495c 100644
--- a/upath/tests/test_core.py
+++ b/upath/tests/test_core.py
@@ -6,21 +6,16 @@ import warnings
 import pytest
 from upath import UPath
 from upath.implementations.s3 import S3Path
-from upath.tests.cases import BaseTests
+from .cases import BaseTests
+from .utils import only_on_windows, skip_on_windows
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"),
-    reason="don't run test on Windows",
-)
+@skip_on_windows
 def test_posix_path(local_testdir):
     assert isinstance(UPath(local_testdir), pathlib.PosixPath)
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("win"),
-    reason="only run test if Windows Machine",
-)
+@only_on_windows
 def test_windows_path(local_testdir):
     assert isinstance(UPath(local_testdir), pathlib.WindowsPath)
 
@@ -107,10 +102,7 @@ def test_new_method(local_testdir):
     assert not isinstance(path, UPath)
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"),
-    reason="don't run test on Windows",
-)  # need to fix windows tests here
+@skip_on_windows
 class TestFSSpecLocal(BaseTests):
     @pytest.fixture(autouse=True)
     def path(self, local_testdir):
@@ -235,3 +227,21 @@ def test_copy_path_append_kwargs():
     assert str(path) == str(copy_path)
     assert not copy_path._kwargs["anon"]
     assert path._kwargs["anon"]
+
+
+def test_relative_to():
+    assert "s3://test_bucket/file.txt" == str(
+        UPath("s3://test_bucket/file.txt").relative_to(
+            UPath("s3://test_bucket")
+        )
+    )
+
+    with pytest.raises(ValueError):
+        UPath("s3://test_bucket/file.txt").relative_to(
+            UPath("gcs://test_bucket")
+        )
+
+    with pytest.raises(ValueError):
+        UPath("s3://test_bucket/file.txt", anon=True).relative_to(
+            UPath("s3://test_bucket", anon=False)
+        )
diff --git a/upath/tests/utils.py b/upath/tests/utils.py
new file mode 100644
index 0000000..f356748
--- /dev/null
+++ b/upath/tests/utils.py
@@ -0,0 +1,18 @@
+import sys
+import pytest
+
+
+def skip_on_windows(func):
+    return pytest.mark.skipif(
+        sys.platform.startswith("win"), reason="Don't run on Windows"
+    )(func)
+
+
+def only_on_windows(func):
+    return pytest.mark.skipif(
+        not sys.platform.startswith("win"), reason="Only run on Windows"
+    )(func)
+
+
+def posixify(path):
+    return str(path).replace("\\", "/")