Repository: ungarj/mapchete
Pull request #632 (2024-06-03 13:54:21)

Cli fsspec kwargs


Patch:
diff --git a/mapchete/path.py b/mapchete/path.py
index ee662cd..f31d1d3 100644
--- a/mapchete/path.py
+++ b/mapchete/path.py
@@ -1,4 +1,5 @@
 """Functions handling paths and file systems."""
+
 from __future__ import annotations
 
 import json
@@ -8,14 +9,15 @@ import warnings
 from collections import defaultdict
 from datetime import datetime
 from functools import cached_property
-from typing import IO, Generator, List, Optional, Set, TextIO, Union
+from io import TextIOWrapper
+from typing import IO, Generator, List, Optional, Set, TextIO, Tuple, Union
 
 import fiona
 import fsspec
 import oyaml as yaml
 import rasterio
 from fiona.session import Session as FioSession
-from fsspec import AbstractFileSystem
+from fsspec.spec import AbstractBufferedFile, AbstractFileSystem
 from rasterio.session import Session as RioSession
 from retry import retry
 
@@ -36,10 +38,13 @@ class MPath(os.PathLike):
     Partially replicates pathlib.Path but with remote file support.
     """
 
+    _storage_options: dict
+    _gdal_options: dict
+
     def __init__(
         self,
         path: Union[str, os.PathLike, MPath],
-        fs: AbstractFileSystem = None,
+        fs: Optional[AbstractFileSystem] = None,
         storage_options: Union[dict, None] = None,
         _info: Union[dict, None] = None,
         **kwargs,
@@ -73,6 +78,7 @@ class MPath(os.PathLike):
             DEFAULT_STORAGE_OPTIONS, **self._kwargs.get("storage_options") or {}
         )
         self._info = _info
+        self._gdal_options = dict()
 
     @staticmethod
     def from_dict(dictionary: dict) -> MPath:
@@ -166,14 +172,13 @@ class MPath(os.PathLike):
                 },
             )
         elif self._path_str.startswith(("http://", "https://")):
-            if self._storage_options.get("username") or self._storage_options.get(
-                "password"
-            ):  # pragma: no cover
+            username = self._storage_options.get("username")
+            if username:
                 from aiohttp import BasicAuth
 
                 auth = BasicAuth(
-                    self._storage_options.get("username"),
-                    self._storage_options.get("password"),
+                    login=username,
+                    password=self._storage_options.get("password", ""),
                 )
             else:
                 auth = None
@@ -191,7 +196,7 @@ class MPath(os.PathLike):
 
     def fs_session(self):
         if hasattr(self.fs, "session"):
-            return self.fs.session
+            return getattr(self.fs, "session")
         else:
             return None
 
@@ -301,9 +306,15 @@ class MPath(os.PathLike):
             return self.new(os.path.relpath(self._path_str, start=start))
 
     def relative_to(self, other: MPathLike) -> str:
-        return str(self.without_protocol().relative_path(other.without_protocol()))
+        return str(
+            self.without_protocol().relative_path(
+                MPath.from_inp(other).without_protocol()
+            )
+        )
 
-    def open(self, mode: str = "r") -> Union[IO, TextIO]:
+    def open(
+        self, mode: str = "r"
+    ) -> Union[IO, TextIO, TextIOWrapper, AbstractBufferedFile]:
         """Open file."""
         return self.fs.open(self._path_str, mode)
 
@@ -312,7 +323,7 @@ class MPath(os.PathLike):
         """Open and return file content as text."""
         try:
             with self.open() as src:
-                return src.read()
+                return str(src.read())
         except FileNotFoundError:
             raise FileNotFoundError(f"{str(self)} not found")
         except Exception as e:  # pragma: no cover
@@ -391,28 +402,29 @@ class MPath(os.PathLike):
         absolute_paths: bool = True,
         **kwargs,
     ) -> Generator[
-        List[Union[MPath, dict], List[Union[MPath, dict]], List[Union[MPath, dict]]],
+        Tuple[Union[MPath, dict], List[Union[MPath, dict]], List[Union[MPath, dict]]],
         None,
         None,
     ]:
         for root, subpaths, files in self.fs.walk(
             str(self), maxdepth=maxdepth, topdown=topdown, **kwargs
         ):
-            yield (
-                self._create_full_path(root, absolute_path=absolute_paths),
-                [
-                    self._create_full_path(
-                        MPath(root) / subpath, absolute_path=absolute_paths
-                    )
-                    for subpath in subpaths
-                ],
-                [
-                    self._create_full_path(
-                        MPath(root) / file, absolute_path=absolute_paths
-                    )
-                    for file in files
-                ],
-            )
+            if isinstance(root, str):
+                yield (
+                    self._create_full_path(root, absolute_path=absolute_paths),
+                    [
+                        self._create_full_path(
+                            MPath(root) / subpath, absolute_path=absolute_paths
+                        )
+                        for subpath in subpaths
+                    ],
+                    [
+                        self._create_full_path(
+                            MPath(root) / file, absolute_path=absolute_paths
+                        )
+                        for file in files
+                    ],
+                )
 
     def rm(self, recursive: bool = False, ignore_errors: bool = False) -> None:
         if (
@@ -432,11 +444,12 @@ class MPath(os.PathLike):
 
     def last_modified(self) -> datetime:
         # for S3 objects
-        if "LastModified" in self.info():
-            return self.info().get("LastModified")
+        last_modified = self.info().get("LastModified")
+        mtime = self.info().get("mtime")
+        if last_modified:
+            return last_modified
         # for local files
-        elif "mtime" in self.info():
-            mtime = self.info().get("mtime")
+        elif mtime:
             return datetime.fromtimestamp(mtime)
         else:  # pragma: no cover
             raise ValueError("Object timestamp could not be determined.")
@@ -448,7 +461,7 @@ class MPath(os.PathLike):
         else:
             return self.fs.isdir(self._path_str)
 
-    def joinpath(self, *other: List[MPathLike]) -> MPath:
+    def joinpath(self, *other: Union[MPathLike, List[MPathLike]]) -> MPath:
         """Join path with other."""
         return self.new(os.path.join(self._path_str, *list(map(str, other))))
 
@@ -502,11 +515,9 @@ class MPath(os.PathLike):
                 default_remote_extensions = gdal_opts.get(
                     "CPL_VSIL_CURL_ALLOWED_EXTENSIONS", ""
                 ).split(", ")
-                if allowed_remote_extensions:
-                    extensions = allowed_remote_extensions.split(",") + (
-                        default_remote_extensions
-                    )
                 extensions = default_remote_extensions + [self.suffix]
+                if allowed_remote_extensions:
+                    extensions += allowed_remote_extensions
                 # make sure current path extension is added to allowed_remote_extensions
                 gdal_opts.update(
                     CPL_VSIL_CURL_ALLOWED_EXTENSIONS=", ".join(
@@ -515,14 +526,16 @@ class MPath(os.PathLike):
                 )
 
             # secure HTTP credentials
-            if self.fs.kwargs.get("auth"):
-                gdal_opts.update(
-                    GDAL_HTTP_USERPWD=f"{self.fs.kwargs['auth'].login}:{self.fs.kwargs['auth'].password}"
-                )
+            auth = getattr(self.fs, "kwargs", {}).get("auth")
+            if auth:
+                gdal_opts.update(GDAL_HTTP_USERPWD=f"{auth.login}:{auth.password}")
 
             # if a custom S3 endpoint is used, we need to deactivate these AWS options
             if self._endpoint_url:
-                gdal_opts.update(AWS_VIRTUAL_HOSTING=False, AWS_HTTPS=False)
+                gdal_opts.update(
+                    AWS_VIRTUAL_HOSTING=False,
+                    AWS_HTTPS=self._gdal_options.get("aws_https", False),
+                )
 
             # merge everything with user options
             gdal_opts.update(user_opts)
@@ -538,21 +551,25 @@ class MPath(os.PathLike):
     def _endpoint_url(self) -> Union[str, None]:
         # GDAL parses the paths in a weird way, so we have to be careful with a custom
         # endpoint
-        if hasattr(self.fs, "endpoint_url") and isinstance(self.fs.endpoint_url, str):
-            return self.fs.endpoint_url.lstrip("http://").lstrip("https://")
+        endpoint_url = getattr(self.fs, "endpoint_url", None)
+        if endpoint_url:
+            self._gdal_options.update(aws_https=endpoint_url.startswith("https://"))
+            return endpoint_url.replace("http://", "").replace("https://", "")
         else:
             return None
 
-    def rio_session(self) -> "rasterio.session.Session":
+    def rio_session(self) -> RioSession:
         if self.fs_session():
             # rasterio accepts a Session object but only a boto3.session.Session
             # object and not a aiobotocore.session.AioSession which we get from fsspec
             return RioSession.from_path(
                 self._path_str,
-                aws_access_key_id=self.fs.key,
-                aws_secret_access_key=self.fs.secret,
+                aws_access_key_id=getattr(self.fs, "key"),
+                aws_secret_access_key=getattr(self.fs, "secret"),
                 endpoint_url=self._endpoint_url,
-                requester_pays=self.fs.storage_options.get("requester_pays", False),
+                requester_pays=getattr(self.fs, "storage_options", {}).get(
+                    "requester_pays", False
+                ),
             )
         else:
             return RioSession.from_path(self._path_str)
@@ -583,16 +600,18 @@ class MPath(os.PathLike):
             )
         )
 
-    def fio_session(self) -> "fiona.session.Session":
+    def fio_session(self) -> FioSession:
         if self.fs_session():
             # fiona accepts a Session object but only a boto3.session.Session
             # object and not a aiobotocore.session.AioSession which we get from fsspec
             return FioSession.from_path(
                 self._path_str,
-                aws_access_key_id=self.fs.key,
-                aws_secret_access_key=self.fs.secret,
+                aws_access_key_id=getattr(self.fs, "key"),
+                aws_secret_access_key=getattr(self.fs, "secret"),
                 endpoint_url=self._endpoint_url,
-                requester_pays=self.fs.storage_options.get("requester_pays", False),
+                requester_pays=getattr(self.fs, "storage_options", {}).get(
+                    "requester_pays", False
+                ),
             )
         else:
             return FioSession.from_path(self._path_str)
@@ -629,7 +648,7 @@ class MPath(os.PathLike):
 
     def __add__(self, other: MPathLike) -> "MPath":
         """Short for self.joinpath()."""
-        return self.new(str(self) + other)
+        return self.new(str(self) + str(other))
 
     def __eq__(self, other: MPathLike):
         if isinstance(other, str):
@@ -689,7 +708,7 @@ def path_exists(
 
 
 def absolute_path(
-    path: Union[MPathLike, None] = None,
+    path: MPathLike,
     base_dir: Union[MPathLike, None] = None,
     **kwargs,
 ) -> MPath:
@@ -709,7 +728,7 @@ def absolute_path(
 
 
 def relative_path(
-    path: Union[MPathLike, None] = None,
+    path: MPathLike,
     base_dir: Union[MPathLike, None] = None,
     **kwargs,
 ) -> MPath:
@@ -742,11 +761,15 @@ def makedirs(
 
 
 def tiles_exist(
-    config=None,
-    output_tiles=None,
-    output_tiles_batches=None,
-    process_tiles=None,
-    process_tiles_batches=None,
+    config,
+    output_tiles: Optional[Generator[BufferedTile, None, None]] = None,
+    output_tiles_batches: Optional[
+        Generator[Generator[BufferedTile, None, None], None, None]
+    ] = None,
+    process_tiles: Optional[Generator[BufferedTile, None, None]] = None,
+    process_tiles_batches: Optional[
+        Generator[Generator[BufferedTile, None, None], None, None]
+    ] = None,
     **kwargs,
 ):
     """
@@ -798,10 +821,15 @@ def tiles_exist(
             )
 
         def _tiles():
-            if process_tiles or output_tiles:  # pragma: no cover
-                yield from process_tiles or output_tiles
-            else:
-                for batch in process_tiles_batches or output_tiles_batches:
+            if process_tiles:  # pragma: no cover
+                yield from process_tiles
+            elif output_tiles:  # pragma: no cover
+                yield from output_tiles
+            elif process_tiles_batches:
+                for batch in process_tiles_batches:
+                    yield from batch
+            elif output_tiles_batches:  # pragma: no cover
+                for batch in output_tiles_batches:
                     yield from batch
 
         for tile in _tiles():
@@ -819,39 +847,52 @@ def tiles_exist(
         logger.debug("sort output tiles by %s, this could take a while", sort_attribute)
         output_tiles_batches = _batch_tiles_by_attribute(output_tiles, sort_attribute)
 
+    if process_tiles_batches:
+        yield from _process_tiles_batches_exist(
+            process_tiles_batches,
+            config,
+            _is_https_without_ls(config.output_reader.path),
+        )
+    elif output_tiles_batches:
+        yield from _output_tiles_batches_exist(
+            output_tiles_batches,
+            config,
+            _is_https_without_ls(config.output_reader.path),
+        )
+
+
+def _is_https_without_ls(path: MPath, default_file: str = "metadata.json") -> bool:
     # Some HTTP endpoints won't allow ls() on them, so we will have to
     # request tile by tile in order to determine whether they exist or not.
     # This flag will trigger this further down.
     is_https_without_ls = False
-    if "https" in config.output_reader.path.protocols:
+    if "https" in path.protocols:
         try:
-            config.output_reader.path.ls()
+            path.ls()
         except FileNotFoundError:  # pragma: no cover
-            metadata_json = config.output_reader.path / "metadata.json"
+            metadata_json = path / default_file
             if not metadata_json.exists():
                 raise FileNotFoundError(
-                    f"TileDirectory does not seem to exist or metadata.json is not available: {config.output_reader.path}"
+                    f"TileDirectory does not seem to exist or {default_file} is not available: {path}"
                 )
             is_https_without_ls = True
+    return is_https_without_ls
 
-    if process_tiles_batches:
-        yield from _process_tiles_batches_exist(
-            process_tiles_batches, config, is_https_without_ls
-        )
-    elif output_tiles_batches:
-        yield from _output_tiles_batches_exist(
-            output_tiles_batches, config, is_https_without_ls
-        )
 
-
-def _batch_tiles_by_attribute(tiles: List[BufferedTile], attribute: str = "row"):
+def _batch_tiles_by_attribute(
+    tiles: Generator[BufferedTile, None, None], attribute: str = "row"
+) -> Generator[Generator[BufferedTile, None, None], None, None]:
     ordered = defaultdict(set)
     for tile in tiles:
         ordered[getattr(tile, attribute)].add(tile)
     return ((t for t in ordered[key]) for key in sorted(list(ordered.keys())))
 
 
-def _output_tiles_batches_exist(output_tiles_batches, config, is_https_without_ls):
+def _output_tiles_batches_exist(
+    output_tiles_batches: Generator[Generator[BufferedTile, None, None], None, None],
+    config,
+    is_https_without_ls,
+):
     with Executor(concurrency=mapchete_options.tiles_exist_concurrency) as executor:
         for batch in executor.as_completed(
             _output_tiles_batch_exists,
@@ -930,9 +971,9 @@ def _process_tiles_batch_exists(tiles, config, is_https_without_ls):
 
 
 def _existing_output_tiles(
-    output_rows=None,
-    output_paths=None,
-    config=None,
+    output_rows: List[BufferedTile],
+    output_paths: dict,
+    config,
     zoom=None,
     is_https_without_ls=False,
 ):
@@ -963,8 +1004,7 @@ def _existing_output_tiles(
 
 def fs_from_path(path: MPathLike, **kwargs) -> fsspec.AbstractFileSystem:
     """Guess fsspec FileSystem from path and initialize using the desired options."""
-    path = path if isinstance(path, MPath) else MPath(path, **kwargs)
-    return path.fs
+    return path.fs if isinstance(path, MPath) else MPath(path, **kwargs).fs
 
 
 def batch_sort_property(tile_path_schema: str) -> str:


Test patch:
diff --git a/test/test_io.py b/test/test_io.py
index fd5abbd..811f10d 100644
--- a/test/test_io.py
+++ b/test/test_io.py
@@ -3,10 +3,12 @@
 import os
 
 import pytest
+from pytest_lazyfixture import lazy_fixture
 from shapely.errors import TopologicalError
 
 import mapchete
 from mapchete.io import (
+    MatchingMethod,
     absolute_path,
     copy,
     get_best_zoom_level,
@@ -60,8 +62,8 @@ def test_absolute_path():
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("s3_metadata_json"),
-        pytest.lazy_fixture("http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
+        lazy_fixture("http_metadata_json"),
     ],
 )
 def test_read_remote_json(path):
@@ -72,14 +74,14 @@ def test_read_remote_json(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("s3_metadata_json"),
-        pytest.lazy_fixture("http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
+        lazy_fixture("http_metadata_json"),
     ],
 )
 def test_read_remote_json_errors(path):
     # keep access credentials but invalidate URI
     path = path + "not_here"
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises(FileNotFoundError):  # type: ignore
         read_json(path)
 
 
@@ -93,7 +95,9 @@ def test_tile_to_zoom_level():
     # at Northern boundary
     assert tile_to_zoom_level(tp_merc.tile(zoom, 0, col), tp_geod) == 9
     assert (
-        tile_to_zoom_level(tp_merc.tile(zoom, 0, col), tp_geod, matching_method="min")
+        tile_to_zoom_level(
+            tp_merc.tile(zoom, 0, col), tp_geod, matching_method=MatchingMethod.min
+        )
         == 12
     )
     # at Equator
@@ -107,7 +111,7 @@ def test_tile_to_zoom_level():
         tile_to_zoom_level(
             tp_merc.tile(zoom, tp_merc.matrix_height(zoom) // 2, col),
             tp_geod,
-            matching_method="min",
+            matching_method=MatchingMethod.min,
         )
         == 9
     )
@@ -122,7 +126,7 @@ def test_tile_to_zoom_level():
         tile_to_zoom_level(
             tp_merc.tile(zoom, tp_merc.matrix_height(zoom) - 1, col),
             tp_geod,
-            matching_method="min",
+            matching_method=MatchingMethod.min,
         )
         == 12
     )
@@ -130,7 +134,7 @@ def test_tile_to_zoom_level():
         tile_to_zoom_level(
             BufferedTilePyramid("mercator", metatiling=2, pixelbuffer=20).tile(4, 0, 7),
             BufferedTilePyramid("geodetic", metatiling=8, pixelbuffer=20),
-            matching_method="gdal",
+            matching_method=MatchingMethod.gdal,
         )
         == 4
     )
@@ -147,7 +151,9 @@ def test_tile_to_zoom_level_geodetic_from_mercator():
     # NOTE: using a newer proj version (8.2.0) will yield different results
     assert tile_to_zoom_level(tp_geod.tile(zoom, 0, col), tp_merc) in [0, 2]
     with pytest.raises(TopologicalError):
-        tile_to_zoom_level(tp_geod.tile(zoom, 0, col), tp_merc, matching_method="min")
+        tile_to_zoom_level(
+            tp_geod.tile(zoom, 0, col), tp_merc, matching_method=MatchingMethod.min
+        )
     # at Equator
     assert (
         tile_to_zoom_level(
@@ -159,7 +165,7 @@ def test_tile_to_zoom_level_geodetic_from_mercator():
         tile_to_zoom_level(
             tp_geod.tile(zoom, tp_geod.matrix_height(zoom) // 2, col),
             tp_merc,
-            matching_method="min",
+            matching_method=MatchingMethod.min,
         )
         == 10
     )
@@ -172,7 +178,7 @@ def test_tile_to_zoom_level_geodetic_from_mercator():
         tile_to_zoom_level(
             tp_geod.tile(zoom, tp_geod.matrix_height(zoom) - 1, col),
             tp_merc,
-            matching_method="min",
+            matching_method=MatchingMethod.min,
         )
 
     # check wrong method
@@ -180,7 +186,7 @@ def test_tile_to_zoom_level_geodetic_from_mercator():
         tile_to_zoom_level(
             tp_geod.tile(zoom, tp_geod.matrix_height(zoom) - 1, col),
             tp_merc,
-            matching_method="invalid_method",
+            matching_method="invalid_method",  # type: ignore
         )
 
 
@@ -266,9 +272,9 @@ def test_tiles_exist_s3(gtiff_s3):
 
         # manually check which tiles exist
         written_output_tiles = set()
-        for t in output_tiles:
-            if mp.config.output_reader.tiles_exist(output_tile=t):
-                written_output_tiles.add(t)
+        for tt in output_tiles:
+            if mp.config.output_reader.tiles_exist(output_tile=tt):  # type: ignore
+                written_output_tiles.add(tt)
         full_process_tiles = set(
             [
                 tile
diff --git a/test/test_io_raster.py b/test/test_io_raster.py
index bebf54e..70370fe 100644
--- a/test/test_io_raster.py
+++ b/test/test_io_raster.py
@@ -1056,6 +1056,9 @@ def test_read_raster_window_aws_s3(path, grid, pixelbuffer, zoom):
 @pytest.mark.parametrize(
     "path",
     [
+        lazy_fixture("raster_4band_s3"),
+        lazy_fixture("raster_4band_http"),
+        lazy_fixture("raster_4band_secure_http"),
         lazy_fixture("raster_4band"),
         lazy_fixture("stacta"),
     ],
diff --git a/test/test_path.py b/test/test_path.py
index f6c3d5a..2121b63 100644
--- a/test/test_path.py
+++ b/test/test_path.py
@@ -2,6 +2,7 @@ import pickle
 from datetime import datetime
 
 import pytest
+from pytest_lazyfixture import lazy_fixture
 
 from mapchete.config import get_hash
 from mapchete.path import MPath, batch_sort_property
@@ -25,17 +26,17 @@ def test_parse(path_str):
 
 def test_parse_error():
     with pytest.raises(TypeError):
-        MPath(None)
+        MPath(None)  # type: ignore
 
 
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_metadata_json"),
-        pytest.lazy_fixture("secure_http_metadata_json"),
-        pytest.lazy_fixture("s3_metadata_json"),
-        pytest.lazy_fixture("vsicurl_metadata_json"),
+        lazy_fixture("http_metadata_json"),
+        lazy_fixture("secure_http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
+        lazy_fixture("vsicurl_metadata_json"),
     ],
 )
 def test_remote_existing_paths(path):
@@ -109,14 +110,14 @@ def test_parse_local_dirpath(path_str):
 
 
 def test_makedirs_filepath(mp_tmpdir):
-    path = MPath(mp_tmpdir).joinpath("path_mkdir_test", "file.ext")
+    path = MPath(mp_tmpdir) / "path_mkdir_test" / "file.ext"
     path.parent.makedirs()
     assert path.parent.exists()
     assert not path.exists()
 
 
 def test_makedirs_dirpath(mp_tmpdir):
-    path = MPath(mp_tmpdir).joinpath("path_mkdir_test", "directory/")
+    path = MPath(mp_tmpdir) / "path_mkdir_test" / "directory/"
     path.makedirs()
     assert path.parent.exists()
     assert path.exists()
@@ -150,7 +151,7 @@ def test_with_suffix():
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("testdata_dir"),
+        lazy_fixture("testdata_dir"),
     ],
 )
 def test_ls(path):
@@ -167,9 +168,9 @@ def test_ls(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_testdata_dir"),
-        pytest.lazy_fixture("secure_http_testdata_dir"),
-        pytest.lazy_fixture("s3_testdata_dir"),
+        lazy_fixture("http_testdata_dir"),
+        lazy_fixture("secure_http_testdata_dir"),
+        lazy_fixture("s3_testdata_dir"),
     ],
 )
 def test_ls_remote(path):
@@ -179,7 +180,7 @@ def test_ls_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("testdata_dir"),
+        lazy_fixture("testdata_dir"),
     ],
 )
 @pytest.mark.parametrize("absolute_paths", [True, False])
@@ -202,9 +203,9 @@ def test_walk(path, absolute_paths):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_testdata_dir"),
-        pytest.lazy_fixture("secure_http_testdata_dir"),
-        pytest.lazy_fixture("s3_testdata_dir"),
+        lazy_fixture("http_testdata_dir"),
+        lazy_fixture("secure_http_testdata_dir"),
+        lazy_fixture("s3_testdata_dir"),
     ],
 )
 @pytest.mark.parametrize("absolute_paths", [True, False])
@@ -215,7 +216,7 @@ def test_walk_remote(path, absolute_paths):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("testdata_dir"),
+        lazy_fixture("testdata_dir"),
     ],
 )
 def test_is_directory(path):
@@ -226,9 +227,9 @@ def test_is_directory(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_testdata_dir"),
-        pytest.lazy_fixture("secure_http_testdata_dir"),
-        pytest.lazy_fixture("s3_testdata_dir"),
+        lazy_fixture("http_testdata_dir"),
+        lazy_fixture("secure_http_testdata_dir"),
+        lazy_fixture("s3_testdata_dir"),
     ],
 )
 def test_is_dir_remote(path):
@@ -238,7 +239,7 @@ def test_is_dir_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("metadata_json"),
+        lazy_fixture("metadata_json"),
     ],
 )
 def test_io_read(path):
@@ -254,9 +255,9 @@ def test_io_read(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_metadata_json"),
-        pytest.lazy_fixture("secure_http_metadata_json"),
-        pytest.lazy_fixture("s3_metadata_json"),
+        lazy_fixture("http_metadata_json"),
+        lazy_fixture("secure_http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
     ],
 )
 def test_io_read_remote(path):
@@ -266,7 +267,7 @@ def test_io_read_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("metadata_json"),
+        lazy_fixture("metadata_json"),
     ],
 )
 def test_size(path):
@@ -277,9 +278,9 @@ def test_size(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_metadata_json"),
-        pytest.lazy_fixture("secure_http_metadata_json"),
-        pytest.lazy_fixture("s3_metadata_json"),
+        lazy_fixture("http_metadata_json"),
+        lazy_fixture("secure_http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
     ],
 )
 def test_size_remote(path):
@@ -289,7 +290,7 @@ def test_size_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("metadata_json"),
+        lazy_fixture("metadata_json"),
     ],
 )
 def test_last_modified(path):
@@ -300,9 +301,9 @@ def test_last_modified(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("http_metadata_json"),
-        pytest.lazy_fixture("secure_http_metadata_json"),
-        pytest.lazy_fixture("s3_metadata_json"),
+        lazy_fixture("http_metadata_json"),
+        lazy_fixture("secure_http_metadata_json"),
+        lazy_fixture("s3_metadata_json"),
     ],
 )
 def test_last_modified_remote(path):
@@ -345,9 +346,9 @@ def test_gdal_env_params(path_str):
     assert path.suffix in remote_extensions
 
     # add custom extensions
-    remote_extensions = path.gdal_env_params(allowed_remote_extensions=".foo,.bar")[
-        "CPL_VSIL_CURL_ALLOWED_EXTENSIONS"
-    ].split(", ")
+    remote_extensions = path.gdal_env_params(
+        allowed_remote_extensions=[".foo", ".bar"]
+    )["CPL_VSIL_CURL_ALLOWED_EXTENSIONS"].split(", ")
     assert path.suffix in remote_extensions
 
 
@@ -415,7 +416,7 @@ def test_transfer_fs(secure_http_testdata_dir):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band"),
+        lazy_fixture("raster_4band"),
     ],
 )
 def test_pickle_path(path):
@@ -428,9 +429,9 @@ def test_pickle_path(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band_s3"),
-        pytest.lazy_fixture("raster_4band_http"),
-        pytest.lazy_fixture("raster_4band_secure_http"),
+        lazy_fixture("raster_4band_s3"),
+        lazy_fixture("raster_4band_http"),
+        lazy_fixture("raster_4band_secure_http"),
     ],
 )
 def test_pickle_path_remote(path):
@@ -441,7 +442,7 @@ def test_pickle_path_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band_aws_s3"),
+        lazy_fixture("raster_4band_aws_s3"),
     ],
 )
 def test_pickle_path_aws_s3(path):
@@ -451,7 +452,7 @@ def test_pickle_path_aws_s3(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band"),
+        lazy_fixture("raster_4band"),
     ],
 )
 def test_pickle_fs(path):
@@ -462,10 +463,10 @@ def test_pickle_fs(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band_s3"),
-        pytest.lazy_fixture("raster_4band_http"),
-        pytest.lazy_fixture("raster_4band_secure_http"),
-        pytest.lazy_fixture("raster_4band_aws_s3"),
+        lazy_fixture("raster_4band_s3"),
+        lazy_fixture("raster_4band_http"),
+        lazy_fixture("raster_4band_secure_http"),
+        lazy_fixture("raster_4band_aws_s3"),
     ],
 )
 def test_pickle_fs_remote(path):
@@ -476,7 +477,7 @@ def test_pickle_fs_remote(path):
 @pytest.mark.parametrize(
     "path",
     [
-        pytest.lazy_fixture("raster_4band_aws_s3"),
+        lazy_fixture("raster_4band_aws_s3"),
     ],
 )
 def test_pickle_fs_aws_s3(path):