Repository: ungarj/mapchete
Pull request #467 (2022-06-08 10:07:50)

don't overwrite optional GDAL environment settings when applying defa…
…ult http options; add more logging


Patch:
diff --git a/mapchete/__init__.py b/mapchete/__init__.py
index 73eb9bf..7fc3a24 100644
--- a/mapchete/__init__.py
+++ b/mapchete/__init__.py
@@ -22,6 +22,5 @@ __all__ = [
 ]
 __version__ = "2022.4.1"
 
-
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py
index 8e05fce..1a7bc9b 100644
--- a/mapchete/commands/_convert.py
+++ b/mapchete/commands/_convert.py
@@ -337,6 +337,7 @@ def _get_input_info(inp):
 
     # assuming single file if path has a file extension
     if os.path.splitext(inp)[1]:
+        logger.debug("assuming single file")
         driver = driver_from_file(inp)
 
         # single file input can be a mapchete file or a rasterio/fiona file
diff --git a/mapchete/formats/__init__.py b/mapchete/formats/__init__.py
index aa5b0c2..3bcfae1 100644
--- a/mapchete/formats/__init__.py
+++ b/mapchete/formats/__init__.py
@@ -144,7 +144,7 @@ def driver_metadata(driver_name):
         raise ValueError(f"driver '{driver_name}' not found")
 
 
-def driver_from_file(input_file):
+def driver_from_file(input_file, quick=True):
     """
     Guess driver from file by opening it.
 
@@ -154,14 +154,27 @@ def driver_from_file(input_file):
         driver name
     """
     file_ext = os.path.splitext(input_file)[1].split(".")[1]
+
+    # mapchete files can immediately be returned:
     if file_ext == "mapchete":
         return "Mapchete"
+
+    # use the most common file extensions to quickly determine input driver for file:
+    if quick:
+        if file_ext in ["tif", "jp2"]:
+            return "raster_file"
+        if file_ext in ["shp", "geojson", "gpkg"]:
+            return "vector_file"
+
+    # brute force by trying to open file with rasterio and fiona:
     try:
-        with rasterio.open(input_file):
+        logger.debug("try to open %s with rasterio...", input_file)
+        with rasterio.open(input_file):  # pragma: no cover
             return "raster_file"
     except Exception as rio_exception:
         try:
-            with fiona.open(input_file):
+            logger.debug("try to open %s with fiona...", input_file)
+            with fiona.open(input_file):  # pragma: no cover
                 return "vector_file"
         except Exception as fio_exception:
             if path_exists(input_file):
diff --git a/mapchete/formats/default/raster_file.py b/mapchete/formats/default/raster_file.py
index 14bb1b6..161ebef 100644
--- a/mapchete/formats/default/raster_file.py
+++ b/mapchete/formats/default/raster_file.py
@@ -80,6 +80,28 @@ class InputData(base.InputData):
             if "abstract" in input_params
             else input_params["path"]
         )
+        with rasterio.open(self.path, "r") as src:
+            self.profile = deepcopy(src.meta)
+            # determine bounding box
+            if src.transform.is_identity:
+                if src.gcps[1] is not None:
+                    with WarpedVRT(src) as dst:
+                        self._src_bounds = dst.bounds
+                        self._src_crs = src.gcps[1]
+                        self._src_transform = dst.transform
+                elif src.rpcs:  # pragma: no cover
+                    with WarpedVRT(src) as dst:
+                        self._src_bounds = dst.bounds
+                        self._src_crs = CRS.from_string("EPSG:4326")
+                        self._src_transform = dst.transform
+                else:  # pragma: no cover
+                    raise TypeError("cannot determine georeference")
+            else:
+                self._src_crs = src.crs
+                self._src_bounds = src.bounds
+                self._src_transform = src.transform
+            self._src_bbox = box(*self._src_bounds)
+
         self._cache_task = f"cache_{self.path}"
         if "abstract" in input_params and "cache" in input_params["abstract"]:
             if isinstance(input_params["abstract"]["cache"], dict):
@@ -118,12 +140,6 @@ class InputData(base.InputData):
                     f"invalid cache configuration given: {input_params['abstract']['cache']}"
                 )
 
-    @cached_property
-    def profile(self):
-        """Return raster metadata."""
-        with rasterio.open(self.path, "r") as src:
-            return deepcopy(src.meta)
-
     def open(self, tile, **kwargs):
         """
         Return InputTile object.
@@ -166,37 +182,20 @@ class InputData(base.InputData):
             Shapely geometry object
         """
         out_crs = self.pyramid.crs if out_crs is None else out_crs
-        with rasterio.open(self.path) as src:
-            if src.transform.is_identity:
-                if src.gcps[1] is not None:
-                    with WarpedVRT(src) as dst:
-                        src_bounds = dst.bounds
-                        src_crs = src.gcps[1]
-                        src_transform = dst.transform
-                elif src.rpcs:  # pragma: no cover
-                    with WarpedVRT(src) as dst:
-                        src_bounds = dst.bounds
-                        src_crs = CRS.from_string("EPSG:4326")
-                        src_transform = dst.transform
-                else:  # pragma: no cover
-                    raise TypeError("cannot determine georeference")
-            else:
-                src_crs = src.crs
-                src_bounds = src.bounds
-                src_transform = src.transform
 
-            out_bbox = bbox = box(*src_bounds)
         # If soucre and target CRSes differ, segmentize and reproject
-        if src_crs != out_crs:
+        if self._src_crs != out_crs:
             # estimate segmentize value (raster pixel size * tile size)
             # and get reprojected bounding box
             return reproject_geometry(
-                segmentize_geometry(bbox, src_transform[0] * self.pyramid.tile_size),
-                src_crs=src_crs,
+                segmentize_geometry(
+                    self._src_bbox, self._src_transform[0] * self.pyramid.tile_size
+                ),
+                src_crs=self._src_crs,
                 dst_crs=out_crs,
             )
         else:
-            return out_bbox
+            return self._src_bbox
 
     def exists(self):
         """
@@ -254,14 +253,7 @@ class InputTile(base.InputTile):
             self._in_memory_raster = in_memory_raster
         else:
             self.path = input_data._cached_path or input_data.path
-            if io.path_is_remote(input_data.path):
-                file_ext = os.path.splitext(self.path)[1]
-                self.gdal_opts = {
-                    "GDAL_DISABLE_READDIR_ON_OPEN": True,
-                    "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": "%s,.ovr" % file_ext,
-                }
-            else:
-                self.gdal_opts = {}
+            self.gdal_opts = {}
 
     def __repr__(self):  # pragma: no cover
         source = (
diff --git a/mapchete/io/_misc.py b/mapchete/io/_misc.py
index 47010a3..2acef79 100644
--- a/mapchete/io/_misc.py
+++ b/mapchete/io/_misc.py
@@ -13,17 +13,23 @@ logger = logging.getLogger(__name__)
 
 
 GDAL_HTTP_OPTS = dict(
-    GDAL_DISABLE_READDIR_ON_OPEN=True,
-    CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".tif, .ovr, .jp2, .png, .xml",
-    GDAL_HTTP_TIMEOUT=30,
-    GDAL_HTTP_MAX_RETRY=3,
-    GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=True,
-    GDAL_HTTP_RETRY_DELAY=5,
+    GDAL_DISABLE_READDIR_ON_OPEN=os.environ.get(
+        "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE"
+    ).upper()
+    == "TRUE",
+    CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".tif, .ovr, .jp2, .png, .xml, .rpc",
+    GDAL_HTTP_TIMEOUT=int(os.environ.get("GDAL_HTTP_TIMEOUT", "30")),
+    GDAL_HTTP_MAX_RETRY=int(os.environ.get("GDAL_HTTP_MAX_RETRY", "3")),
+    GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=os.environ.get(
+        "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE"
+    ).upper()
+    == "TRUE",
+    GDAL_HTTP_RETRY_DELAY=int(os.environ.get("GDAL_HTTP_RETRY_DELAY", "5")),
 )
 MAPCHETE_IO_RETRY_SETTINGS = {
-    "tries": int(os.environ.get("MAPCHETE_IO_RETRY_TRIES", 3)),
-    "delay": float(os.environ.get("MAPCHETE_IO_RETRY_DELAY", 1)),
-    "backoff": float(os.environ.get("MAPCHETE_IO_RETRY_BACKOFF", 1)),
+    "tries": int(os.environ.get("MAPCHETE_IO_RETRY_TRIES", "3")),
+    "delay": float(os.environ.get("MAPCHETE_IO_RETRY_DELAY", "1")),
+    "backoff": float(os.environ.get("MAPCHETE_IO_RETRY_BACKOFF", "1")),
 }
 
 
@@ -210,10 +216,11 @@ def get_gdal_options(opts, is_remote=False, allowed_remote_extensions=[]):
     """
     user_opts = {} if opts is None else dict(**opts)
     if is_remote:
-        out = dict(GDAL_HTTP_OPTS)
+        gdal_opts = dict(GDAL_HTTP_OPTS)
         if allowed_remote_extensions:
-            out.update(CPL_VSIL_CURL_ALLOWED_EXTENSIONS=allowed_remote_extensions)
-        out.update(user_opts)
-        return out
+            gdal_opts.update(CPL_VSIL_CURL_ALLOWED_EXTENSIONS=allowed_remote_extensions)
+        gdal_opts.update(user_opts)
     else:
-        return user_opts
+        gdal_opts = user_opts
+    logger.debug("using GDAL options: %s", gdal_opts)
+    return gdal_opts


Test patch:
diff --git a/test/test_formats.py b/test/test_formats.py
index 3f831e0..ababd61 100644
--- a/test/test_formats.py
+++ b/test/test_formats.py
@@ -64,7 +64,7 @@ def test_driver_from_file_errors(execute_kwargs_py):
         driver_from_file(execute_kwargs_py)
 
     with pytest.raises(FileNotFoundError):
-        driver_from_file("non_existing_file.tif")
+        driver_from_file("non_existing_file.tif", quick=False)
 
 
 def test_mapchete_input(mapchete_input):