Repository: OSGeo/gdal (2026-05-23T00:19:34Z)

Issue:
pipeline materialize to named file + step don't play nicely together
GDAL 3.14.0dev master at b6a838891f438d4181b1f9d0e1af1eb88107e03b

```
$ gdal raster pipeline read autotest/gdrivers/data/small_world.tif ! materialize --output /vsimem/tmp.tif  ! tile /vsimem/
Warning 1: pipeline: Step materialize is not natively streaming compatible, and may cause significant processing time at opening
ERROR 1: pipeline: Step 'materialize' not allowed in stream execution, unless the GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM configuration option is set.
```

Adding ``--config GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM=YES`` pseudo works around the issue, but on bigger images where tiling will be multi-threaded:
```
$ gdal raster pipeline --config GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM=YES read autotest/gdrivers/data/small_world.tif ! resize --size 4096,0 ! materialize --format COG --output /vsimem/tmp.tif ! tile /vsimem/
Warning 1: pipeline: Step materialize is not natively streaming compatible, and may cause significant processing time at opening
ERROR 1: tile: materialize: Dataset '/vsimem/tmp.tif' already exists. You may specify the --overwrite option.
```

Adding ``--overwrite`` is a red flag, and doesn't work on more complex scenarios:
```
$ gdal raster pipeline --config GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM=YES  read s2_rgb_clamped.vrt      !             materialize --format COG --creation-option RESAMPLING=CUBIC --output=mosaic.tif  --overwrite            !             tile --min-zoom 10 s2_tiled_min_zoom10 --format WEBP
Warning 1: pipeline: Step materialize is not natively streaming compatible, and may cause significant processing time at opening
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileOffsets
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileOffsets
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileByteCounts
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileByteCounts
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileOffsets
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileByteCounts
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileOffsets
Warning 1: mosaic.tif: TIFFReadDirectory:Invalid data type for tag TileByteCounts
ERROR 1: Deleting mosaic.tif failed:
No such file or directory
ERROR 1: Deleting mosaic.tif failed:
No such file or directory
[....]
```


Discussion:

Patch:
diff --git a/apps/gdalalg_abstract_pipeline.cpp b/apps/gdalalg_abstract_pipeline.cpp
index 8508d5bcb5b7..5f42d16f5cb8 100644
--- a/apps/gdalalg_abstract_pipeline.cpp
+++ b/apps/gdalalg_abstract_pipeline.cpp
@@ -15,6 +15,7 @@
 #include "cpl_json.h"
 
 #include "gdalalg_abstract_pipeline.h"
+#include "gdalalg_materialize.h"
 #include "gdalalg_raster_read.h"
 #include "gdalalg_raster_write.h"
 #include "gdalalg_vector_read.h"
@@ -1754,10 +1755,14 @@ GDALAbstractPipelineAlgorithm::GetAutoComplete(std::vector<std::string> &args,
 }
 
 /************************************************************************/
-/*            GDALAbstractPipelineAlgorithm::SaveGDALGFile()            */
+/*      GDALAbstractPipelineAlgorithm::SaveGDALGIntoFileOrString()      */
 /************************************************************************/
 
-bool GDALAbstractPipelineAlgorithm::SaveGDALGFile(
+/** Save the pipeline either into the file of name outFilename, if
+ * outFilename is not empty, or into the output string outString if
+ * outFilename is empty.
+ */
+bool GDALAbstractPipelineAlgorithm::SaveGDALGIntoFileOrString(
     const std::string &outFilename, std::string &outString) const
 {
     std::string osCommandLine;
@@ -1869,7 +1874,7 @@ bool GDALAbstractPipelineAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
             }
 
             std::string outStringUnused;
-            return SaveGDALGFile(filename, outStringUnused);
+            return SaveGDALGIntoFileOrString(filename, outStringUnused);
         }
 
         const auto outputFormatArg =
@@ -1951,51 +1956,106 @@ bool GDALAbstractPipelineAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
     }
 
     // Because of multiprocessing in gdal raster tile, make sure that all
-    // steps before it are serialized in a .gdal.json file
+    // steps before it are either materialized or serialized in a .gdal.json file
     if (m_steps.size() >= 2 && m_steps.back()->SupportsInputMultiThreading() &&
         m_steps.back()
                 ->GetArg(GDAL_ARG_NAME_NUM_THREADS_INT_HIDDEN)
                 ->Get<int>() > 1 &&
         !(m_steps.size() == 2 && m_steps[0]->GetName() == "read"))
     {
-        bool ret = false;
-        auto poSrcDS = m_inputDataset.size() == 1
-                           ? m_inputDataset[0].GetDatasetRef()
-                           : nullptr;
-        if (poSrcDS)
-        {
-            auto poSrcDriver = poSrcDS->GetDriver();
-            if (!poSrcDriver || EQUAL(poSrcDriver->GetDescription(), "MEM"))
+        if (m_steps[m_steps.size() - 2]->GetName() ==
+            GDALMaterializeRasterAlgorithm::NAME)
+        {
+            // If the step immediately before the last one is materialize,
+            // then make sure the file it generates can be re-opened
+            auto poArg = m_steps[m_steps.size() - 2]->GetArg(
+                GDALMaterializeRasterAlgorithm::
+                    ARG_NAME_REOPEN_AND_DO_NOT_EARLY_DELETE);
+            CPLAssert(poArg);
+            poArg->Set(true);
+        }
+        else
+        {
+            for (size_t i = m_steps.size() - 2; i > 0;)
+            {
+                --i;
+                if (m_steps[i]->GetName() ==
+                    GDALMaterializeRasterAlgorithm::NAME)
+                {
+                    auto poArg = m_steps[i]->GetArg(GDAL_ARG_NAME_OUTPUT);
+                    CPLAssert(poArg);
+                    if (poArg->IsExplicitlySet())
+                    {
+                        // We could potentially support that scenario but that
+                        // would require executing the pipeline up to that
+                        // step, and serializing the rest to GDALG
+                        ReportError(
+                            CE_Failure, CPLE_AppDefined,
+                            "Cannot execute this pipeline in parallel mode due "
+                            "to the presence of a materialize step that has a "
+                            "'output' argument and is not immediately before "
+                            "the last step. "
+                            "Move/create a materialize step immediately before "
+                            "the last step, or add '-j 1' to the last step "
+                            "'%s'",
+                            m_steps.back()->GetName().c_str());
+                        return false;
+                    }
+                }
+            }
+
+            bool ret = false;
+            auto poSrcDS = m_inputDataset.size() == 1
+                               ? m_inputDataset[0].GetDatasetRef()
+                               : nullptr;
+            if (poSrcDS)
+            {
+                auto poSrcDriver = poSrcDS->GetDriver();
+                if (!poSrcDriver || EQUAL(poSrcDriver->GetDescription(), "MEM"))
+                {
+                    ReportError(
+                        CE_Failure, CPLE_AppDefined,
+                        "Cannot execute this pipeline in parallel mode due to "
+                        "input dataset of last step being a non-materialized "
+                        "dataset. "
+                        "Materialize it first, or add '-j 1' to the last step "
+                        "'%s'",
+                        m_steps.back()->GetName().c_str());
+                    return false;
+                }
+            }
+            std::string outString;
+            if (SaveGDALGIntoFileOrString(std::string(), outString))
+            {
+                const char *const apszAllowedDrivers[] = {"GDALG", nullptr};
+                auto poCurDS = GDALDataset::Open(
+                    outString.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR,
+                    apszAllowedDrivers);
+                if (poCurDS)
+                {
+                    auto &lastAlg = m_steps.back();
+                    lastAlg->m_inputDataset.clear();
+                    lastAlg->m_inputDataset.resize(1);
+                    lastAlg->m_inputDataset[0].Set(poCurDS);
+                    lastAlg->m_inputDataset[0].SetDatasetOpenedByAlgorithm();
+                    poCurDS->Release();
+                    ret = lastAlg->RunStep(ctxt);
+                    lastAlg->m_inputDataset[0].Close();
+                }
+            }
+            else
             {
                 ReportError(
                     CE_Failure, CPLE_AppDefined,
                     "Cannot execute this pipeline in parallel mode due to "
-                    "input dataset being a non-materialized dataset. "
-                    "Materialize it first, or add '-j 1' to the last step "
-                    "'tile'");
+                    "an unexpected error. "
+                    "Trying adding a materialize step before the last step "
+                    "'%s', or add '-j 1' to the last step.",
+                    m_steps.back()->GetName().c_str());
                 return false;
             }
+            return ret;
         }
-        std::string outString;
-        if (SaveGDALGFile(std::string(), outString))
-        {
-            const char *const apszAllowedDrivers[] = {"GDALG", nullptr};
-            auto poCurDS = GDALDataset::Open(
-                outString.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR,
-                apszAllowedDrivers);
-            if (poCurDS)
-            {
-                auto &tileAlg = m_steps.back();
-                tileAlg->m_inputDataset.clear();
-                tileAlg->m_inputDataset.resize(1);
-                tileAlg->m_inputDataset[0].Set(poCurDS);
-                tileAlg->m_inputDataset[0].SetDatasetOpenedByAlgorithm();
-                poCurDS->Release();
-                ret = tileAlg->RunStep(ctxt);
-                tileAlg->m_inputDataset[0].Close();
-            }
-        }
-        return ret;
     }
 
     int countPipelinesWithProgress = 0;
diff --git a/apps/gdalalg_abstract_pipeline.h b/apps/gdalalg_abstract_pipeline.h
index 1131f0588b36..0c9153c46a88 100644
--- a/apps/gdalalg_abstract_pipeline.h
+++ b/apps/gdalalg_abstract_pipeline.h
@@ -505,8 +505,8 @@ class GDALAbstractPipelineAlgorithm CPL_NON_FINAL
                         std::vector<std::string> &nestedPipelineArgs,
                         bool forAutoComplete);
 
-    bool SaveGDALGFile(const std::string &outFilename,
-                       std::string &outString) const;
+    bool SaveGDALGIntoFileOrString(const std::string &outFilename,
+                                   std::string &outString) const;
 
     virtual std::unique_ptr<GDALAbstractPipelineAlgorithm>
     CreateNestedPipeline() const = 0;
diff --git a/apps/gdalalg_materialize.cpp b/apps/gdalalg_materialize.cpp
index b39d90ecb6d9..e9bb1a4ace86 100644
--- a/apps/gdalalg_materialize.cpp
+++ b/apps/gdalalg_materialize.cpp
@@ -45,6 +45,11 @@ GDALMaterializeRasterAlgorithm::GDALMaterializeRasterAlgorithm()
 
     AddCreationOptionsArg(&m_creationOptions);
     AddOverwriteArg(&m_overwrite);
+
+    AddArg(ARG_NAME_REOPEN_AND_DO_NOT_EARLY_DELETE, 0,
+           _("Reopen after materialization and do not early deleted"),
+           &m_reopenAndDoNotEarlyDelete)
+        .SetHidden();
 }
 
 /************************************************************************/
@@ -72,9 +77,10 @@ bool GDALMaterializeRasterAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
     }
 
     std::string filename = m_outputDataset.GetName();
-    const bool autoDeleteFile =
-        filename.empty() && !EQUAL(m_format.c_str(), "MEM");
-    if (autoDeleteFile)
+    const bool autoDeleteFile = !m_reopenAndDoNotEarlyDelete &&
+                                filename.empty() &&
+                                !EQUAL(m_format.c_str(), "MEM");
+    if (filename.empty() && !EQUAL(m_format.c_str(), "MEM"))
     {
         filename = CPLGenerateTempFilenameSafe(nullptr);
 
@@ -118,7 +124,8 @@ bool GDALMaterializeRasterAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
     bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None;
     if (poOutDS)
     {
-        if (poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
+        if (m_reopenAndDoNotEarlyDelete ||
+            poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
         {
             ok = poOutDS->Close() == CE_None;
             poOutDS.reset();
@@ -229,9 +236,10 @@ bool GDALMaterializeVectorAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
     }
 
     std::string filename = m_outputDataset.GetName();
-    const bool autoDeleteFile =
-        filename.empty() && !EQUAL(m_format.c_str(), "MEM");
-    if (autoDeleteFile)
+    const bool autoDeleteFile = !m_reopenAndDoNotEarlyDelete &&
+                                filename.empty() &&
+                                !EQUAL(m_format.c_str(), "MEM");
+    if (filename.empty() && !EQUAL(m_format.c_str(), "MEM"))
     {
         filename = CPLGenerateTempFilenameSafe(nullptr);
 
@@ -301,7 +309,8 @@ bool GDALMaterializeVectorAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
     bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None;
     if (poOutDS)
     {
-        if (poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
+        if (m_reopenAndDoNotEarlyDelete ||
+            poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
         {
             ok = poOutDS->Close() == CE_None;
             poOutDS.reset();
diff --git a/apps/gdalalg_materialize.h b/apps/gdalalg_materialize.h
index 41b9a4a88f49..113e94044df9 100644
--- a/apps/gdalalg_materialize.h
+++ b/apps/gdalalg_materialize.h
@@ -33,6 +33,9 @@ class GDALMaterializeStepAlgorithm /* non final */
         "Materialize a piped dataset on disk to increase the efficiency of the "
         "following steps.";
 
+    static constexpr const char *ARG_NAME_REOPEN_AND_DO_NOT_EARLY_DELETE =
+        "reopen-and-do-not-early-delete";
+
     bool IsNativelyStreamingCompatible() const override
     {
         return false;
@@ -55,6 +58,8 @@ class GDALMaterializeStepAlgorithm /* non final */
     {
         return nDatasetType;
     }
+
+    bool m_reopenAndDoNotEarlyDelete = false;
 };
 
 /************************************************************************/


Test patch:
diff --git a/autotest/utilities/test_gdalalg_raster_tile.py b/autotest/utilities/test_gdalalg_raster_tile.py
index e6060b791f85..b6fc8ea48ad7 100755
--- a/autotest/utilities/test_gdalalg_raster_tile.py
+++ b/autotest/utilities/test_gdalalg_raster_tile.py
@@ -2301,3 +2301,37 @@ def test_gdalalg_raster_tile_overview_selection(tmp_vsimem):
 
     ds = gdal.Open(tmp_vsimem / "0/0/0.png")
     assert ds.GetRasterBand(1).ComputeRasterMinMax() == (127, 127)
+
+
+def test_gdalalg_raster_tile_pipeline_materialize_explicit_filename(tmp_vsimem):
+
+    gdal.alg.pipeline(
+        pipeline=f"read ../gdrivers/data/small_world.tif ! materialize --output {tmp_vsimem}/tmp.tif ! tile {tmp_vsimem}"
+    )
+
+    ds = gdal.Open(tmp_vsimem / "0/0/0.png")
+    assert ds.GetRasterBand(1).ComputeRasterMinMax() == (0, 255)
+
+    ds = gdal.Open(tmp_vsimem / "tmp.tif")
+    assert ds.RasterXSize == 400
+
+
+def test_gdalalg_raster_tile_pipeline_materialize_no_explicit_filename(tmp_vsimem):
+
+    gdal.alg.pipeline(
+        pipeline=f"read ../gdrivers/data/small_world.tif ! materialize ! tile {tmp_vsimem}"
+    )
+
+    ds = gdal.Open(tmp_vsimem / "0/0/0.png")
+    assert ds.GetRasterBand(1).ComputeRasterMinMax() == (0, 255)
+
+
+def test_gdalalg_raster_tile_pipeline_materialize_not_second_to_lat(tmp_vsimem):
+
+    with pytest.raises(
+        Exception,
+        match="Cannot execute this pipeline in parallel mode due to the presence of a materialize step that has a 'output' argument and is not immediately before the last step",
+    ):
+        gdal.alg.pipeline(
+            pipeline=f"read ../gdrivers/data/small_world.tif ! materialize --output {tmp_vsimem}/tmp.tif ! edit ! tile {tmp_vsimem}"
+        )