Repository: 3liz/qgis-wfsOutputExtension
Pull request #82: Fix response streaming
Author: login: dmarteau
Labels: name: enhancement
Created: 2025-03-20T20:42:49Z | Merged: 2025-03-24T16:52:20Z

The goal of this PR is to fix how response is handled. In the actual state, content is read at least twice in memory before being
sent to wire.

The PR comes in several parts:
- [x] Fix consistency in library usage by avoding using both Qt and python library for files and temporary. 
- [x] Isolate request contextual data. 
- [x] Use proper return code on error 
- [x] Stream response data

No api change.


Diff:
diff --git a/Makefile b/Makefile
index dacbf53..6ae7e91 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,22 @@
 .PHONY: tests
 
+PYTHON_PKG=wfsOutputExtension
+
 test:
 	$(MAKE) -C tests test
+
+lint:
+	@ruff check --output-format=concise $(PYTHON_PKG)
+
+lint-preview:
+	@ruff check --preview $(PYTHON_PKG)
+
+lint-fix:
+	@ruff check --preview --fix $(PYTHON_PKG)
+
+install-dev:
+	$(MAKE) -C tests install-requirements
+	pip install -U --upgrade-strategy=eager ruff mypy
+
+typing:
+	mypy --config-file=mypy.ini -p $(PYTHON_PKG)
diff --git a/mypy.ini b/mypy.ini
new file mode 100644
index 0000000..2f644c6
--- /dev/null
+++ b/mypy.ini
@@ -0,0 +1,11 @@
+
+[mypy]
+python_version = 3.9
+allow_redefinition = true
+
+# The following packages do not
+# define proper type annotations
+
+[mypy-qgis.*]
+ignore_missing_imports = true
+
diff --git a/ruff.toml b/ruff.toml
new file mode 100644
index 0000000..9d328ad
--- /dev/null
+++ b/ruff.toml
@@ -0,0 +1,38 @@
+# Ruff configuration
+# See https://docs.astral.sh/ruff/configuration/
+line-length = 120
+target-version = "py39"
+
+[lint]
+extend-select = ["E", "F", "I", "ANN", "W", "T", "COM", "RUF"]
+ignore = ["ANN002", "ANN003"]
+
+[format]
+indent-style = "space"
+
+[lint.per-file-ignores]
+"tests/*" = ["T201"]
+"pyqgisservercontrib/core/componentmanager.py" = ["ANN401"]
+
+[lint.isort]
+lines-between-types = 1
+known-first-party = [
+    "wfsOutputExtension",
+]
+section-order = [
+    "future",
+    "standard-library",
+    "third-party",
+    "qgis",
+    "first-party",
+    "local-folder",
+]
+
+[lint.isort.sections]
+qgis = ["qgis"]
+
+[lint.flake8-annotations]
+ignore-fully-untyped = true
+suppress-none-returning = true
+suppress-dummy-args = true
+
diff --git a/tests/Makefile b/tests/Makefile
index 06ef891..3985d03 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,5 +1,9 @@
 SHELL:=bash
 
+DEPTH=..
+
+topsrcdir=$(shell realpath $(DEPTH))
+
 COMMITID=$(shell git rev-parse --short HEAD)
 
 install-requirements:
@@ -9,7 +13,9 @@ install-requirements:
 test: export QGIS_SERVER_LOG_FILE="/src/tests/qgis_server.log"
 test: export QGIS_SERVER_LOG_LEVEL=0
 test: install-requirements
-	pytest -v --qgis-plugins=..
+	pytest -v --qgis-plugins=$(topsrcdir)
+clean:
+	rm -r ./tmp/test-*
 
 
 # Use docker image
diff --git a/tests/conftest.py b/tests/conftest.py
index 62f55bd..a3884fa 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -6,6 +6,10 @@
 import tempfile
 import warnings
 
+
+from pathlib import Path
+
+
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=DeprecationWarning)
     try:
@@ -18,9 +22,6 @@
 
 from qgis.PyQt import Qt
 
-logging.basicConfig(stream=sys.stderr)
-logging.disable(logging.NOTSET)
-
 LOGGER = logging.getLogger('server')
 LOGGER.setLevel(logging.DEBUG)
 
@@ -71,7 +72,7 @@ def pytest_sessionstart(session):
     qgis_application.initQgis()
 
     # Install logger hook
-    install_logger_hook(verbose=True)
+    install_logger_hook()
 
 
 def pytest_sessionfinish(session, exitstatus):
@@ -95,9 +96,10 @@ def pytest_sessionfinish(session, exitstatus):
 
 class OWSResponse:
 
-    def __init__(self, resp: QgsBufferServerResponse) -> None:
+    def __init__(self, resp: QgsBufferServerResponse, dir: Path) -> None:
         self._resp = resp
         self._xml = None
+        self._dir = dir
 
     @property
     def xml(self) -> 'xml':
@@ -110,7 +112,11 @@ def content(self) -> bytes:
         return bytes(self._resp.body())
 
     def file(self, extension) -> str:
-        path = os.path.join(tempfile.mkdtemp(), 'test.' + extension)
+        _, path = tempfile.mkstemp(
+            prefix="test-",
+            suffix=f".{extension}",
+            dir=str(self._dir.joinpath('tmp')),
+        )
         f = open(path, 'wb')
         f.write(self.content)
         f.close()
@@ -133,6 +139,7 @@ def xpath_text(self, path: str) -> str:
         return ' '.join(e.text for e in self.xpath(path))
 
 
+
 @pytest.fixture(scope='session')
 def client(request):
     """ Return a qgis server instance
@@ -148,6 +155,7 @@ def __init__(self) -> None:
             # Activate debug headers
             os.environ['QGIS_WMTS_CACHE_DEBUG_HEADERS'] = 'true'
 
+            self.rootdir  = request.config.rootdir
             self.datapath = request.config.rootdir.join('data')
             self.server = QgsServer()
 
@@ -175,7 +183,7 @@ def get(self, query: str, project: str=None) -> OWSResponse:
             else:
                 qgsproject = None
             self.server.handleRequest(request, response, project=qgsproject)
-            return OWSResponse(response)
+            return OWSResponse(response, dir=Path(self.rootdir.strpath))
 
     return _Client()
 
@@ -273,7 +281,7 @@ def load_plugins(serverIface: 'QgsServerInterface') -> None:
 # Logger hook
 #
 
-def install_logger_hook( verbose: bool=False ) -> None:
+def install_logger_hook() -> None:
     """ Install message log hook
     """
     from qgis.core import Qgis, QgsApplication
@@ -285,9 +293,7 @@ def writelogmessage(message, tag, level):
             LOGGER.warning(arg)
         elif level == Qgis.Critical:
             LOGGER.error(arg)
-        elif verbose:
-            # Qgis is somehow very noisy
-            # log only if verbose is set
+        else:
             LOGGER.info(arg)
 
     messageLog = QgsApplication.messageLog()
diff --git a/tests/pytest.ini b/tests/pytest.ini
index d7e467b..cb983bd 100644
--- a/tests/pytest.ini
+++ b/tests/pytest.ini
@@ -1,6 +1,8 @@
 [pytest] 
 addopts= --junit-xml=__output__/junit.xml
 junit_family=xunit2
+log_cli=1
+log_cli_level=error
 norecursedirs= 
     tests/data 
     .local 
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 041db5d..aec3ae2 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,3 +1,2 @@
 pytest
 lxml
-
diff --git a/tests/test_feature.py b/tests/test_feature.py
index 61d7b30..0b8227e 100644
--- a/tests/test_feature.py
+++ b/tests/test_feature.py
@@ -85,7 +85,7 @@ def test_getfeature_kml(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/vnd.google-earth.kml+xml' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/vnd.google-earth.kml+xml' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('kml'), 'LIBKML')
     assert layer.crs().authid() == 'EPSG:4326'
 
@@ -122,7 +122,7 @@ def test_getfeature_gpkg(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/geopackage+vnd.sqlite3' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/geopackage+vnd.sqlite3' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('gpkg'), 'GPKG')
     _test_list(
         layer.fields().names(),
@@ -152,7 +152,7 @@ def test_getfeature_gpx(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/gpx+xml' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/gpx+xml' in rv.headers.get('Content-Type'), rv.headers
 
     # Lines is translated as routes
     layer = _test_vector_layer(rv.file('gpx') + '|layername=routes', 'GPX')
@@ -195,7 +195,7 @@ def test_getfeature_ods(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/vnd.oasis.opendocument.spreadsheet' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/vnd.oasis.opendocument.spreadsheet' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('ods'), 'ODS')
     _test_list(
         layer.fields().names(),
@@ -264,7 +264,7 @@ def test_getfeature_excel(client):
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
     expected = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-    assert expected in rv.headers.get('Content-type'), rv.headers
+    assert expected in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('xlsx'), 'XLSX')
     _test_list(
         layer.fields().names(),
@@ -304,7 +304,7 @@ def test_getfeature_csv(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'text/csv' in rv.headers.get('Content-type'), rv.headers
+    assert 'text/csv' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('csv'), 'CSV')
     _test_list(
         layer.fields().names(),
@@ -345,7 +345,7 @@ def test_getfeature_shapefile(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert "application/x-zipped-shp" in rv.headers.get('Content-type'), rv.headers
+    assert "application/x-zipped-shp" in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer('/vsizip/' + rv.file('zip'), 'ESRI Shapefile')
     # shapefile is splitting trailing_zero to trailing_z
     _test_list(layer.fields().names(), ['gml_id', 'id', 'trailing_z', 'name', 'comment', 'date_time', 'date'])
@@ -374,7 +374,7 @@ def test_getfeature_tab(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/x-zipped-tab' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/x-zipped-tab' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer('/vsizip/' + rv.file('zip') + '/lines.tab', 'MapInfo File')
     _test_list(
         layer.fields().names(),
@@ -404,7 +404,7 @@ def test_getfeature_mif(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/x-zipped-mif' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/x-zipped-mif' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer('/vsizip/' + rv.file('zip') + '/lines.mif', 'MapInfo File')
     _test_list(
         layer.fields().names(),
@@ -434,7 +434,7 @@ def test_getfeature_layer_name_with_accent(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'text/csv' in rv.headers.get('Content-type'), rv.headers
+    assert 'text/csv' in rv.headers.get('Content-Type'), rv.headers
     assert 'attachment; filename="éàIncê.csv"' in rv.headers.get('Content-Disposition')
 
 
@@ -480,7 +480,7 @@ def test_getfeature_fgb(client):
     )
     rv = client.get(query_string, PROJECT)
     assert rv.status_code == 200
-    assert 'application/x-fgb' in rv.headers.get('Content-type'), rv.headers
+    assert 'application/x-fgb' in rv.headers.get('Content-Type'), rv.headers
     layer = _test_vector_layer(rv.file('fgb'), storage='FlatGeobuf')
     _test_list(
         layer.fields().names(),
diff --git a/tests/tmp/.gitignore b/tests/tmp/.gitignore
new file mode 100644
index 0000000..ebb0ef1
--- /dev/null
+++ b/tests/tmp/.gitignore
@@ -0,0 +1 @@
+test-*
diff --git a/wfsOutputExtension/__init__.py b/wfsOutputExtension/__init__.py
index 832bd41..68449aa 100644
--- a/wfsOutputExtension/__init__.py
+++ b/wfsOutputExtension/__init__.py
@@ -52,7 +52,7 @@ def __init__(self, server_iface):
             self.logger.log_exception(e)
             self.logger.critical('Error while calling the API stats')
 
-        from .wfs_filter import WFSFilter  # NOQA ABS101
+        from .wfs_filter import WFSFilter
         server_iface.registerFilter(WFSFilter(server_iface), 50)
 
 
diff --git a/wfsOutputExtension/definitions.py b/wfsOutputExtension/definitions.py
index 1ddb2a6..6227ba8 100644
--- a/wfsOutputExtension/definitions.py
+++ b/wfsOutputExtension/definitions.py
@@ -7,8 +7,6 @@
 
 PLUGIN = 'WfsOutputExtension'
 
-# With Python 3.7, switch to dataclass
-
 
 class Format(NamedTuple):
     content_type: str
diff --git a/wfsOutputExtension/tools.py b/wfsOutputExtension/tools.py
index 2bd8763..c3ba471 100644
--- a/wfsOutputExtension/tools.py
+++ b/wfsOutputExtension/tools.py
@@ -30,7 +30,7 @@ def version() -> str:
         return config["general"]["version"]
 
 
-def to_bool(val: Union[str, int, float, bool], default_value: bool = True) -> bool:
+def to_bool(val: Union[str, int, float, bool, None], default_value: bool = True) -> bool:
     """ Convert config value to boolean """
     if isinstance(val, str):
         # For string, compare lower value to True string
diff --git a/wfsOutputExtension/wfs_filter.py b/wfsOutputExtension/wfs_filter.py
index 6df65a1..e4bfa58 100644
--- a/wfsOutputExtension/wfs_filter.py
+++ b/wfsOutputExtension/wfs_filter.py
@@ -4,11 +4,11 @@
 
 import os
 import tempfile
-import time
 
-from os import listdir, makedirs, remove
-from os.path import basename, exists, join, splitext
+from dataclasses import dataclass
+from io import BufferedReader
 from pathlib import Path
+from typing import Optional
 from xml.dom import minidom
 
 from qgis.core import (
@@ -18,15 +18,17 @@
     QgsVectorFileWriter,
     QgsVectorLayer,
 )
-from qgis.PyQt.QtCore import QFile, QTemporaryFile
 from qgis.server import (
     QgsBufferServerRequest,
     QgsBufferServerResponse,
+    QgsRequestHandler,
+    QgsServerException,
     QgsServerFilter,
+    QgsServerInterface,
     QgsServerRequest,
 )
 
-from wfsOutputExtension.definitions import OutputFormats
+from wfsOutputExtension.definitions import Format, OutputFormats
 from wfsOutputExtension.logging import Logger, log_function
 
 
@@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUE_STR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    num_bytes = stream.readinto(data)
+    while num_bytes:
+        handler.appendBody(data[:num_bytes])
+        handler.sendResponse()  # Call flush()
+        num_bytes = stream.readinto(data)
+
+
 class WFSFilter(QgsServerFilter):
     @log_function
-    def __init__(self, server_iface):
+    def __init__(self, server_iface: QgsServerInterface) -> None:
         super().__init__(server_iface)
         self.server_iface = server_iface
         self.logger = Logger()
-
-        self.format = None
-        self.typename = ""
-        self.filename = ""
-        self.base_name_target = None
-        self.all_gml = False
-
-        self.temp_dir = join(tempfile.gettempdir(), 'QGIS_WfsOutputExtension')
-        # self.temp_dir = '/src/'  # Use ONLY in debug for docker
-
-        self.debug_mode = os.environ.get(
-            'DEBUG_WFSOUTPUTEXTENSION', 'false').upper() in ('TRUE', '1')
-
-        # Fix race-condition if multiple servers are run concurrently
-        makedirs(self.temp_dir, exist_ok=True)
-        self.logger.info(f'Temporary directory is {self.temp_dir}')
+        self.debug_mode = os.getenv("DEBUG_WFSOUTPUTEXTENSION", "").lower() in TRUE_STR
+        # NOTE: we need to hold a reference to the context
+        # because of the QgsServerFilter implementation
+        self.context: Optional[Context] = None
 
     @log_function
     def requestReady(self):
-        self.format = None
-        self.all_gml = False
+
+        self.context = None
 
         handler = self.serverInterface().requestHandler()
         params = handler.parameterMap()
@@ -78,38 +104,67 @@ def requestReady(self):
 
         # verifying format
         output_format = params.get('OUTPUTFORMAT', '').lower()
-
         format_definition = OutputFormats.find(output_format)
         if not format_definition:
+            # Fallback to default
             return
 
         handler.setParameter('OUTPUTFORMAT', 'GML2')
-        self.format = output_format
-        self.typename = params.get('TYPENAME', '')
-        self.filename = f'gml_features_{time.time()}'
+
+        # Create temporary directory
+        if self.debug_mode:
+            lock_dir = None
+            temp_dir = Path(tempfile.mkdtemp(prefix=TMPDIR_PREFIX))
+        else:
+            # Removed when deleted
+            lock_dir = tempfile.TemporaryDirectory(prefix=TMPDIR_PREFIX)
+            temp_dir = Path(lock_dir.name)
+
+        base_name_target = f"to-{output_format}"
+        request_id = handler.requestHeader("X-Request-Id")
+
+        # Create the request context
+        self.context = Context(
+            output_format=output_format,
+            format_definition=format_definition,
+            # FIXME: typename is used as base name for files
+            # but TYPENAME may be a comma separated list of names
+            # which is not appropriate nor is the empty string.
+            typename=params.get('TYPENAME', ''),
+            filename='gml_features',
+            base_name_target=base_name_target,
+            lock_dir=lock_dir,
+            temp_dir=temp_dir,
+            request_id=request_id,
+        )
+
+        self.logger.info(f"REQ_ID:{request_id or '-'}\t request accepted")
 
         # set headers
         handler.clear()
         handler.setResponseHeader('Content-Type', format_definition.content_type)
         if format_definition.zip:
             handler.setResponseHeader(
-                'Content-Disposition', f'attachment; filename="{self.typename}.zip"')
+                'Content-Disposition', f'attachment; filename="{self.context.typename}.zip"')
         else:
             handler.setResponseHeader(
                 'Content-Disposition',
-                f'attachment; filename="{self.typename}.{format_definition.filename_ext}"')
+                f'attachment; filename="{self.context.typename}.{format_definition.filename_ext}"')
 
-    def sendResponse(self):
-        # if the format is null, nothing to do
-        if not self.format:
+    def sendResponse(self) -> None:
+        # if the context is null, nothing to do
+        if not self.context or self.context.has_errors:
             return
 
+        context = self.context
+
         handler = self.serverInterface().requestHandler()
 
         # write body in GML temp file
         data = handler.body().data().decode('utf8')
-        output_file = join(self.temp_dir, f'{self.filename}.gml')
-        with open(output_file, 'ab') as f:
+        output_file = context.temp_dir.joinpath(f'{context.filename}.gml')
+
+        with output_file.open('ab') as f:
             if data.find('xsi:schemaLocation') == -1:
                 # noinspection PyTypeChecker
                 f.write(handler.body())
@@ -119,155 +174,147 @@ def sendResponse(self):
                 data = re.sub(r'xsi:schemaLocation=\".*\"', 'xsi:schemaLocation=""', data)
                 f.write(data.encode('utf8'))
 
-        format_definition = OutputFormats.find(self.format)
+        format_definition = context.format_definition
 
         # change the headers
         # update content-type and content-disposition
         if not handler.headersSent():
             handler.clear()
-            handler.setResponseHeader('Content-type', format_definition.content_type)
+            handler.setResponseHeader('Content-Type', format_definition.content_type)
             if format_definition.zip:
                 handler.setResponseHeader(
-                    'Content-Disposition', f'attachment; filename="{self.typename}.zip"')
+                    'Content-Disposition', f'attachment; filename="{context.typename}.zip"')
             else:
                 handler.setResponseHeader(
                     'Content-Disposition',
-                    f'attachment; filename="{self.typename}.{format_definition.filename_ext}"')
+                    f'attachment; filename="{context.typename}.{format_definition.filename_ext}"')
         else:
             handler.clearBody()
 
         if data.rstrip().endswith('</wfs:FeatureCollection>'):
-            # all the gml has been intercepted
-            self.all_gml = True
-            self.send_output_file(handler)
+            try:
+                # all the gml has been intercepted
+                context.all_gml = True
+                self.send_output_file(handler, context)
+            except Exception as e:
+                self.logger.log_exception(e)
+                context.has_errors = True
+                handler.clearBody()
+                handler.setServiceException(QgsServerException("Internal error", 500))
 
     @log_function
-    def send_output_file(self, handler):
+    def send_output_file(self, handler: QgsRequestHandler, context: Context) -> bool:
         """ Process the request.
 
         :raise ProcessingRequestException when there is an error
         """
-        format_definition = OutputFormats.find(self.format)
+        format_definition = context.format_definition
         self.logger.info(f"WFS request to get format {format_definition.ogr_provider}")
 
         # Fetch the XSD
-        type_name = handler.parameterMap().get('TYPENAME', '')
+        type_name = context.typename
         headers = handler.requestHeaders()
-        result = self.xsd_for_layer(type_name, headers)
+        result = self.xsd_for_layer(type_name, headers, context)
 
         # read the GML
-        gml_path = join(self.temp_dir, f'{self.filename}.gml')
-        if result:
-            gml_path += '|option:FORCE_SRS_DETECTION=YES'
-        output_layer = QgsVectorLayer(gml_path, 'qgis_server_wfs_features', 'ogr')
+        gml_path = context.temp_dir.joinpath(f'{context.filename}.gml')
+        gml_url = f"{gml_path}|option:FORCE_SRS_DETECTION=YES" if result else f"{gml_path}"
+        output_layer = QgsVectorLayer(gml_url, 'qgis_server_wfs_features', 'ogr')
 
-        self.logger.info(f"Temporary GML file is {gml_path}")
+        self.logger.info(f"Temporary GML file is {gml_url}")
 
         if not output_layer.isValid():
-            handler.appendBody(b'')
-            raise ProcessingRequestException(f'Output layer {gml_path} is not valid.')
+            raise ProcessingRequestException(f'Output layer {gml_url} is not valid.')
 
         # Temporary file where to write the output
-        temporary = QTemporaryFile(
-            join(self.temp_dir, f'to-{self.format}-XXXXXX.{format_definition.filename_ext}'))
-        temporary.open()
-        output_file = temporary.fileName()
-        temporary.remove()  # Fix issue #18
+        output_file = context.temp_dir.joinpath(
+            f"{context.base_name_target}.{format_definition.filename_ext}",
+        )
+
         self.logger.info(f"Temporary {format_definition.filename_ext} file is {output_file}")
-        self.base_name_target = basename(splitext(output_file)[0])
-
-        try:
-            # create save options
-            options = QgsVectorFileWriter.SaveVectorOptions()
-            # driver name
-            options.driverName = format_definition.ogr_provider
-            # file encoding
-            options.fileEncoding = 'utf-8'
-
-            # coordinate transformation
-            if format_definition.force_crs:
-                # noinspection PyArgumentList
-                options.ct = QgsCoordinateTransform(
-                    output_layer.crs(),
-                    QgsCoordinateReferenceSystem(format_definition.force_crs),
-                    QgsProject.instance())
-
-            # datasource options
-            if format_definition.ogr_datasource_options:
-                options.datasourceOptions = format_definition.ogr_datasource_options
-
-            # write file
+
+        # create save options
+        options = QgsVectorFileWriter.SaveVectorOptions()
+        # driver name
+        options.driverName = format_definition.ogr_provider
+        # file encoding
+        options.fileEncoding = 'utf-8'
+
+        # coordinate transformation
+        if format_definition.force_crs:
             # noinspection PyArgumentList
-            write_result, error_message, _, _ = QgsVectorFileWriter.writeAsVectorFormatV3(
-                output_layer,
-                output_file,
-                QgsProject.instance().transformContext(),
-                options)
-
-            # noinspection PyUnresolvedReferences
-            if write_result != QgsVectorFileWriter.NoError:
-                handler.appendBody(b'')
-                self.logger.critical(error_message)
-                return False
-
-        except Exception:
+            options.ct = QgsCoordinateTransform(
+                output_layer.crs(),
+                QgsCoordinateReferenceSystem(format_definition.force_crs),
+                QgsProject.instance())
+
+        # datasource options
+        if format_definition.ogr_datasource_options:
+            options.datasourceOptions = format_definition.ogr_datasource_options
+
+        # write file
+        # noinspection PyArgumentList
+        write_result, error_message, _, _ = QgsVectorFileWriter.writeAsVectorFormatV3(
+            output_layer,
+            str(output_file),
+            QgsProject.instance().transformContext(),
+            options)
+
+        # noinspection PyUnresolvedReferences
+        if write_result != QgsVectorFileWriter.NoError:
             handler.appendBody(b'')
-            raise
+            self.logger.critical(error_message)
+            return False
 
         if format_definition == OutputFormats.Shp:
             # For SHP, we add the CPG, #55
-            cpg_file = Path(self.temp_dir).joinpath(self.base_name_target + '.cpg')
-            with open(cpg_file, 'w', encoding='utf8') as f:
+            cpg_file = context.temp_dir.joinpath(f"{context.base_name_target}.cpg")
+            with cpg_file.open('w', encoding='utf8') as f:
                 f.write(f"{options.fileEncoding}\n")
 
         if format_definition.zip:
             # compress files
             import zipfile
             try:
-                import zlib  # NOQA
+                import zlib  # noqa
                 compression = zipfile.ZIP_DEFLATED
             except ImportError:
                 compression = zipfile.ZIP_STORED
 
             # create the zip file
-            base_filename = splitext(output_file)[0]
-            zip_file_path = join(self.temp_dir, f'{base_filename}.zip')
+            zip_file_path = context.temp_dir.joinpath(f"{context.base_name_target}.zip")
             self.logger.info(f"Zipping the output in {zip_file_path}")
             with zipfile.ZipFile(zip_file_path, 'w') as zf:
 
                 # Add the main file
-                arc_filename = f'{self.typename}.{format_definition.filename_ext}'
+                arc_filename = f'{context.typename}.{format_definition.filename_ext}'
                 zf.write(
                     output_file,
                     compress_type=compression,
-                    arcname=arc_filename)
+                    arcname=arc_filename,
+                )
 
                 for extension in format_definition.ext_to_zip:
-                    file_path = join(self.temp_dir, f'{base_filename}.{extension}')
-                    if exists(file_path):
-                        arc_filename = f'{self.typename}.{extension}'
+                    file_path = context.temp_dir.joinpath(f'{context.base_name_target}.{extension}')
+                    if file_path.exists():
+                        arc_filename = f'{context.typename}.{extension}'
                         zf.write(
                             file_path,
                             compress_type=compression,
-                            arcname=arc_filename)
+                            arcname=arc_filename,
+                        )
 
                 zf.close()
 
-            f = QFile(zip_file_path)
-            # noinspection PyUnresolvedReferences
-            if f.open(QFile.ReadOnly):
-                ba = f.readAll()
-                handler.appendBody(ba)
+            with zip_file_path.open("rb") as f:
+                stream_bytes(handler, f)
                 return True
 
         else:
             self.logger.info("Sending the output file")
             # return the file created without zip
-            f = QFile(output_file)
-            # noinspection PyUnresolvedReferences
-            if f.open(QFile.ReadOnly):
-                ba = f.readAll()
-                handler.appendBody(ba)
+            with output_file.open("rb") as f:
+                stream_bytes(handler, f)
                 return True
 
         handler.appendBody(b'')
@@ -275,7 +322,8 @@ def send_output_file(self, handler):
         return False
 
     @log_function
-    def xsd_for_layer(self, type_name: str, headers: dict) -> bool:
+    def xsd_for_layer(self, type_name: str, headers: dict, context: Context) -> bool:
+
         """ Get the XSD describing the layer. """
         # noinspection PyArgumentList
         project = QgsProject.instance()
@@ -294,7 +342,7 @@ def xsd_for_layer(self, type_name: str, headers: dict) -> bool:
             query_string,
             QgsServerRequest.GetMethod,
             headers,
-            None
+            None,
         )
         service = self.server_iface.serviceRegistry().getService('WFS', '1.0.0')
         response = QgsBufferServerResponse()
@@ -312,13 +360,22 @@ def xsd_for_layer(self, type_name: str, headers: dict) -> bool:
             self.logger.critical(f"HTTP error when requesting the XSD : return {response.statusCode()}")
             return False
 
-        with open(join(self.temp_dir, f'{self.filename}.xsd'), 'w') as f:
+        with context.temp_dir.joinpath(f'{context.filename}.xsd').open('w') as f:
             f.write(content)
 
         return True
 
     @log_function
-    def responseComplete(self):
+    def responseComplete(self) -> None:
+
+        context = self.context
+
+        # Remove current context
+        self.context = None
+
+        if context and context.has_errors:
+            return
+
         # Update the WFS capabilities
         # by adding ResultFormat to GetFeature
         handler = self.serverInterface().requestHandler()
@@ -332,40 +389,19 @@ def responseComplete(self):
         if request not in ('GETCAPABILITIES', 'GETFEATURE'):
             return
 
-        if request == 'GETFEATURE' and self.format:
-            if not self.all_gml:
+        if request == 'GETFEATURE' and context:
+            if not context.all_gml:
                 try:
                     # all the gml has not been intercepted in sendResponse
                     handler.clearBody()
-                    with open(join(self.temp_dir, f'{self.filename}.gml'), 'a') as f:
+                    with context.temp_dir.joinpath(f'{context.filename}.gml').open('a') as f:
                         f.write('</wfs:FeatureCollection>')
-                    self.send_output_file(handler)
+                    self.send_output_file(handler, context)
                 except Exception as e:
                     self.logger.critical("Critical exception when processing the request :")
                     self.logger.log_exception(e)
-                finally:
-                    # Find all files associated with the request and remove them
-                    for file in listdir(self.temp_dir):
-                        if file.startswith(self.filename):  # GML, GFS
-                            file_path = join(self.temp_dir, file)
-                            if self.debug_mode:
-                                self.logger.info(
-                                    f"DEBUG_WFSOUTPUTEXTENSION is on, not removing {file_path}")
-                            else:
-                                remove(file_path)
-
-                        if file.startswith(self.base_name_target):  # Target extension, ZIP
-                            file_path = join(self.temp_dir, file)
-                            if self.debug_mode:
-                                self.logger.info(
-                                    f"DEBUG_WFSOUTPUTEXTENSION is on, not removing {file_path}")
-                            else:
-                                remove(file_path)
-
-            self.format = None
-            self.all_gml = False
-            self.filename = None
-            self.base_name_target = None
+                    handler.clearBody()
+                    handler.setServiceException(QgsServerException("Internal error", 500))
             return
 
         if request == 'GETCAPABILITIES':


Reviews:
user: login: Gustry
body: Minor review (just to avoid warnings about typo with spell checker)
LGTM for the code cleanup, thanks.

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T11:02:11Z
user: login: dmarteau
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T13:54:14Z
user: login: dmarteau
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T14:00:20Z
user: login: dmarteau
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T14:00:34Z
user: login: Gustry
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T14:22:00Z
user: login: dmarteau
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T15:26:01Z

Review comments:
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
path: wfsOutputExtension/wfs_filter.py
user: login: Gustry
body: ```suggestion
TRUE_STR = ('yes', 'true', '1')
```
To avoid a warning about typo (and other occurences)
created_at: 2025-03-24T10:49:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
path: wfsOutputExtension/wfs_filter.py
user: login: Gustry
body: ```suggestion
    n_bytes = stream.readinto(data)
```
Same as above, to avoid some warnings about typo
created_at: 2025-03-24T10:52:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
+    while nbytes:
+        handler.appendBody(data[:nbytes])
+        handler.sendResponse()  # Call flush()
+        nbytes = stream.readinto(data)
+
+
 class WFSFilter(QgsServerFilter):
     @log_function
-    def __init__(self, server_iface):
+    def __init__(self, server_iface: QgsServerInterface) -> None:
         super().__init__(server_iface)
         self.server_iface = server_iface
         self.logger = Logger()
-
-        self.format = None
-        self.typename = ""
-        self.filename = ""
-        self.base_name_target = None
-        self.all_gml = False
-
-        self.temp_dir = join(tempfile.gettempdir(), 'QGIS_WfsOutputExtension')
-        # self.temp_dir = '/src/'  # Use ONLY in debug for docker
-
-        self.debug_mode = os.environ.get(
-            'DEBUG_WFSOUTPUTEXTENSION', 'false').upper() in ('TRUE', '1')
-
-        # Fix race-condition if multiple servers are run concurrently
-        makedirs(self.temp_dir, exist_ok=True)
-        self.logger.info(f'Temporary directory is {self.temp_dir}')
+        self.debug_mode = os.getenv("DEBUG_WFSOUTPUTEXTENSION", "").lower() in TRUESTR
+        # XXX: we need to hold a reference to the context
path: wfsOutputExtension/wfs_filter.py
user: login: Gustry
body: ```suggestion
        # We need to hold a reference to the context
```
?
created_at: 2025-03-24T10:52:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
+    while nbytes:
+        handler.appendBody(data[:nbytes])
+        handler.sendResponse()  # Call flush()
+        nbytes = stream.readinto(data)
+
+
 class WFSFilter(QgsServerFilter):
     @log_function
-    def __init__(self, server_iface):
+    def __init__(self, server_iface: QgsServerInterface) -> None:
         super().__init__(server_iface)
         self.server_iface = server_iface
         self.logger = Logger()
-
-        self.format = None
-        self.typename = ""
-        self.filename = ""
-        self.base_name_target = None
-        self.all_gml = False
-
-        self.temp_dir = join(tempfile.gettempdir(), 'QGIS_WfsOutputExtension')
-        # self.temp_dir = '/src/'  # Use ONLY in debug for docker
-
-        self.debug_mode = os.environ.get(
-            'DEBUG_WFSOUTPUTEXTENSION', 'false').upper() in ('TRUE', '1')
-
-        # Fix race-condition if multiple servers are run concurrently
-        makedirs(self.temp_dir, exist_ok=True)
-        self.logger.info(f'Temporary directory is {self.temp_dir}')
+        self.debug_mode = os.getenv("DEBUG_WFSOUTPUTEXTENSION", "").lower() in TRUESTR
+        # XXX: we need to hold a reference to the context
path: wfsOutputExtension/wfs_filter.py
user: login: dmarteau
body: This should not be removed: `XXX`, `NOTE`, `TODO`, ... are legitimate comment annotations used to visually mark specific portion of code which requires a particular attention. They are treated as special markers and highlighted by code editeurs (Vim, etc...).


created_at: 2025-03-24T13:54:14Z
author_association: MEMBER
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
path: wfsOutputExtension/wfs_filter.py
user: login: dmarteau
body: n_bytes -> num_bytes
created_at: 2025-03-24T14:00:19Z
author_association: MEMBER
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
path: wfsOutputExtension/wfs_filter.py
user: login: dmarteau
body: Done
created_at: 2025-03-24T14:00:34Z
author_association: MEMBER
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
+    while nbytes:
+        handler.appendBody(data[:nbytes])
+        handler.sendResponse()  # Call flush()
+        nbytes = stream.readinto(data)
+
+
 class WFSFilter(QgsServerFilter):
     @log_function
-    def __init__(self, server_iface):
+    def __init__(self, server_iface: QgsServerInterface) -> None:
         super().__init__(server_iface)
         self.server_iface = server_iface
         self.logger = Logger()
-
-        self.format = None
-        self.typename = ""
-        self.filename = ""
-        self.base_name_target = None
-        self.all_gml = False
-
-        self.temp_dir = join(tempfile.gettempdir(), 'QGIS_WfsOutputExtension')
-        # self.temp_dir = '/src/'  # Use ONLY in debug for docker
-
-        self.debug_mode = os.environ.get(
-            'DEBUG_WFSOUTPUTEXTENSION', 'false').upper() in ('TRUE', '1')
-
-        # Fix race-condition if multiple servers are run concurrently
-        makedirs(self.temp_dir, exist_ok=True)
-        self.logger.info(f'Temporary directory is {self.temp_dir}')
+        self.debug_mode = os.getenv("DEBUG_WFSOUTPUTEXTENSION", "").lower() in TRUESTR
+        # XXX: we need to hold a reference to the context
path: wfsOutputExtension/wfs_filter.py
user: login: Gustry
body: Yes ok, indeed...
I would have preferred a more descriptive keyword, like `Note : We need to....`.
`XXX` is not self-explanatory.
created_at: 2025-03-24T14:22:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,33 +37,57 @@ class ProcessingRequestException(Exception):
     pass
 
 
+# Execution context, created for each request
+@dataclass
+class Context:
+    output_format: str
+    typename: str
+    filename: str
+    base_name_target: str
+    temp_dir: Path
+    format_definition: Format
+    # Optional for debugging and keep the intermediate files
+    lock_dir: Optional[tempfile.TemporaryDirectory] = None
+    all_gml: bool = False
+    has_errors: bool = False
+    request_id: str = ""
+
+
+TRUESTR = ('yes', 'true', '1')
+TMPDIR_PREFIX = "QGIS_WfsOutputExtension-"
+
+# Chunk size in bytes set to 1Mo
+CHUNK_SIZE = 1024 * 1024
+
+
+# Stream bytes
+def stream_bytes(handler: QgsRequestHandler, stream: BufferedReader):
+    # Pre-allocate input buffer and use readinto(...)
+    # NOTE: we should be able to read content directly into the internal
+    # QByteArray
+    data = bytearray(CHUNK_SIZE)
+    nbytes = stream.readinto(data)
+    while nbytes:
+        handler.appendBody(data[:nbytes])
+        handler.sendResponse()  # Call flush()
+        nbytes = stream.readinto(data)
+
+
 class WFSFilter(QgsServerFilter):
     @log_function
-    def __init__(self, server_iface):
+    def __init__(self, server_iface: QgsServerInterface) -> None:
         super().__init__(server_iface)
         self.server_iface = server_iface
         self.logger = Logger()
-
-        self.format = None
-        self.typename = ""
-        self.filename = ""
-        self.base_name_target = None
-        self.all_gml = False
-
-        self.temp_dir = join(tempfile.gettempdir(), 'QGIS_WfsOutputExtension')
-        # self.temp_dir = '/src/'  # Use ONLY in debug for docker
-
-        self.debug_mode = os.environ.get(
-            'DEBUG_WFSOUTPUTEXTENSION', 'false').upper() in ('TRUE', '1')
-
-        # Fix race-condition if multiple servers are run concurrently
-        makedirs(self.temp_dir, exist_ok=True)
-        self.logger.info(f'Temporary directory is {self.temp_dir}')
+        self.debug_mode = os.getenv("DEBUG_WFSOUTPUTEXTENSION", "").lower() in TRUESTR
+        # XXX: we need to hold a reference to the context
path: wfsOutputExtension/wfs_filter.py
user: login: dmarteau
body: Changed to NOTE (uppercase is significant)
created_at: 2025-03-24T15:26:01Z
author_association: MEMBER

PR comments:
user: login: rldhont
created_at: 2025-03-21T12:10:30Z
body: @dmarteau Is this refactoring good for `qgisMinimumVersion=3.28` ?
author_association: COLLABORATOR
user: login: dmarteau
created_at: 2025-03-21T12:15:05Z
body: > @dmarteau Is this refactoring good for `qgisMinimumVersion=3.28` ?

The tests are running against 3.28 (see checks results)
author_association: MEMBER