Repository: datalad/datalad-next

Issue:
http-downloads via `download` (`HttpUrlOperations`) result in gzipped files on disk

Comments:

Patch:
diff --git a/datalad_next/url_operations/http.py b/datalad_next/url_operations/http.py
index 8d31c03..63ddb89 100644
--- a/datalad_next/url_operations/http.py
+++ b/datalad_next/url_operations/http.py
@@ -9,7 +9,6 @@ import sys
 from typing import Dict
 import requests
 from requests_toolbelt import user_agent
-from requests_toolbelt.downloadutils.tee import tee as requests_tee
 import www_authenticate
 
 import datalad
@@ -247,24 +246,50 @@ class HttpUrlOperations(UrlOperations):
         progress_id = self._get_progress_id(from_url, to_path)
         # get download size, but not every server provides it
         try:
+            # for compressed downloads the content length refers to the
+            # compressed content
             expected_size = int(r.headers.get('content-length'))
         except (ValueError, TypeError):
+            # some responses do not have a `content-length` header,
+            # even though they HTTP200 and deliver the content.
+            # example:
+            # https://github.com/datalad/datalad-next/pull/365#issuecomment-1557114109
             expected_size = None
         self._progress_report_start(
             progress_id,
             ('Download %s to %s', from_url, to_path),
             'downloading',
+            # can be None, and that is OK
             expected_size,
         )
 
         fp = None
         props = {}
         try:
+            # we can only write to file-likes opened in bytes mode
             fp = sys.stdout.buffer if to_path is None else open(to_path, 'wb')
-            # TODO make chunksize a config item
-            for chunk in requests_tee(r, fp):
+            # we need to track how much came down the pipe for progress
+            # reporting
+            downloaded_bytes = 0
+            # TODO make chunksize a config item, 65536 is the default in
+            # requests_toolbelt
+            for chunk in r.raw.stream(amt=65536, decode_content=True):
+                # update how much data was transferred from the remote server,
+                # but we cannot use the size of the chunk for that,
+                # because content might be downloaded with transparent
+                # (de)compression. ask the download stream itself for its
+                # "position"
+                if expected_size:
+                    tell = r.raw.tell()
+                else:
+                    tell = downloaded_bytes + len(chunk)
                 self._progress_report_update(
-                    progress_id, ('Downloaded chunk',), len(chunk))
+                    progress_id,
+                    ('Downloaded chunk',),
+                    tell - downloaded_bytes,
+                )
+                fp.write(chunk)
+                downloaded_bytes = tell
                 # compute hash simultaneously
                 hasher.update(chunk)
             props.update(hasher.get_hexdigest())


Test patch:
diff --git a/datalad_next/url_operations/tests/test_http.py b/datalad_next/url_operations/tests/test_http.py
index 817d756..9850b00 100644
--- a/datalad_next/url_operations/tests/test_http.py
+++ b/datalad_next/url_operations/tests/test_http.py
@@ -1,4 +1,4 @@
-from pathlib import Path
+import gzip
 import pytest
 
 from ..any import AnyUrlOperations
@@ -57,3 +57,42 @@ def test_custom_http_headers_via_config(datalad_cfg):
     auo = AnyUrlOperations()
     huo = auo._get_handler(f'http://example.com')
     assert huo._headers['X-Funky'] == 'Stuff'
+
+
+def test_transparent_decompression(tmp_path):
+    # this file is offered with transparent compression/decompression
+    # by the github webserver
+    url = 'https://raw.githubusercontent.com/datalad/datalad-next/' \
+          'd0c4746425a48ef20e3b1c218e68954db9412bee/pyproject.toml'
+    dpath = tmp_path / 'test.txt'
+    ops = HttpUrlOperations()
+    ops.download(from_url=url, to_path=dpath)
+
+    # make sure it ends up on disk uncompressed
+    assert dpath.read_text() == \
+        '[build-system]\nrequires = ["setuptools >= 43.0.0", "wheel"]\n'
+
+
+def test_compressed_file_stay_compressed(tmp_path):
+    # this file is offered with transparent compression/decompression
+    # by the github webserver, but is also actually gzip'ed
+    url = \
+        'https://github.com/datalad/datalad-neuroimaging/raw/' \
+        '05b45c8c15d24b6b894eb59544daa17159a88945/' \
+        'datalad_neuroimaging/tests/data/files/nifti1.nii.gz'
+
+    # first confirm validity of the test approach, opening an
+    # uncompressed file should raise an exception
+    with pytest.raises(gzip.BadGzipFile):
+        testpath = tmp_path / 'uncompressed'
+        testpath.write_text('some')
+        with gzip.open(testpath, 'rb') as f:
+            f.read(1000)
+
+    # and now with a compressed file
+    dpath = tmp_path / 'test.nii.gz'
+    ops = HttpUrlOperations()
+    ops.download(from_url=url, to_path=dpath)
+    # make sure it ends up on disk compressed!
+    with gzip.open(dpath, 'rb') as f:
+        f.read(1000)