Repository: CS-SI/eodag
Pull request #1072 (2024-03-25 16:38:11)

fix: completed progress_callback with unknown total
Mark progress bar as completed (green in notebooks) even if total size is unknown:
- use `total=None` instead of `total=0` in `download` `progress_callback` parameters

github-actions[bot]: ## Test Results
    4 files  ±0      4 suites  ±0   6m 2s :stopwatch: -5s
  507 tests ±0    504 :white_check_mark: ±0   3 :zzz: ±0  0 :x: ±0 
2 028 runs  ±0  1 942 :white_check_mark: ±0  86 :zzz: ±0  0 :x: ±0 

Results for commit ad9dcffe. ± Comparison against base commit 8132c6c1.

[test-results]:data:application/gzip;base64,H4sIAHSpAWYC/03MTQ7CIBCG4as0rF2MdFrAyxgcICG2xfCzMt5dWlvs8n0m872Z85NN7NbhpWOp+NzClKizD0vNfuQV6imvxwHEUfdUiDbCPz39a/1p4LSfKkADG2OIu8SyrJscuNzr2Lwq5I1+m3I84LS59XmSwjz7XINpoww5Z8WAKAmU06gehKCUkVpCz0lbCUKwzxdZCV+vCAEAAA==

github-actions[bot]: ![badge](https://raw.githubusercontent.com/CS-SI/eodag/_xml_coverage_reports/data/progress_callback_nototal/./badge.svg)


## Code Coverage (Ubuntu)

<details>


```
Filename                                     Stmts    Miss  Cover    Missing
-----------------------------------------  -------  ------  -------  ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
__init__.py                                     11       0  100.00%
cli.py                                         300      47  84.33%   61, 647-686, 788-839, 843
config.py                                      315      27  91.43%   80-82, 91, 99, 103-105, 178, 190, 388-390, 454-457, 504-505, 514-515, 594, 663-668, 670
crunch.py                                        6       6  0.00%    18-24
api/__init__.py                                  1       0  100.00%
api/core.py                                    731      76  89.60%   86-95, 368, 581, 625-628, 666, 771, 775-780, 806, 876, 946, 1063-1068, 1154-1166, 1206, 1208, 1212, 1233-1235, 1239-1250, 1263-1269, 1359-1362, 1391-1411, 1459, 1465-1468, 1477, 1843, 1876-1882, 2147, 2151-2154, 2168-2170, 2205
api/search_result.py                            44       6  86.36%   33-35, 70, 79, 86, 100
api/product/__init__.py                          5       0  100.00%
api/product/_assets.py                          44       5  88.64%   27-29, 79, 129
api/product/_product.py                        208      32  84.62%   48-55, 59-61, 165-172, 256-257, 347, 383, 444, 458-461, 474, 498-501, 544-550
api/product/metadata_mapping.py                649      84  87.06%   66-68, 129-131, 232, 264-265, 317-329, 331, 342, 348-360, 401-404, 441, 462-465, 488, 496-497, 570-571, 595-596, 602-605, 620-621, 770, 816, 889-890, 933-938, 1069, 1083-1103, 1123, 1128, 1238, 1260, 1274, 1287-1306, 1345, 1397, 1435-1439, 1458
api/product/drivers/__init__.py                  6       0  100.00%
api/product/drivers/base.py                      8       2  75.00%   23, 41
plugins/__init__.py                              1       0  100.00%
plugins/base.py                                 23       3  86.96%   25, 48, 55
plugins/manager.py                             127      14  88.98%   49-51, 95-100, 146, 185, 205, 231, 270-271
plugins/apis/__init__.py                         1       0  100.00%
plugins/apis/base.py                             4       0  100.00%
plugins/apis/cds.py                            206      49  76.21%   72-80, 126-129, 176-188, 244, 309, 317-322, 335-342, 360-362, 383-385, 395-397, 411-412, 457, 460, 470, 487, 515, 517
plugins/apis/ecmwf.py                           99      15  84.85%   47-55, 156-158, 205-206, 232-234
plugins/apis/usgs.py                           168      36  78.57%   58-63, 124, 197, 231, 266-268, 273, 299-300, 305, 335-342, 353-358, 380-386, 388-394, 417
plugins/authentication/__init__.py               7       1  85.71%   31
plugins/authentication/aws_auth.py              20       2  90.00%   25-27
plugins/authentication/base.py                  19       2  89.47%   26, 47
plugins/authentication/generic.py               16       3  81.25%   28, 40, 50
plugins/authentication/header.py                17       1  94.12%   27
plugins/authentication/keycloak.py              88      17  80.68%   32-34, 159-160, 190-212, 238-243
plugins/authentication/oauth.py                 15       8  46.67%   25, 32-34, 38-41
plugins/authentication/openid_connect.py       103      64  37.86%   39-41, 138-150, 154-172, 180-222, 228-237, 246-286, 291-299, 304-305
plugins/authentication/qsauth.py                36       2  94.44%   32, 83
plugins/authentication/sas_auth.py              49       2  95.92%   32, 76
plugins/authentication/token.py                 81      13  83.95%   35-37, 86, 99, 101, 114-117, 165-168
plugins/crunch/__init__.py                       1       0  100.00%
plugins/crunch/base.py                          10       2  80.00%   25, 38
plugins/crunch/filter_date.py                   62      15  75.81%   30, 53-58, 72, 81, 90, 93, 105-107, 116-118, 125
plugins/crunch/filter_latest_intersect.py       50      10  80.00%   32-34, 51-52, 71, 80-83, 85, 92-95
plugins/crunch/filter_latest_tpl_name.py        33       2  93.94%   28, 86
plugins/crunch/filter_overlap.py                68      17  75.00%   28-30, 33, 82-85, 91, 99, 110-126
plugins/crunch/filter_property.py               33       8  75.76%   29, 60-65, 68-69, 85-89
plugins/download/__init__.py                     1       0  100.00%
plugins/download/aws.py                        491     165  66.40%   77-83, 272, 285, 352-355, 369-373, 419-421, 425, 458-459, 465-469, 502, 537, 541, 548, 578-586, 590, 628-636, 643-645, 686-760, 778-839, 850-855, 871-884, 913, 928-930, 933, 943-951, 959-972, 982-1001, 1008-1020, 1061, 1087, 1132-1134, 1354
plugins/download/base.py                       261      57  78.16%   58-64, 145, 180, 319-320, 340-346, 377-381, 387-388, 432, 435-449, 461, 465, 538-542, 572-573, 581-598, 605-613, 615-619, 666, 689, 711, 719
plugins/download/creodias_s3.py                 17       9  47.06%   44-58
plugins/download/http.py                       435     102  76.55%   81-88, 122, 171, 192-199, 209-222, 262-267, 291, 293, 365, 392-394, 404, 410, 412, 434, 473-477, 534, 607-663, 677, 702-711, 737-738, 746, 749, 765, 788, 796-801, 819-820, 827, 888-894, 949-950, 956, 966, 1032, 1050-1066
plugins/download/s3rest.py                     117      27  76.92%   55-58, 124, 165, 199, 229-236, 239-241, 245, 258-264, 272-273, 276-280, 303, 324-327
plugins/search/__init__.py                       1       0  100.00%
plugins/search/base.py                         127       9  92.91%   49-54, 108, 112, 275, 295, 378
plugins/search/build_search_result.py           56       6  89.29%   67, 112-113, 121, 132, 176
plugins/search/creodias_s3.py                   55       3  94.55%   56, 74, 108
plugins/search/csw.py                          107      83  22.43%   43-45, 57-58, 62-63, 74-122, 128-141, 149-181, 199-240
plugins/search/data_request_search.py          198      65  67.17%   52, 89-92, 108, 120, 124-125, 139, 144, 149, 156, 169-172, 226-227, 231, 241-247, 252, 280-283, 291-302, 319, 321, 328-329, 331-332, 350-354, 387, 394, 405, 418, 424-436, 441
plugins/search/qssearch.py                     560      65  88.39%   89, 361-367, 375-376, 483-489, 544-547, 620-621, 662, 680, 695, 748, 769, 772-773, 782, 793, 802, 825, 885-890, 894-895, 923, 994, 1013-1030, 1069, 1145-1146, 1210-1211, 1232, 1257, 1263, 1294-1295, 1305-1311, 1354, 1368, 1388, 1398, 1478
plugins/search/static_stac_search.py            47       3  93.62%   39-40, 82
rest/__init__.py                                 1       0  100.00%
rest/core.py                                   169      10  94.08%   242, 513, 515, 518-520, 592, 599-603
rest/server.py                                 298      54  81.88%   80-81, 105, 128-129, 240-242, 258, 298-299, 311-327, 409-414, 444, 601-608, 639, 682-683, 771-773, 790-795, 825, 827, 831-832, 836-837
rest/stac.py                                   436      95  78.21%   59-61, 229-231, 273, 286-295, 314-320, 365, 402-404, 427, 462-463, 549-594, 639, 659-660, 840, 905-907, 1126, 1136-1148, 1161-1183, 1197-1242, 1401-1402
rest/types/__init__.py                           1       0  100.00%
rest/types/eodag_search.py                     185       9  95.14%   52-55, 231-235, 288, 291, 359
rest/types/stac_queryables.py                   30       2  93.33%   28, 114
rest/types/stac_search.py                      122      10  91.80%   48-51, 167, 182-184, 192, 196
rest/utils/__init__.py                         116      14  87.93%   53, 108-109, 128-130, 180, 190-204, 236
rest/utils/cql_evaluate.py                      48       5  89.58%   69, 76, 90, 97, 105
rest/utils/rfc3339.py                           31       3  90.32%   78, 90, 92
types/__init__.py                               76       6  92.11%   53, 87, 152, 172, 177, 185
types/bbox.py                                   43      19  55.81%   46-61, 72-74, 85-87, 99-101, 113-115, 123
types/download_args.py                           9       0  100.00%
types/queryables.py                             81       0  100.00%
types/search_args.py                            70      18  74.29%   60-64, 71-88, 104
utils/__init__.py                              495      43  91.31%   83, 88, 109-111, 189-190, 199-226, 229, 243, 325-329, 405-409, 430-432, 514, 519, 529, 567-568, 964-967, 975-976, 1017-1018, 1098, 1182, 1200, 1375
utils/constraints.py                           123      42  65.85%   31, 84-93, 134, 139, 143, 154, 177-178, 189-197, 206, 220-236, 245-256
utils/exceptions.py                             37       2  94.59%   23, 93
utils/import_system.py                          30      20  33.33%   27, 67-81, 93-103
utils/logging.py                                29       1  96.55%   123
utils/notebook.py                               44      23  47.73%   25-29, 36-41, 58-62, 72-78, 83-87
utils/stac_reader.py                            91      28  69.23%   55-56, 63-86, 93-95, 99, 141, 155-158
TOTAL                                         8703    1569  81.97%
```

</details>


### Diff against develop

```
Filename      Stmts    Miss  Cover
----------  -------  ------  --------
TOTAL             0       0  +100.00%
```


Results for commit: ad9dcffe75448c09fa49bc4099d8a8032cae8077


_Minimum allowed coverage is `70%`_


:recycle: This comment has been updated with latest results


<!-- Sticky Pull Request Commentartifacts/unit-test-results-python3.12-ubuntu-latest/coverage.xml -->

Patch:
diff --git a/docs/notebooks/tutos/tuto_wekeo.ipynb b/docs/notebooks/tutos/tuto_wekeo.ipynb
index e32583a6..9df658bf 100644
--- a/docs/notebooks/tutos/tuto_wekeo.ipynb
+++ b/docs/notebooks/tutos/tuto_wekeo.ipynb
@@ -20,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {
     "tags": []
    },
@@ -75,7 +75,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {
     "tags": []
    },
@@ -86,7 +86,7 @@
        "81"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -98,7 +98,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {
     "tags": []
    },
@@ -189,7 +189,7 @@
        " 'UERRA_EUROPE_SL']"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -236,7 +236,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Download found product to `/tmp`:"
+    "Download found product to `/tmp`, changing wait time to 12s (0.2') between product order and download retries:"
    ]
   },
   {
@@ -247,7 +247,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "73f7efd0ef104cb8932124d531430b0f",
+       "model_id": "9925bcf0ed804fd0bbec2f5efe4ea519",
        "version_major": 2,
        "version_minor": 0
       },
@@ -261,7 +261,7 @@
     {
      "data": {
       "text/html": [
-       "[Retry #3] Waiting 118.618292s until next download try for ordered product (retry every 2' for 20')"
+       "[Retry #1] Waiting 11.999954s until next download try for ordered product (retry every 0.2' for 20')"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
@@ -270,13 +270,6 @@
      "metadata": {},
      "output_type": "display_data"
     },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Downloaded product is not a Zip File. Please check its file type before using it\n"
-     ]
-    },
     {
      "data": {
       "text/plain": [
@@ -289,7 +282,7 @@
     }
    ],
    "source": [
-    "path = products[0].download(outputs_prefix=\"/tmp\")\n",
+    "path = products[0].download(outputs_prefix=\"/tmp\", wait=0.2)\n",
     "path"
    ]
   },
@@ -319,7 +312,118 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.10.12"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {
+     "21473516ad1d4c068c3a5e545c8dfcf1": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "36968842fea54ba98d0cfd6c58e526b4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "6aff6dba5b9d48bc8c1fc085987d5b44": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {
+       "display": "inline-flex",
+       "flex_flow": "row wrap",
+       "width": "100%"
+      }
+     },
+     "7ddb60a045a6443ba92e17175e30656a": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {
+       "flex": "2"
+      }
+     },
+     "8769c5368c334dd69b0c04bdbcc06812": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "91cd62f3d8e941b9ac8b7750d43b9fcf": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_e0b39382c4764717b02317df3dfa7c87",
+       "style": "IPY_MODEL_8769c5368c334dd69b0c04bdbcc06812",
+       "value": "4bd37b17b45d146f378e349ab3f90464: "
+      }
+     },
+     "9925bcf0ed804fd0bbec2f5efe4ea519": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "children": [
+        "IPY_MODEL_91cd62f3d8e941b9ac8b7750d43b9fcf",
+        "IPY_MODEL_f6d232de7f064cc7885528a9b41acb3a",
+        "IPY_MODEL_f38e4e2ab3c149a680153e368eb198ab"
+       ],
+       "layout": "IPY_MODEL_6aff6dba5b9d48bc8c1fc085987d5b44"
+      }
+     },
+     "c2ae4ebd11714f98b099dcd87ee08af6": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "e0b39382c4764717b02317df3dfa7c87": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "f38e4e2ab3c149a680153e368eb198ab": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_c2ae4ebd11714f98b099dcd87ee08af6",
+       "style": "IPY_MODEL_36968842fea54ba98d0cfd6c58e526b4",
+       "value": " 1.08k/? [00:00&lt;00:00, 809kB/s]"
+      }
+     },
+     "f6d232de7f064cc7885528a9b41acb3a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "bar_style": "success",
+       "layout": "IPY_MODEL_7ddb60a045a6443ba92e17175e30656a",
+       "max": 1,
+       "style": "IPY_MODEL_21473516ad1d4c068c3a5e545c8dfcf1",
+       "value": 1
+      }
+     }
+    },
+    "version_major": 2,
+    "version_minor": 0
+   }
   }
  },
  "nbformat": 4,
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 7760902a..453a5584 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -341,7 +341,9 @@ class UsgsApi(Api):
                             error_message = str(e)
                         raise NotAvailableError(error_message)
                     else:
-                        stream_size = int(stream.headers.get("content-length", 0))
+                        stream_size = (
+                            int(stream.headers.get("content-length", 0)) or None
+                        )
                         progress_callback.reset(total=stream_size)
                         with open(fs_path, "wb") as fhandle:
                             for chunk in stream.iter_content(chunk_size=64 * 1024):
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index e553d990..eb131e5f 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -337,7 +337,7 @@ class AwsDownload(Download):
             product,
         )
 
-        total_size = sum([p.size for p in unique_product_chunks])
+        total_size = sum([p.size for p in unique_product_chunks]) or None
 
         # download
         progress_callback.reset(total=total_size)
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 93e852a1..0a449608 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -799,7 +799,7 @@ class HTTPDownload(Download):
                     if self.stream.status_code == status_running:
                         product.properties["storageStatus"] = "ORDERED"
                         self._process_exception(None, product, ordered_message)
-                stream_size = self._check_stream_size(product)
+                stream_size = self._check_stream_size(product) or None
                 product.headers = self.stream.headers
                 progress_callback.reset(total=stream_size)
                 for chunk in self.stream.iter_content(chunk_size=64 * 1024):
@@ -831,7 +831,7 @@ class HTTPDownload(Download):
             self.config, "dl_url_params", {}
         )
 
-        total_size = self._get_asset_sizes(assets_values, auth, params)
+        total_size = self._get_asset_sizes(assets_values, auth, params) or None
 
         progress_callback.reset(total=total_size)
 


Test patch:
diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
index 960448fc..da331be5 100644
--- a/tests/units/test_download_plugins.py
+++ b/tests/units/test_download_plugins.py
@@ -922,6 +922,23 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
             plugin.download(self.product, outputs_prefix=temp_dir)
         mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=4 + 4)
 
+        # unknown size
+        mock_requests_get.return_value.__enter__.return_value.headers.pop(
+            "content-disposition"
+        )
+        mock_progress_callback_reset.reset_mock()
+        self.product.location = "http://somewhere"
+        self.product.assets.clear()
+        self.product.assets.update(
+            {
+                "foo": {"href": "http://somewhere/a"},
+                "bar": {"href": "http://somewhere/b"},
+            }
+        )
+        with TemporaryDirectory() as temp_dir:
+            plugin.download(self.product, outputs_prefix=temp_dir)
+        mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=None)
+
     def test_plugins_download_http_one_local_asset(
         self,
     ):