Repository: Senth/minecraft-mod-manager (python)

Issue:
Retry if downloading fails due to overload or some other issue
### Describe the bug 
Sometimes mods fail to download from Modrinth due to the server being overloaded.
This makes tests fail sometimes, but probably also makes installation and updates fail.

### Steps to reproduce
Ran tests

### Expected Behavior
mcman should retry to download the file, either directly or a bit later.

### Info
 - OS: linux
 - Python --version: 3.8
 - minecraft-mod-manager --version: 1.2.7

### Test output
```
........................................................................ [ 51%]
.................................................................F.      [100%]
=================================== FAILURES ===================================
__________________________ test_install_from_modrinth __________________________

helper = <tests.util.helper.Helper object at 0x7fa36904c970>

    def test_install_from_modrinth(helper: Helper):
        code = helper.run("install", "lithium=modrinth:lithium")
        lithium_mod = helper.get_mod_in_dir_like("*lithium*.jar")
>       assert lithium_mod is not None
E       assert None is not None

tests/install_test.py:49: AssertionError
----------------------------- Captured stdout call -----------------------------
lithium
----------------------------- Captured stderr call -----------------------------
Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/3.[8](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:8).12/x64/lib/python3.8/site-packages/requests/models.py", line [9](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:9)[10](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:10), in json
    return complexjson.loads(self.text, **kwargs)
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/json/__init__.py", line 370, in loads
    return cls(**kw).decode(s)
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/__main__.py", line 43, in <module>
    main()
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/__main__.py", line 29, in main
    install.execute(config.arg_mods)
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/app/install/install.py", line 24, in execute
    self.find_download_and_install(mods)
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/app/download/download.py", line 27, in find_download_and_install
    mod.sites = self._repo.search_for_mod(mod)
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/adapters/repo_impl.py", line 73, in search_for_mod
    site = api.find_mod_id(mod)
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/gateways/api/api.py", line 25, in find_mod_id
    info = self._find_mod_id_by_slug(existing_site.slug, set([existing_site.slug]))
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/gateways/api/modrinth_api.py", line 33, in _find_mod_id_by_slug
    json = self.downloader.get(ModrinthApi._make_search_url(search))
  File "/home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/gateways/downloader.py", line 20, in get
    return response.json(strict=False)
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/requests/models.py", line 917, in json
    raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: [Errno Expecting value] <!DOCTYPE html>
<!--[if lt IE 7]> <html class="no-js ie6 oldie" lang="en-US"> <![endif]-->
<!--[if IE 7]>    <html class="no-js ie7 oldie" lang="en-US"> <![endif]-->
<!--[if IE 8]>    <html class="no-js ie8 oldie" lang="en-US"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en-US"> <!--<![endif]-->
<head>
<meta http-equiv="refresh" content="0">

<title>api.modrinth.com | 524: A timeout occurred</title>
<meta charset="UTF-8" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" />
<meta name="robots" content="noindex, nofollow" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<link rel="stylesheet" id="cf_styles-css" href="/cdn-cgi/styles/main.css" type="text/css" media="screen,projection" />


<script type="text/javascript">
(function(){if(document.addEventListener&&window.XMLHttpRequest&&JSON&&JSON.stringify){var e=function(a){var c=document.getElementById("error-feedback-survey"),d=document.getElementById("error-feedback-success"),b=new XMLHttpRequest;a={event:"feedback clicked",properties:{errorCode:524,helpful:a,version:1}};b.open("POST","https://sparrow.cloudflare.com/api/v1/event");b.setRequestHeader("Content-Type","application/json");b.setRequestHeader("Sparrow-Source-Key","c771f0e4b54944bebf4261d44bd79a1e");
b.send(JSON.stringify(a));c.classList.add("feedback-hidden");d.classList.remove("feedback-hidden")};document.addEventListener("DOMContentLoaded",function(){var a=document.getElementById("error-feedback"),c=document.getElementById("feedback-button-yes"),d=document.getElementById("feedback-button-no");"classList"in a&&(a.classList.remove("feedback-hidden"),c.addEventListener("click",function(){e(!0)}),d.addEventListener("click",function(){e(!1)}))})}})();
</script>

</head>
<body>
<div id="cf-wrapper">

    

    <div id="cf-error-details" class="p-0">
        <header class="mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8">
            <h1 class="inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2">
              
              <span class="cf-error-type">Error</span>
              <span class="cf-error-code">524</span>
            </h1>
            <span class="inline-block sm:block font-mono text-15 lg:text-sm lg:leading-relaxed">Ray ID: 6f32e[11](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:11)e4a0566ef &bull;</span>
            <span class="inline-block sm:block font-mono text-15 lg:text-sm lg:leading-relaxed">2022-03-28 19:49:25 UTC</span>
            <h2 class="text-gray-600 leading-1.3 text-3xl font-light">A timeout occurred</h2>
        </header>
        
        <div class="my-8 bg-gradient-gray">
            <div class="w-240 lg:w-full mx-auto">
                <div class="clearfix md:px-8">
                  
<div id="cf-browser-status" class=" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center">
  <div class="relative mb-10 md:m-0">
    <span class="cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat"></span>
    <span class="cf-icon-ok w-[12](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:12) h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4"></span>
  </div>
  <span class="md:block w-full truncate">You</span>
  <h3 class="md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3">Browser</h3>
  <span class="leading-1.3 text-2xl text-green-success">Working</span>
</div>

<div id="cf-cloudflare-status" class=" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center">
  <div class="relative mb-10 md:m-0">
    <span class="cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat"></span>
    <span class="cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4"></span>
  </div>
  <span class="md:block w-full truncate">Dallas</span>
  <h3 class="md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3">Cloudflare</h3>
  <span class="leading-1.3 text-2xl text-green-success">Working</span>
</div>

<div id="cf-host-status" class="cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center">
  <div class="relative mb-10 md:m-0">
    <span class="cf-icon-server block md:hidden h-20 bg-center bg-no-repeat"></span>
    <span class="cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4"></span>
  </div>
  <span class="md:block w-full truncate">api.modrinth.com</span>
  <h3 class="md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3">Host</h3>
  <span class="leading-1.3 text-2xl text-red-error">Error</span>
</div>

                </div>
              
            </div>
        </div>

        <div class="w-240 lg:w-full mx-auto mb-8 lg:px-8">
            <div class="clearfix">
                <div class="w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed">
                    <h2 class="text-3xl font-normal leading-1.3 mb-4">What happened?</h2>
                    <p>The origin web server timed out responding to this request.</p>
                </div>
              
                <div class="w-1/2 md:w-full float-left leading-relaxed">
                    <h2 class="text-3xl font-normal leading-1.3 mb-4">What can I do?</h2>
                          <h3 class="text-15 font-semibold mb-2">If you're a visitor of this website:</h3>
      <p class="mb-6">Please try again in a few minutes.</p>

      <h3 class="text-15 font-semibold mb-2">If you're the owner of this website:</h3>
      <p><span>The connection to the origin web server was made, but the origin web server timed out before responding. The likely cause is an overloaded background task, database or application, stressing the resources on your web server. To resolve, please work with your hosting provider or web development team to free up resources for your database or overloaded application.</span> <a rel="noopener noreferrer" href="https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524">Additional troubleshooting information here.</a></p>
                </div>
            </div>
              
        </div>

        <div class="feedback-hidden py-8 text-center" id="error-feedback">
    <div id="error-feedback-survey" class="footer-line-wrapper">
        Was this page helpful?
        <button class="border border-solid bg-white cf-button cursor-pointer ml-4 px-4 py-2 rounded" id="feedback-button-yes" type="button">Yes</button>
        <button class="border border-solid bg-white cf-button cursor-pointer ml-4 px-4 py-2 rounded" id="feedback-button-no" type="button">No</button>
    </div>
    <div class="feedback-success feedback-hidden" id="error-feedback-success">
        Thank you for your feedback!
    </div>
</div>


        <div class="cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300">
  <p class="text-[13](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:13)">
    <span class="cf-footer-item sm:block sm:mb-1">Cloudflare Ray ID: <strong class="font-semibold">6f32e11e4a0566ef</strong></span>
    <span class="cf-footer-separator sm:hidden">&bull;</span>
    <span class="cf-footer-item sm:block sm:mb-1"><span>Your IP</span>: 168.61.[15](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:15)4.112</span>
    <span class="cf-footer-separator sm:hidden">&bull;</span>
    <span class="cf-footer-item sm:block sm:mb-1"><span>Performance &amp; security by</span> <a rel="noopener noreferrer" href="https://www.cloudflare.com/5xx-error-landing" id="brand_link" target="_blank">Cloudflare</a></span>
    
  </p>
</div><!-- /.error-footer -->


    </div>
</div>
</body>
</html>

: 0
=============================== warnings summary ===============================
minecraft_mod_manager/adapters/repo_impl_test.py:46
  /home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/adapters/repo_impl_test.py:46: PytestCollectionWarning: cannot collect test class 'TestSearchForMod' because it has a __init__ constructor (from: minecraft_mod_manager/adapters/repo_impl_test.py)
    class TestSearchForMod:

minecraft_mod_manager/adapters/repo_impl_test.py:134
  /home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/adapters/repo_impl_test.py:134: PytestCollectionWarning: cannot collect test class 'TestGetVersions' because it has a __init__ constructor (from: minecraft_mod_manager/adapters/repo_impl_test.py)
    class TestGetVersions:

minecraft_mod_manager/core/utils/latest_version_finder_test.py:[22](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:22)
  /home/runner/work/minecraft-mod-manager/minecraft-mod-manager/minecraft_mod_manager/core/utils/latest_version_finder_test.py:22: PytestCollectionWarning: cannot collect test class 'Test' because it has a __init__ constructor (from: minecraft_mod_manager/core/utils/latest_version_finder_test.py)
    class Test:

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/install_test.py::test_install_from_modrinth - assert None is not...
1 failed, 138 passed, 3 warnings in 108.[30](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:30)s (0:01:[48](https://github.com/Senth/minecraft-mod-manager/runs/5726552412?check_suite_focus=true#step:5:48))
```

Pull request:
Retry fetch/download if timed out or other reason
### What changed?
- Downloader now automatically retries 5 times with exponential backoff

### Testing
- [X] Added unit tests
- [ ] Added integration tests
- [ ] ...Your own tests...

### Checklist
- [x] I have reviewed my own code

### Related Issues
Fixes #169


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 71e835b..ece4185 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Colored output continued to next messages [#141](https://github.com/Senth/minecraft-mod-manager/issues/141)
 - Handles Ctrl-C by stopping the program without printing a stack trace.
   It can still break the program in an invalid state [#140](https://github.com/Senth/minecraft-mod-manager/issues/140)
+- Now retries to fetch/download file 5 times before skipping [#169](https://github.com/Senth/minecraft-mod-manager/issues/169)
 
 ## [1.2.7] - 2022-03-06
 
diff --git a/minecraft_mod_manager/app/download/download.py b/minecraft_mod_manager/app/download/download.py
index 072e60b..e0affb1 100644
--- a/minecraft_mod_manager/app/download/download.py
+++ b/minecraft_mod_manager/app/download/download.py
@@ -1,5 +1,6 @@
 from typing import List, Sequence
 
+from minecraft_mod_manager.gateways.downloader import MaxRetriesExceeded
 from tealprint import TealPrint
 
 from ...core.entities.mod import Mod, ModArg
@@ -36,9 +37,9 @@ class Download:
                         self._update_mod_from_file(downloaded_mod)
                         self._repo.update_mod(downloaded_mod)
                         self.on_version_found(mod, downloaded_mod)
-                    except DownloadFailed as e:
+                    except (DownloadFailed, MaxRetriesExceeded) as e:
                         TealPrint.error(
-                            f"🔺 Download failed from {latest_version.site_name}. Might be user-agent error.",
+                            f"🔺 Download failed from {latest_version.site_name}",
                         )
                         TealPrint.error(str(e))
                     except ModFileInvalid:
@@ -46,14 +47,15 @@ class Download:
                         self._repo.remove_mod_file(latest_version.filename)
                         TealPrint.error("❌ Corrupted file.")
                         corrupt_mods.append(mod)
-                        continue
-
                 else:
                     self.on_version_not_found(mod, versions)
 
             except ModNotFoundException as exception:
                 TealPrint.warning("🔺 Mod not found on any site...")
                 mods_not_found.append(exception)
+            except MaxRetriesExceeded:
+                TealPrint.warning("🔺 Max retries exceeded. Skipping...")
+                mods_not_found.append(ModNotFoundException(mod))
 
             TealPrint.pop_indent()
 
diff --git a/minecraft_mod_manager/gateways/downloader.py b/minecraft_mod_manager/gateways/downloader.py
index af21842..6249903 100644
--- a/minecraft_mod_manager/gateways/downloader.py
+++ b/minecraft_mod_manager/gateways/downloader.py
@@ -1,10 +1,12 @@
 import re
+import time
 from os import path
 from typing import Any
 
 import latest_user_agents
 import requests
 from requests.models import Response
+from tealprint import TealPrint
 
 from ..config import config
 from ..core.errors.download_failed import DownloadFailed
@@ -12,11 +14,24 @@ from ..core.errors.download_failed import DownloadFailed
 _headers = {"User-Agent": latest_user_agents.get_random_user_agent()}
 
 
+class MaxRetriesExceeded(Exception):
+    def __init__(self, url: str, retries: int, status_code: int, reason: str, content: str):
+        self.url = url
+        self.retries = retries
+        self.status_code = status_code
+        self.reason = reason
+        self.content = content
+
+    def __str__(self) -> str:
+        return f"{self.status_code}: {self.reason}"
+
+
 class Downloader:
-    def get(self, url: str) -> Any:
-        headers = _headers
+    retries_max = 5
+    retry_backoff_factor = 1.5
 
-        with requests.get(url, headers=headers) as response:
+    def get(self, url: str) -> Any:
+        with Downloader._get_with_retries(url) as response:
             return response.json(strict=False)
 
     def download(self, url: str, filename: str) -> str:
@@ -30,7 +45,7 @@ class Downloader:
         if config.pretend:
             return filename
 
-        with requests.get(url, headers=_headers) as response:
+        with Downloader._get_with_retries(url) as response:
             if response.status_code != 200:
                 raise DownloadFailed(response.status_code, response.reason, str(response.content))
 
@@ -48,6 +63,24 @@ class Downloader:
 
         return filename
 
+    @staticmethod
+    def _get_with_retries(url: str) -> Response:
+        response: Response = Response()
+        for retry in range(Downloader.retries_max):
+            response = requests.get(url, headers=_headers)
+            if response.status_code < 500 or response.status_code >= 600:
+                return response
+            elif retry < Downloader.retries_max:
+                response.close()
+                delay = Downloader.retry_backoff_factor**retry
+                TealPrint.warning(f"{(retry+1)}: Failed to connect to {url}. Retrying in {delay} seconds...")
+                time.sleep(delay)
+        TealPrint.error(f"{Downloader.retries_max}: Failed to connect to {url}. Giving up.")
+        TealPrint.error(f"{response.status_code}: {response.reason}")
+        raise MaxRetriesExceeded(
+            url, Downloader.retries_max, response.status_code, response.reason, str(response.content)
+        )
+
     @staticmethod
     def _get_filename(response: Response) -> str:
         content_disposition = response.headers.get("content-disposition")


Test patch:
diff --git a/minecraft_mod_manager/gateways/downloader_test.py b/minecraft_mod_manager/gateways/downloader_test.py
index 73809d6..eb4bb82 100644
--- a/minecraft_mod_manager/gateways/downloader_test.py
+++ b/minecraft_mod_manager/gateways/downloader_test.py
@@ -9,8 +9,8 @@ from requests.models import Response
 from requests.structures import CaseInsensitiveDict
 
 from ..config import config
-from ..gateways.downloader import Downloader
 from ..core.errors.download_failed import DownloadFailed
+from ..gateways.downloader import Downloader, MaxRetriesExceeded
 
 
 @pytest.fixture
@@ -115,7 +115,7 @@ def test_get_when_non_strict_json(downloader):
 
 
 def test_download_failed(downloader, mock_response):
-    mock_response.status_code = "404"  # type:ignore
+    mock_response.status_code = 404  # type:ignore
     mock_response.reason = "Not found"  # type:ignore
     mock_response.content = "404 not found"  # type:ignore
     when(requests).get(...).thenReturn(mock_response)
@@ -124,3 +124,17 @@ def test_download_failed(downloader, mock_response):
         downloader.download("", "")
 
     unstub()
+
+
+def test_download_retry(downloader, mock_response):
+    mock_response.status_code = 524  # type:ignore
+    mock_response.reason = "Timed out"  # type:ignore
+    mock_response.content = "524 Timed out"  # type:ignore
+    when(mock_response).close()
+
+    when(requests).get(...).thenReturn(mock_response)
+
+    with pytest.raises(MaxRetriesExceeded):
+        downloader.download("", "")
+
+    unstub()