Repository: miurahr/aqtinstall
Pull request #493 (2022-02-27 19:58:00)

[Security] Use SHA-256 checksums from trusted mirrors only
This change causes `aqt` to use SHA-256 checksums for all downloads, and only download them from a list of trusted mirrors. By default, this list includes only https://download.qt.io, but the settings.ini file may be used to modify this list. The settings.ini file can also be used to set the number of times to retry the failed download of a checksum.

Partially addresses #488

Todo:

- [x] Require checksums for Updates.xml files


Patch:
diff --git a/aqt/archives.py b/aqt/archives.py
index b0f385a..72e7842 100644
--- a/aqt/archives.py
+++ b/aqt/archives.py
@@ -19,15 +19,15 @@
 # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
+import binascii
 import posixpath
 import xml.etree.ElementTree as ElementTree
 from dataclasses import dataclass, field
 from logging import getLogger
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from aqt.exceptions import ArchiveDownloadError, ArchiveListError, NoPackageFound
-from aqt.helper import Settings, getUrl, ssplit
+from aqt.exceptions import ArchiveDownloadError, ArchiveListError, ChecksumDownloadFailure, NoPackageFound
+from aqt.helper import Settings, get_hash, getUrl, ssplit
 from aqt.metadata import QtRepoProperty, Version
 
 
@@ -45,10 +45,10 @@ class TargetConfig:
 @dataclass
 class QtPackage:
     name: str
-    archive_url: str
+    base_url: str
+    archive_path: str
     archive: str
     package_desc: str
-    hashurl: str
     pkg_update_name: str
     version: Optional[Version] = field(default=None)
 
@@ -59,9 +59,9 @@ class QtPackage:
     def __str__(self):
         v_info = f", version={self.version}" if self.version else ""
         return (
-            f"QtPackage(name={self.name}, url={self.archive_url}, "
+            f"QtPackage(name={self.name}, url={self.archive_path}, "
             f"archive={self.archive}, desc={self.package_desc}"
-            f"hashurl={self.hashurl}{v_info})"
+            f"{v_info})"
         )
 
 
@@ -140,7 +140,7 @@ class QtArchives:
         self.os_name: str = os_name
         self.all_extra: bool = all_extra
         self.arch_list: List[str] = [item.get("arch") for item in Settings.qt_combinations]
-        self.base: str = posixpath.join(base, "online/qtsdkrepository")
+        self.base: str = base
         self.logger = getLogger("aqt.archives")
         self.archives: List[QtPackage] = []
         self.subarchives: Optional[Iterable[str]] = subarchives
@@ -217,22 +217,26 @@ class QtArchives:
     def _get_archives(self):
         # Get packages index
 
-        # archive_path: windows_x86/desktop/qt5_59_src_doc_examples
-        archive_path = posixpath.join(
+        # os_target_folder: online/qtsdkrepository/windows_x86/desktop/qt5_59_src_doc_examples
+        os_target_folder = posixpath.join(
+            "online/qtsdkrepository",
             self.os_name + ("_x86" if self.os_name == "windows" else "_x64"),
             self.target,
             f"qt{self.version.major}_{self._version_str()}{self._arch_ext()}",
         )
-        update_xml_url = posixpath.join(self.base, archive_path, "Updates.xml")
-        archive_url = posixpath.join(self.base, archive_path)
-        self._download_update_xml(update_xml_url)
-        self._parse_update_xml(archive_url, self._target_packages())
+        update_xml_path = posixpath.join(os_target_folder, "Updates.xml")
+        self._download_update_xml(update_xml_path)
+        self._parse_update_xml(os_target_folder, self._target_packages())
 
-    def _download_update_xml(self, update_xml_url):
+    def _download_update_xml(self, update_xml_path):
         """Hook for unit test."""
-        self.update_xml_text = getUrl(update_xml_url, self.timeout)
+        xml_hash = binascii.unhexlify(get_hash(update_xml_path, "sha256", self.timeout))
+        if xml_hash == "":
+            raise ChecksumDownloadFailure(f"Checksum for '{update_xml_path}' is empty")
+        update_xml_text = getUrl(posixpath.join(self.base, update_xml_path), self.timeout, xml_hash)
+        self.update_xml_text = update_xml_text
 
-    def _parse_update_xml(self, archive_url, target_packages: Optional[ModuleToPackage]):
+    def _parse_update_xml(self, os_target_folder, target_packages: Optional[ModuleToPackage]):
         if not target_packages:
             target_packages = ModuleToPackage({})
         try:
@@ -270,22 +274,21 @@ class QtArchives:
                 archive_name = archive.split("-", maxsplit=1)[0]
                 if should_filter_archives and archive_name not in self.subarchives:
                     continue
-                package_url = posixpath.join(
-                    # https://download.qt.io/online/qtsdkrepository/linux_x64/desktop/qt5_5150/
-                    archive_url,
+                archive_path = posixpath.join(
+                    # online/qtsdkrepository/linux_x64/desktop/qt5_5150/
+                    os_target_folder,
                     # qt.qt5.5150.gcc_64/
                     pkg_name,
                     # 5.15.0-0-202005140804qtbase-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z
                     full_version + archive,
                 )
-                hashurl = package_url + ".sha1"
                 self.archives.append(
                     QtPackage(
                         name=archive_name,
-                        archive_url=package_url,
+                        base_url=self.base,
+                        archive_path=archive_path,
                         archive=archive,
                         package_desc=package_desc,
-                        hashurl=hashurl,
                         pkg_update_name=pkg_name,  # For testing purposes
                     )
                 )
@@ -431,25 +434,20 @@ class ToolArchives(QtArchives):
         raise ArchiveListError(msg, suggested_action=[help_msg]) from e
 
     def _get_archives(self):
-        _a = "_x64"
-        if self.os_name == "windows":
-            _a = "_x86"
-
-        archive_url = posixpath.join(
-            # https://download.qt.io/online/qtsdkrepository/
-            self.base,
+        os_target_folder = posixpath.join(
+            "online/qtsdkrepository",
             # linux_x64/
-            self.os_name + _a,
+            self.os_name + ("_x86" if self.os_name == "windows" else "_x64"),
             # desktop/
             self.target,
             # tools_ifw/
             self.tool_name,
         )
-        update_xml_url = posixpath.join(archive_url, "Updates.xml")
+        update_xml_url = posixpath.join(os_target_folder, "Updates.xml")
         self._download_update_xml(update_xml_url)  # call super method.
-        self._parse_update_xml(archive_url, None)
+        self._parse_update_xml(os_target_folder, None)
 
-    def _parse_update_xml(self, archive_url, *ignored):
+    def _parse_update_xml(self, os_target_folder, *ignored):
         try:
             self.update_xml = ElementTree.fromstring(self.update_xml_text)
         except ElementTree.ParseError as perror:
@@ -472,22 +470,21 @@ class ToolArchives(QtArchives):
             message = f"The package '{self.arch}' contains no downloadable archives!"
             raise NoPackageFound(message)
         for archive in ssplit(downloadable_archives):
-            package_url = posixpath.join(
-                # https://download.qt.io/online/qtsdkrepository/linux_x64/desktop/tools_ifw/
-                archive_url,
+            archive_path = posixpath.join(
+                # online/qtsdkrepository/linux_x64/desktop/tools_ifw/
+                os_target_folder,
                 # qt.tools.ifw.41/
                 name,
                 #  4.1.1-202105261130ifw-linux-x64.7z
                 f"{named_version}{archive}",
             )
-            hashurl = package_url + ".sha1"
             self.archives.append(
                 QtPackage(
                     name=name,
-                    archive_url=package_url,
+                    base_url=self.base,
+                    archive_path=archive_path,
                     archive=archive,
                     package_desc=package_desc,
-                    hashurl=hashurl,
                     pkg_update_name=name,  # Redundant
                 )
             )
diff --git a/aqt/exceptions.py b/aqt/exceptions.py
index 8c46d25..f02f032 100644
--- a/aqt/exceptions.py
+++ b/aqt/exceptions.py
@@ -20,6 +20,8 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 from typing import Iterable
 
+DOCS_CONFIG = "https://aqtinstall.readthedocs.io/en/stable/configuration.html#configuration"
+
 
 class AqtException(Exception):
     def __init__(self, *args, **kwargs):
@@ -47,6 +49,19 @@ class ArchiveChecksumError(ArchiveDownloadError):
     pass
 
 
+class ChecksumDownloadFailure(ArchiveDownloadError):
+    def __init__(self, *args, **kwargs):
+        kwargs["suggested_action"] = kwargs.pop("suggested_action", []).extend(
+            [
+                "Check your internet connection",
+                "Consider modifying `requests.max_retries_to_retrieve_hash` in settings.ini",
+                f"Consider modifying `mirrors.trusted_mirrors` in settings.ini (see {DOCS_CONFIG})",
+            ]
+        )
+        kwargs["should_show_help"] = True
+        super(ChecksumDownloadFailure, self).__init__(*args, **kwargs)
+
+
 class ArchiveConnectionError(AqtException):
     pass
 
diff --git a/aqt/helper.py b/aqt/helper.py
index dbf8edd..ea56f0c 100644
--- a/aqt/helper.py
+++ b/aqt/helper.py
@@ -24,18 +24,26 @@ import hashlib
 import json
 import logging.config
 import os
+import posixpath
+import secrets
 import sys
 import xml.etree.ElementTree as ElementTree
 from logging import getLogger
 from logging.handlers import QueueListener
 from pathlib import Path
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, Generator, List, Optional, Tuple
 from urllib.parse import urlparse
 
 import requests
 import requests.adapters
 
-from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ArchiveListError
+from aqt.exceptions import (
+    ArchiveChecksumError,
+    ArchiveConnectionError,
+    ArchiveDownloadError,
+    ArchiveListError,
+    ChecksumDownloadFailure,
+)
 
 
 def _get_meta(url: str):
@@ -47,7 +55,13 @@ def _check_content_type(ct: str) -> bool:
     return any(ct.startswith(t) for t in candidate)
 
 
-def getUrl(url: str, timeout) -> str:
+def getUrl(url: str, timeout, expected_hash: Optional[bytes] = None) -> str:
+    """
+    Gets a file from `url` via HTTP GET.
+
+    No caller should call this function without providing an expected_hash, unless
+    the caller is `get_hash`, which cannot know what the expected hash should be.
+    """
     logger = getLogger("aqt.helper")
     with requests.sessions.Session() as session:
         retries = requests.adapters.Retry(
@@ -76,6 +90,14 @@ def getUrl(url: str, timeout) -> str:
                 msg = f"Failed to retrieve file at {url}\nServer response code: {r.status_code}, reason: {r.reason}"
                 raise ArchiveDownloadError(msg)
         result = r.text
+        filename = url.split("/")[-1]
+        actual_hash = hashlib.sha256(bytes(result, "utf-8")).digest()
+        if expected_hash is not None and expected_hash != actual_hash:
+            raise ArchiveChecksumError(
+                f"Downloaded file {filename} is corrupted! Detect checksum error.\n"
+                f"Expect {expected_hash.hex()}: {url}\n"
+                f"Actual {actual_hash.hex()}: {filename}"
+            )
     return result
 
 
@@ -134,6 +156,42 @@ def retry_on_errors(action: Callable[[], any], acceptable_errors: Tuple, num_ret
             raise e from e
 
 
+def retry_on_bad_connection(function: Callable[[str], any], base_url: str):
+    logger = getLogger("aqt.helper")
+    fallback_url = secrets.choice(Settings.fallbacks)
+    try:
+        return function(base_url)
+    except ArchiveConnectionError:
+        logger.warning(f"Connection to '{base_url}' failed. Retrying with fallback '{fallback_url}'.")
+        return function(fallback_url)
+
+
+def iter_list_reps(_list: List, num_reps: int) -> Generator:
+    list_index = 0
+    for i in range(num_reps):
+        yield _list[list_index]
+        list_index += 1
+        if list_index >= len(_list):
+            list_index = 0
+
+
+def get_hash(archive_path: str, algorithm: str, timeout) -> str:
+    logger = getLogger("aqt.helper")
+    for base_url in iter_list_reps(Settings.trusted_mirrors, Settings.max_retries_to_retrieve_hash):
+        url = posixpath.join(base_url, f"{archive_path}.{algorithm}")
+        logger.debug(f"Attempt to download checksum at {url}")
+        try:
+            r = getUrl(url, timeout)
+            # sha256 & md5 files are: "some_hash archive_filename"
+            return r.split(" ")[0]
+        except (ArchiveConnectionError, ArchiveDownloadError):
+            pass
+    filename = archive_path.split("/")[-1]
+    raise ChecksumDownloadFailure(
+        f"Failed to download checksum for the file '{filename}' from mirrors '{Settings.trusted_mirrors}"
+    )
+
+
 def altlink(url: str, alt: str):
     """
     Blacklisting redirected(alt) location based on Settings.blacklist configuration.
@@ -225,7 +283,9 @@ def xml_to_modules(
 
 class MyConfigParser(configparser.ConfigParser):
     def getlist(self, section: str, option: str, fallback=[]) -> List[str]:
-        value = self.get(section, option)
+        value = self.get(section, option, fallback=None)
+        if value is None:
+            return fallback
         try:
             result = list(filter(None, (x.strip() for x in value.splitlines())))
         except Exception:
@@ -361,10 +421,18 @@ class SettingsClass:
     def max_retries_on_checksum_error(self):
         return self.config.getint("requests", "max_retries_on_checksum_error", fallback=int(self.max_retries))
 
+    @property
+    def max_retries_to_retrieve_hash(self):
+        return self.config.getint("requests", "max_retries_to_retrieve_hash", fallback=int(self.max_retries))
+
     @property
     def backoff_factor(self):
         return self.config.getfloat("requests", "retry_backoff", fallback=0.1)
 
+    @property
+    def trusted_mirrors(self):
+        return self.config.getlist("mirrors", "trusted_mirrors", fallback=[self.baseurl])
+
     @property
     def fallbacks(self):
         return self.config.getlist("mirrors", "fallbacks", fallback=[])
diff --git a/aqt/installer.py b/aqt/installer.py
index 31bfc25..e37de32 100644
--- a/aqt/installer.py
+++ b/aqt/installer.py
@@ -27,7 +27,7 @@ import gc
 import multiprocessing
 import os
 import platform
-import random
+import posixpath
 import signal
 import subprocess
 import sys
@@ -36,14 +36,13 @@ from logging import getLogger
 from logging.handlers import QueueHandler
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any, Callable, List, Optional
+from typing import List, Optional
 
 import aqt
 from aqt.archives import QtArchives, QtPackage, SrcDocExamplesArchives, ToolArchives
 from aqt.exceptions import (
     AqtException,
     ArchiveChecksumError,
-    ArchiveConnectionError,
     ArchiveDownloadError,
     ArchiveExtractionError,
     ArchiveListError,
@@ -51,7 +50,15 @@ from aqt.exceptions import (
     CliKeyboardInterrupt,
     OutOfMemory,
 )
-from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, getUrl, retry_on_errors, setup_logging
+from aqt.helper import (
+    MyQueueListener,
+    Settings,
+    downloadBinaryFile,
+    get_hash,
+    retry_on_bad_connection,
+    retry_on_errors,
+    setup_logging,
+)
 from aqt.metadata import ArchiveId, MetadataFactory, QtRepoProperty, SimpleSpec, Version, show_list, suggested_follow_up
 from aqt.updater import Updater
 
@@ -301,7 +308,7 @@ class Cli:
         if not all_extra and not self._check_modules_arg(qt_version, modules):
             self.logger.warning("Some of specified modules are unknown.")
 
-        qt_archives = self.retry_on_bad_connection(
+        qt_archives = retry_on_bad_connection(
             lambda base_url: QtArchives(
                 os_name,
                 target,
@@ -366,7 +373,7 @@ class Cli:
         if not self._check_qt_arg_versions(qt_version):
             self.logger.warning("Specified Qt version is unknown: {}.".format(qt_version))
 
-        srcdocexamples_archives: SrcDocExamplesArchives = self.retry_on_bad_connection(
+        srcdocexamples_archives: SrcDocExamplesArchives = retry_on_bad_connection(
             lambda base_url: SrcDocExamplesArchives(
                 flavor,
                 os_name,
@@ -460,7 +467,7 @@ class Cli:
             if not self._check_tools_arg_combination(os_name, tool_name, arch):
                 self.logger.warning("Specified target combination is not valid: {} {} {}".format(os_name, tool_name, arch))
 
-            tool_archives: ToolArchives = self.retry_on_bad_connection(
+            tool_archives: ToolArchives = retry_on_bad_connection(
                 lambda base_url: ToolArchives(
                     os_name=os_name,
                     tool_name=tool_name,
@@ -917,14 +924,6 @@ class Cli:
         except ValueError as e:
             raise CliInputError(f"Invalid version: '{version_str}'! Please use the form '5.X.Y'.") from e
 
-    def retry_on_bad_connection(self, function: Callable[[str], Any], base_url: str):
-        fallback_url = random.choice(Settings.fallbacks)
-        try:
-            return function(base_url)
-        except ArchiveConnectionError:
-            self.logger.warning(f"Connection to '{base_url}' failed. Retrying with fallback '{fallback_url}'.")
-            return function(fallback_url)
-
 
 def is_64bit() -> bool:
     """check if running platform is 64bit python."""
@@ -988,7 +987,7 @@ def init_worker_sh():
 
 
 def installer(
-    qt_archive: QtPackage,
+    qt_package: QtPackage,
     base_dir: str,
     command: Optional[str],
     queue: multiprocessing.Queue,
@@ -1000,10 +999,9 @@ def installer(
     Installer function to download archive files and extract it.
     It is called through multiprocessing.Pool()
     """
-    name = qt_archive.name
-    url = qt_archive.archive_url
-    hashurl = qt_archive.hashurl
-    archive: Path = archive_dest / qt_archive.archive
+    name = qt_package.name
+    base_url = qt_package.base_url
+    archive: Path = archive_dest / qt_package.archive
     start_time = time.perf_counter()
     # set defaults
     Settings.load_settings()
@@ -1016,14 +1014,19 @@ def installer(
         logger.removeHandler(handler)
     logger.addHandler(qh)
     #
-    logger.debug("Download URL: {}".format(url))
     if response_timeout is None:
         timeout = (Settings.connection_timeout, Settings.response_timeout)
     else:
         timeout = (Settings.connection_timeout, response_timeout)
-    hash = binascii.unhexlify(getUrl(hashurl, timeout))
+    hash = binascii.unhexlify(get_hash(qt_package.archive_path, algorithm="sha256", timeout=timeout))
+
+    def download_bin(_base_url):
+        url = posixpath.join(_base_url, qt_package.archive_path)
+        logger.debug("Download URL: {}".format(url))
+        return downloadBinaryFile(url, archive, "sha256", hash, timeout)
+
     retry_on_errors(
-        action=lambda: downloadBinaryFile(url, archive, "sha1", hash, timeout),
+        action=lambda: retry_on_bad_connection(download_bin, base_url),
         acceptable_errors=(ArchiveChecksumError,),
         num_retries=Settings.max_retries_on_checksum_error,
         name=f"Downloading {name}",
diff --git a/aqt/metadata.py b/aqt/metadata.py
index f5044d4..eb408d3 100644
--- a/aqt/metadata.py
+++ b/aqt/metadata.py
@@ -18,7 +18,7 @@
 # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
+import binascii
 import itertools
 import operator
 import posixpath
@@ -35,7 +35,7 @@ from semantic_version import Version as SemanticVersion
 from texttable import Texttable
 
 from aqt.exceptions import ArchiveConnectionError, ArchiveDownloadError, ArchiveListError, CliInputError, EmptyMetadata
-from aqt.helper import Settings, getUrl, xml_to_modules
+from aqt.helper import Settings, get_hash, getUrl, xml_to_modules
 
 
 class SimpleSpec(SemanticSimpleSpec):
@@ -452,7 +452,7 @@ class MetadataFactory:
 
     def fetch_extensions(self, version: Version) -> List[str]:
         versions_extensions = MetadataFactory.get_versions_extensions(
-            self.fetch_http(self.archive_id.to_url()), self.archive_id.category
+            self.fetch_http(self.archive_id.to_url(), False), self.archive_id.category
         )
         filtered = filter(
             lambda ver_ext: ver_ext[0] == version and ver_ext[1],
@@ -469,7 +469,7 @@ class MetadataFactory:
             return ver_ext[0]
 
         versions_extensions = MetadataFactory.get_versions_extensions(
-            self.fetch_http(self.archive_id.to_url()), self.archive_id.category
+            self.fetch_http(self.archive_id.to_url(), False), self.archive_id.category
         )
         versions = sorted(filter(None, map(get_version, filter(filter_by, versions_extensions))))
         iterables = itertools.groupby(versions, lambda version: version.minor)
@@ -479,7 +479,7 @@ class MetadataFactory:
         return self.fetch_versions().latest()
 
     def fetch_tools(self) -> List[str]:
-        html_doc = self.fetch_http(self.archive_id.to_url())
+        html_doc = self.fetch_http(self.archive_id.to_url(), False)
         return list(MetadataFactory.iterate_folders(html_doc, "tools"))
 
     def fetch_tool_modules(self, tool_name: str) -> List[str]:
@@ -572,15 +572,14 @@ class MetadataFactory:
         return version
 
     @staticmethod
-    def fetch_http(rest_of_url: str) -> str:
+    def fetch_http(rest_of_url: str, is_check_hash: bool = True) -> str:
+        timeout = (Settings.connection_timeout, Settings.response_timeout)
+        expected_hash = binascii.unhexlify(get_hash(rest_of_url, "sha256", timeout)) if is_check_hash else None
         base_urls = Settings.baseurl, random.choice(Settings.fallbacks)
         for i, base_url in enumerate(base_urls):
             try:
                 url = posixpath.join(base_url, rest_of_url)
-                return getUrl(
-                    url=url,
-                    timeout=(Settings.connection_timeout, Settings.response_timeout),
-                )
+                return getUrl(url=url, timeout=timeout, expected_hash=expected_hash)
 
             except (ArchiveDownloadError, ArchiveConnectionError) as e:
                 if i == len(base_urls) - 1:
diff --git a/aqt/settings.ini b/aqt/settings.ini
index c05986e..9b37968 100644
--- a/aqt/settings.ini
+++ b/aqt/settings.ini
@@ -15,8 +15,11 @@ response_timeout: 30
 max_retries_on_connection_error: 5
 retry_backoff: 0.1
 max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
 
 [mirrors]
+trusted_mirrors:
+    https://download.qt.io
 blacklist:
     http://mirrors.ocf.berkeley.edu
     http://mirrors.tuna.tsinghua.edu.cn
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 9657516..9d78b00 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -30,8 +30,11 @@ A file is like as follows:
     max_retries_on_connection_error: 5
     retry_backoff: 0.1
     max_retries_on_checksum_error: 5
+    max_retries_to_retrieve_hash: 5
 
     [mirrors]
+    trusted_mirrors:
+        https://download.qt.io
     blacklist:
         http://mirrors.ustc.edu.cn
         http://mirrors.tuna.tsinghua.edu.cn
@@ -130,6 +133,24 @@ max_retries_on_checksum_error:
 
 The ``[mirrors]`` section is a configuration for mirror handling.
 
+trusted_mirrors:
+    ``trusted_mirrors`` is a list of URLs that you trust to provide accurate
+    checksums for all downloaded archives.
+    This is a security feature; please do not change this value unless you know
+    what you're doing!
+
+    ``aqtinstall`` downloads all checksums from mirrors in this list.
+    These checksums are used to verify that every other file that ``aqtinstall``
+    downloads is, in fact, the correct file, and not a corrupt or malicious copy
+    of the file.
+    You may need to modify this list if the default mirrors are unreachable,
+    or if you do not trust that they have not been compromised.
+
+    ``aqtinstall`` can safely download archive files from the fallback mirror
+    list, and ensure that they are not malicious files, by checking them against
+    the checksums downloaded from the ``trusted_mirrors`` list.
+    ``aqtinstall`` uses the SHA-256 algorithm to perform this check.
+
 blacklist:
     It is a list of URL where is a problematic mirror site.
     Some mirror sites ignore a connection from IP addresses out of their preffered one.


Test patch:
diff --git a/tests/test_archives.py b/tests/test_archives.py
index ea4add5..88300af 100644
--- a/tests/test_archives.py
+++ b/tests/test_archives.py
@@ -1,6 +1,6 @@
+import hashlib
 import json
 import os
-import posixpath
 import re
 from itertools import groupby
 from pathlib import Path
@@ -50,8 +50,8 @@ def test_parse_update_xml(monkeypatch, os_name, version, arch, datafile):
     assert qt_archives_all_modules.archives is not None
 
     # Extract all urls
-    url_list = [item.archive_url for item in qt_archives.archives]
-    url_all_modules_list = [item.archive_url for item in qt_archives_all_modules.archives]
+    url_list = [item.archive_path for item in qt_archives.archives]
+    url_all_modules_list = [item.archive_path for item in qt_archives_all_modules.archives]
 
     # Check the difference list contains only extra modules urls for target specified
     list_diff = [item for item in url_all_modules_list if item not in url_list]
@@ -74,7 +74,10 @@ def corrupt_xmlfile():
     ),
 )
 def test_qtarchive_parse_corrupt_xmlfile(monkeypatch, corrupt_xmlfile, archives_class, init_args):
-    monkeypatch.setattr("aqt.archives.getUrl", lambda self, url: corrupt_xmlfile)
+    monkeypatch.setattr("aqt.archives.getUrl", lambda *args, **kwargs: corrupt_xmlfile)
+    monkeypatch.setattr(
+        "aqt.archives.get_hash", lambda *args, **kwargs: hashlib.sha256(bytes(corrupt_xmlfile, "utf-8")).hexdigest()
+    )
 
     with pytest.raises(ArchiveListError) as error:
         archives_class(*init_args)
@@ -124,7 +127,7 @@ def test_qt_archives_modules(monkeypatch, arch, requested_module_names, has_none
         # https://download.qt.io/online/qtsdkrepository/windows_x86/desktop/qt5_5140/
         # qt.qt5.5140.qtcharts.win32_mingw73/5.14.0-0-202108190846qtcharts-windows-win32_mingw73.7z
 
-        url_begin = posixpath.join(base, "online/qtsdkrepository/windows_x86/desktop/qt5_5140")
+        url_begin = "online/qtsdkrepository/windows_x86/desktop/qt5_5140"
         expected_archive_url_pattern = re.compile(
             r"^" + re.escape(url_begin) + "/(" + expect["Name"] + ")/" + re.escape(expect["Version"]) + r"(.+\.7z)$"
         )
@@ -135,11 +138,10 @@ def test_qt_archives_modules(monkeypatch, arch, requested_module_names, has_none
                 assert not pkg.package_desc
             else:
                 assert pkg.package_desc == expect["Description"]
-            url_match = expected_archive_url_pattern.match(pkg.archive_url)
+            url_match = expected_archive_url_pattern.match(pkg.archive_path)
             assert url_match
             mod_name, archive_name = url_match.groups()
             assert pkg.archive == archive_name
-            assert pkg.hashurl == pkg.archive_url + ".sha1"
             assert archive_name in expected_7z_files
             expected_7z_files.remove(archive_name)
         assert len(expected_7z_files) == 0, "Actual number of packages was fewer than expected"
@@ -212,7 +214,7 @@ def test_tools_variants(monkeypatch, tool_name, tool_variant_name, is_expect_fai
     expect = next(filter(lambda x: x["Name"] == tool_variant_name, expect_json["variants_metadata"]))
     expected_7z_files = set(expect["DownloadableArchives"])
     qt_pkgs = ToolArchives(host, target, tool_name, base, arch=tool_variant_name).archives
-    url_begin = posixpath.join(base, f"online/qtsdkrepository/mac_x64/{target}/{tool_name}")
+    url_begin = f"online/qtsdkrepository/mac_x64/{target}/{tool_name}"
     expected_archive_url_pattern = re.compile(
         r"^" + re.escape(url_begin) + "/(" + expect["Name"] + ")/" + re.escape(expect["Version"]) + r"(.+\.7z)$"
     )
@@ -222,12 +224,11 @@ def test_tools_variants(monkeypatch, tool_name, tool_variant_name, is_expect_fai
             assert not pkg.package_desc
         else:
             assert pkg.package_desc == expect["Description"]
-        url_match = expected_archive_url_pattern.match(pkg.archive_url)
+        url_match = expected_archive_url_pattern.match(pkg.archive_path)
         assert url_match
         actual_variant_name, archive_name = url_match.groups()
         assert actual_variant_name == tool_variant_name
         assert pkg.archive == archive_name
-        assert pkg.hashurl == pkg.archive_url + ".sha1"
         assert archive_name in expected_7z_files
         expected_7z_files.remove(archive_name)
     assert len(expected_7z_files) == 0, f"Failed to produce QtPackages for {expected_7z_files}"
@@ -446,6 +447,7 @@ def test_archives_weird_module_7z_name(
     expect_archives: Set[str],
 ):
     monkeypatch.setattr("aqt.archives.getUrl", lambda *args: xml)
+    monkeypatch.setattr("aqt.archives.get_hash", lambda *args, **kwargs: hashlib.sha256(bytes(xml, "utf-8")).hexdigest())
 
     qt_archives = make_archives_fn(subarchives, modules, is_include_base)
     archives = {pkg.archive for pkg in qt_archives.archives}
diff --git a/tests/test_cli.py b/tests/test_cli.py
index ba4dbfa..2a8f53e 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -96,7 +96,7 @@ def test_cli_determine_qt_version(
     monkeypatch, host, target, arch, version_or_spec: str, expected_version: Version, is_bad_spec: bool
 ):
     _html = (Path(__file__).parent / "data" / f"{host}-{target}.html").read_text("utf-8")
-    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda self, _: _html)
+    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda *args, **kwargs: _html)
     cli = Cli()
     cli._setup_settings()
 
diff --git a/tests/test_doc_archives.py b/tests/test_doc_archives.py
index 7ad13b2..656407a 100644
--- a/tests/test_doc_archives.py
+++ b/tests/test_doc_archives.py
@@ -41,8 +41,8 @@ def test_parse_update_xml(monkeypatch, os_name, version, flavor, datafile):
     assert qt_archives_all_modules.archives is not None
 
     # Extract all urls
-    url_list = [item.archive_url for item in qt_archives.archives]
-    url_all_modules_list = [item.archive_url for item in qt_archives_all_modules.archives]
+    url_list = [item.archive_path for item in qt_archives.archives]
+    url_all_modules_list = [item.archive_path for item in qt_archives_all_modules.archives]
 
     # Check the difference list contains only extra modules urls for target specified
     list_diff = [item for item in url_all_modules_list if item not in url_list]
diff --git a/tests/test_helper.py b/tests/test_helper.py
index bc27068..f846f3e 100644
--- a/tests/test_helper.py
+++ b/tests/test_helper.py
@@ -9,11 +9,18 @@ import requests
 from requests.models import Response
 
 from aqt import helper
-from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError
-from aqt.helper import getUrl, retry_on_errors
+from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ChecksumDownloadFailure
+from aqt.helper import Settings, get_hash, getUrl, retry_on_errors
 from aqt.metadata import Version
 
 
+@pytest.fixture(autouse=True)
+def load_default_settings(use_defaults: bool = True):
+    """For each test, first load the default settings file, unless marked otherwise"""
+    if use_defaults:
+        helper.Settings.load_settings()
+
+
 def test_helper_altlink(monkeypatch):
     class Message:
         headers = {"content-type": "text/plain", "length": 300}
@@ -56,6 +63,7 @@ def test_helper_altlink(monkeypatch):
     assert newurl.startswith("http://ftp.jaist.ac.jp/")
 
 
+@pytest.mark.load_default_settings(False)
 def test_settings(tmp_path):
     helper.Settings.load_settings(os.path.join(os.path.dirname(__file__), "data", "settings.ini"))
     assert helper.Settings.concurrency == 3
@@ -183,6 +191,41 @@ def test_helper_retry_on_error(num_attempts_before_success, num_retries_allowed)
         assert retry_on_errors(action, (RuntimeError,), num_retries_allowed, "do something")
 
 
+@pytest.mark.parametrize(
+    "num_tries_required, num_retries_allowed",
+    (
+        (2, 5),
+        (5, 5),
+        (6, 5),
+    ),
+)
+def test_helper_get_hash_retries(monkeypatch, num_tries_required, num_retries_allowed):
+    num_tries = 0
+
+    def mock_getUrl(url, *args, **kwargs):
+        nonlocal num_tries
+        num_tries += 1
+        if num_tries < num_tries_required:
+            raise ArchiveConnectionError(f"Must retry {num_tries_required - num_tries} more times before success")
+        parsed = urlparse(url)
+        base = f"{parsed.scheme}://{parsed.netloc}"
+        assert base in Settings.trusted_mirrors
+
+        hash_filename = str(parsed.path.split("/")[-1])
+        assert hash_filename == "archive.7z.sha256"
+        return "MOCK_HASH archive.7z"
+
+    monkeypatch.setattr("aqt.helper.getUrl", mock_getUrl)
+
+    if num_tries_required > num_retries_allowed:
+        with pytest.raises(ChecksumDownloadFailure) as e:
+            result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
+        assert e.type == ChecksumDownloadFailure
+    else:
+        result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
+        assert result == "MOCK_HASH"
+
+
 @pytest.mark.parametrize(
     "version, expect",
     [
@@ -288,3 +331,12 @@ def test_helper_getUrl_conn_error(monkeypatch):
         getUrl(url, timeout)
     assert e.type == ArchiveConnectionError
     assert expect_re.match(format(e.value))
+
+
+def test_helper_getUrl_checksum_error(monkeypatch):
+    mocked_get, mocked_session_get = mock_get_redirect(0)
+    monkeypatch.setattr(requests, "get", mocked_get)
+    monkeypatch.setattr(requests.Session, "get", mocked_session_get)
+    with pytest.raises(ArchiveChecksumError) as e:
+        getUrl("some_url", timeout=(5, 5), expected_hash=b"AAAAAAAAAAA")
+    assert e.type == ArchiveChecksumError
diff --git a/tests/test_install.py b/tests/test_install.py
index 6c2ebf7..b14f335 100644
--- a/tests/test_install.py
+++ b/tests/test_install.py
@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import os
 import re
@@ -134,11 +135,14 @@ def make_mock_geturl_download_archive(
     for _arc in archives:
         assert _arc.filename_7z.endswith(".7z")
 
+    xml = "<Updates>\n{}\n</Updates>".format("\n".join([archive.xml_package_update() for archive in archives]))
+
     def mock_getUrl(url: str, *args) -> str:
         if url.endswith(updates_url):
-            return "<Updates>\n{}\n</Updates>".format("\n".join([archive.xml_package_update() for archive in archives]))
-        elif url.endswith(".sha1"):
-            return ""  # Skip the checksum
+            return xml
+        elif url.endswith(".sha256"):
+            filename = url.split("/")[-1][: -len(".sha256")]
+            return f"{hashlib.sha256(bytes(xml, 'utf-8')).hexdigest()} {filename}"
         assert False
 
     def mock_download_archive(url: str, out: str, *args):
@@ -598,7 +602,7 @@ def test_install(
 
     mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
     monkeypatch.setattr("aqt.installer.downloadBinaryFile", mock_download_archive)
 
     with TemporaryDirectory() as output_dir:
@@ -707,13 +711,16 @@ def test_install(
     ),
 )
 def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Optional[str], expected):
+    xml = (Path(__file__).parent / "data" / xml_file).read_text("utf-8") if xml_file else ""
+
     def mock_get_url(url, *args, **kwargs):
         if not xml_file:
             raise ArchiveDownloadError(f"Failed to retrieve file at {url}\nServer response code: 404, reason: Not Found")
-        return (Path(__file__).parent / "data" / xml_file).read_text("utf-8")
+        return xml
 
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.archives.get_hash", lambda *args, **kwargs: hashlib.sha256(bytes(xml, "utf-8")).hexdigest())
+    monkeypatch.setattr("aqt.metadata.get_hash", lambda *args, **kwargs: hashlib.sha256(bytes(xml, "utf-8")).hexdigest())
     monkeypatch.setattr("aqt.metadata.getUrl", mock_get_url)
 
     cli = Cli()
@@ -779,7 +786,7 @@ def test_install_pool_exception(monkeypatch, capsys, make_exception, settings_fi
     cmd = ["install-qt", host, target, ver, arch]
     mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
     monkeypatch.setattr("aqt.installer.installer", mock_installer_func)
 
     Settings.load_settings(str(Path(__file__).parent / settings_file))
@@ -793,18 +800,18 @@ def test_install_installer_archive_extraction_err(monkeypatch):
     def mock_extractor_that_fails(*args, **kwargs):
         raise subprocess.CalledProcessError(returncode=1, cmd="some command", output="out", stderr="err")
 
-    monkeypatch.setattr("aqt.installer.getUrl", lambda *args: "")
+    monkeypatch.setattr("aqt.installer.get_hash", lambda *args, **kwargs: "")
     monkeypatch.setattr("aqt.installer.downloadBinaryFile", lambda *args: None)
     monkeypatch.setattr("aqt.installer.subprocess.run", mock_extractor_that_fails)
 
     with pytest.raises(ArchiveExtractionError) as err, TemporaryDirectory() as temp_dir:
         installer(
-            qt_archive=QtPackage(
+            qt_package=QtPackage(
                 "name",
-                "archive-url",
+                "base_url",
+                "archive_path",
                 "archive",
                 "package_desc",
-                "hashurl",
                 "pkg_update_name",
             ),
             base_dir=temp_dir,
diff --git a/tests/test_list.py b/tests/test_list.py
index bdffee3..57e308e 100644
--- a/tests/test_list.py
+++ b/tests/test_list.py
@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import re
@@ -131,7 +132,7 @@ def spec_regex():
 )
 def test_list_versions_tools(monkeypatch, spec_regex, os_name, target, in_file, expect_out_file):
     _html = (Path(__file__).parent / "data" / in_file).read_text("utf-8")
-    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda self, _: _html)
+    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda *args, **kwargs: _html)
 
     expected = json.loads((Path(__file__).parent / "data" / expect_out_file).read_text("utf-8"))
 
@@ -433,7 +434,7 @@ def test_list_qt_cli(
         expect_set = expect
     assert isinstance(expect_set, set)
 
-    def _mock_fetch_http(_, rest_of_url: str) -> str:
+    def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str:
         htmltext = (Path(__file__).parent / "data" / htmlfile).read_text("utf-8")
         if not rest_of_url.endswith("Updates.xml"):
             return htmltext
@@ -722,7 +723,7 @@ def test_list_describe_filters(meta: MetadataFactory, expect: str):
 )
 def test_list_to_version(monkeypatch, archive_id, spec, version_str, expect):
     _html = (Path(__file__).parent / "data" / "mac-desktop.html").read_text("utf-8")
-    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda self, _: _html)
+    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda *args, **kwargs: _html)
 
     if isinstance(expect, Exception):
         with pytest.raises(CliInputError) as error:
@@ -846,7 +847,7 @@ def test_show_list_versions(monkeypatch, capsys):
 
 def test_show_list_tools(monkeypatch, capsys):
     page = (Path(__file__).parent / "data" / "mac-desktop.html").read_text("utf-8")
-    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda self, _: page)
+    monkeypatch.setattr(MetadataFactory, "fetch_http", lambda *args, **kwargs: page)
 
     expect_file = Path(__file__).parent / "data" / "mac-desktop-expect.json"
     expect = "\n".join(json.loads(expect_file.read_text("utf-8"))["tools"]) + "\n"
@@ -917,7 +918,7 @@ def test_list_tool_cli(monkeypatch, capsys, host: str, target: str, tool_name: s
     xml_data = json.loads(xmljson)
     expected_tool_modules = set(xml_data["modules"])
 
-    def _mock_fetch_http(_, rest_of_url: str) -> str:
+    def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str:
         if not rest_of_url.endswith("Updates.xml"):
             return htmltext
         folder = urlparse(rest_of_url).path.split("/")[-2]
@@ -953,6 +954,7 @@ def test_list_tool_cli(monkeypatch, capsys, host: str, target: str, tool_name: s
 
 
 def test_fetch_http_ok(monkeypatch):
+    monkeypatch.setattr("aqt.metadata.get_hash", lambda *args, **kwargs: hashlib.sha256(b"some_html_content").hexdigest())
     monkeypatch.setattr("aqt.metadata.getUrl", lambda **kwargs: "some_html_content")
     assert MetadataFactory.fetch_http("some_url") == "some_html_content"
 
@@ -966,6 +968,7 @@ def test_fetch_http_failover(monkeypatch):
             raise ArchiveDownloadError()
         return "some_html_content"
 
+    monkeypatch.setattr("aqt.metadata.get_hash", lambda *args, **kwargs: hashlib.sha256(b"some_html_content").hexdigest())
     monkeypatch.setattr("aqt.metadata.getUrl", _mock)
 
     # Require that the first attempt failed, but the second did not
@@ -973,33 +976,19 @@ def test_fetch_http_failover(monkeypatch):
     assert len(urls_requested) == 2
 
 
-def test_fetch_http_download_error(monkeypatch):
+@pytest.mark.parametrize("exception_on_error", (ArchiveDownloadError, ArchiveConnectionError))
+def test_fetch_http_download_error(monkeypatch, exception_on_error):
     urls_requested = set()
 
     def _mock(url, **kwargs):
         urls_requested.add(url)
-        raise ArchiveDownloadError()
+        raise exception_on_error()
 
+    monkeypatch.setattr("aqt.metadata.get_hash", lambda *args, **kwargs: hashlib.sha256(b"some_html_content").hexdigest())
     monkeypatch.setattr("aqt.metadata.getUrl", _mock)
-    with pytest.raises(ArchiveDownloadError) as e:
+    with pytest.raises(exception_on_error) as e:
         MetadataFactory.fetch_http("some_url")
-    assert e.type == ArchiveDownloadError
-
-    # Require that a fallback url was tried
-    assert len(urls_requested) == 2
-
-
-def test_fetch_http_conn_error(monkeypatch):
-    urls_requested = set()
-
-    def _mock(url, **kwargs):
-        urls_requested.add(url)
-        raise ArchiveConnectionError()
-
-    monkeypatch.setattr("aqt.metadata.getUrl", _mock)
-    with pytest.raises(ArchiveConnectionError) as e:
-        MetadataFactory.fetch_http("some_url")
-    assert e.type == ArchiveConnectionError
+    assert e.type == exception_on_error
 
     # Require that a fallback url was tried
     assert len(urls_requested) == 2