Repository: miurahr/aqtinstall
Pull request #196 (2021-01-20 23:21:52)

Select fallback mirror site when main site connection failed
Add feature to select fallback mirror site randomly from known sites.

ref #194


Patch:
diff --git a/README.rst b/README.rst
index dab4db4..ffa2365 100644
--- a/README.rst
+++ b/README.rst
@@ -78,7 +78,8 @@ General usage looks like this:
 .. code-block:: bash
 
     aqt [-h][--help][-O | --outputdir <directory>][-b | --base <mirror url>][-E | --external <7zip command>] \
-        install <qt-version> <host> <target> [<arch>] [-m all | -m [extra module] [extra module]...] [--internal] [--archives archive]
+        install <qt-version> <host> <target> [<arch>] [-m all | -m [extra module] [extra module]...] [--internal]
+        [--archives archive] [ --fallback <mirror url> ]
 
 You can also call with ``python -m aqt`` syntax as well as command script ``aqt``.
 
@@ -109,7 +110,7 @@ You can install tools and utilities using following syntax;
 .. code-block:: bash
 
     python -m aqt [-h][--help][-O | --outputdir <directory>][-b | --base <mirror url>][-E | --external <7zip command>] \
-        tool <host> <tool_name> <tool-version> <arch>
+        tool <host> <tool_name> <tool-version> <arch>  [ --fallback <mirror url> ]
 
 * tool_name is one of `tools_ifw`, `tools_vcredist`, and `tools_openssl`.
 * arch is full qualified tool name such as `qt.tools.ifw.31` which values can be seen on Qt `archive_site`_
diff --git a/aqt/archives.py b/aqt/archives.py
index b7cda57..18efff5 100644
--- a/aqt/archives.py
+++ b/aqt/archives.py
@@ -28,6 +28,10 @@ import requests
 from aqt.helper import Settings
 
 
+class ArchiveConnectionError(Exception):
+    pass
+
+
 class ArchiveListError(Exception):
     pass
 
@@ -48,12 +52,11 @@ class QtPackage:
     """
       Hold package information.
     """
-    def __init__(self, name, archive_url, archive, package_desc, has_mirror=False):
+    def __init__(self, name, archive_url, archive, package_desc):
         self.name = name
         self.url = archive_url
         self.archive = archive
         self.desc = package_desc
-        self.has_mirror = has_mirror
 
 
 class ListInfo:
@@ -72,14 +75,13 @@ class PackagesList:
         Hold packages list information.
     """
 
-    BASE_URL = 'https://download.qt.io/online/qtsdkrepository/'
-
-    def __init__(self, version, os_name, target):
+    def __init__(self, version, os_name, target, base):
         self.version = version
         self.os_name = os_name
         self.target = target
-        self.base = self.BASE_URL
         self.archives = []
+        self.base = base
+        self.logger = getLogger('aqt')
         self._get_archives()
 
     def _get_archives(self):
@@ -88,25 +90,20 @@ class PackagesList:
         # Get packages index
         archive_path = "{0}{1}{2}/qt5_{3}/".format(self.os_name, '_x86/' if self.os_name == 'windows' else '_x64/',
                                                    self.target, qt_ver_num)
-        update_xml_url = "{0}{1}Updates.xml".format(self.BASE_URL, archive_path)
-        try:
-            r = requests.get(update_xml_url)
-        except requests.exceptions.ConnectionError as e:
-            self.logger.error('Download error: %s\n' % e.args, exc_info=True)
-            raise e
-        else:
-            self.update_xml = ElementTree.fromstring(r.text)
-            for packageupdate in self.update_xml.iter("PackageUpdate"):
-                name = packageupdate.find("Name").text
-                if packageupdate.find("DownloadableArchives").text is not None:
-                    package_desc = packageupdate.findtext("Description")
-                    display_name = packageupdate.findtext("DisplayName")
-                    virtual_str = packageupdate.findtext("Virtual")
-                    if virtual_str is None or virtual_str == 'false':
-                        virtual = False
-                    else:
-                        virtual = True
-                    self.archives.append(ListInfo(name, display_name, package_desc, virtual))
+        update_xml_url = "{0}{1}Updates.xml".format(self.base, archive_path)
+        r = requests.get(update_xml_url)
+        self.update_xml = ElementTree.fromstring(r.text)
+        for packageupdate in self.update_xml.iter("PackageUpdate"):
+            name = packageupdate.find("Name").text
+            if packageupdate.find("DownloadableArchives").text is not None:
+                package_desc = packageupdate.findtext("Description")
+                display_name = packageupdate.findtext("DisplayName")
+                virtual_str = packageupdate.findtext("Virtual")
+                if virtual_str is None or virtual_str == 'false':
+                    virtual = False
+                else:
+                    virtual = True
+                self.archives.append(ListInfo(name, display_name, package_desc, virtual))
         if len(self.archives) == 0:
             self.logger.error("Error while parsing package information!")
             exit(1)
@@ -118,24 +115,16 @@ class PackagesList:
 class QtArchives:
     """Hold Qt archive packages list."""
 
-    BASE_URL = 'https://download.qt.io/online/qtsdkrepository/'
-
-    def __init__(self, os_name, target, version, arch, subarchives=None,
-                 modules=None, mirror=None, logging=None, all_extra=False):
+    def __init__(self, os_name, target, version, arch, base, subarchives=None,
+                 modules=None, logging=None, all_extra=False):
         self.version = version
         self.target = target
         self.arch = arch
-        self.mirror = mirror
         self.os_name = os_name
         self.all_extra = all_extra
         self.arch_list = [item.get('arch') for item in Settings().qt_combinations]
         all_archives = (subarchives is None)
-        if mirror is not None:
-            self.has_mirror = True
-            self.base = mirror + '/online/qtsdkrepository/'
-        else:
-            self.has_mirror = False
-            self.base = self.BASE_URL
+        self.base = base
         if logging:
             self.logger = logging
         else:
@@ -172,9 +161,8 @@ class QtArchives:
     def _download_update_xml(self, update_xml_url):
         try:
             r = requests.get(update_xml_url)
-        except requests.exceptions.ConnectionError as e:
-            self.logger.error('Download error: %s\n' % e.args, exc_info=True)
-            raise ArchiveDownloadError("Download error!")
+        except (ConnectionResetError, requests.exceptions.ConnectionError):
+            raise ArchiveConnectionError()
         else:
             if r.status_code == 200:
                 self.update_xml_text = r.text
@@ -211,8 +199,7 @@ class QtArchives:
                         for archive in downloadable_archives:
                             archive_name = archive.split('-', maxsplit=1)[0]
                             package_url = archive_url + name + "/" + full_version + archive
-                            self.archives.append(QtPackage(archive_name, package_url, archive, package_desc,
-                                                           has_mirror=self.has_mirror))
+                            self.archives.append(QtPackage(archive_name, package_url, archive, package_desc))
         if len(self.archives) == 0:
             self.logger.error("Specified packages are not found while parsing XML of package information!")
             self.logger.debug(self.update_xml_text)
@@ -238,13 +225,14 @@ class QtArchives:
 class SrcDocExamplesArchives(QtArchives):
     """Hold doc/src/example archive package list."""
 
-    def __init__(self, flavor, os_name, target, version, subarchives=None,
-                 modules=None, mirror=None, logging=None, all_extra=False):
+    def __init__(self, flavor, os_name, target, version, base, subarchives=None,
+                 modules=None, logging=None, all_extra=False):
         self.flavor = flavor
         self.target = target
         self.os_name = os_name
-        super(SrcDocExamplesArchives, self).__init__(os_name, target, version, self.flavor, subarchives=subarchives,
-                                                     modules=modules, mirror=mirror, logging=logging,
+        self.base = base
+        super(SrcDocExamplesArchives, self).__init__(os_name, target, version, self.flavor, base,
+                                                     subarchives=subarchives, modules=modules, logging=logging,
                                                      all_extra=all_extra)
 
     def _get_archives(self, qt_ver_num):
@@ -276,10 +264,10 @@ class ToolArchives(QtArchives):
         ToolArchive(linux, desktop, 3.1.1, ifw)
     """
 
-    def __init__(self, os_name, tool_name, version, arch, mirror=None, logging=None):
+    def __init__(self, os_name, tool_name, version, arch, base, logging=None):
         self.tool_name = tool_name
         self.os_name = os_name
-        super(ToolArchives, self).__init__(os_name, 'desktop', version, arch, mirror=mirror, logging=logging)
+        super(ToolArchives, self).__init__(os_name, 'desktop', version, arch, base, logging=logging)
 
     def _get_archives(self, qt_ver_num):
         if self.os_name == 'windows':
@@ -315,8 +303,7 @@ class ToolArchives(QtArchives):
                 package_desc = packageupdate.find("Description").text
                 for archive in downloadable_archives:
                     package_url = archive_url + self.tool_name + "/" + name + "/" + named_version + archive
-                    self.archives.append(QtPackage(name, package_url, archive, package_desc,
-                                                   has_mirror=(self.mirror is not None)))
+                    self.archives.append(QtPackage(name, package_url, archive, package_desc))
 
     def get_target_config(self) -> TargetConfig:
         """Get target configuration.
diff --git a/aqt/installer.py b/aqt/installer.py
index 012dede..70c4f84 100644
--- a/aqt/installer.py
+++ b/aqt/installer.py
@@ -28,6 +28,7 @@ import multiprocessing
 import os
 import pathlib
 import platform
+import random
 import subprocess
 import sys
 import time
@@ -40,8 +41,9 @@ from requests.adapters import HTTPAdapter
 from texttable import Texttable
 from urllib3.util.retry import Retry
 
-from aqt.archives import (ArchiveDownloadError, ArchiveListError, PackagesList,
-                          QtArchives, SrcDocExamplesArchives, ToolArchives)
+from aqt.archives import (ArchiveConnectionError, ArchiveDownloadError,
+                          ArchiveListError, PackagesList, QtArchives,
+                          SrcDocExamplesArchives, ToolArchives)
 from aqt.helper import Settings, Updater, altlink, versiontuple
 
 try:
@@ -54,6 +56,14 @@ class ExtractionError(Exception):
     pass
 
 
+BASE_URL = 'https://download.qt.io/online/qtsdkrepository/'
+FALLBACK_URLS = ['https://mirrors.ocf.berkeley.edu/qt/online/qtsdkrepository/',
+                 'https://ftp.jaist.ac.jp/pub/qtproject/online/qtsdkrepository/',
+                 'http://ftp1.nluug.nl/languages/qt/online/qtsdkrepository/',
+                 'https://mirrors.dotsrc.org/qtproject/online/qtsdkrepository/'
+                 ]
+
+
 class Cli:
     """CLI main class to parse command line argument and launch proper functions."""
 
@@ -164,9 +174,12 @@ class Cli:
         arch = self._set_arch(args, arch, os_name, target, qt_version)
         modules = args.modules
         sevenzip = self._set_sevenzip(args)
-        mirror = args.base
+        if args.base is not None:
+            base = args.base + '/online/qtsdkrepository/'
+        else:
+            base = BASE_URL
         archives = args.archives
-        self._run_common_part(output_dir, mirror)
+        self._run_common_part(output_dir, base)
         if not self._check_qt_arg_versions(qt_version):
             self.logger.warning("Specified Qt version is unknown: {}.".format(qt_version))
         if not self._check_qt_arg_combination(qt_version, os_name, target, arch):
@@ -176,14 +189,22 @@ class Cli:
         if not all_extra and not self._check_modules_arg(qt_version, modules):
             self.logger.warning("Some of specified modules are unknown.")
         try:
-            qt_archives = QtArchives(os_name, target, qt_version, arch, subarchives=archives, modules=modules,
-                                     mirror=mirror, logging=self.logger, all_extra=all_extra)
+            qt_archives = QtArchives(os_name, target, qt_version, arch, base, subarchives=archives, modules=modules,
+                                     logging=self.logger, all_extra=all_extra)
+        except ArchiveConnectionError:
+            try:
+                self.logger.warning("Connection to the download site failed and fallback to mirror site.")
+                qt_archives = QtArchives(os_name, target, qt_version, arch, random.choice(FALLBACK_URLS),
+                                         subarchives=archives, modules=modules, logging=self.logger,
+                                         all_extra=all_extra)
+            except Exception:
+                self.logger.error("Connection to the download site failed. Aborted...")
+                exit(1)
         except ArchiveDownloadError or ArchiveListError:
             exit(1)
-        else:
-            target_config = qt_archives.get_target_config()
-            self.call_installer(qt_archives, output_dir, sevenzip)
-            finisher(target_config, base_dir, self.logger)
+        target_config = qt_archives.get_target_config()
+        self.call_installer(qt_archives, output_dir, sevenzip)
+        finisher(target_config, base_dir, self.logger)
         self.logger.info("Finished installation")
         self.logger.info("Time elasped: {time:.8f} second".format(time=time.perf_counter() - start_time))
 
@@ -193,24 +214,36 @@ class Cli:
         os_name = args.host
         qt_version = args.qt_version
         output_dir = args.outputdir
-        mirror = args.base
+        if args.base is not None:
+            base = args.base + '/online/qtsdkrepository/'
+        else:
+            base = BASE_URL
         sevenzip = self._set_sevenzip(args)
         modules = args.modules
         archives = args.archives
-        self._run_common_part(output_dir, mirror)
+        self._run_common_part(output_dir, base)
         all_extra = True if modules is not None and 'all' in modules else False
         if not self._check_qt_arg_versions(qt_version):
             self.logger.warning("Specified Qt version is unknown: {}.".format(qt_version))
         try:
-            srcdocexamples_archives = SrcDocExamplesArchives(flavor, os_name, target, qt_version, subarchives=archives,
-                                                             modules=modules, mirror=mirror, logging=self.logger,
+            srcdocexamples_archives = SrcDocExamplesArchives(flavor, os_name, target, qt_version, base,
+                                                             subarchives=archives, modules=modules, logging=self.logger,
                                                              all_extra=all_extra)
+        except ArchiveConnectionError:
+            try:
+                self.logger.warning("Connection to the download site failed and fallback to mirror site.")
+                srcdocexamples_archives = SrcDocExamplesArchives(flavor, os_name, target, qt_version,
+                                                                 random.choice(FALLBACK_URLS),
+                                                                 subarchives=archives, modules=modules,
+                                                                 logging=self.logger, all_extra=all_extra)
+            except Exception:
+                self.logger.error("Connection to the download site failed. Aborted...")
+                exit(1)
         except ArchiveDownloadError or ArchiveListError:
             exit(1)
-        else:
-            self.call_installer(srcdocexamples_archives, output_dir, sevenzip)
+        self.call_installer(srcdocexamples_archives, output_dir, sevenzip)
         self.logger.info("Finished installation")
-        self.logger.info("Time elasped: {time:.8f} second".format(time=time.perf_counter() - start_time))
+        self.logger.info("Time elapsed: {time:.8f} second".format(time=time.perf_counter() - start_time))
 
     def run_src(self, args):
         self._run_src_doc_examples('src', args)
@@ -229,22 +262,35 @@ class Cli:
         output_dir = args.outputdir
         sevenzip = self._set_sevenzip(args)
         version = args.version
-        mirror = args.base
-        self._run_common_part(output_dir, mirror)
+        if args.base is not None:
+            base = args.base + '/online/qtsdkrepository/'
+        else:
+            base = BASE_URL
+        self._run_common_part(output_dir, base)
         if not self._check_tools_arg_combination(os_name, tool_name, arch):
             self.logger.warning("Specified target combination is not valid: {} {} {}".format(os_name, tool_name, arch))
         try:
-            tool_archives = ToolArchives(os_name, tool_name, version, arch, mirror=mirror, logging=self.logger)
+            tool_archives = ToolArchives(os_name, tool_name, version, arch, base, logging=self.logger)
+        except ArchiveConnectionError:
+            try:
+                self.logger.warning("Connection to the download site failed and fallback to mirror site.")
+                tool_archives = ToolArchives(os_name, tool_name, version, arch, random.choice(FALLBACK_URLS),
+                                             logging=self.logger)
+            except Exception:
+                self.logger.error("Connection to the download site failed. Aborted...")
+                exit(1)
         except ArchiveDownloadError or ArchiveListError:
             exit(1)
-        else:
-            self.call_installer(tool_archives, output_dir, sevenzip)
+        self.call_installer(tool_archives, output_dir, sevenzip)
         self.logger.info("Finished installation")
-        self.logger.info("Time elasped: {time:.8f} second".format(time=time.perf_counter() - start_time))
+        self.logger.info("Time elapsed: {time:.8f} second".format(time=time.perf_counter() - start_time))
 
     def run_list(self, args):
         self.show_aqt_version()
-        pl = PackagesList(args.qt_version, args.host, args.target)
+        try:
+            pl = PackagesList(args.qt_version, args.host, args.target, BASE_URL)
+        except requests.exceptions.ConnectionError:
+            pl = PackagesList(args.qt_version, args.host, args.target, random.choice(FALLBACK_URLS))
         print('List Qt packages in %s for %s' % (args.qt_version, args.host))
         table = Texttable()
         table.set_deco(Texttable.HEADER)
@@ -373,7 +419,7 @@ class Cli:
         return args.func(args)
 
 
-def installer(qt_archive, base_dir, command):
+def installer(qt_archive, base_dir, command, response_timeout=30):
     name = qt_archive.name
     url = qt_archive.url
     archive = qt_archive.archive
@@ -381,51 +427,52 @@ def installer(qt_archive, base_dir, command):
     logger = getLogger('aqt')
     logger.info("Downloading {}...".format(name))
     logger.debug("Download URL: {}".format(url))
-    session = requests.Session()
-    retry = Retry(connect=5, backoff_factor=0.5)
-    adapter = HTTPAdapter(max_retries=retry)
-    session.mount('http://', adapter)
-    session.mount('https://', adapter)
-    try:
-        r = session.get(url, allow_redirects=False, stream=True)
-        if r.status_code == 302:
-            newurl = altlink(r.url, r.headers['Location'], logger=logger)
-            logger.info('Redirected URL: {}'.format(newurl))
-            r = session.get(newurl, stream=True)
-    except requests.exceptions.ConnectionError as e:
-        logger.error("Connection error: %s" % e.args)
-        raise e
-    else:
+    timeout = (3.5, response_timeout)
+    with requests.Session() as session:
+        retry = Retry(connect=5, backoff_factor=0.5)
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
         try:
-            with open(archive, 'wb') as fd:
-                for chunk in r.iter_content(chunk_size=8196):
-                    fd.write(chunk)
-                    fd.flush()
-            if command is None:
-                with py7zr.SevenZipFile(archive, 'r') as szf:
-                    szf.extractall(path=base_dir)
-        except Exception as e:
-            exc = sys.exc_info()
-            logger.error("Download error: %s" % exc[1])
+            r = session.get(url, allow_redirects=False, stream=True, timeout=timeout)
+            if r.status_code == 302:
+                newurl = altlink(r.url, r.headers['Location'], logger=logger)
+                logger.info('Redirected URL: {}'.format(newurl))
+                r = session.get(newurl, stream=True, timeout=timeout)
+        except requests.exceptions.ConnectionError as e:
+            logger.error("Connection error: %s" % e.args)
             raise e
         else:
-            if command is not None:
-                if base_dir is not None:
-                    command_args = [command, 'x', '-aoa', '-bd', '-y', '-o{}'.format(base_dir), archive]
-                else:
-                    command_args = [command, 'x', '-aoa', '-bd', '-y', archive]
-                try:
-                    proc = subprocess.run(command_args, stdout=subprocess.PIPE, check=True)
-                    logger.debug(proc.stdout)
-                except subprocess.CalledProcessError as cpe:
-                    logger.error("Extraction error: %d" % cpe.returncode)
-                    if cpe.stdout is not None:
-                        logger.error(cpe.stdout)
-                    if cpe.stderr is not None:
-                        logger.error(cpe.stderr)
-                    raise cpe
+            try:
+                with open(archive, 'wb') as fd:
+                    for chunk in r.iter_content(chunk_size=8196):
+                        fd.write(chunk)
+                        fd.flush()
+                if command is None:
+                    with py7zr.SevenZipFile(archive, 'r') as szf:
+                        szf.extractall(path=base_dir)
+            except Exception as e:
+                exc = sys.exc_info()
+                logger.error("Download error: %s" % exc[1])
+                raise e
+            else:
+                if command is not None:
+                    if base_dir is not None:
+                        command_args = [command, 'x', '-aoa', '-bd', '-y', '-o{}'.format(base_dir), archive]
+                    else:
+                        command_args = [command, 'x', '-aoa', '-bd', '-y', archive]
+                    try:
+                        proc = subprocess.run(command_args, stdout=subprocess.PIPE, check=True)
+                        logger.debug(proc.stdout)
+                    except subprocess.CalledProcessError as cpe:
+                        logger.error("Extraction error: %d" % cpe.returncode)
+                        if cpe.stdout is not None:
+                            logger.error(cpe.stdout)
+                        if cpe.stderr is not None:
+                            logger.error(cpe.stderr)
+                        raise cpe
     os.unlink(archive)
-    logger.info("Finished installation of {} in {}".format(archive, time.perf_counter() - start_time))
+    logger.info("Finish installation of {} in {}".format(archive, time.perf_counter() - start_time))
 
 
 def finisher(target, base_dir, logger):


Test patch:
diff --git a/tests/test_archives.py b/tests/test_archives.py
index 3753da1..90f98d5 100644
--- a/tests/test_archives.py
+++ b/tests/test_archives.py
@@ -3,6 +3,7 @@ import os
 import pytest
 
 from aqt.archives import QtArchives
+from aqt.installer import BASE_URL
 
 
 @pytest.mark.parametrize("os_name,version,target,datafile", [
@@ -18,7 +19,7 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile):
 
     monkeypatch.setattr(QtArchives, "_download_update_xml", _mock)
 
-    qt_archives = QtArchives(os_name, 'desktop', version, target)
+    qt_archives = QtArchives(os_name, 'desktop', version, target, BASE_URL)
     assert qt_archives.archives is not None
 
     # Get packages with all extra modules
diff --git a/tests/test_doc_archives.py b/tests/test_doc_archives.py
index 8afed6d..93f45f7 100644
--- a/tests/test_doc_archives.py
+++ b/tests/test_doc_archives.py
@@ -3,6 +3,7 @@ import os
 import pytest
 
 from aqt.archives import QtArchives
+from aqt.installer import BASE_URL
 
 
 @pytest.mark.parametrize("os_name,version,target,datafile", [
@@ -16,11 +17,11 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile):
 
     monkeypatch.setattr(QtArchives, "_download_update_xml", _mock)
 
-    qt_archives = QtArchives(os_name, 'desktop', version, target)
+    qt_archives = QtArchives(os_name, 'desktop', version, target, BASE_URL)
     assert qt_archives.archives is not None
 
     # Get packages with all extra modules
-    qt_archives_all_modules = QtArchives(os_name, 'desktop', version, target, None, ['all'], None, None, True)
+    qt_archives_all_modules = QtArchives(os_name, 'desktop', version, target, BASE_URL, None, ['all'], None, True)
     assert qt_archives_all_modules.archives is not None
 
     # Extract all urls