Repository: miurahr/aqtinstall
Pull request #304 (2021-07-10 18:55:52)

List tool versions
This adds the ability to list tool module names alongside some extra metadata, including version, release date, display name, and description. Here's some sample output:

```
$ python -m aqt list tools windows desktop --tool-long tools_conan

 Tool Variant Name           Version         Release Date     Display Name              Description         
============================================================================================================
qt.tools.conan         1.33-202102101246     2021-02-10     Conan 1.33          Conan command line tool 1.33
qt.tools.conan.cmake   0.16.0-202102101246   2021-02-10     Conan conan.cmake   Conan conan.cmake (0.16.0)
```

This kind of output is really helpful for me personally, since I'm trying to work on `aqt tool`, and that requires looking for the version and release date of packages in Updates.xml files. This command makes that a lot easier. I think that if it's useful to me, it might be pretty useful for others as well.

The only part of this I'm unsure about is the inclusion of Display Name and Description. I think that part could be really useful for some people, but it makes the output really wide. I'm using `texttable` to print the tables, and when the tables are set to the default width (80 characters), many entries become completely unreadable. I think it would make sense to add some cli flags that change this kind of output, but I don't want to make the `aqt list` interface too complicated. 


Patch:
diff --git a/aqt/archives.py b/aqt/archives.py
index 07b087d..4dd71f7 100644
--- a/aqt/archives.py
+++ b/aqt/archives.py
@@ -20,15 +20,27 @@
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 import itertools
+import operator
 import posixpath
 import random
 import re
 import xml.etree.ElementTree as ElementTree
 from logging import getLogger
-from typing import Callable, Generator, Iterable, Iterator, List, Optional, Tuple, Union
+from typing import (
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import bs4
 from semantic_version import SimpleSpec, Version
+from texttable import Texttable
 
 from aqt import helper
 from aqt.exceptions import (
@@ -119,6 +131,20 @@ class ListCommand:
         def pretty_print(self) -> str:
             return "\n".join(self.strings)
 
+    class Table:
+        def __init__(self, head: List[str], rows: List[List[str]], max_width: int = 0):
+            # max_width is set to 0 by default: this disables wrapping of text table cells
+            self.head = head
+            self.rows = rows
+            self.max_width = max_width
+
+        def pretty_print(self) -> str:
+            table = Texttable(max_width=self.max_width)
+            table.set_deco(Texttable.HEADER)
+            table.header(self.head)
+            table.add_rows(self.rows, header=False)
+            return table.draw()
+
     def __init__(
         self,
         archive_id: ArchiveId,
@@ -129,6 +155,7 @@ class ListCommand:
         extensions_ver: Optional[str] = None,
         architectures_ver: Optional[str] = None,
         tool_name: Optional[str] = None,
+        tool_long_listing: Optional[str] = None,
     ):
         """
         Construct ListCommand.
@@ -149,6 +176,9 @@ class ListCommand:
             if tool_name:
                 self.request_type = "tool variant names"
                 self._action = lambda: self.fetch_tool_modules(tool_name)
+            elif tool_long_listing:
+                self.request_type = "tool long listing"
+                self._action = lambda: self.fetch_tool_long_listing(tool_long_listing)
             else:
                 self.request_type = "tools"
                 self._action = self.fetch_tools
@@ -240,15 +270,71 @@ class ListCommand:
         html_doc = self.fetch_http(self.archive_id.to_url())
         return ListCommand.Tools(list(ListCommand.iterate_folders(html_doc, "tools")))
 
-    def fetch_tool_modules(self, tool_name: str) -> ListOfStr:
+    def _fetch_tool_data(
+        self, tool_name: str, keys_to_keep: Optional[Iterable[str]] = None
+    ) -> Dict[str, Dict[str, str]]:
+        # raises ArchiveDownloadError, ArchiveConnectionError
         rest_of_url = self.archive_id.to_url() + tool_name + "/Updates.xml"
-        xml = self.fetch_http(rest_of_url)  # raises RequestException
+        xml = self.fetch_http(rest_of_url)
         modules = xml_to_modules(
             xml,
             predicate=ListCommand._has_nonempty_downloads,
-            keys_to_keep=(),  # Just want names
+            keys_to_keep=keys_to_keep,
         )
-        return ListCommand.ListOfStr(strings=list(modules.keys()))
+        return modules
+
+    def fetch_tool_modules(self, tool_name: str) -> ListOfStr:
+        tool_data = self._fetch_tool_data(tool_name, keys_to_keep=())
+        return ListCommand.ListOfStr(strings=list(tool_data.keys()))
+
+    def fetch_tool_by_simple_spec(
+        self, tool_name: str, simple_spec: SimpleSpec
+    ) -> Optional[Dict[str, str]]:
+        # Get data for all the tool modules
+        all_tools_data = self._fetch_tool_data(tool_name)
+        return ListCommand.choose_highest_version_in_spec(all_tools_data, simple_spec)
+
+    def fetch_tool_long_listing(self, tool_name: str) -> Table:
+        head = [
+            "Tool Variant Name",
+            "Version",
+            "Release Date",
+            "Display Name",
+            "Description",
+        ]
+        keys = ("Version", "ReleaseDate", "DisplayName", "Description")
+        tool_data = self._fetch_tool_data(tool_name, keys_to_keep=keys)
+        rows = [
+            [name, *[content[key] for key in keys]]
+            for name, content in tool_data.items()
+        ]
+        return ListCommand.Table(head, rows)
+
+    @staticmethod
+    def choose_highest_version_in_spec(
+        all_tools_data: Dict[str, Dict[str, str]], simple_spec: SimpleSpec
+    ) -> Optional[Dict[str, str]]:
+        # Get versions of all modules. Fail if version cannot be determined.
+        try:
+            tools_versions = [
+                (name, tool_data, helper.to_version_permissive(tool_data["Version"]))
+                for name, tool_data in all_tools_data.items()
+            ]
+        except ValueError:
+            return None
+
+        # Remove items that don't conform to simple_spec
+        tools_versions = filter(
+            lambda tool_item: tool_item[2] in simple_spec, tools_versions
+        )
+
+        try:
+            # Return the conforming item with the highest version.
+            # If there are multiple items with the same version, the result will not be predictable.
+            return max(tools_versions, key=operator.itemgetter(2))[1]
+        except ValueError:
+            # There were no tools that fit the simple_spec
+            return None
 
     def _to_version(self, qt_ver: str) -> Version:
         """
diff --git a/aqt/helper.py b/aqt/helper.py
index b293158..26c0b39 100644
--- a/aqt/helper.py
+++ b/aqt/helper.py
@@ -297,6 +297,30 @@ def to_version(qt_ver: str) -> Version:
     )
 
 
+def to_version_permissive(version_str: str) -> Version:
+    """Converts a version string with dots (5.X.Y, etc) into a semantic version.
+    If the version omits either the patch or minor versions, they will be filled in with zeros,
+    and the remaining version string becomes part of the prerelease component.
+    If the version cannot be converted to a Version, a ValueError is raised.
+
+    This function is intended to be used on Version tags in an Updates.xml file.
+
+    to_version_permissive('1.33.1-202102101246') => Version('1.33.1-202102101246')
+    to_version_permissive('1.33-202102101246') => Version('1.33.0-202102101246')    # tools_conan
+    to_version_permissive('2020-05-19-1') => Version('2020.0.0-05-19-1')            # tools_vcredist
+    """
+    match = re.match(r"^(\d+)(\.(\d+)(\.(\d+))?)?(.*)$", version_str)
+    if not match:
+        raise ValueError("Invalid version detected: '{}'".format(version_str))
+
+    major, dot_minor, minor, dot_patch, patch, prerelease = match.groups()
+    return Version(
+        "{}.{}.{}{}".format(
+            major, minor if minor else 0, patch if patch else 0, prerelease
+        )
+    )
+
+
 def get_semantic_version(qt_ver: str, is_preview: bool) -> Optional[Version]:
     """Converts a Qt version string (596, 512, 5132, etc) into a semantic version.
     This makes a lot of assumptions based on established patterns:
@@ -332,11 +356,16 @@ def get_semantic_version(qt_ver: str, is_preview: bool) -> Optional[Version]:
 def xml_to_modules(
     xml_text: str,
     predicate: Callable[[ElementTree.Element], bool],
-    keys_to_keep: Iterable[str],
+    keys_to_keep: Optional[Iterable[str]] = None,
 ) -> Dict[str, Dict[str, str]]:
     """Converts an XML document to a dict of `PackageUpdate` dicts, indexed by `Name` attribute.
     Only report elements that satisfy `predicate(element)`.
     Only report keys in the list `keys_to_keep`.
+    :param xml_text: The entire contents of an xml file
+    :param predicate: A function that decides which elements to keep or discard
+    :param keys_to_keep: A list of which tags in the element should be kept.
+                        If the list is empty, then no tags will be kept.
+                        If the list is None, then all tags will be kept.
     """
     try:
         parsed_xml = ElementTree.fromstring(xml_text)
@@ -348,8 +377,12 @@ def xml_to_modules(
             continue
         name = packageupdate.find("Name").text
         packages[name] = {}
-        for key in keys_to_keep:
-            packages[name][key] = getattr(packageupdate.find(key), "text", None)
+        if keys_to_keep is None:
+            for child in packageupdate:
+                packages[name][child.tag] = child.text
+        else:
+            for key in keys_to_keep:
+                packages[name][key] = getattr(packageupdate.find(key), "text", None)
     return packages
 
 
diff --git a/aqt/installer.py b/aqt/installer.py
index b508bc2..9958464 100644
--- a/aqt/installer.py
+++ b/aqt/installer.py
@@ -510,6 +510,7 @@ class Cli:
             extensions_ver=args.extensions,
             architectures_ver=args.arch,
             tool_name=args.tool,
+            tool_long_listing=args.tool_long,
         )
         return command.run()
 
@@ -600,6 +601,16 @@ class Cli:
             # TODO: find a better word ^^^^^^^^^^^^^^^^^^^^; this is a mysterious help message
             "The output of this command is intended to be used with `aqt tool`.",
         )
+        output_modifier_exclusive_group.add_argument(
+            "--tool-long",
+            type=str,
+            metavar="TOOL_NAME",
+            help="The name of a tool. Use 'aqt list tools <host> <target>' to see accepted values. "
+            "This flag only works with the 'tools' category, and cannot be combined with any other flags. "
+            "When set, this prints all 'tool variant names' available, along with versions and release dates. "
+            # TODO: find a better word ^^^^^^^^^^^^^^^^^^^^; this is a mysterious help message
+            "The output of this command is formatted as a table.",
+        )
         list_parser.set_defaults(func=self.run_list)
 
     def show_help(self, args=None):
diff --git a/ci/steps.yml b/ci/steps.yml
index 3c65cb6..43bbcab 100644
--- a/ci/steps.yml
+++ b/ci/steps.yml
@@ -76,6 +76,11 @@ steps:
         aqt list $qtmajor $(HOST)                                               # print all targets for host
         aqt list tools $(HOST) $(TARGET)                                        # print all tools for host/target
         aqt list tools $(HOST) $(TARGET) --tool tools_qt3dstudio_runtime_240    # print all tool variant names for qt3dstudio
+        aqt list tools $(HOST) $(TARGET) --tool-long tools_qt3dstudio_runtime_240    # print tool variant names, versions, release dates
+        if [[ "$(TARGET)" == "desktop" ]]; then
+          aqt list tools $(HOST) $(TARGET) --tool tools_qtcreator               # print all tool variant names for qtcreator
+          aqt list tools $(HOST) $(TARGET) --tool-long tools_qtcreator          # print tool variant names, versions, release dates
+        fi
         aqt list $qtmajor $(HOST) $(TARGET)                                     # print all versions of Qt
         if [[ $(HAS_WASM_EXTENSION) == "True" ]]; then
           aqt list $qtmajor $(HOST) $(TARGET) --extension wasm                  # print all wasm versions of Qt5


Test patch:
diff --git a/tests/data/mac-desktop-tools_cmake-expect.json b/tests/data/mac-desktop-tools_cmake-expect.json
index 90eb011..13d1395 100644
--- a/tests/data/mac-desktop-tools_cmake-expect.json
+++ b/tests/data/mac-desktop-tools_cmake-expect.json
@@ -1,5 +1,14 @@
 {
   "modules": [
     "qt.tools.cmake"
+  ],
+  "long_listing": [
+    [
+      "qt.tools.cmake",
+      "3.19.2-202101071155",
+      "2021-01-07",
+      "CMake 3.19.2",
+      "CMake tools 3.19.2"
+    ]
   ]
 }
\ No newline at end of file
diff --git a/tests/data/mac-desktop-tools_ifw-expect.json b/tests/data/mac-desktop-tools_ifw-expect.json
index fa4e0f3..bd30d14 100644
--- a/tests/data/mac-desktop-tools_ifw-expect.json
+++ b/tests/data/mac-desktop-tools_ifw-expect.json
@@ -1,5 +1,14 @@
 {
   "modules": [
     "qt.tools.ifw.41"
+  ],
+  "long_listing": [
+    [
+      "qt.tools.ifw.41",
+      "4.1.1-202105261132",
+      "2021-05-26",
+      "Qt Installer Framework 4.1",
+      "The Qt Installer Framework provides a set of tools and utilities to create installers for the supported desktop Qt platforms: Linux, Microsoft Windows, and macOS."
+    ]
   ]
 }
\ No newline at end of file
diff --git a/tests/data/mac-desktop-tools_qtcreator-expect.json b/tests/data/mac-desktop-tools_qtcreator-expect.json
index 9f83d2c..f628fdd 100644
--- a/tests/data/mac-desktop-tools_qtcreator-expect.json
+++ b/tests/data/mac-desktop-tools_qtcreator-expect.json
@@ -3,5 +3,28 @@
     "qt.tools.qtcreator",
     "qt.tools.qtcreatordbg",
     "qt.tools.qtcreatordev"
+  ],
+  "long_listing": [
+    [
+      "qt.tools.qtcreator",
+      "4.15.1-0-202106081242",
+      "2021-06-08",
+      "Qt Creator 4.15.1",
+      "IDE for Qt application development"
+    ],
+    [
+      "qt.tools.qtcreatordbg",
+      "4.15.1-0-202106081242",
+      "2021-06-08",
+      "Qt Creator 4.15.1 Debug Symbols",
+      "Additional symbol files required to debug Qt Creator"
+    ],
+    [
+      "qt.tools.qtcreatordev",
+      "4.15.1-0-202106081242",
+      "2021-06-08",
+      "Qt Creator 4.15.1 Plugin Development",
+      "Headers and libs required to develop additional plugins"
+    ]
   ]
 }
\ No newline at end of file
diff --git a/tests/test_helper.py b/tests/test_helper.py
index 924ee29..0535994 100644
--- a/tests/test_helper.py
+++ b/tests/test_helper.py
@@ -1,8 +1,10 @@
 import binascii
 import os
 
+import pytest
 import requests
 from requests.models import Response
+from semantic_version import Version
 
 from aqt import helper
 
@@ -95,3 +97,16 @@ def test_helper_downloadBinary_sha256(tmp_path, monkeypatch):
     helper.downloadBinaryFile(
         "http://example.com/test.xml", out, "sha256", expected, 60
     )
+
+
+@pytest.mark.parametrize(
+    "version, expect",
+    [
+        ("1.33.1", Version("1.33.1")),
+        ("1.33.1-202102101246", Version("1.33.1-202102101246")),
+        ("1.33-202102101246", Version("1.33.0-202102101246")),
+        ("2020-05-19-1", Version("2020.0.0-05-19-1")),
+    ],
+)
+def test_helper_to_version_permissive(version, expect):
+    assert helper.to_version_permissive(version) == expect
diff --git a/tests/test_list.py b/tests/test_list.py
index e25c0ba..1b3e993 100644
--- a/tests/test_list.py
+++ b/tests/test_list.py
@@ -3,7 +3,7 @@ import re
 from pathlib import Path
 
 import pytest
-from semantic_version import Version
+from semantic_version import SimpleSpec, Version
 
 from aqt import archives, installer
 from aqt.archives import ListCommand
@@ -129,6 +129,29 @@ def test_tool_modules(monkeypatch, host: str, target: str, tool_name: str):
     assert modules.strings == expect["modules"]
 
 
+@pytest.mark.parametrize(
+    "host, target, tool_name",
+    [
+        ("mac", "desktop", "tools_cmake"),
+        ("mac", "desktop", "tools_ifw"),
+        ("mac", "desktop", "tools_qtcreator"),
+    ],
+)
+def test_tool_long_listing(monkeypatch, host: str, target: str, tool_name: str):
+    archive_id = ArchiveId("tools", host, target)
+    in_file = "{}-{}-{}-update.xml".format(host, target, tool_name)
+    expect_out_file = "{}-{}-{}-expect.json".format(host, target, tool_name)
+    _xml = (Path(__file__).parent / "data" / in_file).read_text("utf-8")
+    expect = json.loads(
+        (Path(__file__).parent / "data" / expect_out_file).read_text("utf-8")
+    )
+
+    monkeypatch.setattr(archives.ListCommand, "fetch_http", lambda self, _: _xml)
+
+    table = ListCommand(archive_id).fetch_tool_long_listing(tool_name)
+    assert table.rows == expect["long_listing"]
+
+
 @pytest.mark.parametrize(
     "cat, host, target, minor_ver, ver, ext, xmlfile, xmlexpect, htmlfile, htmlexpect",
     [
@@ -232,3 +255,29 @@ def test_list_cli(
     check_arches()
     cli.run(["list", cat, host, target, *_ext, "--arch", ver])
     check_arches()
+
+
+@pytest.mark.parametrize(
+    "simple_spec, expected_name",
+    (
+        (SimpleSpec("*"), "mytool.999"),
+        (SimpleSpec(">3.5"), "mytool.999"),
+        (SimpleSpec("3.5.5"), "mytool.355"),
+        (SimpleSpec("<3.5"), "mytool.300"),
+        (SimpleSpec("<=3.5"), "mytool.355"),
+        (SimpleSpec("<=3.5.0"), "mytool.350"),
+        (SimpleSpec(">10"), None),
+    ),
+)
+def test_list_choose_tool_by_version(simple_spec, expected_name):
+    tools_data = {
+        "mytool.999": {"Version": "9.9.9", "Name": "mytool.999"},
+        "mytool.355": {"Version": "3.5.5", "Name": "mytool.355"},
+        "mytool.350": {"Version": "3.5.0", "Name": "mytool.350"},
+        "mytool.300": {"Version": "3.0.0", "Name": "mytool.300"},
+    }
+    item = ListCommand.choose_highest_version_in_spec(tools_data, simple_spec)
+    if item is not None:
+        assert item["Name"] == expected_name
+    else:
+        assert expected_name is None