Repository: python-cmd2/cmd2

Issue:
Support formatted tab-completion output
Add support for newlines and colored text in tab-completion hints.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24dbc4a1..95c3f71a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,11 +15,15 @@
     * Removed `--silent` flag from `alias/macro create` since startup scripts can be run silently.
     * Removed `--with_silent` flag from `alias/macro list` since startup scripts can be run silently.
     * Removed `with_argparser_and_unknown_args` since it was deprecated in 1.3.0.
+    * Replaced `cmd2.Cmd.completion_header` with `cmd2.Cmd.formatted_completions`. See Enhancements
+      for description of this new class member.
 * Enhancements
     * Added support for custom tab completion and up-arrow input history to `cmd2.Cmd2.read_input`.
       See [read_input.py](https://github.com/python-cmd2/cmd2/blob/master/examples/read_input.py)
       for an example.
     * Added `cmd2.exceptions.PassThroughException` to raise unhandled command exceptions instead of printing them.
+    * Added support for ANSI styles and newlines in tab completion results using `cmd2.Cmd.formatted_completions`.
+      `cmd2` provides this capability automatically if you return argparse completion matches as `CompletionItems`.
 
 ## 1.5.0 (January 31, 2021)
 * Bug Fixes
diff --git a/cmd2/ansi.py b/cmd2/ansi.py
index 59e25483..0f34016d 100644
--- a/cmd2/ansi.py
+++ b/cmd2/ansi.py
@@ -184,15 +184,39 @@ def strip_style(text: str) -> str:
 
 def style_aware_wcswidth(text: str) -> int:
     """
-    Wrap wcswidth to make it compatible with strings that contains ANSI style sequences
+    Wrap wcswidth to make it compatible with strings that contain ANSI style sequences.
+    This is intended for single line strings. If text contains a newline, this
+    function will return -1. For multiline strings, call widest_line() instead.
 
     :param text: the string being measured
-    :return: the width of the string when printed to the terminal
+    :return: The width of the string when printed to the terminal if no errors occur.
+             If text contains characters with no absolute width (i.e. tabs),
+             then this function returns -1. Replace tabs with spaces before calling this.
     """
     # Strip ANSI style sequences since they cause wcswidth to return -1
     return wcswidth(strip_style(text))
 
 
+def widest_line(text: str) -> int:
+    """
+    Return the width of the widest line in a multiline string. This wraps style_aware_wcswidth()
+    so it handles ANSI style sequences and has the same restrictions on non-printable characters.
+
+    :param text: the string being measured
+    :return: The width of the string when printed to the terminal if no errors occur.
+             If text contains characters with no absolute width (i.e. tabs),
+             then this function returns -1. Replace tabs with spaces before calling this.
+    """
+    if not text:
+        return 0
+
+    lines_widths = [style_aware_wcswidth(line) for line in text.splitlines()]
+    if -1 in lines_widths:
+        return -1
+
+    return max(lines_widths)
+
+
 def style_aware_write(fileobj: IO, msg: str) -> None:
     """
     Write a string to a fileobject and strip its ANSI style sequences if required by allow_style setting
diff --git a/cmd2/argparse_completer.py b/cmd2/argparse_completer.py
index fbaa6107..21007289 100644
--- a/cmd2/argparse_completer.py
+++ b/cmd2/argparse_completer.py
@@ -9,7 +9,6 @@ See the header of argparse_custom.py for instructions on how to use these featur
 import argparse
 import inspect
 import numbers
-import shutil
 from collections import (
     deque,
 )
@@ -527,6 +526,7 @@ class ArgparseCompleter:
     def _format_completions(self, arg_state: _ArgumentState, completions: List[Union[str, CompletionItem]]) -> List[str]:
         # Check if the results are CompletionItems and that there aren't too many to display
         if 1 < len(completions) <= self._cmd2_app.max_completion_items and isinstance(completions[0], CompletionItem):
+            four_spaces = 4 * ' '
 
             # If the user has not already sorted the CompletionItems, then sort them before appending the descriptions
             if not self._cmd2_app.matches_sorted:
@@ -549,30 +549,27 @@ class ArgparseCompleter:
             if desc_header is None:
                 desc_header = DEFAULT_DESCRIPTIVE_HEADER
 
+            # Replace tabs with 4 spaces so we can calculate width
+            desc_header = desc_header.replace('\t', four_spaces)
+
             # Calculate needed widths for the token and description columns of the table
             token_width = ansi.style_aware_wcswidth(destination)
-            desc_width = ansi.style_aware_wcswidth(desc_header)
+            desc_width = ansi.widest_line(desc_header)
 
             for item in completions:
                 token_width = max(ansi.style_aware_wcswidth(item), token_width)
-                desc_width = max(ansi.style_aware_wcswidth(item.description), desc_width)
-
-            # Create a table that's over half the width of the terminal.
-            # This will force readline to place each entry on its own line.
-            min_width = int(shutil.get_terminal_size().columns * 0.6)
-            base_width = SimpleTable.base_width(2)
-            initial_width = base_width + token_width + desc_width
 
-            if initial_width < min_width:
-                desc_width += min_width - initial_width
+                # Replace tabs with 4 spaces so we can calculate width
+                item.description = item.description.replace('\t', four_spaces)
+                desc_width = max(ansi.widest_line(item.description), desc_width)
 
             cols = list()
             cols.append(Column(destination.upper(), width=token_width))
             cols.append(Column(desc_header, width=desc_width))
 
             hint_table = SimpleTable(cols, divider_char=None)
-            self._cmd2_app.completion_header = hint_table.generate_header()
-            self._cmd2_app.display_matches = [hint_table.generate_data_row([item, item.description]) for item in completions]
+            table_data = [[item, item.description] for item in completions]
+            self._cmd2_app.formatted_completions = hint_table.generate_table(table_data, row_spacing=0)
 
         return completions
 
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 2d60206c..2e4a4cb5 100644
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -467,8 +467,10 @@ class Cmd(cmd.Cmd):
         # An optional hint which prints above tab completion suggestions
         self.completion_hint = ''
 
-        # Header which prints above CompletionItem tables
-        self.completion_header = ''
+        # Already formatted completion results. If this is populated, then cmd2 will print it instead
+        # of using readline's columnized results. ANSI style sequences and newlines in tab completion
+        # results are supported by this member. ArgparseCompleter uses this to print tab completion tables.
+        self.formatted_completions = ''
 
         # Used by complete() for readline tab completion
         self.completion_matches = []
@@ -1118,7 +1120,7 @@ class Cmd(cmd.Cmd):
         self.allow_appended_space = True
         self.allow_closing_quote = True
         self.completion_hint = ''
-        self.completion_header = ''
+        self.formatted_completions = ''
         self.completion_matches = []
         self.display_matches = []
         self.matches_delimited = False
@@ -1645,22 +1647,6 @@ class Cmd(cmd.Cmd):
 
         return [cur_match + padding for cur_match in matches_to_display], len(padding)
 
-    def _build_completion_metadata_string(self) -> str:  # pragma: no cover
-        """Build completion metadata string which can contain a hint and CompletionItem table header"""
-        metadata = ''
-
-        # Add hint if one exists and we are supposed to display it
-        if self.always_show_hint and self.completion_hint:
-            metadata += '\n' + self.completion_hint
-
-        # Add table header if one exists
-        if self.completion_header:
-            if not metadata:
-                metadata += '\n'
-            metadata += '\n' + self.completion_header
-
-        return metadata
-
     def _display_matches_gnu_readline(
         self, substitution: str, matches: List[str], longest_match_length: int
     ) -> None:  # pragma: no cover
@@ -1673,45 +1659,55 @@ class Cmd(cmd.Cmd):
         """
         if rl_type == RlType.GNU:
 
-            # Check if we should show display_matches
-            if self.display_matches:
-                matches_to_display = self.display_matches
+            # Print hint if one exists and we are supposed to display it
+            hint_printed = False
+            if self.always_show_hint and self.completion_hint:
+                hint_printed = True
+                sys.stdout.write('\n' + self.completion_hint)
 
-                # Recalculate longest_match_length for display_matches
-                longest_match_length = 0
+            # Check if we already have formatted results to print
+            if self.formatted_completions:
+                if not hint_printed:
+                    sys.stdout.write('\n')
+                sys.stdout.write('\n' + self.formatted_completions + '\n\n')
 
-                for cur_match in matches_to_display:
-                    cur_length = ansi.style_aware_wcswidth(cur_match)
-                    if cur_length > longest_match_length:
-                        longest_match_length = cur_length
+            # Otherwise use readline's formatter
             else:
-                matches_to_display = matches
+                # Check if we should show display_matches
+                if self.display_matches:
+                    matches_to_display = self.display_matches
 
-            # Add padding for visual appeal
-            matches_to_display, padding_length = self._pad_matches_to_display(matches_to_display)
-            longest_match_length += padding_length
+                    # Recalculate longest_match_length for display_matches
+                    longest_match_length = 0
 
-            # We will use readline's display function (rl_display_match_list()), so we
-            # need to encode our string as bytes to place in a C array.
-            encoded_substitution = bytes(substitution, encoding='utf-8')
-            encoded_matches = [bytes(cur_match, encoding='utf-8') for cur_match in matches_to_display]
+                    for cur_match in matches_to_display:
+                        cur_length = ansi.style_aware_wcswidth(cur_match)
+                        if cur_length > longest_match_length:
+                            longest_match_length = cur_length
+                else:
+                    matches_to_display = matches
+
+                # Add padding for visual appeal
+                matches_to_display, padding_length = self._pad_matches_to_display(matches_to_display)
+                longest_match_length += padding_length
 
-            # rl_display_match_list() expects matches to be in argv format where
-            # substitution is the first element, followed by the matches, and then a NULL.
-            # noinspection PyCallingNonCallable,PyTypeChecker
-            strings_array = (ctypes.c_char_p * (1 + len(encoded_matches) + 1))()
+                # We will use readline's display function (rl_display_match_list()), so we
+                # need to encode our string as bytes to place in a C array.
+                encoded_substitution = bytes(substitution, encoding='utf-8')
+                encoded_matches = [bytes(cur_match, encoding='utf-8') for cur_match in matches_to_display]
 
-            # Copy in the encoded strings and add a NULL to the end
-            strings_array[0] = encoded_substitution
-            strings_array[1:-1] = encoded_matches
-            strings_array[-1] = None
+                # rl_display_match_list() expects matches to be in argv format where
+                # substitution is the first element, followed by the matches, and then a NULL.
+                # noinspection PyCallingNonCallable,PyTypeChecker
+                strings_array = (ctypes.c_char_p * (1 + len(encoded_matches) + 1))()
 
-            # Print any metadata like a hint or table header
-            sys.stdout.write(self._build_completion_metadata_string())
+                # Copy in the encoded strings and add a NULL to the end
+                strings_array[0] = encoded_substitution
+                strings_array[1:-1] = encoded_matches
+                strings_array[-1] = None
 
-            # Call readline's display function
-            # rl_display_match_list(strings_array, number of completion matches, longest match length)
-            readline_lib.rl_display_match_list(strings_array, len(encoded_matches), longest_match_length)
+                # rl_display_match_list(strings_array, number of completion matches, longest match length)
+                readline_lib.rl_display_match_list(strings_array, len(encoded_matches), longest_match_length)
 
             # Redraw prompt and input line
             rl_force_redisplay()
@@ -1724,20 +1720,34 @@ class Cmd(cmd.Cmd):
         """
         if rl_type == RlType.PYREADLINE:
 
-            # Check if we should show display_matches
-            if self.display_matches:
-                matches_to_display = self.display_matches
-            else:
-                matches_to_display = matches
+            # Print hint if one exists and we are supposed to display it
+            hint_printed = False
+            if self.always_show_hint and self.completion_hint:
+                hint_printed = True
+                readline.rl.mode.console.write('\n' + self.completion_hint)
 
-            # Add padding for visual appeal
-            matches_to_display, _ = self._pad_matches_to_display(matches_to_display)
+            # Check if we already have formatted results to print
+            if self.formatted_completions:
+                if not hint_printed:
+                    readline.rl.mode.console.write('\n')
+                readline.rl.mode.console.write('\n' + self.formatted_completions + '\n\n')
+
+                # Redraw the prompt and input lines
+                rl_force_redisplay()
+
+            # Otherwise use pyreadline's formatter
+            else:
+                # Check if we should show display_matches
+                if self.display_matches:
+                    matches_to_display = self.display_matches
+                else:
+                    matches_to_display = matches
 
-            # Print any metadata like a hint or table header
-            readline.rl.mode.console.write(self._build_completion_metadata_string())
+                # Add padding for visual appeal
+                matches_to_display, _ = self._pad_matches_to_display(matches_to_display)
 
-            # Display matches using actual display function. This also redraws the prompt and line.
-            orig_pyreadline_display(matches_to_display)
+                # Display matches using actual display function. This also redraws the prompt and input lines.
+                orig_pyreadline_display(matches_to_display)
 
     def _perform_completion(
         self, text: str, line: str, begidx: int, endidx: int, custom_settings: Optional[utils.CustomCompletionSettings] = None
diff --git a/cmd2/table_creator.py b/cmd2/table_creator.py
index 3b0db7b3..d9377bc4 100644
--- a/cmd2/table_creator.py
+++ b/cmd2/table_creator.py
@@ -139,9 +139,7 @@ class TableCreator:
             # For headers with the width not yet set, use the width of the
             # widest line in the header or 1 if the header has no width
             if col.width is None:
-                line_widths = [ansi.style_aware_wcswidth(line) for line in col.header.splitlines()]
-                line_widths.append(1)
-                col.width = max(line_widths)
+                col.width = max(1, ansi.widest_line(col.header))
 
     @staticmethod
     def _wrap_long_word(word: str, max_width: int, max_lines: Union[int, float], is_last_word: bool) -> Tuple[str, int, int]:
@@ -396,7 +394,7 @@ class TableCreator:
         aligned_text = utils.align_text(wrapped_text, fill_char=fill_char, width=col.width, alignment=text_alignment)
 
         lines = deque(aligned_text.splitlines())
-        cell_width = max([ansi.style_aware_wcswidth(line) for line in lines])
+        cell_width = ansi.widest_line(aligned_text)
         return lines, cell_width
 
     def generate_row(
diff --git a/examples/argparse_completion.py b/examples/argparse_completion.py
index fa821b77..7350f46c 100644
--- a/examples/argparse_completion.py
+++ b/examples/argparse_completion.py
@@ -14,6 +14,7 @@ from cmd2 import (
     Cmd2ArgumentParser,
     CompletionError,
     CompletionItem,
+    ansi,
     with_argparser,
 )
 
@@ -45,7 +46,9 @@ class ArgparseCompletion(Cmd):
     # noinspection PyMethodMayBeStatic
     def choices_completion_item(self) -> List[CompletionItem]:
         """Return CompletionItem instead of strings. These give more context to what's being tab completed."""
-        items = {1: "My item", 2: "Another item", 3: "Yet another item"}
+        fancy_item = "These things can\ncontain newlines and\n"
+        fancy_item += ansi.style("styled text!!", fg=ansi.fg.bright_yellow, underline=True)
+        items = {1: "My item", 2: "Another item", 3: "Yet another item", 4: fancy_item}
         return [CompletionItem(item_id, description) for item_id, description in items.items()]
 
     # noinspection PyMethodMayBeStatic
@@ -73,8 +76,6 @@ class ArgparseCompletion(Cmd):
     # want the entire choices list showing in the usage text for this command.
     example_parser.add_argument('--choices', choices=food_item_strs, metavar="CHOICE", help="tab complete using choices")
 
-    example_parser.add_argument('--choices', choices=food_item_strs, metavar="CHOICE", help="tab complete using choices")
-
     # Tab complete from choices provided by a choices_provider
     example_parser.add_argument(
         '--choices_provider', choices_provider=choices_provider, help="tab complete using a choices_provider"


Test patch:
diff --git a/tests/test_ansi.py b/tests/test_ansi.py
index 7ebda497..1797a047 100644
--- a/tests/test_ansi.py
+++ b/tests/test_ansi.py
@@ -20,7 +20,20 @@ def test_strip_style():
 def test_style_aware_wcswidth():
     base_str = HELLO_WORLD
     ansi_str = ansi.style(base_str, fg='green')
-    assert ansi.style_aware_wcswidth(ansi_str) != len(ansi_str)
+    assert ansi.style_aware_wcswidth(HELLO_WORLD) == ansi.style_aware_wcswidth(ansi_str)
+
+    assert ansi.style_aware_wcswidth('i have a tab\t') == -1
+    assert ansi.style_aware_wcswidth('i have a newline\n') == -1
+
+
+def test_widest_line():
+    text = ansi.style('i have\n3 lines\nThis is the longest one', fg='green')
+    assert ansi.widest_line(text) == ansi.style_aware_wcswidth("This is the longest one")
+
+    text = "I'm just one line"
+    assert ansi.widest_line(text) == ansi.style_aware_wcswidth(text)
+
+    assert ansi.widest_line('i have a tab\t') == -1
 
 
 def test_style_none():
diff --git a/tests/test_argparse_completer.py b/tests/test_argparse_completer.py
index efda7660..75f24b3e 100644
--- a/tests/test_argparse_completer.py
+++ b/tests/test_argparse_completer.py
@@ -24,6 +24,7 @@ from cmd2.utils import (
 
 from .conftest import (
     complete_tester,
+    normalize,
     run_cmd,
 )
 
@@ -720,8 +721,12 @@ def test_completion_items(ac_app, num_aliases, show_description):
     assert len(ac_app.completion_matches) == num_aliases
     assert len(ac_app.display_matches) == num_aliases
 
-    # If show_description is True, the alias's value will be in the display text
-    assert ('help' in ac_app.display_matches[0]) == show_description
+    assert bool(ac_app.formatted_completions) == show_description
+    if show_description:
+        # If show_description is True, the table will show both the alias name and result
+        first_result_line = normalize(ac_app.formatted_completions)[1]
+        assert 'fake_alias0' in first_result_line
+        assert 'help' in first_result_line
 
 
 @pytest.mark.parametrize(
@@ -842,7 +847,7 @@ def test_completion_items_arg_header(ac_app):
     begidx = endidx - len(text)
 
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert "DESC_HEADER" in ac_app.completion_header
+    assert "DESC_HEADER" in normalize(ac_app.formatted_completions)[0]
 
     # Test when metavar is a string
     text = ''
@@ -851,7 +856,7 @@ def test_completion_items_arg_header(ac_app):
     begidx = endidx - len(text)
 
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert ac_app.STR_METAVAR in ac_app.completion_header
+    assert ac_app.STR_METAVAR in normalize(ac_app.formatted_completions)[0]
 
     # Test when metavar is a tuple
     text = ''
@@ -861,7 +866,7 @@ def test_completion_items_arg_header(ac_app):
 
     # We are completing the first argument of this flag. The first element in the tuple should be the column header.
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert ac_app.TUPLE_METAVAR[0].upper() in ac_app.completion_header
+    assert ac_app.TUPLE_METAVAR[0].upper() in normalize(ac_app.formatted_completions)[0]
 
     text = ''
     line = 'choices --tuple_metavar token_1 {}'.format(text)
@@ -870,7 +875,7 @@ def test_completion_items_arg_header(ac_app):
 
     # We are completing the second argument of this flag. The second element in the tuple should be the column header.
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert ac_app.TUPLE_METAVAR[1].upper() in ac_app.completion_header
+    assert ac_app.TUPLE_METAVAR[1].upper() in normalize(ac_app.formatted_completions)[0]
 
     text = ''
     line = 'choices --tuple_metavar token_1 token_2 {}'.format(text)
@@ -880,7 +885,7 @@ def test_completion_items_arg_header(ac_app):
     # We are completing the third argument of this flag. It should still be the second tuple element
     # in the column header since the tuple only has two strings in it.
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert ac_app.TUPLE_METAVAR[1].upper() in ac_app.completion_header
+    assert ac_app.TUPLE_METAVAR[1].upper() in normalize(ac_app.formatted_completions)[0]
 
 
 def test_completion_items_descriptive_header(ac_app):
@@ -895,7 +900,7 @@ def test_completion_items_descriptive_header(ac_app):
     begidx = endidx - len(text)
 
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert ac_app.CUSTOM_DESC_HEADER in ac_app.completion_header
+    assert ac_app.CUSTOM_DESC_HEADER in normalize(ac_app.formatted_completions)[0]
 
     # This argument did not provide a descriptive header, so it should be DEFAULT_DESCRIPTIVE_HEADER
     text = ''
@@ -904,7 +909,7 @@ def test_completion_items_descriptive_header(ac_app):
     begidx = endidx - len(text)
 
     complete_tester(text, line, begidx, endidx, ac_app)
-    assert DEFAULT_DESCRIPTIVE_HEADER in ac_app.completion_header
+    assert DEFAULT_DESCRIPTIVE_HEADER in normalize(ac_app.formatted_completions)[0]
 
 
 @pytest.mark.parametrize(