Repository: xonsh/xonsh
Pull request: Fixes spaces in readline tab completer
Should fix #2485

Problem statement:
Tab completion breaks on paths with spaces
```
de3lxl-106789 /tmp  $ xonfig
+------------------+----------------------+
| xonsh            | 0.5.12.dev47.dev102  |
| Git SHA          | c787f9ab             |
| Commit Date      | Aug 16 00:10:57 2017 |
| Python           | 3.4.3                |
| PLY              | 3.8                  |
| have readline    | True                 |
| prompt toolkit   | 1.0.0                |
| shell type       | readline             |
| pygments         | 1.6                  |
| on posix         | True                 |
| on linux         | True                 |
| distro           | Ubuntu               |
| on darwin        | False                |
| on windows       | False                |
| on cygwin        | False                |
| is superuser     | False                |
| default encoding | utf-8                |
| xonsh encoding   | utf-8                |
| encoding errors  | surrogateescape      |
+------------------+----------------------+
de3lxl-106789 /tmp  $ mkdir 'a space'

de3lxl-106789 /tmp  $ touch 'a space/test.txt'
de3lxl-106789 /tmp  $ cat 'a space'<tab>
de3lxl-106789 /tmp  $ cat 'a 'a space/test.txt'
```
The last two lines demonstrate the problem: Pressing <tab> in the second-to-last line at the where indicated produces the last line (note the double 'a), this doesn't seem to be a mere visual/refreshing issue, the produced command is actually not understood by xonsh (pressing return prompts for more input because of the resulting uneven number of ').

OS is Ubuntu 14.04.5 LTS, terminal is rxvt-unicode-256color.

Hints from issue discussion:
Thanks for reporting @Droggelbecher ! This seems like a regression.
I've occasionally seen similar issues with partial completes not substituting correctly.
I had the same issue but it was fixed when I upgraded to prompt_toolkit version 1.0.15
Ok! I am not seeing this issue anymore on PTK v1.015, however, I am still seeing it on readline.
Should be fixed by #2530

Commit message: indents

Patch to review:
diff --git a/docs/intro.rst b/docs/intro.rst
index 6bc36d9bb8..43aec33c65 100644
--- a/docs/intro.rst
+++ b/docs/intro.rst
@@ -42,6 +42,7 @@ the xonsh shell
         "Conches for the xonsh god!",
         "Python-powered, cross-platform, Unix-gazing shell",
         "Tab completion in Alderaan places",
+        "This fix was trickier expected",
         "Exploiting the workers and hanging on to outdated imperialist dogma since 2015."
         ];
     document.write(taglines[Math.floor(Math.random() * taglines.length)]);
diff --git a/news/aspace.rst b/news/aspace.rst
new file mode 100644
index 0000000000..42d66abcf8
--- /dev/null
+++ b/news/aspace.rst
@@ -0,0 +1,16 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed issues with readline completer tab completing entries
+  with spaces.
+* Fixed ``xonsh.tools.columnize()`` bug the prevented single-row
+  input from being columnized correctly.
+
+**Security:** None
diff --git a/xonsh/completers/bash_completion.py b/xonsh/completers/bash_completion.py
index 77550bead0..f5f50b61cc 100644
--- a/xonsh/completers/bash_completion.py
+++ b/xonsh/completers/bash_completion.py
@@ -190,7 +190,7 @@ def _bash_quote_paths(paths, start, end):
         if end in s:
             s = s.replace(end, ''.join('\\%s' % i for i in end))
         out.add(start + s + end)
-    return out
+    return out, need_quotes
 
 
 BASH_COMPLETE_SCRIPT = r"""
@@ -270,7 +270,8 @@ def bash_completions(prefix, line, begidx, endidx, env=None, paths=None,
         from the environment and platform.
     quote_paths : callable, optional
         A functions that quotes file system paths. You shouldn't normally need
-        this as the default is acceptable 99+% of the time.
+        this as the default is acceptable 99+% of the time. This function should
+        a set of the new paths and a boolean for whether the paths were quoted.
 
     Returns
     -------
@@ -335,7 +336,8 @@ def bash_completions(prefix, line, begidx, endidx, env=None, paths=None,
         strip_len += 1
 
     if '-o noquote' not in complete_stmt:
-        out = quote_paths(out, '', '')
+        out, need_qoutes = quote_paths(out, '', '')
+        strip_len += int(need_qoutes)
     if '-o nospace' in complete_stmt:
         out = set([x.rstrip() for x in out])
 
diff --git a/xonsh/completers/path.py b/xonsh/completers/path.py
index 373e8317dc..268a623516 100644
--- a/xonsh/completers/path.py
+++ b/xonsh/completers/path.py
@@ -164,7 +164,7 @@ def _quote_paths(paths, start, end, append_end=True):
             s = s.replace(end, ''.join('\\%s' % i for i in end))
         s = start + s + end if append_end else start + s
         out.add(s)
-    return out
+    return out, need_quotes
 
 
 def _joinpath(path):
@@ -298,10 +298,10 @@ def complete_path(prefix, line, start, end, ctx, cdpath=True, filtfunc=None):
     if cdpath:
         _add_cdpaths(paths, prefix)
     paths = set(filter(filtfunc, paths))
-    paths = _quote_paths({_normpath(s) for s in paths},
-                         path_str_start,
-                         path_str_end,
-                         append_end)
+    paths, _ = _quote_paths({_normpath(s) for s in paths},
+                            path_str_start,
+                            path_str_end,
+                            append_end)
     paths.update(filter(filtfunc, _dots(prefix)))
     paths.update(filter(filtfunc, _env(prefix)))
     return paths, lprefix
diff --git a/xonsh/readline_shell.py b/xonsh/readline_shell.py
index 849055a1ef..bb76391e76 100644
--- a/xonsh/readline_shell.py
+++ b/xonsh/readline_shell.py
@@ -268,6 +268,43 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, **kwargs):
         self._current_indent = ''
         self._current_prompt = ''
         self._force_hide = None
+        self._complete_only_last_table = {
+            # Truth table for completions, keys are:
+            # (prefix_begs_quote, prefix_ends_quote, i_ends_quote,
+            #  last_starts_with_prefix, i_has_space)
+            (True, True, True, True, True): True,
+            (True, True, True, True, False): True,
+            (True, True, True, False, True): False,
+            (True, True, True, False, False): True,
+            (True, True, False, True, True): False,
+            (True, True, False, True, False): False,
+            (True, True, False, False, True): False,
+            (True, True, False, False, False): False,
+            (True, False, True, True, True): True,
+            (True, False, True, True, False): False,
+            (True, False, True, False, True): False,
+            (True, False, True, False, False): True,
+            (True, False, False, True, True): False,
+            (True, False, False, True, False): False,
+            (True, False, False, False, True): False,
+            (True, False, False, False, False): False,
+            (False, True, True, True, True): True,
+            (False, True, True, True, False): True,
+            (False, True, True, False, True): True,
+            (False, True, True, False, False): True,
+            (False, True, False, True, True): False,
+            (False, True, False, True, False): False,
+            (False, True, False, False, True): False,
+            (False, True, False, False, False): False,
+            (False, False, True, True, True): True,
+            (False, False, True, True, False): False,
+            (False, False, True, False, True): False,
+            (False, False, True, False, False): True,
+            (False, False, False, True, True): True,
+            (False, False, False, True, False): False,
+            (False, False, False, False, True): False,
+            (False, False, False, False, False): False,
+            }
         self.cmdqueue = collections.deque()
 
     def __del__(self):
@@ -296,11 +333,15 @@ def parseline(self, line):
         return '', line, line
 
     def _querycompletions(self, completions, loc):
-        """Returns whether or not we should show completions"""
+        """Returns whether or not we should show completions. 0 means that prefixes
+        should not be shown, 1 means that there is a common prefix among all completions
+        and they should be shown, while 2 means that there is no common prefix but
+        we are under the query limit and they should be shown.
+        """
         if os.path.commonprefix([c[loc:] for c in completions]):
-            return True
+            return 1
         elif len(completions) <= builtins.__xonsh_env__.get('COMPLETION_QUERY_LIMIT'):
-            return True
+            return 2
         msg = '\nDisplay all {} possibilities? '.format(len(completions))
         msg += '({GREEN}y{NO_COLOR} or {RED}n{NO_COLOR})'
         self.print_color(msg, end='', flush=True, file=sys.stderr)
@@ -311,7 +352,7 @@ def _querycompletions(self, completions, loc):
         print()
         if not show_completions:
             rl_on_new_line()
-            return False
+            return 0
         w, h = shutil.get_terminal_size()
         lines = columnize(completions, width=w)
         more_msg = self.format_color('{YELLOW}==={NO_COLOR} more or '
@@ -325,10 +366,10 @@ def _querycompletions(self, completions, loc):
             print(flush=True, file=sys.stderr)
             if q == 'q':
                 rl_on_new_line()
-                return False
+                return 0
         print(''.join(lines), end='', flush=True, file=sys.stderr)
         rl_on_new_line()
-        return False
+        return 0
 
     def completedefault(self, prefix, line, begidx, endidx):
         """Implements tab-completion for text."""
@@ -341,9 +382,31 @@ def completedefault(self, prefix, line, begidx, endidx):
                                                  begidx, endidx,
                                                  ctx=self.ctx)
         chopped = prefix[:-l]
-        rtn_completions = [chopped + i for i in completions]
+        if chopped:
+            rtn_completions = [chopped + i for i in completions]
+        else:
+            rtn_completions = completions
+        rtn = []
+        prefix_begs_quote = prefix.startswith("'") or prefix.startswith('"')
+        prefix_ends_quote = prefix.endswith("'") or prefix.endswith('"')
+        for i in rtn_completions:
+            i_ends_quote = i.endswith("'") or i.endswith('"')
+            last = i.rsplit(' ', 1)[-1]
+            last_starts_prefix = last.startswith(prefix)
+            i_has_space = ' ' in i
+            key = (prefix_begs_quote, prefix_ends_quote, i_ends_quote,
+                   last_starts_prefix, i_has_space)
+            rtn.append(last if self._complete_only_last_table[key] else i)
+        # return based on show completions
         show_completions = self._querycompletions(completions, endidx - begidx)
-        return rtn_completions if show_completions else []
+        if show_completions == 0:
+            return []
+        elif show_completions == 1:
+            return rtn
+        elif show_completions == 2:
+            return completions
+        else:
+            raise ValueError('query compeltions flag not understood.')
 
     # tab complete on first index too
     completenames = completedefault
diff --git a/xonsh/tools.py b/xonsh/tools.py
index 1c2338f174..2001ab63fe 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -1923,12 +1923,21 @@ def columnize(elems, width=80, newline='\n'):
     The newline character will be appended to the end of each line.
     """
     sizes = [len(e) + 1 for e in elems]
+    total = sum(sizes)
     nelem = len(elems)
-    ncols = 1
-    nrows = len(sizes)
-    columns = [sizes]
-    last_longest_row = max(sizes)
-    while True:
+    if total - 1 <= width:
+        ncols = len(sizes)
+        nrows = 1
+        columns = [sizes]
+        last_longest_row = total
+        enter_loop = False
+    else:
+        ncols = 1
+        nrows = len(sizes)
+        columns = [sizes]
+        last_longest_row = max(sizes)
+        enter_loop = True
+    while enter_loop:
         longest_row = sum(map(max, columns))
         if longest_row - 1 <= width:
             # we might be able to fit another column.
diff --git a/xonsh/xonfig.py b/xonsh/xonfig.py
index b457a77ffe..ed2da45e87 100644
--- a/xonsh/xonfig.py
+++ b/xonsh/xonfig.py
@@ -577,6 +577,7 @@ def TAGLINES():
         "Conches for the xonsh god!",
         "Python-powered, cross-platform, Unix-gazing shell",
         "Tab completion in Alderaan places",
+        "This fix was trickier expected",
     ]
 
 


Code review comments:
diff_hunk: @@ -335,7 +336,8 @@ def bash_completions(prefix, line, begidx, endidx, env=None, paths=None,
         strip_len += 1
 
     if '-o noquote' not in complete_stmt:
-        out = quote_paths(out, '', '')
+        out, need_qoutes = quote_paths(out, '', '')
+        strip_len += int(need_qoutes)
path: xonsh/completers/bash_completion.py
text: @user1:
`need_quotes`

diff_hunk: @@ -335,7 +336,8 @@ def bash_completions(prefix, line, begidx, endidx, env=None, paths=None,
         strip_len += 1
 
     if '-o noquote' not in complete_stmt:
-        out = quote_paths(out, '', '')
+        out, need_qoutes = quote_paths(out, '', '')
path: xonsh/completers/bash_completion.py
text: @user1:
`need_quotes`