Repository: xonsh/xonsh

Issue:
Multiline comments do not work within @()
Example:

```
$ print("""Line
break""")
Line
break

$ echo @("""Line
break""")
xonsh: To log full traceback to a file set: $XONSH_TRACEBACK_LOGFILE = <filename>
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/xonsh/ptk/shell.py", line 137, in _push
    locs=None)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/__amalgam__.py", line 20054, in compile
    transform=transform)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/__amalgam__.py", line 20023, in parse
    tree, input = self._parse_ctx_free(input, mode=mode, filename=filename)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/__amalgam__.py", line 20123, in _parse_ctx_free
    raise original_error from None
  File "/usr/local/lib/python3.5/dist-packages/xonsh/__amalgam__.py", line 20110, in _parse_ctx_free
    debug_level=(self.debug_level > 2))
  File "/usr/local/lib/python3.5/dist-packages/xonsh/parsers/base.py", line 349, in parse
    tree = self.parser.parse(input=s, lexer=self.lexer, debug=debug_level)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/ply/ply/yacc.py", line 331, in parse
    return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/ply/ply/yacc.py", line 1199, in parseopt_notrack
    tok = call_errorfunc(self.errorfunc, errtoken, self)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/ply/ply/yacc.py", line 193, in call_errorfunc
    r = errorfunc(token)
  File "/usr/local/lib/python3.5/dist-packages/xonsh/parsers/base.py", line 2760, in p_error
    column=p.lexpos))
  File "/usr/local/lib/python3.5/dist-packages/xonsh/parsers/base.py", line 482, in _parse_error
    raise err
  File "<string>", line None
SyntaxError: <xonsh-code>:1:5: ('code: @(',)
echo @("""Line
     ^
```

Adding a backslash pre-line break prevents a crash, but as a result doesn't parse the newline (which is the correct pythonic behavior)

```
$ print("""No line\
break""")
No linebreak

$ echo @("""No line\
break""")
No linebreak
```

My xonfig is:

```
+------------------+----------------------+
| xonsh            | 0.7.0                |
| Git SHA          | 3779e8b5             |
| Commit Date      | Jul 14 14:40:04 2018 |
| Python           | 3.5.2                |
| PLY              | 3.9                  |
| have readline    | True                 |
| prompt toolkit   | 1.0.15               |
| shell type       | prompt_toolkit1      |
| pygments         | None                 |
| on posix         | True                 |
| on linux         | True                 |
| distro           | neon                 |
| on darwin        | False                |
| on windows       | False                |
| on cygwin        | False                |
| on msys2         | False                |
| is superuser     | False                |
| default encoding | utf-8                |
| xonsh encoding   | utf-8                |
| encoding errors  | surrogateescape      |
+------------------+----------------------+
```


Discussion:
Thanks for pointing this out @battaglia01! Note that it does work, if you provide an explicit subprocess:

```console
$ ![echo @("""hello 
  mom""")]
hello 
mom
```

But this is certainly  a bug.
Also, strangely enough, this works:

```
$ echo """lol
asd"""
lol
asd
```
Yeah strings in subproc mode act the same way they do in Python mode

Resolving patch:
diff --git a/xonsh/tools.py b/xonsh/tools.py
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -342,6 +342,11 @@ def find_next_break(line, mincol=0, lexer=None):
     return maxcol
 
 
+def _offset_from_prev_lines(line, last):
+    lines = line.splitlines(keepends=True)[:last]
+    return sum(map(len, lines))
+
+
 def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False,
                  greedy=False):
     """Encapsulates tokens in a source code line in a uncaptured
@@ -431,6 +436,9 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False,
         return  # handle comment lines
     elif saw_macro or greedy:
         end_offset = len(toks[-1].value.rstrip()) + 1
+    if toks[0].lineno != toks[-1].lineno:
+        # handle multiline cases
+        end_offset += _offset_from_prev_lines(line, toks[-1].lineno)
     beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset)
     end = len(line[:end].rstrip())
     rtn = '![' + line[beg:end] + ']'


Test patch:
diff --git a/tests/test_execer.py b/tests/test_execer.py
--- a/tests/test_execer.py
+++ b/tests/test_execer.py
@@ -98,6 +98,12 @@ def test_pyeval_redirect():
     code = 'echo @("foo") > bar\n'
     assert check_parse(code)
 
+
+def test_pyeval_multiline_str():
+    code = 'echo @("""hello\nmom""")\n'
+    assert check_parse(code)
+
+
 def test_echo_comma():
     code = 'echo ,\n'
     assert check_parse(code)
@@ -114,3 +120,5 @@ def test_echo_comma_2val():
 def test_echo_line_cont():
     code = 'echo "1 \\\n2"\n'
     assert check_parse(code)
+
+
diff --git a/tests/test_tools.py b/tests/test_tools.py
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -292,6 +292,13 @@ def test_subproc_toks_pyeval():
     assert exp == obs
 
 
+def test_subproc_toks_pyeval_multiline_string():
+    s = 'echo @("""hello\nmom""")'
+    exp = '![{0}]'.format(s)
+    obs = subproc_toks(s, lexer=LEXER, returnline=True)
+    assert exp == obs
+
+
 def test_subproc_toks_twopyeval():
     s = 'echo @(1+1) @(40 + 2)'
     exp = '![{0}]'.format(s)