Repository: xonsh/xonsh

Issue:
String literal concatenation does not work for formatted strings and Path strings
[String literal concatenation](https://docs.python.org/3/reference/lexical_analysis.html#string-literal-concatenation) is a feature where multiple adjacent string literals can be concatenated at compile time into a single string.

In xonsh, string literal concatenation works for regular string literals and byte literals:
```xonsh
"hello" "world" # 'helloworld'
b"hello" b"world" # b'helloworld'
```
but it does not work for formatted strings:
```xonsh
f"hello" f"world"
# Expected Behaviour: "helloworld"
# Current Behaviour: AttributeError: 'JoinedStr' object has no attribute 's'
```
and it also does not work for Path strings:
```xonsh
p"hello" p"world"
# Expected Behaviour: PosixPath("helloworld")
# Current Behaviour: AttributeError: 'Call' object has no attribute 's'
```

I wrote a short script to test the behavior of various string literal concatenations that should work:
```xonsh
def test_literals(markers):
    for m1 in markers:
        for m2 in markers:
            expr=m1+r"'11{a}22\n'"+' '+m2+r"'33{b}44\n'"
            print(f"EXPR: {expr}")
            try:
                print(f"RESULT: {repr(__xonsh__.execer.eval(expr))}\n")
            except Exception as e:
                print(f"EXCEPTION: {e}\n")

print("STRINGS --------------------\n")
test_literals(['','f','u','r','fr']) # strings
print("PATH STRINGS ---------------\n")
test_literals(['p','pr','pf']) # path strings
print("BYTES ----------------------\n")
test_literals(['b','br']) # bytes
```
This is the output:

```
STRINGS --------------------

EXPR: '11{a}22\n' '33{b}44\n'
RESULT: '11{a}22\n33{b}44\n'

EXPR: '11{a}22\n' f'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: '11{a}22\n' u'33{b}44\n'
RESULT: '11{a}22\n33{b}44\n'

EXPR: '11{a}22\n' r'33{b}44\n'
RESULT: '11{a}22\n33{b}44\\n'

EXPR: '11{a}22\n' fr'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: f'11{a}22\n' '33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: f'11{a}22\n' f'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: f'11{a}22\n' u'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: f'11{a}22\n' r'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: f'11{a}22\n' fr'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: u'11{a}22\n' '33{b}44\n'
RESULT: '11{a}22\n33{b}44\n'

EXPR: u'11{a}22\n' f'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: u'11{a}22\n' u'33{b}44\n'
RESULT: '11{a}22\n33{b}44\n'

EXPR: u'11{a}22\n' r'33{b}44\n'
RESULT: '11{a}22\n33{b}44\\n'

EXPR: u'11{a}22\n' fr'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: r'11{a}22\n' '33{b}44\n'
RESULT: '11{a}22\\n33{b}44\n'

EXPR: r'11{a}22\n' f'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: r'11{a}22\n' u'33{b}44\n'
RESULT: '11{a}22\\n33{b}44\n'

EXPR: r'11{a}22\n' r'33{b}44\n'
RESULT: '11{a}22\\n33{b}44\\n'

EXPR: r'11{a}22\n' fr'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: fr'11{a}22\n' '33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: fr'11{a}22\n' f'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: fr'11{a}22\n' u'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: fr'11{a}22\n' r'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

EXPR: fr'11{a}22\n' fr'33{b}44\n'
EXCEPTION: 'JoinedStr' object has no attribute 's'

PATH STRINGS ---------------

EXPR: p'11{a}22\n' p'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: p'11{a}22\n' pr'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: p'11{a}22\n' pf'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pr'11{a}22\n' p'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pr'11{a}22\n' pr'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pr'11{a}22\n' pf'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pf'11{a}22\n' p'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pf'11{a}22\n' pr'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

EXPR: pf'11{a}22\n' pf'33{b}44\n'
EXCEPTION: 'Call' object has no attribute 's'

BYTES ----------------------

EXPR: b'11{a}22\n' b'33{b}44\n'
RESULT: b'11{a}22\n33{b}44\n'

EXPR: b'11{a}22\n' br'33{b}44\n'
RESULT: b'11{a}22\n33{b}44\\n'

EXPR: br'11{a}22\n' b'33{b}44\n'
RESULT: b'11{a}22\\n33{b}44\n'

EXPR: br'11{a}22\n' br'33{b}44\n'
RESULT: b'11{a}22\\n33{b}44\\n'

```
As you can see, the xonsh parser throws an exception whenever `f""` or `p""` is involved in the string literal concatenation.

## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**



Discussion:

Resolving patch:
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -2573,7 +2573,82 @@ def p_string_literal_list(self, p):
                             | string_literal_list string_literal
         """
         if len(p) == 3:
-            p[1].s += p[2].s
+
+            def literal_type(node):
+                """
+                Determines if a node is a string literal, a bytes literal,
+                or a path literal.
+                """
+                if isinstance(node, ast.Constant):
+                    if isinstance(node.value, bytes):
+                        return "bytes"
+                    else:
+                        return "str"  # constant string literal
+                elif isinstance(node, ast.Call):
+                    return "path"
+                else:
+                    return "str"  # formatted string literal
+
+            def join_joinedstr_values(x, y):
+                """
+                Joins two lists that may contain a mix of ast.Constant and
+                ast.FormattedValue values (child nodes of ast.JoinedStr).
+
+                If the first list ends with an ast.Constant and the second list
+                starts with an ast.Constant, then the two ast.Constants are
+                merged into a single value.
+                """
+                if (
+                    len(x) > 0
+                    and len(y) > 0
+                    and isinstance(x[-1], ast.Constant)
+                    and isinstance(y[0], ast.Constant)
+                ):
+                    # merge the Constant at the head of y
+                    # with the Constant at the tail of x
+                    x[-1].value += y[0].value
+                    x += y[1:]
+                else:
+                    x += y
+                return x
+
+            def join_str_nodes(x, y):
+                """
+                Joins two str literal nodes into a single node. Each node can
+                either be an ast.Constant or an ast.JoinedStr.
+
+                If both nodes are ast.Constants, the resultant node is an
+                ast.Constant. Otherwise, the resultant node is an
+                ast.JoinedStr.
+                """
+                if isinstance(x, ast.Constant):
+                    if isinstance(y, ast.Constant):
+                        x.value += y.value
+                        return x
+                    else:
+                        y.values = join_joinedstr_values([x], y.values)
+                        return y
+                elif isinstance(y, ast.Constant):
+                    x.values = join_joinedstr_values(x.values, [y])
+                    return x
+                else:
+                    x.values = join_joinedstr_values(x.values, y.values)
+                    return x
+
+            p1_type = literal_type(p[1])
+            p2_type = literal_type(p[2])
+            if p1_type == p2_type:
+                if p1_type == "path":
+                    # the nodes are function calls to __xonsh__.path_literal()
+                    # so we need to join their first arguments
+                    p[1].args[0] = join_str_nodes(p[1].args[0], p[2].args[0])
+                else:
+                    p[1] = join_str_nodes(p[1], p[2])
+            else:
+                self._set_error(
+                    "cannot mix literals of different types",
+                    self.currloc(lineno=p[1].lineno, column=p[1].col_offset),
+                )
         p[0] = p[1]
 
     def p_number(self, p):
diff --git a/xonsh/pytest/tools.py b/xonsh/pytest/tools.py
--- a/xonsh/pytest/tools.py
+++ b/xonsh/pytest/tools.py
@@ -110,6 +110,10 @@ def nodes_equal(x, y):
         type(x),
         type(y),
     )
+    if isinstance(x, ast.Constant):
+        assert (
+            x.value == y.value
+        ), f"Constant ast nodes do not have the same value: {repr(x.value)} != {repr(y.value)}"
     if isinstance(x, (ast.Expr, ast.FunctionDef, ast.ClassDef)):
         assert (
             x.lineno == y.lineno


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -144,6 +144,7 @@ def test_str_literal(check_ast):
 def test_bytes_literal(check_ast):
     check_ast('b"hello"')
     check_ast('B"hello"')
+    check_ast('b"hello" b"world"')
 
 
 def test_raw_literal(check_ast):
@@ -156,6 +157,16 @@ def test_f_literal(check_ast):
     check_ast('F"{yo}"', run=False)
 
 
+@pytest.mark.parametrize(
+    "first_prefix, second_prefix",
+    itertools.product(["", "f", "r", "fr"], repeat=2),
+)
+def test_string_literal_concat(first_prefix, second_prefix, check_ast):
+    check_ast(
+        first_prefix + r"'11{a}22\n'" + " " + second_prefix + r"'33{b}44\n'", run=False
+    )
+
+
 def test_f_env_var(check_xonsh_ast):
     check_xonsh_ast({}, 'f"{$HOME}"', run=False)
     check_xonsh_ast({}, "f'{$XONSH_DEBUG}'", run=False)
@@ -2202,6 +2213,16 @@ def test_path_fstring_literal(check_xonsh_ast):
     check_xonsh_ast({}, 'Fp"/foo{1+1}"', False)
 
 
+@pytest.mark.parametrize(
+    "first_prefix, second_prefix",
+    itertools.product(["p", "pf", "pr"], repeat=2),
+)
+def test_path_literal_concat(first_prefix, second_prefix, check_xonsh_ast):
+    check_xonsh_ast(
+        {}, first_prefix + r"'11{a}22\n'" + " " + second_prefix + r"'33{b}44\n'", False
+    )
+
+
 def test_dollar_name(check_xonsh_ast):
     check_xonsh_ast({"WAKKA": 42}, "$WAKKA")
 
@@ -3262,6 +3283,14 @@ def test_syntax_error_lambda_posonly_nondefault_follows_default(parser):
         parser.parse("lambda x, y=1, /, z: x", mode="exec")
 
 
+@pytest.mark.parametrize(
+    "first_prefix, second_prefix", itertools.permutations(["", "p", "b"], 2)
+)
+def test_syntax_error_literal_concat_different(first_prefix, second_prefix, parser):
+    with pytest.raises(SyntaxError):
+        parser.parse(f"{first_prefix}'hello' {second_prefix}'world'")
+
+
 def test_get_repo_url(parser):
     parser.parse(
         "def get_repo_url():\n"