Repository: psf/black

Issue:
ESP: a standalone comment causes strings to be merged into one far too long (and requiring two passes to do so)
**To Reproduce**

For example, take this code:

```python
def a():
    CompiledSQL(
        "addresses.id AS addresses_id, "
        "addresses.email_address AS addresses_email_address "
        # note the comma-separated FROM clause
        "IN (__[POSTCOMPILE_primary_keys]) "
        "AND addresses.id = anon_1.id ORDER BY addresses.id",
        ["hmm"]
    )
```

And run it with these arguments:

```sh
$ black file.py --experimental-string-processing
```

The resulting error is:

```
error: cannot format test.py: INTERNAL ERROR: Black produced different code on the second pass of the formatter.  Please report a bug on https://github.com/psf/black/issues.  This diff might be helpful: /tmp/blk_wqp6guhd.log
Oh no! 💥 💔 💥
1 file failed to reformat.
```

```diff
Mode(target_versions={<TargetVersion.PY36: 6>, <TargetVersion.PY38: 8>, <TargetVersion.PY37: 7>}, line_length=88, string_normalization=True, is_pyi=False, is_ipynb=False, magic_trailing_comma=True, experimental_string_processing=True)
--- source
+++ first pass
@@ -1,9 +1,9 @@
 def a():
     CompiledSQL(
         "addresses.id AS addresses_id, "
         "addresses.email_address AS addresses_email_address "
         # note the comma-separated FROM clause
-        "IN (__[POSTCOMPILE_primary_keys]) "
-        "AND addresses.id = anon_1.id ORDER BY addresses.id",
-        ["hmm"]
+        "IN (__[POSTCOMPILE_primary_keys]) AND addresses.id = anon_1.id ORDER BY"
+        " addresses.id",
+        ["hmm"],
     )
--- first pass
+++ second pass
@@ -1,9 +1,8 @@
 def a():
     CompiledSQL(
         "addresses.id AS addresses_id, "
         "addresses.email_address AS addresses_email_address "
         # note the comma-separated FROM clause
-        "IN (__[POSTCOMPILE_primary_keys]) AND addresses.id = anon_1.id ORDER BY"
-        " addresses.id",
+        "IN (__[POSTCOMPILE_primary_keys]) AND addresses.id = anon_1.id ORDER BY addresses.id",
         ["hmm"],
     )
```

**Expected behavior**

It not crashing :slightly_smiling_face: 

**Environment**

<!-- Please complete the following information: -->

- Black's version: main on b8df7e4b10bca2d7e478e224502975ec8f220e21
- OS and Python version: Ubuntu 20.04.03 LTS w/ CPython 3.8.5

**Additional context**

This was discovered by primer (in sqlalchemy, test/orm/test_relationship_criteria.py) and I wasn't able to fix it quickly so instead of letting primer fail on every build unproductively, I'm excluding it and filing this issue so we don't forget about it.


Discussion:

Resolving patch:
diff --git a/src/black/trans.py b/src/black/trans.py
--- a/src/black/trans.py
+++ b/src/black/trans.py
@@ -553,6 +553,9 @@ def make_naked(string: str, string_prefix: str) -> str:
 
             next_str_idx += 1
 
+        # Take a note on the index of the non-STRING leaf.
+        non_string_idx = next_str_idx
+
         S_leaf = Leaf(token.STRING, S)
         if self.normalize_strings:
             S_leaf.value = normalize_string_quotes(S_leaf.value)
@@ -572,7 +575,18 @@ def make_naked(string: str, string_prefix: str) -> str:
         string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
 
         if atom_node is not None:
-            replace_child(atom_node, string_leaf)
+            # If not all children of the atom node are merged (this can happen
+            # when there is a standalone comment in the middle) ...
+            if non_string_idx - string_idx < len(atom_node.children):
+                # We need to replace the old STRING leaves with the new string leaf.
+                first_child_idx = LL[string_idx].remove()
+                for idx in range(string_idx + 1, non_string_idx):
+                    LL[idx].remove()
+                if first_child_idx is not None:
+                    atom_node.insert_child(first_child_idx, string_leaf)
+            else:
+                # Else replace the atom node with the new string leaf.
+                replace_child(atom_node, string_leaf)
 
         # Build the final line ('new_line') that this method will later return.
         new_line = line.clone()


Test patch:
diff --git a/tests/data/preview/long_strings.py b/tests/data/preview/long_strings.py
--- a/tests/data/preview/long_strings.py
+++ b/tests/data/preview/long_strings.py
@@ -72,6 +72,25 @@
     zzz,
 )
 
+inline_comments_func1(
+    "if there are inline "
+    "comments in the middle "
+    # Here is the standard alone comment.
+    "of the implicitly concatenated "
+    "string, we should handle "
+    "them correctly",
+    xxx,
+)
+
+inline_comments_func2(
+    "what if the string is very very very very very very very very very very long and this part does "
+    "not fit into a single line? "
+    # Here is the standard alone comment.
+    "then the string should still be properly handled by merging and splitting "
+    "it into parts that fit in line length.",
+    xxx,
+)
+
 raw_string = r"This is a long raw string. When re-formatting this string, black needs to make sure it prepends the 'r' onto the new string."
 
 fmt_string1 = "We also need to be sure to preserve any and all {} which may or may not be attached to the string in question.".format("method calls")
@@ -395,6 +414,22 @@ def foo():
     zzz,
 )
 
+inline_comments_func1(
+    "if there are inline comments in the middle "
+    # Here is the standard alone comment.
+    "of the implicitly concatenated string, we should handle them correctly",
+    xxx,
+)
+
+inline_comments_func2(
+    "what if the string is very very very very very very very very very very long and"
+    " this part does not fit into a single line? "
+    # Here is the standard alone comment.
+    "then the string should still be properly handled by merging and splitting "
+    "it into parts that fit in line length.",
+    xxx,
+)
+
 raw_string = (
     r"This is a long raw string. When re-formatting this string, black needs to make"
     r" sure it prepends the 'r' onto the new string."