Repository: google/yapf (python)

Issue:
Breaks code using line continuation with backslashes
This:

    text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "\
           "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "\
           "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "\
           "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "\
           "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet, "\
           "consectetur adipiscing elit. Aenean ut gravida lorem. Ut turpis felis, pulvinar ..."

gets convert to this:

    text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "
    "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "
    "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "
    "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "
    "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet, "
    "consectetur adipiscing elit. Aenean ut gravida lorem. Ut turpis felis, pulvinar ..."

That is, the backslashes causing the string literals to be considered one statement are removed, effectively cutting off the string after the first line.

Pull request description:
Add support for line continuations.
If we encounter a line continuation, we make an assumption that the programmer
knew what they were doing and then we leave the line alone. Failing to do so
could lead to syntactically correct, but semantically wrong code.

Closes #58

Patch:
diff --git a/yapf/yapflib/continuation_splicer.py b/yapf/yapflib/continuation_splicer.py
new file mode 100644
index 0000000..26a1702
--- /dev/null
+++ b/yapf/yapflib/continuation_splicer.py
@@ -0,0 +1,53 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Insert "continuation" nodes into lib2to3 tree.
+
+The "backslash-newline" continuation marker is shoved into the node's prefix.
+Pull them out and make it into nodes of their own.
+
+  SpliceContinuations(): the main funciton exported by this module.
+"""
+
+from lib2to3 import pygram
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+
+from yapf.yapflib import format_token
+from yapf.yapflib import pytree_utils
+
+
+def SpliceContinuations(tree):
+  """Given a pytree, splice the continuation marker into nodes.
+
+  Arguments:
+    tree: (pytree.Node) The tree to work on. The tree is modified by this
+      function.
+  """
+
+  def RecSplicer(node):
+    if isinstance(node, pytree.Leaf):
+      if node.prefix.lstrip().startswith('\\\n'):
+        new_lineno = node.lineno - node.prefix.count('\n')
+        return pytree.Leaf(type=format_token.CONTINUATION,
+                           value=node.prefix,
+                           context=('', (new_lineno, 0)))
+      return None
+    num_inserted = 0
+    for index, child in enumerate(node.children[:]):
+      continuation_node = RecSplicer(child)
+      if continuation_node:
+        node.children.insert(index + num_inserted, continuation_node)
+        num_inserted += 1
+
+  RecSplicer(tree)
diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py
index e96b197..e99c130 100644
--- a/yapf/yapflib/format_token.py
+++ b/yapf/yapflib/format_token.py
@@ -25,6 +25,9 @@ from lib2to3.pgen2 import token
 from yapf.yapflib import pytree_utils
 from yapf.yapflib import style
 
+CONTINUATION = token.N_TOKENS
+token.N_TOKENS += 1
+
 
 class Subtype(object):
   """Subtype information about tokens.
@@ -193,6 +196,10 @@ class FormatToken(object):
   def is_comment(self):
     return self._node.type == token.COMMENT
 
+  @property
+  def is_continuation(self):
+    return self._node.type == CONTINUATION
+
   @property
   def is_keyword(self):
     return keyword.iskeyword(self.value)
diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py
index 5bf9d1f..3898305 100644
--- a/yapf/yapflib/reformatter.py
+++ b/yapf/yapflib/reformatter.py
@@ -52,8 +52,10 @@ def Reformat(uwlines, verify=True):
     state = format_decision_state.FormatDecisionState(uwline, indent_amt)
     if _LineContainsI18n(uwline):
       _EmitLineUnformatted(state)
-    elif _CanPlaceOnSingleLine(uwline):
-      # The unwrapped line fits on one line.
+    elif _CanPlaceOnSingleLine(uwline) or _LineHasContinuationMarkers(uwline):
+      # The unwrapped line fits on one line. Or the line contains continuation
+      # markers, in which case we assume the programmer formatted the code this
+      # way intentionally.
       while state.next_token:
         state.AddTokenToState(newline=False, dry_run=False)
     else:
@@ -127,6 +129,11 @@ def _LineContainsI18n(uwline):
   return False
 
 
+def _LineHasContinuationMarkers(uwline):
+  """Return true if the line has continuation markers in it."""
+  return any(tok.is_continuation for tok in uwline.tokens)
+
+
 def _CanPlaceOnSingleLine(uwline):
   """Determine if the unwrapped line can go on a single line.
 
diff --git a/yapf/yapflib/unwrapped_line.py b/yapf/yapflib/unwrapped_line.py
index 6e7ea88..90d1c93 100644
--- a/yapf/yapflib/unwrapped_line.py
+++ b/yapf/yapflib/unwrapped_line.py
@@ -193,6 +193,9 @@ def _IsBinaryOperator(tok):
 
 def _SpaceRequiredBetween(left, right):
   """Return True if a space is required between the left and right token."""
+  if left.is_continuation or right.is_continuation:
+    # The continuation node's value has all of the spaces it needs.
+    return False
   if right.name in pytree_utils.NONSEMANTIC_TOKENS:
     # No space before a non-semantic token.
     return False
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
index f1c0b34..41fb9e8 100644
--- a/yapf/yapflib/yapf_api.py
+++ b/yapf/yapflib/yapf_api.py
@@ -40,6 +40,7 @@ import sys
 
 from yapf.yapflib import blank_line_calculator
 from yapf.yapflib import comment_splicer
+from yapf.yapflib import continuation_splicer
 from yapf.yapflib import pytree_unwrapper
 from yapf.yapflib import pytree_utils
 from yapf.yapflib import reformatter
@@ -99,6 +100,7 @@ def FormatCode(unformatted_source,
 
   # Run passes on the tree, modifying it in place.
   comment_splicer.SpliceComments(tree)
+  continuation_splicer.SpliceContinuations(tree)
   subtype_assigner.AssignSubtypes(tree)
   split_penalty.ComputeSplitPenalties(tree)
   blank_line_calculator.CalculateBlankLines(tree)


Test patch:
diff --git a/yapftests/reformatter_test.py b/yapftests/reformatter_test.py
index 5fbdb14..f88a14e 100644
--- a/yapftests/reformatter_test.py
+++ b/yapftests/reformatter_test.py
@@ -19,6 +19,7 @@ import unittest
 
 from yapf.yapflib import blank_line_calculator
 from yapf.yapflib import comment_splicer
+from yapf.yapflib import continuation_splicer
 from yapf.yapflib import py3compat
 from yapf.yapflib import pytree_unwrapper
 from yapf.yapflib import pytree_utils
@@ -123,7 +124,7 @@ class BasicReformatterTest(unittest.TestCase):
     uwlines = _ParseAndUnwrap(unformatted_code)
     self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
 
-  def testLineContinuation(self):
+  def testMultipleUgliness(self):
     unformatted_code = textwrap.dedent("""\
         x = {  'a':37,'b':42,
 
@@ -134,8 +135,7 @@ class BasicReformatterTest(unittest.TestCase):
         a = 'hello {}'.format('world')
         class foo  (     object  ):
           def f    (self   ):
-            return       \\
-        37*-+2
+            return       37*-+2
           def g(self, x,y=42):
               return y
         def f  (   a ) :
@@ -981,6 +981,32 @@ format_token.Subtype.NONE))
     uwlines = _ParseAndUnwrap(code)
     self.assertEqual(code, reformatter.Reformat(uwlines))
 
+  def testContinuationMarkers(self):
+    code = textwrap.dedent("""\
+        text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "\\
+               "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "\\
+               "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "\\
+               "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "\\
+               "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet"
+        """)
+    uwlines = _ParseAndUnwrap(code)
+    self.assertEqual(code, reformatter.Reformat(uwlines))
+
+    code = textwrap.dedent("""\
+        from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \\
+            print_function, unicode_literals
+        """)
+    uwlines = _ParseAndUnwrap(code)
+    self.assertEqual(code, reformatter.Reformat(uwlines))
+
+    code = textwrap.dedent("""\
+        if aaaaaaaaa == 42 and bbbbbbbbbbbbbb == 42 and \\
+           cccccccc == 42:
+          pass
+        """)
+    uwlines = _ParseAndUnwrap(code)
+    self.assertEqual(code, reformatter.Reformat(uwlines))
+
 
 class BuganizerFixes(unittest.TestCase):
 
@@ -1681,6 +1707,7 @@ def _ParseAndUnwrap(code, dumptree=False):
   """
   tree = pytree_utils.ParseCodeToTree(code)
   comment_splicer.SpliceComments(tree)
+  continuation_splicer.SpliceContinuations(tree)
   subtype_assigner.AssignSubtypes(tree)
   split_penalty.ComputeSplitPenalties(tree)
   blank_line_calculator.CalculateBlankLines(tree)