Repository: google/yapf
Pull request #986 (2021-12-30 16:14:55)

Add support for Python 3.10
And updates for lower Python 3.x supported version.

gwelymernans: Thanks! Please add an update in CHANGELOG.
hugovk: CHANGELOG updated!

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 280af43..c219c38 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [2.7, 3.7, 3.8, 3.9]
+        python-version: ["2.7", "3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macos-latest]
 
     steps:
diff --git a/CHANGELOG b/CHANGELOG
index 1394619..fb38ff5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,7 @@
 ## [0.40.0] UNRELEASED
 ### Changes
 - Moved 'pytree' parsing tools into its own subdirectory.
+- Add support for Python 3.10.
 
 ## [0.32.0] 2021-12-26
 ### Added
diff --git a/setup.py b/setup.py
index 70e57da..8b00835 100644
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,10 @@ with codecs.open('README.rst', 'r', 'utf-8') as fd:
           'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
           'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+          'Programming Language :: Python :: 3.8',
+          'Programming Language :: Python :: 3.9',
+          'Programming Language :: Python :: 3.10',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Software Development :: Quality Assurance',
       ],
diff --git a/tox.ini b/tox.ini
index f42b884..5134916 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist=py27,py34,py35,py36,py37,py38
+envlist=py27,py36,py37,py38,py39,py310
 
 [testenv]
 commands=
diff --git a/yapf/yapflib/blank_line_calculator.py b/yapf/pytree/blank_line_calculator.py
similarity index 100%
rename from yapf/yapflib/blank_line_calculator.py
rename to yapf/pytree/blank_line_calculator.py
diff --git a/yapf/yapflib/comment_splicer.py b/yapf/pytree/comment_splicer.py
similarity index 100%
rename from yapf/yapflib/comment_splicer.py
rename to yapf/pytree/comment_splicer.py
diff --git a/yapf/yapflib/continuation_splicer.py b/yapf/pytree/continuation_splicer.py
similarity index 100%
rename from yapf/yapflib/continuation_splicer.py
rename to yapf/pytree/continuation_splicer.py
diff --git a/yapf/pytree/pytree_unwrapper.py b/yapf/pytree/pytree_unwrapper.py
index 3ae6cd6..39dceac 100644
--- a/yapf/pytree/pytree_unwrapper.py
+++ b/yapf/pytree/pytree_unwrapper.py
@@ -33,12 +33,16 @@ from lib2to3.pgen2 import token as grammar_token
 
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
+from yapf.pytree import split_penalty
+from yapf.yapflib import format_token
 from yapf.yapflib import logical_line
 from yapf.yapflib import object_state
-from yapf.yapflib import split_penalty
 from yapf.yapflib import style
 from yapf.yapflib import subtypes
 
+_OPENING_BRACKETS = frozenset({'(', '[', '{'})
+_CLOSING_BRACKETS = frozenset({')', ']', '}'})
+
 
 def UnwrapPyTree(tree):
   """Create and return a list of logical lines from the given pytree.
@@ -293,7 +297,7 @@ class PyTreeUnwrapper(pytree_visitor.PyTreeVisitor):
       self._StartNewLine()
     elif leaf.type != grammar_token.COMMENT or leaf.value.strip():
       # Add non-whitespace tokens and comments that aren't empty.
-      self._cur_logical_line.AppendNode(leaf)
+      self._cur_logical_line.AppendToken(format_token.FormatToken(leaf))
 
 
 _BRACKET_MATCH = {')': '(', '}': '{', ']': '['}
@@ -311,9 +315,9 @@ def _MatchBrackets(line):
   """
   bracket_stack = []
   for token in line.tokens:
-    if token.value in pytree_utils.OPENING_BRACKETS:
+    if token.value in _OPENING_BRACKETS:
       bracket_stack.append(token)
-    elif token.value in pytree_utils.CLOSING_BRACKETS:
+    elif token.value in _CLOSING_BRACKETS:
       bracket_stack[-1].matching_bracket = token
       token.matching_bracket = bracket_stack[-1]
       bracket_stack.pop()
@@ -372,9 +376,9 @@ def _AdjustSplitPenalty(line):
       pytree_utils.SetNodeAnnotation(token.node,
                                      pytree_utils.Annotation.SPLIT_PENALTY,
                                      split_penalty.UNBREAKABLE)
-    if token.value in pytree_utils.OPENING_BRACKETS:
+    if token.value in _OPENING_BRACKETS:
       bracket_level += 1
-    elif token.value in pytree_utils.CLOSING_BRACKETS:
+    elif token.value in _CLOSING_BRACKETS:
       bracket_level -= 1
 
 
diff --git a/yapf/pytree/pytree_utils.py b/yapf/pytree/pytree_utils.py
index 8762032..66a54e6 100644
--- a/yapf/pytree/pytree_utils.py
+++ b/yapf/pytree/pytree_utils.py
@@ -39,9 +39,6 @@ from lib2to3.pgen2 import token
 # unwrapper.
 NONSEMANTIC_TOKENS = frozenset(['DEDENT', 'INDENT', 'NEWLINE', 'ENDMARKER'])
 
-OPENING_BRACKETS = frozenset({'(', '[', '{'})
-CLOSING_BRACKETS = frozenset({')', ']', '}'})
-
 
 class Annotation(object):
   """Annotation names associated with pytrees."""
diff --git a/yapf/yapflib/split_penalty.py b/yapf/pytree/split_penalty.py
similarity index 100%
rename from yapf/yapflib/split_penalty.py
rename to yapf/pytree/split_penalty.py
diff --git a/yapf/yapflib/subtype_assigner.py b/yapf/pytree/subtype_assigner.py
similarity index 100%
rename from yapf/yapflib/subtype_assigner.py
rename to yapf/pytree/subtype_assigner.py
diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py
index cbf11b7..c299d1c 100644
--- a/yapf/yapflib/format_decision_state.py
+++ b/yapf/yapflib/format_decision_state.py
@@ -26,9 +26,9 @@ through the code to commit the whitespace formatting.
   FormatDecisionState: main class exported by this module.
 """
 
+from yapf.pytree import split_penalty
 from yapf.yapflib import logical_line
 from yapf.yapflib import object_state
-from yapf.yapflib import split_penalty
 from yapf.yapflib import style
 from yapf.yapflib import subtypes
 
diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py
index d01a7bc..2c38e96 100644
--- a/yapf/yapflib/format_token.py
+++ b/yapf/yapflib/format_token.py
@@ -11,10 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Pytree nodes with extra formatting information.
-
-This is a thin wrapper around a pytree.Leaf node.
-"""
+"""Enhanced token information for formatting."""
 
 import keyword
 import re
@@ -28,6 +25,9 @@ from yapf.yapflib import subtypes
 
 CONTINUATION = token.N_TOKENS
 
+_OPENING_BRACKETS = frozenset({'(', '[', '{'})
+_CLOSING_BRACKETS = frozenset({')', ']', '}'})
+
 
 def _TabbedContinuationAlignPadding(spaces, align_style, tab_width):
   """Build padding string for continuation alignment in tabbed indentation.
@@ -48,13 +48,13 @@ def _TabbedContinuationAlignPadding(spaces, align_style, tab_width):
 
 
 class FormatToken(object):
-  """A wrapper around pytree Leaf nodes.
+  """Enhanced token information for formatting.
 
   This represents the token plus additional information useful for reformatting
   the code.
 
   Attributes:
-    node: The PyTree node this token represents.
+    node: The original token node.
     next_token: The token in the logical line after this token or None if this
       is the last token in the logical line.
     previous_token: The token in the logical line before this token or None if
@@ -210,10 +210,10 @@ class FormatToken(object):
     self.spaces_required_before = cur_column - (prev_column + prev_len)
 
   def OpensScope(self):
-    return self.value in pytree_utils.OPENING_BRACKETS
+    return self.value in _OPENING_BRACKETS
 
   def ClosesScope(self):
-    return self.value in pytree_utils.CLOSING_BRACKETS
+    return self.value in _CLOSING_BRACKETS
 
   def AddSubtype(self, subtype):
     self.subtypes.add(subtype)
diff --git a/yapf/yapflib/logical_line.py b/yapf/yapflib/logical_line.py
index b91cfef..53c345a 100644
--- a/yapf/yapflib/logical_line.py
+++ b/yapf/yapflib/logical_line.py
@@ -20,9 +20,9 @@ perform the wrapping required to comply with the style guide.
 """
 
 from yapf.pytree import pytree_utils
+from yapf.pytree import split_penalty
 from yapf.yapflib import format_token
 from yapf.yapflib import py3compat
-from yapf.yapflib import split_penalty
 from yapf.yapflib import style
 from yapf.yapflib import subtypes
 
@@ -135,16 +135,6 @@ class LogicalLine(object):
       self.last.next_token = token
     self._tokens.append(token)
 
-  def AppendNode(self, node):
-    """Convenience method to append a pytree node directly.
-
-    Wraps the node with a FormatToken.
-
-    Arguments:
-      node: the node to append
-    """
-    self.AppendToken(format_token.FormatToken(node))
-
   @property
   def first(self):
     """Returns the first non-whitespace token."""
@@ -312,9 +302,9 @@ def _SpaceRequiredBetween(left, right, is_line_disabled):
     return True
   if style.Get('SPACE_INSIDE_BRACKETS'):
     # Supersede the "no space before a colon or comma" check.
-    if lval in pytree_utils.OPENING_BRACKETS and rval == ':':
+    if left.OpensScope() and rval == ':':
       return True
-    if rval in pytree_utils.CLOSING_BRACKETS and lval == ':':
+    if right.ClosesScope() and lval == ':':
       return True
   if (style.Get('SPACES_AROUND_SUBSCRIPT_COLON') and
       (_IsSubscriptColonAndValuePair(left, right) or
@@ -365,7 +355,7 @@ def _SpaceRequiredBetween(left, right, is_line_disabled):
       # A string followed by something other than a subscript, closing bracket,
       # dot, or a binary op should have a space after it.
       return True
-    if rval in pytree_utils.CLOSING_BRACKETS:
+    if right.ClosesScope():
       # A string followed by closing brackets should have a space after it
       # depending on SPACE_INSIDE_BRACKETS.  A string followed by opening
       # brackets, however, should not.
@@ -449,28 +439,26 @@ def _SpaceRequiredBetween(left, right, is_line_disabled):
         (rval == ')' and
          _IsDictListTupleDelimiterTok(right, is_opening=False)))):
       return True
-  if (lval in pytree_utils.OPENING_BRACKETS and
-      rval in pytree_utils.OPENING_BRACKETS):
+  if left.OpensScope() and right.OpensScope():
     # Nested objects' opening brackets shouldn't be separated, unless enabled
     # by SPACE_INSIDE_BRACKETS.
     return style.Get('SPACE_INSIDE_BRACKETS')
-  if (lval in pytree_utils.CLOSING_BRACKETS and
-      rval in pytree_utils.CLOSING_BRACKETS):
+  if left.ClosesScope() and right.ClosesScope():
     # Nested objects' closing brackets shouldn't be separated, unless enabled
     # by SPACE_INSIDE_BRACKETS.
     return style.Get('SPACE_INSIDE_BRACKETS')
-  if lval in pytree_utils.CLOSING_BRACKETS and rval in '([':
+  if left.ClosesScope() and rval in '([':
     # A call, set, dictionary, or subscript that has a call or subscript after
     # it shouldn't have a space between them.
     return False
-  if lval in pytree_utils.OPENING_BRACKETS and _IsIdNumberStringToken(right):
+  if left.OpensScope() and _IsIdNumberStringToken(right):
     # Don't separate the opening bracket from the first item, unless enabled
     # by SPACE_INSIDE_BRACKETS.
     return style.Get('SPACE_INSIDE_BRACKETS')
   if left.is_name and rval in '([':
     # Don't separate a call or array access from the name.
     return False
-  if rval in pytree_utils.CLOSING_BRACKETS:
+  if right.ClosesScope():
     # Don't separate the closing bracket from the last item, unless enabled
     # by SPACE_INSIDE_BRACKETS.
     # FIXME(morbo): This might be too permissive.
@@ -478,11 +466,11 @@ def _SpaceRequiredBetween(left, right, is_line_disabled):
   if lval == 'print' and rval == '(':
     # Special support for the 'print' function.
     return False
-  if lval in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right):
+  if left.OpensScope() and _IsUnaryOperator(right):
     # Don't separate a unary operator from the opening bracket, unless enabled
     # by SPACE_INSIDE_BRACKETS.
     return style.Get('SPACE_INSIDE_BRACKETS')
-  if (lval in pytree_utils.OPENING_BRACKETS and
+  if (left.OpensScope() and
       (subtypes.VARARGS_STAR in right.subtypes or
        subtypes.KWARGS_STAR_STAR in right.subtypes)):
     # Don't separate a '*' or '**' from the opening bracket, unless enabled
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
index 804c9d5..c174514 100644
--- a/yapf/yapflib/yapf_api.py
+++ b/yapf/yapflib/yapf_api.py
@@ -38,17 +38,17 @@ import sys
 
 from yapf.pytree import pytree_unwrapper
 from yapf.pytree import pytree_utils
-from yapf.yapflib import blank_line_calculator
-from yapf.yapflib import comment_splicer
-from yapf.yapflib import continuation_splicer
+from yapf.pytree import blank_line_calculator
+from yapf.pytree import comment_splicer
+from yapf.pytree import continuation_splicer
+from yapf.pytree import split_penalty
+from yapf.pytree import subtype_assigner
 from yapf.yapflib import errors
 from yapf.yapflib import file_resources
 from yapf.yapflib import identify_container
 from yapf.yapflib import py3compat
 from yapf.yapflib import reformatter
-from yapf.yapflib import split_penalty
 from yapf.yapflib import style
-from yapf.yapflib import subtype_assigner
 
 
 def FormatFile(filename,
@@ -198,12 +198,12 @@ def FormatCode(unformatted_source,
 
 
 def _CheckPythonVersion():  # pragma: no cover
-  errmsg = 'yapf is only supported for Python 2.7 or 3.4+'
+  errmsg = 'yapf is only supported for Python 2.7 or 3.6+'
   if sys.version_info[0] == 2:
     if sys.version_info[1] < 7:
       raise RuntimeError(errmsg)
   elif sys.version_info[0] == 3:
-    if sys.version_info[1] < 4:
+    if sys.version_info[1] < 6:
       raise RuntimeError(errmsg)
 
 


Test patch:
diff --git a/yapftests/comment_splicer_test.py b/yapftests/comment_splicer_test.py
index 58bae9e..2e0141b 100644
--- a/yapftests/comment_splicer_test.py
+++ b/yapftests/comment_splicer_test.py
@@ -17,7 +17,8 @@ import textwrap
 import unittest
 
 from yapf.pytree import pytree_utils
-from yapf.yapflib import comment_splicer
+from yapf.pytree import comment_splicer
+
 from yapf.yapflib import py3compat
 
 
diff --git a/yapftests/format_decision_state_test.py b/yapftests/format_decision_state_test.py
index 3bff578..63961f3 100644
--- a/yapftests/format_decision_state_test.py
+++ b/yapftests/format_decision_state_test.py
@@ -17,6 +17,7 @@ import textwrap
 import unittest
 
 from yapf.pytree import pytree_utils
+
 from yapf.yapflib import format_decision_state
 from yapf.yapflib import logical_line
 from yapf.yapflib import style
diff --git a/yapftests/logical_line_test.py b/yapftests/logical_line_test.py
index 6876efe..ca921d2 100644
--- a/yapftests/logical_line_test.py
+++ b/yapftests/logical_line_test.py
@@ -19,9 +19,9 @@ import unittest
 from lib2to3 import pytree
 from lib2to3.pgen2 import token
 
+from yapf.pytree import split_penalty
 from yapf.yapflib import format_token
 from yapf.yapflib import logical_line
-from yapf.yapflib import split_penalty
 
 from yapftests import yapf_test_helper
 
@@ -54,12 +54,6 @@ class LogicalLineBasicTest(unittest.TestCase):
     lline.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')'))
     self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])
 
-  def testAppendNode(self):
-    lline = logical_line.LogicalLine(0)
-    lline.AppendNode(pytree.Leaf(token.LPAR, '('))
-    lline.AppendNode(pytree.Leaf(token.RPAR, ')'))
-    self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])
-
 
 class LogicalLineFormattingInformationTest(yapf_test_helper.YAPFTest):
 
diff --git a/yapftests/split_penalty_test.py b/yapftests/split_penalty_test.py
index 1c47132..f7474a3 100644
--- a/yapftests/split_penalty_test.py
+++ b/yapftests/split_penalty_test.py
@@ -21,7 +21,7 @@ from lib2to3 import pytree
 
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
-from yapf.yapflib import split_penalty
+from yapf.pytree import split_penalty
 from yapf.yapflib import style
 
 from yapftests import yapf_test_helper
diff --git a/yapftests/yapf_test_helper.py b/yapftests/yapf_test_helper.py
index 5c084c6..cb56ec8 100644
--- a/yapftests/yapf_test_helper.py
+++ b/yapftests/yapf_test_helper.py
@@ -17,17 +17,17 @@ import difflib
 import sys
 import unittest
 
+from yapf.pytree import blank_line_calculator
+from yapf.pytree import comment_splicer
+from yapf.pytree import continuation_splicer
 from yapf.pytree import pytree_unwrapper
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
-from yapf.yapflib import blank_line_calculator
-from yapf.yapflib import comment_splicer
-from yapf.yapflib import continuation_splicer
+from yapf.pytree import split_penalty
+from yapf.pytree import subtype_assigner
 from yapf.yapflib import identify_container
 from yapf.yapflib import py3compat
-from yapf.yapflib import split_penalty
 from yapf.yapflib import style
-from yapf.yapflib import subtype_assigner
 
 
 class YAPFTest(unittest.TestCase):