Repository: google/yapf
Pull request #1040 (2022-11-30 02:56:31)

Prevent crashes against valid 'pyproject.toml'.
* Replace 'toml' dependency with 'tomli', which fully supports the current TOML specification.

The `toml` library does not fully support the current [TOML specification](https://toml.io/en/v1.0.0)... in particular, [arrays with items of heterogeneous types](https://toml.io/en/v1.0.0#array). When `toml` encounters things that it does not support, during load, it raises an exception which causes Yapf to crash in turn. Other tools in `pyproject.toml` can handle (and even expect) some of the things on which `toml` crashes.

The `tomli` library is noted as the inspiration for the `tomllib` module in Python 3.11+ standard library, according to [PEP 680](https://peps.python.org/pep-0680/#rationale). Furthermore, the author of `tomli` (who is one of the PEP authors) is actively maintaining `tomli` for backward compatibility at least until Python 3.10 reaches end-of-life, according to a [statement in the `tomli` repository](https://github.com/hukkin/tomli#intro):
> Tomli continues to provide a backport on PyPI for Python versions where the standard library module is not available and that have not yet reached their end-of-life.
Given this, `tomli` seems like the ideal replacement for `toml`.

Where the package is actually being used in the code, I have elected to use `import tomli as tomllib` to make it easier to switch to `tomllib` once Python 3.11 is the baseline supported version. In cases where the import is just being used a feature test, I simply added an "i" to the existing import with the belief that the feature tests can be removed once Python 3.11 is baseline.

In accordance with `HACKING.rst`, I ran:
* `PYTHONPATH=$PWD/yapf python3 -m yapf -i -r .`
* `python3 setup.py test`

before submitting this PR. Also verified directly that the command still runs correctly.

google-cla[bot]: Thanks for your pull request! It looks like this may be your first contribution to a Google open source project. Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA).

View this [failed invocation](https://github.com/google/yapf/pull/1040/checks?check_run_id=9786679815) of the CLA check for more information.

For the most up to date status, view the checks section at the bottom of the pull request.
emcd: I use my Github cloaked emails for commits but the CLA agreement manager detected my Gmail address. Will look into registering my cloaked email tomorrow.

Patch:
diff --git a/setup.py b/setup.py
index 5ba8b7a..013a23d 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,6 @@ with codecs.open('README.rst', 'r', 'utf-8') as fd:
           'test': RunTests,
       },
       extras_require={
-          'pyproject': ['toml'],
+          'pyproject': ['tomli'],
       },
   )
diff --git a/yapf/yapflib/file_resources.py b/yapf/yapflib/file_resources.py
index b5e2612..6809ca9 100644
--- a/yapf/yapflib/file_resources.py
+++ b/yapf/yapflib/file_resources.py
@@ -49,14 +49,15 @@ def _GetExcludePatternsFromPyprojectToml(filename):
   """Get a list of file patterns to ignore from pyproject.toml."""
   ignore_patterns = []
   try:
-    import toml
+    import tomli as tomllib
   except ImportError:
     raise errors.YapfError(
-        "toml package is needed for using pyproject.toml as a "
+        "tomli package is needed for using pyproject.toml as a "
         "configuration file")
 
   if os.path.isfile(filename) and os.access(filename, os.R_OK):
-    pyproject_toml = toml.load(filename)
+    with open(filename, 'rb') as fd:
+      pyproject_toml = tomllib.load(fd)
     ignore_patterns = pyproject_toml.get('tool',
                                          {}).get('yapfignore',
                                                  {}).get('ignore_patterns', [])
@@ -127,19 +128,19 @@ def GetDefaultStyleForDir(dirname, default_style=style.DEFAULT_STYLE):
     # See if we have a pyproject.toml file with a '[tool.yapf]'  section.
     config_file = os.path.join(dirname, style.PYPROJECT_TOML)
     try:
-      fd = open(config_file)
+      fd = open(config_file, 'rb')
     except IOError:
       pass  # It's okay if it's not there.
     else:
       with fd:
         try:
-          import toml
+          import tomli as tomllib
         except ImportError:
           raise errors.YapfError(
-              "toml package is needed for using pyproject.toml as a "
+              "tomli package is needed for using pyproject.toml as a "
               "configuration file")
 
-        pyproject_toml = toml.load(config_file)
+        pyproject_toml = tomllib.load(fd)
         style_dict = pyproject_toml.get('tool', {}).get('yapf', None)
         if style_dict is not None:
           return config_file
diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py
index 233a64e..c8397b3 100644
--- a/yapf/yapflib/style.py
+++ b/yapf/yapflib/style.py
@@ -746,17 +746,18 @@ def _CreateConfigParserFromConfigFile(config_filename):
     # Provide a more meaningful error here.
     raise StyleConfigError(
         '"{0}" is not a valid style or file path'.format(config_filename))
-  with open(config_filename) as style_file:
-    config = py3compat.ConfigParser()
-    if config_filename.endswith(PYPROJECT_TOML):
-      try:
-        import toml
-      except ImportError:
-        raise errors.YapfError(
-            "toml package is needed for using pyproject.toml as a "
-            "configuration file")
-
-      pyproject_toml = toml.load(style_file)
+  config = py3compat.ConfigParser()
+
+  if config_filename.endswith(PYPROJECT_TOML):
+    try:
+      import tomli as tomllib
+    except ImportError:
+      raise errors.YapfError(
+          "tomli package is needed for using pyproject.toml as a "
+          "configuration file")
+
+    with open(config_filename, 'rb') as style_file:
+      pyproject_toml = tomllib.load(style_file)
       style_dict = pyproject_toml.get("tool", {}).get("yapf", None)
       if style_dict is None:
         raise StyleConfigError(
@@ -766,7 +767,9 @@ def _CreateConfigParserFromConfigFile(config_filename):
         config.set('style', k, str(v))
       return config
 
+  with open(config_filename) as style_file:
     config.read_file(style_file)
+
     if config_filename.endswith(SETUP_CONFIG):
       if not config.has_section('yapf'):
         raise StyleConfigError(


Test patch:
diff --git a/yapftests/file_resources_test.py b/yapftests/file_resources_test.py
index 31184c4..f54f393 100644
--- a/yapftests/file_resources_test.py
+++ b/yapftests/file_resources_test.py
@@ -75,7 +75,7 @@ class GetExcludePatternsForDir(unittest.TestCase):
 
   def test_get_exclude_file_patterns_from_pyproject(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return
     local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml')
@@ -93,7 +93,7 @@ class GetExcludePatternsForDir(unittest.TestCase):
   @unittest.skipUnless(py3compat.PY36, 'Requires Python 3.6')
   def test_get_exclude_file_patterns_from_pyproject_with_wrong_syntax(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return
     local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml')
@@ -109,7 +109,7 @@ class GetExcludePatternsForDir(unittest.TestCase):
 
   def test_get_exclude_file_patterns_from_pyproject_no_ignore_section(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return
     local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml')
@@ -122,7 +122,7 @@ class GetExcludePatternsForDir(unittest.TestCase):
 
   def test_get_exclude_file_patterns_from_pyproject_ignore_section_empty(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return
     local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml')
@@ -192,7 +192,7 @@ class GetDefaultStyleForDirTest(unittest.TestCase):
   def test_pyproject_toml(self):
     # An empty pyproject.toml file should not be used
     try:
-      import toml
+      import tomli
     except ImportError:
       return
 
diff --git a/yapftests/style_test.py b/yapftests/style_test.py
index 8a37f95..d2203d9 100644
--- a/yapftests/style_test.py
+++ b/yapftests/style_test.py
@@ -230,7 +230,7 @@ class StyleFromFileTest(yapf_test_helper.YAPFTest):
 
   def testPyprojectTomlNoYapfSection(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return
 
@@ -242,7 +242,7 @@ class StyleFromFileTest(yapf_test_helper.YAPFTest):
 
   def testPyprojectTomlParseYapfSection(self):
     try:
-      import toml
+      import tomli
     except ImportError:
       return