Repository: JelleZijlstra/typeshed_client

Issue:
Switch to using ast for parsing
typed-ast is being phased out (python/typed_ast#170). As far as I can tell, stubs as used in typeshed should always be parsable with Python 3.6 and higher, because 3.7 introduced no new features relevant to stubs and 3.8 (which brought positional-only param syntax) was never ported to typed-ast. Therefore, we should be able to get away with just using `ast.parse`.

This will break the major consumer of typeshed_client (pyanalyze), because it does a lot of isinstance checks on ast3 objects. Therefore, my tentative plan is:
- Make pyanalyze pin typeshed-client to <2 and make a release.
- Switch to ast in typeshed-client and release typeshed-client 2.0.
- Switch pyanalyze to the new types and require typeshed-client 2.0.

Discussion:

Patch:
diff --git a/README.rst b/README.rst
index 8fb5ca8..5da841e 100644
--- a/README.rst
+++ b/README.rst
@@ -39,7 +39,7 @@ Functions provided:
   ``Path('/path/to/typeshed/stdlib/@python2/typing.pyi')``. If there is no stub for the
   module, returns None.
 - ``typeshed_client.get_stub_ast`` has the same interface, but returns an AST
-  object (parsed using `typed_ast <https://www.github.com/python/typed_ast>`_).
+  object (parsed using the standard library ``ast`` module).
 
 Collecting names from stubs
 ---------------------------
@@ -81,6 +81,10 @@ call ``resolver.get_fully_qualified_name('collections.Set')`` to retrieve the
 Changelog
 ---------
 
+Unreleased
+
+- Breaking change: Use `ast` instead of `typed_ast` for parsing
+
 Version 1.2.3 (December 12, 2021)
 
 - Update bundled typeshed
diff --git a/setup.py b/setup.py
index 932aae1..97271c9 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ setup(
     },
     license="MIT",
     packages=["typeshed_client"],
-    install_requires=["typed_ast >= 1.0.3", "importlib_resources >= 1.4.0"],
+    install_requires=["importlib_resources >= 1.4.0"],
     package_data={"typeshed_client": list(find_bundled_files())},
     classifiers=[
         "Development Status :: 3 - Alpha",
diff --git a/tox.ini b/tox.ini
index 95d4eef..af9433d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,8 +5,7 @@ isolated_build = True
 
 [testenv]
 deps =
-    typed_ast >= 1.0.3
-    black == 21.5b0
+    black == 21.12b0
 commands =
     python tests/test.py
 
diff --git a/typeshed_client/__init__.py b/typeshed_client/__init__.py
index ca4eae8..ade19cc 100644
--- a/typeshed_client/__init__.py
+++ b/typeshed_client/__init__.py
@@ -23,4 +23,4 @@ from .parser import (
 from .resolver import ImportedInfo, Resolver
 
 
-__version__ = "1.2.3"
+__version__ = "2.0.0"
diff --git a/typeshed_client/finder.py b/typeshed_client/finder.py
index 11f4258..fe4606d 100644
--- a/typeshed_client/finder.py
+++ b/typeshed_client/finder.py
@@ -6,7 +6,7 @@ import os
 from pathlib import Path
 import subprocess
 import sys
-from typed_ast import ast3
+import ast
 from typing import (
     Dict,
     Generator,
@@ -87,7 +87,7 @@ def get_stub_file(
 
 def get_stub_ast(
     module_name: str, *, search_context: Optional[SearchContext] = None
-) -> Optional[ast3.AST]:
+) -> Optional[ast.AST]:
     """Return the AST for the stub for the given module name."""
     path = get_stub_file(module_name, search_context=search_context)
     if path is None:
@@ -313,10 +313,9 @@ def find_typeshed() -> Path:
     return importlib_resources.files("typeshed_client") / "typeshed"
 
 
-def parse_stub_file(path: Path) -> ast3.AST:
+def parse_stub_file(path: Path) -> ast.AST:
     text = path.read_text()
-    # Always parse stubs as Python 3.6
-    return ast3.parse(text, filename=str(path), feature_version=6)
+    return ast.parse(text, filename=str(path))
 
 
 def _path_to_module(path: Path) -> str:
diff --git a/typeshed_client/parser.py b/typeshed_client/parser.py
index d887965..065d7c7 100644
--- a/typeshed_client/parser.py
+++ b/typeshed_client/parser.py
@@ -1,7 +1,7 @@
 """This module is responsible for parsing a stub AST into a dictionary of names."""
 
 import logging
-from typed_ast import ast3
+import ast
 from typing import (
     Any,
     Dict,
@@ -30,13 +30,13 @@ class ImportedName(NamedTuple):
 
 
 class OverloadedName(NamedTuple):
-    definitions: List[ast3.AST]
+    definitions: List[ast.AST]
 
 
 class NameInfo(NamedTuple):
     name: str
     is_exported: bool
-    ast: Union[ast3.AST, ImportedName, OverloadedName]
+    ast: Union[ast.AST, ImportedName, OverloadedName]
     # should be Optional[NameDict] but that needs a recursive type
     child_nodes: Optional[Dict[str, Any]] = None
 
@@ -61,7 +61,7 @@ def get_stub_names(
 
 
 def parse_ast(
-    ast: ast3.AST,
+    ast: ast.AST,
     search_context: SearchContext,
     module_name: ModulePath,
     *,
@@ -113,20 +113,20 @@ def parse_ast(
 
 
 _CMP_OP_TO_FUNCTION = {
-    ast3.Eq: lambda x, y: x == y,
-    ast3.NotEq: lambda x, y: x != y,
-    ast3.Lt: lambda x, y: x < y,
-    ast3.LtE: lambda x, y: x <= y,
-    ast3.Gt: lambda x, y: x > y,
-    ast3.GtE: lambda x, y: x >= y,
-    ast3.Is: lambda x, y: x is y,
-    ast3.IsNot: lambda x, y: x is not y,
-    ast3.In: lambda x, y: x in y,
-    ast3.NotIn: lambda x, y: x not in y,
+    ast.Eq: lambda x, y: x == y,
+    ast.NotEq: lambda x, y: x != y,
+    ast.Lt: lambda x, y: x < y,
+    ast.LtE: lambda x, y: x <= y,
+    ast.Gt: lambda x, y: x > y,
+    ast.GtE: lambda x, y: x >= y,
+    ast.Is: lambda x, y: x is y,
+    ast.IsNot: lambda x, y: x is not y,
+    ast.In: lambda x, y: x in y,
+    ast.NotIn: lambda x, y: x not in y,
 }
 
 
-class _NameExtractor(ast3.NodeVisitor):
+class _NameExtractor(ast.NodeVisitor):
     """Extract names from a stub module."""
 
     def __init__(
@@ -136,16 +136,16 @@ class _NameExtractor(ast3.NodeVisitor):
         self.module_name = module_name
         self.is_init = is_init
 
-    def visit_Module(self, node: ast3.Module) -> List[NameInfo]:
+    def visit_Module(self, node: ast.Module) -> List[NameInfo]:
         return [info for child in node.body for info in self.visit(child)]
 
-    def visit_FunctionDef(self, node: ast3.FunctionDef) -> Iterable[NameInfo]:
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> Iterable[NameInfo]:
         yield NameInfo(node.name, not node.name.startswith("_"), node)
 
-    def visit_AsyncFunctionDef(self, node: ast3.AsyncFunctionDef) -> Iterable[NameInfo]:
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Iterable[NameInfo]:
         yield NameInfo(node.name, not node.name.startswith("_"), node)
 
-    def visit_ClassDef(self, node: ast3.ClassDef) -> Iterable[NameInfo]:
+    def visit_ClassDef(self, node: ast.ClassDef) -> Iterable[NameInfo]:
         children = [info for child in node.body for info in self.visit(child)]
         child_dict: NameDict = {}
         for info in children:
@@ -164,23 +164,23 @@ class _NameExtractor(ast3.NodeVisitor):
                 child_dict[info.name] = info
         yield NameInfo(node.name, not node.name.startswith("_"), node, child_dict)
 
-    def visit_Assign(self, node: ast3.Assign) -> Iterable[NameInfo]:
+    def visit_Assign(self, node: ast.Assign) -> Iterable[NameInfo]:
         for target in node.targets:
-            if not isinstance(target, ast3.Name):
+            if not isinstance(target, ast.Name):
                 raise InvalidStub(
-                    f"Assignment should only be to a simple name: {ast3.dump(node)}"
+                    f"Assignment should only be to a simple name: {ast.dump(node)}"
                 )
             yield NameInfo(target.id, not target.id.startswith("_"), node)
 
-    def visit_AnnAssign(self, node: ast3.AnnAssign) -> Iterable[NameInfo]:
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> Iterable[NameInfo]:
         target = node.target
-        if not isinstance(target, ast3.Name):
+        if not isinstance(target, ast.Name):
             raise InvalidStub(
-                f"Assignment should only be to a simple name: {ast3.dump(node)}"
+                f"Assignment should only be to a simple name: {ast.dump(node)}"
             )
         yield NameInfo(target.id, not target.id.startswith("_"), node)
 
-    def visit_If(self, node: ast3.If) -> Iterable[NameInfo]:
+    def visit_If(self, node: ast.If) -> Iterable[NameInfo]:
         visitor = _LiteralEvalVisitor(self.ctx)
         value = visitor.visit(node.test)
         if value:
@@ -190,7 +190,7 @@ class _NameExtractor(ast3.NodeVisitor):
             for stmt in node.orelse:
                 yield from self.visit(stmt)
 
-    def visit_Assert(self, node: ast3.Assert) -> Iterable[NameInfo]:
+    def visit_Assert(self, node: ast.Assert) -> Iterable[NameInfo]:
         visitor = _LiteralEvalVisitor(self.ctx)
         value = visitor.visit(node.test)
         if value:
@@ -198,7 +198,7 @@ class _NameExtractor(ast3.NodeVisitor):
         else:
             raise _AssertFailed
 
-    def visit_Import(self, node: ast3.Import) -> Iterable[NameInfo]:
+    def visit_Import(self, node: ast.Import) -> Iterable[NameInfo]:
         for alias in node.names:
             if alias.asname is not None:
                 yield NameInfo(
@@ -211,7 +211,7 @@ class _NameExtractor(ast3.NodeVisitor):
                 name = alias.name.split(".", 1)[0]
                 yield NameInfo(name, False, ImportedName(ModulePath((name,))))
 
-    def visit_ImportFrom(self, node: ast3.ImportFrom) -> Iterable[NameInfo]:
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> Iterable[NameInfo]:
         module: Tuple[str, ...]
         if node.module is None:
             module = ()
@@ -253,77 +253,77 @@ class _NameExtractor(ast3.NodeVisitor):
                     alias.name, False, ImportedName(source_module, alias.name)
                 )
 
-    def visit_Expr(self, node: ast3.Expr) -> Iterable[NameInfo]:
-        if not isinstance(node.value, (ast3.Ellipsis, ast3.Str)):
-            raise InvalidStub(f"Cannot handle node {ast3.dump(node)}")
+    def visit_Expr(self, node: ast.Expr) -> Iterable[NameInfo]:
+        if not isinstance(node.value, (ast.Ellipsis, ast.Str)):
+            raise InvalidStub(f"Cannot handle node {ast.dump(node)}")
         return []
 
-    def visit_Pass(self, node: ast3.Pass) -> Iterable[NameInfo]:
+    def visit_Pass(self, node: ast.Pass) -> Iterable[NameInfo]:
         return []
 
-    def generic_visit(self, node: ast3.AST) -> NoReturn:
-        raise InvalidStub(f"Cannot handle node {ast3.dump(node)}")
+    def generic_visit(self, node: ast.AST) -> NoReturn:
+        raise InvalidStub(f"Cannot handle node {ast.dump(node)}")
 
 
-class _LiteralEvalVisitor(ast3.NodeVisitor):
+class _LiteralEvalVisitor(ast.NodeVisitor):
     def __init__(self, ctx: SearchContext) -> None:
         self.ctx = ctx
 
-    def visit_Num(self, node: ast3.Num) -> Union[int, float]:
+    def visit_Num(self, node: ast.Num) -> Union[int, float]:
         return node.n
 
-    def visit_Str(self, node: ast3.Str) -> str:
+    def visit_Str(self, node: ast.Str) -> str:
         return node.s
 
-    def visit_Index(self, node: ast3.Index) -> int:
+    def visit_Index(self, node: ast.Index) -> int:
         return self.visit(node.value)
 
-    def visit_Tuple(self, node: ast3.Tuple) -> Tuple[Any, ...]:
+    def visit_Tuple(self, node: ast.Tuple) -> Tuple[Any, ...]:
         return tuple(self.visit(elt) for elt in node.elts)
 
-    def visit_Subscript(self, node: ast3.Subscript) -> Any:
+    def visit_Subscript(self, node: ast.Subscript) -> Any:
         value = self.visit(node.value)
         slc = self.visit(node.slice)
         return value[slc]
 
-    def visit_Compare(self, node: ast3.Compare) -> bool:
+    def visit_Compare(self, node: ast.Compare) -> bool:
         if len(node.ops) != 1:
-            raise InvalidStub(f"Cannot evaluate chained comparison {ast3.dump(node)}")
+            raise InvalidStub(f"Cannot evaluate chained comparison {ast.dump(node)}")
         fn = _CMP_OP_TO_FUNCTION[type(node.ops[0])]
         return fn(self.visit(node.left), self.visit(node.comparators[0]))
 
-    def visit_BoolOp(self, node: ast3.BoolOp) -> bool:
+    def visit_BoolOp(self, node: ast.BoolOp) -> bool:
         for val_node in node.values:
             val = self.visit(val_node)
-            if (isinstance(node.op, ast3.Or) and val) or (
-                isinstance(node.op, ast3.And) and not val
+            if (isinstance(node.op, ast.Or) and val) or (
+                isinstance(node.op, ast.And) and not val
             ):
                 return val
         return val
 
-    def visit_Slice(self, node: ast3.Slice) -> slice:
+    def visit_Slice(self, node: ast.Slice) -> slice:
         lower = self.visit(node.lower) if node.lower is not None else None
         upper = self.visit(node.upper) if node.upper is not None else None
         step = self.visit(node.step) if node.step is not None else None
         return slice(lower, upper, step)
 
-    def visit_Attribute(self, node: ast3.Attribute) -> Any:
+    def visit_Attribute(self, node: ast.Attribute) -> Any:
         val = node.value
-        if not isinstance(val, ast3.Name):
-            raise InvalidStub(f"Invalid code in stub: {ast3.dump(node)}")
+        if not isinstance(val, ast.Name):
+            raise InvalidStub(f"Invalid code in stub: {ast.dump(node)}")
         if val.id != "sys":
             raise InvalidStub(
-                f"Attribute access must be on the sys module: {ast3.dump(node)}"
+                f"Attribute access must be on the sys module: {ast.dump(node)}"
             )
         if node.attr == "platform":
             return self.ctx.platform
         elif node.attr == "version_info":
             return self.ctx.version
         else:
-            raise InvalidStub(f"Invalid attribute on {ast3.dump(node)}")
+            raise InvalidStub(f"Invalid attribute on {ast.dump(node)}")
 
-    def generic_visit(self, node: ast3.AST) -> NoReturn:
-        raise InvalidStub(f"Cannot evaluate node {ast3.dump(node)}")
+    def generic_visit(self, node: ast.AST) -> NoReturn:
+        raise InvalidStub(f"Cannot evaluate node {ast.dump(node)}")
 
 
 class _AssertFailed(Exception):


Test patch:
diff --git a/tests/test.py b/tests/test.py
index bdcafaf..30a53ca 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from typed_ast import ast3
+import ast
 import typeshed_client
 from typeshed_client.finder import (
     get_search_context,
@@ -92,11 +92,11 @@ class TestParser(unittest.TestCase):
         )
 
         # Simple assignments
-        self.check_nameinfo(names, "var", ast3.AnnAssign)
-        self.check_nameinfo(names, "old_var", ast3.Assign)
-        self.check_nameinfo(names, "_private", ast3.AnnAssign, is_exported=False)
-        self.check_nameinfo(names, "multiple", ast3.Assign)
-        self.check_nameinfo(names, "assignment", ast3.Assign)
+        self.check_nameinfo(names, "var", ast.AnnAssign)
+        self.check_nameinfo(names, "old_var", ast.Assign)
+        self.check_nameinfo(names, "_private", ast.AnnAssign, is_exported=False)
+        self.check_nameinfo(names, "multiple", ast.Assign)
+        self.check_nameinfo(names, "assignment", ast.Assign)
 
         # Imports
         path = typeshed_client.ModulePath(("other",))
@@ -127,15 +127,15 @@ class TestParser(unittest.TestCase):
         )
 
         # Functions
-        self.check_nameinfo(names, "func", ast3.FunctionDef)
-        self.check_nameinfo(names, "async_func", ast3.AsyncFunctionDef)
+        self.check_nameinfo(names, "func", ast.FunctionDef)
+        self.check_nameinfo(names, "async_func", ast.AsyncFunctionDef)
 
         # Classes
-        self.check_nameinfo(names, "Cls", ast3.ClassDef, has_child_nodes=True)
+        self.check_nameinfo(names, "Cls", ast.ClassDef, has_child_nodes=True)
         cls_names = names["Cls"].child_nodes
         self.assertEqual(set(cls_names.keys()), {"attr", "method"})
-        self.check_nameinfo(cls_names, "attr", ast3.AnnAssign)
-        self.check_nameinfo(cls_names, "method", ast3.FunctionDef)
+        self.check_nameinfo(cls_names, "attr", ast.AnnAssign)
+        self.check_nameinfo(cls_names, "method", ast.FunctionDef)
 
     def test_starimport(self) -> None:
         ctx = get_context((3, 5))
@@ -232,16 +232,16 @@ class TestParser(unittest.TestCase):
         definitions = names["overloaded"].ast.definitions
         self.assertEqual(len(definitions), 2)
         for defn in definitions:
-            self.assertIsInstance(defn, ast3.FunctionDef)
+            self.assertIsInstance(defn, ast.FunctionDef)
 
         classdef = names["OverloadClass"]
-        self.assertIsInstance(classdef.ast, ast3.ClassDef)
+        self.assertIsInstance(classdef.ast, ast.ClassDef)
         children = classdef.child_nodes
         self.assertEqual(set(children.keys()), {"overloaded"})
         definitions = children["overloaded"].ast.definitions
         self.assertEqual(len(definitions), 2)
         for defn in definitions:
-            self.assertIsInstance(defn, ast3.FunctionDef)
+            self.assertIsInstance(defn, ast.FunctionDef)
 
 
 class TestResolver(unittest.TestCase):
@@ -256,7 +256,7 @@ class TestResolver(unittest.TestCase):
         name_info = typeshed_client.NameInfo("exported", True, mock.ANY)
         resolved = res.get_name(path, "exported")
         self.assertEqual(resolved, typeshed_client.ImportedInfo(other_path, name_info))
-        self.assertIsInstance(resolved.info.ast, ast3.AnnAssign)
+        self.assertIsInstance(resolved.info.ast, ast.AnnAssign)
 
         self.assertIsInstance(res.get_name(path, "var"), typeshed_client.NameInfo)