Repository: quora/pyanalyze
Pull request #615 (2023-05-02 01:56:42)

Fix various issues exposed by typeshed-client update


Patch:
diff --git a/docs/changelog.md b/docs/changelog.md
index 42f28fa..b1013ad 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+- Add support for `__new__` methods returning `typing.Self`, fixing
+  various failures with the latest release of `typeshed-client` (#615)
+- Add support for importing stub-only modules in other stubs (#615)
+- Fix signature compatibility bug involving `**kwargs` and positional-only
+  arguments (#615)
 - Fix type narrowing with `in` on enum types in the negative case (#606)
 - Fix crash when `getattr()` on a module object throws an error (#603)
 - Fix handling of positional-only arguments using `/` syntax in stubs (#601)
diff --git a/pyanalyze/analysis_lib.py b/pyanalyze/analysis_lib.py
index ca3bad7..6b616c1 100644
--- a/pyanalyze/analysis_lib.py
+++ b/pyanalyze/analysis_lib.py
@@ -6,6 +6,7 @@ Commonly useful components for static analysis tools.
 import ast
 import linecache
 import os
+from pathlib import Path
 import secrets
 import sys
 import types
@@ -14,7 +15,7 @@ from typing import Callable, List, Mapping, Optional, Set
 
 
 def _all_files(
-    root: str, filter_function: Optional[Callable[[str], bool]] = None
+    root: Path, filter_function: Optional[Callable[[str], bool]] = None
 ) -> Set[str]:
     """Returns the set of all files at the given root.
 
@@ -30,7 +31,7 @@ def _all_files(
     return all_files
 
 
-def files_with_extension_from_directory(extension: str, dirname: str) -> Set[str]:
+def files_with_extension_from_directory(extension: str, dirname: Path) -> Set[str]:
     """Finds all files in a given directory with this extension."""
     return _all_files(dirname, filter_function=lambda fn: fn.endswith("." + extension))
 
diff --git a/pyanalyze/arg_spec.py b/pyanalyze/arg_spec.py
index f48401b..7a8ef7f 100644
--- a/pyanalyze/arg_spec.py
+++ b/pyanalyze/arg_spec.py
@@ -644,12 +644,13 @@ class ArgSpecCache:
             signature = self._cached_get_argspec(
                 _ENUM_CALL, impl, is_asynq, in_overload_resolution
             )
-            bound_sig = make_bound_method(
-                signature, Composite(SubclassValue(TypedValue(obj)))
-            )
+            self_value = SubclassValue(TypedValue(obj))
+            bound_sig = make_bound_method(signature, Composite(self_value))
             if bound_sig is None:
                 return None
-            sig = bound_sig.get_signature(preserve_impl=True, ctx=self.ctx)
+            sig = bound_sig.get_signature(
+                preserve_impl=True, ctx=self.ctx, self_annotation_value=self_value
+            )
             if sig is not None:
                 return sig
             return bound_sig
@@ -739,6 +740,7 @@ class ArgSpecCache:
         if inspect.isclass(obj):
             obj = UnwrapClass.unwrap(obj, self.options)
             override = ConstructorHooks.get_constructor(obj, self.options)
+            is_dunder_new = False
             if isinstance(override, Signature):
                 signature = override
             else:
@@ -760,6 +762,7 @@ class ArgSpecCache:
                     # doesn't have a useful signature.
                     # In practice, we saw this make a difference with NamedTuples.
                     elif isinstance(obj.__new__, FunctionType):
+                        is_dunder_new = True
                         constructor = obj.__new__
                     else:
                         constructor = obj.__init__
@@ -783,7 +786,15 @@ class ArgSpecCache:
             bound_sig = make_bound_method(signature, Composite(TypedValue(obj)))
             if bound_sig is None:
                 return None
-            sig = bound_sig.get_signature(preserve_impl=True, ctx=self.ctx)
+            if is_dunder_new:
+                self_annotation_value = KnownValue(obj)
+            else:
+                self_annotation_value = TypedValue(obj)
+            sig = bound_sig.get_signature(
+                preserve_impl=True,
+                ctx=self.ctx,
+                self_annotation_value=self_annotation_value,
+            )
             if sig is not None:
                 return sig
             return bound_sig
diff --git a/pyanalyze/attributes.py b/pyanalyze/attributes.py
index af33620..0bf9df3 100644
--- a/pyanalyze/attributes.py
+++ b/pyanalyze/attributes.py
@@ -30,6 +30,7 @@ from .value import (
     KnownValue,
     KnownValueWithTypeVars,
     MultiValuedValue,
+    SyntheticModuleValue,
     set_self,
     SubclassValue,
     TypedValue,
@@ -67,6 +68,9 @@ class AttrContext:
     def get_property_type_from_argspec(self, obj: property) -> Value:
         return AnyValue(AnySource.inference)
 
+    def resolve_name_from_typeshed(self, module: str, name: str) -> Value:
+        return UNINITIALIZED_VALUE
+
     def get_attribute_from_typeshed(self, typ: type, *, on_class: bool) -> Value:
         return UNINITIALIZED_VALUE
 
@@ -130,6 +134,9 @@ def get_attribute(ctx: AttrContext) -> Value:
         attribute_value = AnyValue(AnySource.from_another)
     elif isinstance(root_value, MultiValuedValue):
         raise TypeError("caller should unwrap MultiValuedValue")
+    elif isinstance(root_value, SyntheticModuleValue):
+        module = ".".join(root_value.module_path)
+        attribute_value = ctx.resolve_name_from_typeshed(module, ctx.attr)
     else:
         attribute_value = UNINITIALIZED_VALUE
     if (
diff --git a/pyanalyze/checker.py b/pyanalyze/checker.py
index d08757a..cc087f1 100644
--- a/pyanalyze/checker.py
+++ b/pyanalyze/checker.py
@@ -239,6 +239,8 @@ class Checker:
             sig = self.arg_spec_cache.get_argspec(value.val)
         elif isinstance(value, UnboundMethodValue):
             sig = value.get_signature(self)
+        elif isinstance(value, SubclassValue) and value.exactly:
+            sig = self.signature_from_value(value)
         else:
             sig = None
         if sig is not None:
@@ -439,6 +441,9 @@ def _extract_protocol_members(typ: type) -> Set[str]:
 class CheckerAttrContext(AttrContext):
     checker: Checker
 
+    def resolve_name_from_typeshed(self, module: str, name: str) -> Value:
+        return self.checker.ts_finder.resolve_name(module, name)
+
     def get_attribute_from_typeshed(self, typ: type, *, on_class: bool) -> Value:
         return self.checker.ts_finder.get_attribute(typ, self.attr, on_class=on_class)
 
diff --git a/pyanalyze/node_visitor.py b/pyanalyze/node_visitor.py
index 76bb817..0f3545e 100644
--- a/pyanalyze/node_visitor.py
+++ b/pyanalyze/node_visitor.py
@@ -14,6 +14,7 @@ import json
 import logging
 import os
 import os.path
+from pathlib import Path
 import re
 import subprocess
 import sys
@@ -1116,7 +1117,7 @@ def _get_all_files(lst: Iterable[str]) -> Iterable[str]:
     for entry in lst:
         if os.path.isdir(entry):
             yield from sorted(
-                analysis_lib.files_with_extension_from_directory("py", entry)
+                analysis_lib.files_with_extension_from_directory("py", Path(entry))
             )
         else:
             yield entry
diff --git a/pyanalyze/options.py b/pyanalyze/options.py
index 7dba986..f063bc6 100644
--- a/pyanalyze/options.py
+++ b/pyanalyze/options.py
@@ -362,7 +362,9 @@ def parse_config_file(
     if path in seen_paths:
         raise InvalidConfigOption("Recursive config inclusion detected")
     with path.open("rb") as f:
-        data = tomli.load(f)
+        # tomli annotates the arg as BinaryIO, and we don't treat BufferedReader
+        # as a BinaryIO
+        data = tomli.load(f)  # static analysis: ignore[incompatible_argument]
     data = data.get("tool", {}).get("pyanalyze", {})
     yield from _parse_config_section(
         data, path=path, priority=priority, seen_paths={path, *seen_paths}
diff --git a/pyanalyze/signature.py b/pyanalyze/signature.py
index 8f8a75f..6edd970 100644
--- a/pyanalyze/signature.py
+++ b/pyanalyze/signature.py
@@ -61,6 +61,7 @@ from .type_evaluation import (
 )
 from .typevar import resolve_bounds_map
 from .value import (
+    SelfT,
     annotate_value,
     AnnotatedValue,
     AnySource,
@@ -1453,6 +1454,7 @@ class Signature:
         if their_ellipsis is not None:
             args_annotation = kwargs_annotation = AnyValue(AnySource.ellipsis_callable)
         consumed_positional = set()
+        consumed_required_pos_only = set()
         consumed_keyword = set()
         consumed_paramspec = False
         for i, my_param in enumerate(self.parameters.values()):
@@ -1479,6 +1481,8 @@ class Signature:
                         )
                     tv_maps.append(tv_map)
                     consumed_positional.add(their_params[i].name)
+                    if their_params[i].default is None:
+                        consumed_required_pos_only.add(their_params[i].name)
                 elif args_annotation is not None:
                     new_tv_maps = can_assign_var_positional(
                         my_param, args_annotation, i - their_args_index, ctx
@@ -1608,6 +1612,7 @@ class Signature:
                     if param.name not in consumed_keyword
                     and param.kind
                     in (ParameterKind.KEYWORD_ONLY, ParameterKind.POSITIONAL_OR_KEYWORD)
+                    and param.name not in consumed_required_pos_only
                 ]
                 for extra_param in extra_keyword:
                     tv_map = extra_param.get_annotation().can_assign(my_annotation, ctx)
@@ -1885,6 +1890,7 @@ class Signature:
         self,
         *,
         preserve_impl: bool = False,
+        self_annotation_value: Optional[Value] = None,
         self_value: Optional[Value] = None,
         ctx: CanAssignContext,
     ) -> Optional["Signature"]:
@@ -1910,12 +1916,14 @@ class Signature:
             self_annotation = params[0].annotation
         else:
             return None
-        if self_value is not None:
-            tv_map = get_tv_map(self_annotation, self_value, ctx)
+        if self_annotation_value is not None:
+            tv_map = get_tv_map(self_annotation, self_annotation_value, ctx)
             if isinstance(tv_map, CanAssignError):
                 return None
         else:
             tv_map = {}
+        if self_value is not None:
+            tv_map = {**tv_map, SelfT: self_value}
         if tv_map:
             new_params = {
                 param.name: param.substitute_typevars(tv_map) for param in new_params
@@ -2414,10 +2422,16 @@ class OverloadedSignature:
         *,
         preserve_impl: bool = False,
         self_value: Optional[Value] = None,
+        self_annotation_value: Optional[Value] = None,
         ctx: CanAssignContext,
     ) -> Optional["ConcreteSignature"]:
         bound_sigs = [
-            sig.bind_self(preserve_impl=preserve_impl, self_value=self_value, ctx=ctx)
+            sig.bind_self(
+                preserve_impl=preserve_impl,
+                self_value=self_value,
+                self_annotation_value=self_annotation_value,
+                ctx=ctx,
+            )
             for sig in self.signatures
         ]
         bound_sigs = [sig for sig in bound_sigs if isinstance(sig, Signature)]
@@ -2496,10 +2510,19 @@ class BoundMethodSignature:
         return ret
 
     def get_signature(
-        self, *, preserve_impl: bool = False, ctx: CanAssignContext
+        self,
+        *,
+        preserve_impl: bool = False,
+        ctx: CanAssignContext,
+        self_annotation_value: Optional[Value] = None,
     ) -> Optional[ConcreteSignature]:
+        if self_annotation_value is None:
+            self_annotation_value = self.self_composite.value
         return self.signature.bind_self(
-            preserve_impl=preserve_impl, self_value=self.self_composite.value, ctx=ctx
+            preserve_impl=preserve_impl,
+            self_value=self.self_composite.value,
+            ctx=ctx,
+            self_annotation_value=self_annotation_value,
         )
 
     def has_return_value(self) -> bool:
diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py
index 24734ad..13d08d2 100644
--- a/pyanalyze/typeshed.py
+++ b/pyanalyze/typeshed.py
@@ -71,6 +71,7 @@ from .value import (
     CanAssignContext,
     DeprecatedExtension,
     Extension,
+    SyntheticModuleValue,
     annotate_value,
     extract_typevars,
     GenericValue,
@@ -111,6 +112,8 @@ class _AnnotationContext(Context):
         if isinstance(root_value, KnownValue):
             if isinstance(root_value.val, ModuleType):
                 return self.finder.resolve_name(root_value.val.__name__, node.attr)
+        elif isinstance(root_value, SyntheticModuleValue):
+            return self.finder.resolve_name(".".join(root_value.module_path), node.attr)
         return super().get_attribute(root_value, node)
 
 
@@ -762,12 +765,15 @@ class TypeshedFinder:
                         self_val = TypedValue(typ)
                     if from_init:
                         sig = sig.replace_return_value(self_val)
+                        self_annotation_value = self_val
                     else:
-                        self_val = SubclassValue(self_val)
+                        self_annotation_value = SubclassValue(self_val)
                     bound_sig = make_bound_method(sig, Composite(self_val))
                     if bound_sig is None:
                         return None
-                    sig = bound_sig.get_signature(ctx=self.ctx)
+                    sig = bound_sig.get_signature(
+                        ctx=self.ctx, self_annotation_value=self_annotation_value
+                    )
                     return sig
 
                 return None
@@ -1162,8 +1168,7 @@ class TypeshedFinder:
                 __import__(module_path)
                 return KnownValue(sys.modules[module_path])
             except Exception:
-                self.log("Unable to import", module_path)
-                return AnyValue(AnySource.inference)
+                return SyntheticModuleValue(info)
         else:
             self.log("Ignoring info", info)
             return AnyValue(AnySource.inference)
diff --git a/pyanalyze/value.py b/pyanalyze/value.py
index 1a9ee91..7f786d6 100644
--- a/pyanalyze/value.py
+++ b/pyanalyze/value.py
@@ -517,6 +517,13 @@ class KnownValueWithTypeVars(KnownValue):
     """TypeVars substituted on this value."""
 
 
+@dataclass(frozen=True)
+class SyntheticModuleValue(Value):
+    """Represents a module that exists only in stub files."""
+
+    module_path: Sequence[str]
+
+
 @dataclass(frozen=True)
 class UnboundMethodValue(Value):
     """Value that represents a method on an underlying :class:`Value`.


Test patch:
diff --git a/pyanalyze/stubs/_pyanalyze_tests-stubs/self.pyi b/pyanalyze/stubs/_pyanalyze_tests-stubs/self.pyi
index 80e3990..4d87eb3 100644
--- a/pyanalyze/stubs/_pyanalyze_tests-stubs/self.pyi
+++ b/pyanalyze/stubs/_pyanalyze_tests-stubs/self.pyi
@@ -4,5 +4,6 @@ class X:
     def ret(self) -> Self: ...
     @classmethod
     def from_config(cls) -> Self: ...
+    def __new__(cls) -> Self: ...
 
 class Y(X): ...
diff --git a/pyanalyze/stubs/_pyanalyze_tests-stubs/tsself.pyi b/pyanalyze/stubs/_pyanalyze_tests-stubs/tsself.pyi
new file mode 100644
index 0000000..7efefe7
--- /dev/null
+++ b/pyanalyze/stubs/_pyanalyze_tests-stubs/tsself.pyi
@@ -0,0 +1,5 @@
+import _typeshed
+
+class X:
+    def __new__(cls: type[_typeshed.Self]) -> _typeshed.Self: ...
+    def method(self) -> int: ...
diff --git a/pyanalyze/test_pep673.py b/pyanalyze/test_pep673.py
index 3783a49..563daf0 100644
--- a/pyanalyze/test_pep673.py
+++ b/pyanalyze/test_pep673.py
@@ -152,10 +152,13 @@ class TestPEP673(TestNameCheckVisitorBase):
     @assert_passes()
     def test_stub(self):
         def capybara():
+            from typing_extensions import assert_type
             from _pyanalyze_tests.self import X, Y
 
             x = X()
             y = Y()
+            assert_type(x, X)
+            assert_type(y, Y)
 
             def want_x(x: X):
                 pass
@@ -168,3 +171,12 @@ class TestPEP673(TestNameCheckVisitorBase):
 
             want_x(X.from_config())
             want_y(Y.from_config())
+
+    @assert_passes()
+    def test_typeshed_self(self):
+        def capybara():
+            from typing_extensions import assert_type
+            from _pyanalyze_tests.tsself import X
+
+            x = X()
+            assert_type(x, X)
diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py
index 82cb47d..4a0fd30 100644
--- a/pyanalyze/test_typeshed.py
+++ b/pyanalyze/test_typeshed.py
@@ -649,7 +649,6 @@ class TestGetGenericBases:
                 collections.abc.Sequence: [AnyValue(AnySource.generic_argument)],
                 urllib.parse.ParseResult: [],
                 urllib.parse._ParseResultBase: [],
-                "urllib.parse._ResultMixinBase": [TypedValue(str)],
                 tuple: [AnyValue(AnySource.generic_argument)],
                 urllib.parse._ResultMixinStr: [],
                 urllib.parse._NetlocResultMixinBase: [TypedValue(str)],
@@ -659,6 +658,20 @@ class TestGetGenericBases:
             urllib.parse.ParseResult,
         )
 
+    def test_buffered_reader(self):
+        self.check(
+            {
+                io.IOBase: [],
+                io.BufferedIOBase: [],
+                collections.abc.Iterable: [TypedValue(bytes)],
+                collections.abc.Iterator: [TypedValue(bytes)],
+                io.BufferedReader: [],
+                typing.BinaryIO: [],
+                typing.IO: [TypedValue(bytes)],
+            },
+            io.BufferedReader,
+        )
+
 
 class TestAttribute:
     def test_basic(self) -> None: