Repository: twisted/twisted

Issue:
DeferredLock and DeferredSemaphore run method is incorrectly type annotated
**Describe the incorrect behavior you saw**
given

```python
from twisted.internet import defer

def bar() -> defer.Deferred[int]:
    return defer.succeed(1)


l = defer.DeferredLock()
d = l.run(bar)
reveal_type(d)  # Revealed type is "twisted.internet.defer.Deferred[Any]"
```

the revealed type should be  "twisted.internet.defer.Deferred[int]"



Discussion:

Resolving patch:
diff --git a/src/twisted/internet/defer.py b/src/twisted/internet/defer.py
--- a/src/twisted/internet/defer.py
+++ b/src/twisted/internet/defer.py
@@ -43,7 +43,7 @@
 
 import attr
 from incremental import Version
-from typing_extensions import Concatenate, Literal, ParamSpec
+from typing_extensions import Concatenate, Literal, ParamSpec, Self
 
 from twisted.internet.interfaces import IDelayedCall, IReactorTime
 from twisted.logger import Logger
@@ -161,7 +161,7 @@ def maybeDeferred(
 
 @overload
 def maybeDeferred(
-    f: Callable[_P, Coroutine[Deferred[_T], object, _T]],
+    f: Callable[_P, Coroutine[Deferred[Any], Any, _T]],
     *args: _P.args,
     **kwargs: _P.kwargs,
 ) -> "Deferred[_T]":
@@ -176,7 +176,7 @@ def maybeDeferred(
 
 
 def maybeDeferred(
-    f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[_T], object, _T], _T]],
+    f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]],
     *args: _P.args,
     **kwargs: _P.kwargs,
 ) -> "Deferred[_T]":
@@ -1277,8 +1277,8 @@ def uncancel(
     def fromCoroutine(
         cls,
         coro: Union[
-            Coroutine["Deferred[_T]", Any, _T],
-            Generator["Deferred[_T]", Any, _T],
+            Coroutine[Deferred[Any], Any, _T],
+            Generator[Deferred[Any], Any, _T],
         ],
     ) -> "Deferred[_T]":
         """
@@ -1326,8 +1326,8 @@ def main(reactor):
 
 def ensureDeferred(
     coro: Union[
-        Coroutine[Deferred[_T], Any, _T],
-        Generator[Deferred[_T], Any, _T],
+        Coroutine[Deferred[Any], Any, _T],
+        Generator[Deferred[Any], Any, _T],
         Deferred[_T],
     ]
 ) -> Deferred[_T]:
@@ -1923,8 +1923,8 @@ def _gotResultInlineCallbacks(
     r: object,
     waiting: List[Any],
     gen: Union[
-        Generator[Deferred[object], object, _T],
-        Coroutine[Deferred[object], object, _T],
+        Generator[Deferred[Any], Any, _T],
+        Coroutine[Deferred[Any], Any, _T],
     ],
     status: _CancellationStatus[_T],
     context: _Context,
@@ -1950,8 +1950,8 @@ def _gotResultInlineCallbacks(
 def _inlineCallbacks(
     result: object,
     gen: Union[
-        Generator[Deferred[object], object, _T],
-        Coroutine[Deferred[object], object, _T],
+        Generator[Deferred[Any], Any, _T],
+        Coroutine[Deferred[Any], Any, _T],
     ],
     status: _CancellationStatus[_T],
     context: _Context,
@@ -2149,8 +2149,8 @@ def _handleCancelInlineCallbacks(
 
 def _cancellableInlineCallbacks(
     gen: Union[
-        Generator["Deferred[object]", object, _T],
-        Coroutine["Deferred[object]", object, _T],
+        Generator[Deferred[Any], object, _T],
+        Coroutine[Deferred[Any], object, _T],
     ]
 ) -> Deferred[_T]:
     """
@@ -2178,7 +2178,7 @@ class _InternalInlineCallbacksCancelledError(Exception):
 
 
 def inlineCallbacks(
-    f: Callable[_P, Generator[Deferred[object], object, _T]]
+    f: Callable[_P, Generator[Deferred[Any], Any, _T]]
 ) -> Callable[_P, Deferred[_T]]:
     """
     L{inlineCallbacks} helps you write L{Deferred}-using code that looks like a
@@ -2261,26 +2261,47 @@ def unwindGenerator(*args: _P.args, **kwargs: _P.kwargs) -> Deferred[_T]:
 ## DeferredLock/DeferredQueue
 
 
-_ConcurrencyPrimitiveT = TypeVar(
-    "_ConcurrencyPrimitiveT", bound="_ConcurrencyPrimitive[Any]"
-)
-
-
-class _ConcurrencyPrimitive(ABC, Generic[_SelfResultT]):
-    def __init__(self: _ConcurrencyPrimitiveT) -> None:
-        self.waiting: List[Deferred[_ConcurrencyPrimitiveT]] = []
+class _ConcurrencyPrimitive(ABC):
+    def __init__(self: Self) -> None:
+        self.waiting: List[Deferred[Self]] = []
 
     def _releaseAndReturn(self, r: _T) -> _T:
         self.release()
         return r
 
+    @overload
     def run(
-        self: _ConcurrencyPrimitiveT,
+        self: Self,
         /,
-        f: Callable[..., _SelfResultT],
-        *args: object,
-        **kwargs: object,
-    ) -> Deferred[_SelfResultT]:
+        f: Callable[_P, Deferred[_T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
+        ...
+
+    @overload
+    def run(
+        self: Self,
+        /,
+        f: Callable[_P, Coroutine[Deferred[Any], Any, _T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
+        ...
+
+    @overload
+    def run(
+        self: Self, /, f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
+    ) -> Deferred[_T]:
+        ...
+
+    def run(
+        self: Self,
+        /,
+        f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]],
+        *args: _P.args,
+        **kwargs: _P.kwargs,
+    ) -> Deferred[_T]:
         """
         Acquire, run, release.
 
@@ -2295,12 +2316,16 @@ def run(
         @return: L{Deferred} of function result.
         """
 
-        def execute(ignoredResult: object) -> Deferred[_SelfResultT]:
-            return maybeDeferred(f, *args, **kwargs).addBoth(self._releaseAndReturn)
+        def execute(ignoredResult: object) -> Deferred[_T]:
+            # maybeDeferred arg type requires one of the possible union members
+            # and won't accept all possible union members
+            return maybeDeferred(f, *args, **kwargs).addBoth(
+                self._releaseAndReturn
+            )  # type: ignore[return-value]
 
         return self.acquire().addCallback(execute)
 
-    def __aenter__(self: _ConcurrencyPrimitiveT) -> Deferred[_ConcurrencyPrimitiveT]:
+    def __aenter__(self: Self) -> Deferred[Self]:
         """
         We can be used as an asynchronous context manager.
         """
@@ -2318,7 +2343,7 @@ def __aexit__(
         return succeed(False)
 
     @abstractmethod
-    def acquire(self: _ConcurrencyPrimitiveT) -> Deferred[_ConcurrencyPrimitiveT]:
+    def acquire(self: Self) -> Deferred[Self]:
         pass
 
     @abstractmethod
@@ -2326,10 +2351,7 @@ def release(self) -> None:
         pass
 
 
-_DeferredLockT = TypeVar("_DeferredLockT", bound="DeferredLock")
-
-
-class DeferredLock(_ConcurrencyPrimitive[Any]):
+class DeferredLock(_ConcurrencyPrimitive):
     """
     A lock for event driven systems.
 
@@ -2340,7 +2362,7 @@ class DeferredLock(_ConcurrencyPrimitive[Any]):
 
     locked = False
 
-    def _cancelAcquire(self: _DeferredLockT, d: Deferred[_DeferredLockT]) -> None:
+    def _cancelAcquire(self: Self, d: Deferred[Self]) -> None:
         """
         Remove a deferred d from our waiting list, as the deferred has been
         canceled.
@@ -2354,7 +2376,7 @@ def _cancelAcquire(self: _DeferredLockT, d: Deferred[_DeferredLockT]) -> None:
         """
         self.waiting.remove(d)
 
-    def acquire(self: _DeferredLockT) -> Deferred[_DeferredLockT]:
+    def acquire(self: Self) -> Deferred[Self]:
         """
         Attempt to acquire the lock.  Returns a L{Deferred} that fires on
         lock acquisition with the L{DeferredLock} as the value.  If the lock
@@ -2363,7 +2385,7 @@ def acquire(self: _DeferredLockT) -> Deferred[_DeferredLockT]:
         @return: a L{Deferred} which fires on lock acquisition.
         @rtype: a L{Deferred}
         """
-        d: Deferred[_DeferredLockT] = Deferred(canceller=self._cancelAcquire)
+        d: Deferred[Self] = Deferred(canceller=self._cancelAcquire)
         if self.locked:
             self.waiting.append(d)
         else:
@@ -2371,7 +2393,7 @@ def acquire(self: _DeferredLockT) -> Deferred[_DeferredLockT]:
             d.callback(self)
         return d
 
-    def release(self: _DeferredLockT) -> None:
+    def release(self: Self) -> None:
         """
         Release the lock.  If there is a waiting list, then the first
         L{Deferred} in that waiting list will be called back.
@@ -2388,10 +2410,7 @@ def release(self: _DeferredLockT) -> None:
             d.callback(self)
 
 
-_DeferredSemaphoreT = TypeVar("_DeferredSemaphoreT", bound="DeferredSemaphore")
-
-
-class DeferredSemaphore(_ConcurrencyPrimitive[Any]):
+class DeferredSemaphore(_ConcurrencyPrimitive):
     """
     A semaphore for event driven systems.
 
@@ -2415,9 +2434,7 @@ def __init__(self, tokens: int) -> None:
         self.tokens = tokens
         self.limit = tokens
 
-    def _cancelAcquire(
-        self: _DeferredSemaphoreT, d: Deferred[_DeferredSemaphoreT]
-    ) -> None:
+    def _cancelAcquire(self: Self, d: Deferred[Self]) -> None:
         """
         Remove a deferred d from our waiting list, as the deferred has been
         canceled.
@@ -2431,7 +2448,7 @@ def _cancelAcquire(
         """
         self.waiting.remove(d)
 
-    def acquire(self: _DeferredSemaphoreT) -> Deferred[_DeferredSemaphoreT]:
+    def acquire(self: Self) -> Deferred[Self]:
         """
         Attempt to acquire the token.
 
@@ -2440,7 +2457,7 @@ def acquire(self: _DeferredSemaphoreT) -> Deferred[_DeferredSemaphoreT]:
         assert (
             self.tokens >= 0
         ), "Internal inconsistency??  tokens should never be negative"
-        d: Deferred[_DeferredSemaphoreT] = Deferred(canceller=self._cancelAcquire)
+        d: Deferred[Self] = Deferred(canceller=self._cancelAcquire)
         if not self.tokens:
             self.waiting.append(d)
         else:
@@ -2448,7 +2465,7 @@ def acquire(self: _DeferredSemaphoreT) -> Deferred[_DeferredSemaphoreT]:
             d.callback(self)
         return d
 
-    def release(self: _DeferredSemaphoreT) -> None:
+    def release(self: Self) -> None:
         """
         Release the token.
 


Test patch:
diff --git a/src/twisted/test/test_defer.py b/src/twisted/test/test_defer.py
--- a/src/twisted/test/test_defer.py
+++ b/src/twisted/test/test_defer.py
@@ -43,6 +43,7 @@
 from hamcrest import assert_that, empty, equal_to
 from hypothesis import given
 from hypothesis.strategies import integers
+from typing_extensions import assert_type
 
 from twisted.internet import defer, reactor
 from twisted.internet.defer import (
@@ -67,7 +68,7 @@
 
 
 def ensuringDeferred(
-    f: Callable[..., Coroutine[Deferred[_T], Any, _T]]
+    f: Callable[..., Coroutine[Deferred[Any], Any, _T]]
 ) -> Callable[..., Deferred[_T]]:
     @functools.wraps(f)
     def wrapper(*args: object, **kwargs: object) -> Deferred[_T]:
@@ -847,6 +848,7 @@ def plusFive(x: int) -> int:
         results: List[int] = []
         errors: List[Failure] = []
         d = defer.maybeDeferred(plusFive, 10)
+        assert_type(d, Deferred[int])
         d.addCallbacks(results.append, errors.append)
         self.assertEqual(errors, [])
         self.assertEqual(results, [15])
@@ -916,6 +918,20 @@ def test_maybeDeferredCoroutineSuccess(self) -> None:
         by the function.
         """
 
+        async def f() -> int:
+            return 7
+
+        coroutineDeferred = defer.maybeDeferred(f)
+        assert_type(coroutineDeferred, Deferred[int])
+        self.assertEqual(self.successResultOf(coroutineDeferred), 7)
+
+    def test_maybeDeferredCoroutineSuccessIndirect(self) -> None:
+        """
+        When called with a coroutine function L{defer.maybeDeferred} returns a
+        L{defer.Deferred} which has the same result as the coroutine returned
+        by the function.
+        """
+
         async def f() -> int:
             return 7
 
@@ -930,6 +946,7 @@ def typedCallback(result: int) -> int:
             return result + 1
 
         coroutineDeferred = defer.maybeDeferred(g)
+        assert_type(coroutineDeferred, Deferred[Any])
         modifiedDeferred = coroutineDeferred.addCallback(typedCallback)
         self.assertEqual(self.successResultOf(modifiedDeferred), 8)
 
@@ -1685,6 +1702,77 @@ def raiseError(ignored: object) -> None:
         self.assertNotEqual([], localz)
         self.assertNotEqual([], globalz)
 
+    def test_inlineCallbacksTracebacks(self) -> None:
+        """
+        L{defer.inlineCallbacks} that re-raise tracebacks into their deferred
+        should not lose their tracebacks.
+        """
+        f = getDivisionFailure()
+        d: Deferred[None] = Deferred()
+        try:
+            f.raiseException()
+        except BaseException:
+            d.errback()
+
+        def ic(d: object) -> Generator[Any, Any, None]:  # pragma: no cover
+            """
+            This is never called.
+            It is only used as the decorated function.
+            The resulting function is never called in this test.
+            This is used to make sure that if we wrap
+            an already failed deferred, inlineCallbacks
+            will not add any extra traceback frames.
+            """
+            yield d
+
+        defer.inlineCallbacks(ic)
+        newFailure = self.failureResultOf(d)
+        tb = traceback.extract_tb(newFailure.getTracebackObject())
+
+        self.assertEqual(len(tb), 3)
+        self.assertIn("test_defer", tb[2][0])
+        self.assertEqual("getDivisionFailure", tb[2][2])
+        self.assertEqual("1 / 0", tb[2][3])
+
+        self.assertIn("test_defer", tb[0][0])
+        self.assertEqual("test_inlineCallbacksTracebacks", tb[0][2])
+        self.assertEqual("f.raiseException()", tb[0][3])
+
+    def test_fromCoroutine(self) -> None:
+        """
+        It can discover the return type of a first level coroutine.
+        """
+
+        async def returnsInt() -> int:
+            return 1
+
+        d = Deferred.fromCoroutine(returnsInt())
+        assert_type(d, Deferred[int])
+        self.assertEqual(1, self.successResultOf(d))
+
+    def test_fromCoroutineRequiresCoroutine(self) -> None:
+        """
+        L{Deferred.fromCoroutine} requires a coroutine object or a generator,
+        and will reject things that are not that.
+        """
+        thingsThatAreNotCoroutines = [
+            # Lambda
+            lambda x: x,
+            # Int
+            1,
+            # Boolean
+            True,
+            # Function
+            self.test_fromCoroutineRequiresCoroutine,
+            # None
+            None,
+            # Module
+            defer,
+        ]
+
+        for thing in thingsThatAreNotCoroutines:
+            self.assertRaises(defer.NotACoroutineError, Deferred.fromCoroutine, thing)
+
     @pyunit.skipIf(_PYPY, "GC works differently on PyPy.")
     def test_canceller_circular_reference_callback(self) -> None:
         """
@@ -2651,6 +2739,15 @@ def helper(resultValue: object, returnValue: object = None) -> object:
         lock.release()
         self.assertFalse(lock.locked)
 
+        def returnsInt() -> Deferred[int]:
+            return defer.succeed(2)
+
+        async def returnsCoroInt() -> int:
+            return 1
+
+        assert_type(lock.run(returnsInt), Deferred[int])
+        assert_type(lock.run(returnsCoroInt), Deferred[int])
+
     def test_cancelLockAfterAcquired(self) -> None:
         """
         When canceling a L{Deferred} from a L{DeferredLock} that already
@@ -3478,6 +3575,7 @@ async def run() -> str:
 
         # Now it's a Deferred.
         d = ensureDeferred(r)
+        assert_type(d, Deferred[str])
         self.assertIsInstance(d, Deferred)
 
         # The Deferred has the result we want.
@@ -3499,7 +3597,8 @@ def run() -> Generator[Deferred[str], None, str]:
         self.assertIsInstance(r, types.GeneratorType)
 
         # Now it's a Deferred.
-        d: Deferred[str] = ensureDeferred(r)
+        d = ensureDeferred(r)
+        assert_type(d, Deferred[str])
         self.assertIsInstance(d, Deferred)
 
         # The Deferred has the result we want.
@@ -3685,6 +3784,8 @@ def yieldingDeferred() -> Generator[Deferred[Any], Any, None]:
             yield d
             var.set(3)
 
+        assert_type(yieldingDeferred, Callable[[], Deferred[None]])
+
         # context is 1 when the function is defined
         @defer.inlineCallbacks
         def testFunction() -> Generator[Deferred[Any], Any, None]:
@@ -3723,6 +3824,7 @@ def testFunction() -> Generator[Deferred[Any], Any, None]:
 
             defer.returnValue(True)
 
+        assert_type(testFunction, Callable[[], Deferred[None]])
         # The inlineCallbacks context is 2 when it's called
         var.set(2)
         d = testFunction()