Repository: Parsl/parsl

Issue:
python_app.timeout() does not cancel timer when function excepts
**Describe the bug**
When using `parsl.app.python_app.timeout` to time-limit a function, the internal timer is not canceled if the function raises.  This means that a subsequent attempt to execute a function on that same thread will fail as the injected AppTimeout exception has not been dealt with.

Recreation:
```python
from parsl.app.python_app import timeout

def some_kernel():
    raise Exception("Doh!")

time_limited_kernel = timeout(some_kernel, 0.1)
try:
    time_limited_kernel()
except Exception:
    pass  # ignoring for example

time.sleep(0.1)  # when this returns from c-land, we get parsl.app.errors.AppTimeout
# Boom
```

**Expected behavior**
I expect that when the kernel returns, for good or ill, I will not get a seemingly random `AppTimeout`

**Environment**
 - OS: ubuntu
 - Python 3.10
 - Parsl HEAD (b26c04ff - Update python_apps/test_type5 (#2766))

**Distributed Environment**
- Where are you running the Parsl script from? Laptop
- Where do you need the workers to run ? Same as Parsl script, Compute nodes, Cloud nodes



Discussion:

Resolving patch:
diff --git a/parsl/app/python.py b/parsl/app/python.py
--- a/parsl/app/python.py
+++ b/parsl/app/python.py
@@ -1,4 +1,5 @@
 import logging
+import threading
 
 import tblib.pickling_support
 tblib.pickling_support.install()
@@ -8,15 +9,15 @@
 from parsl.app.app import AppBase
 from parsl.app.errors import wrap_error
 from parsl.dataflow.dflow import DataFlowKernelLoader
+from parsl.utils import AutoCancelTimer
 
 
 logger = logging.getLogger(__name__)
 
 
-def timeout(f, seconds):
+def timeout(f, seconds: float):
     @wraps(f)
     def wrapper(*args, **kwargs):
-        import threading
         import ctypes
         import parsl.app.errors
 
@@ -27,11 +28,8 @@ def inject_exception(thread):
             )
 
         thread = threading.current_thread().ident
-        timer = threading.Timer(seconds, inject_exception, args=[thread])
-        timer.start()
-        result = f(*args, **kwargs)
-        timer.cancel()
-        return result
+        with AutoCancelTimer(seconds, inject_exception, args=[thread]):
+            return f(*args, **kwargs)
     return wrapper
 
 
diff --git a/parsl/utils.py b/parsl/utils.py
--- a/parsl/utils.py
+++ b/parsl/utils.py
@@ -5,10 +5,14 @@
 import subprocess
 import threading
 import time
+from types import TracebackType
+
 import typeguard
 from contextlib import contextmanager
 from typing import Any, Callable, List, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
 
+from typing_extensions import Type
+
 import parsl
 from parsl.version import VERSION
 
@@ -347,3 +351,28 @@ def close(self) -> None:
         """
         self._kill_event.set()
         self._thread.join()
+
+
+class AutoCancelTimer(threading.Timer):
+    """
+    Extend threading.Timer for use as a context manager
+
+    Example:
+
+        with AutoCancelTimer(delay, your_callback):
+            some_func()
+
+    If `some_func()` returns before the delay is up, the timer will
+    be cancelled.
+    """
+    def __enter__(self) -> "AutoCancelTimer":
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType]
+    ) -> None:
+        self.cancel()


Test patch:
diff --git a/parsl/tests/test_python_apps/test_timeout.py b/parsl/tests/test_python_apps/test_timeout.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_python_apps/test_timeout.py
@@ -0,0 +1,36 @@
+import time
+
+import pytest
+
+from parsl.app.errors import AppTimeout
+from parsl.app.python import timeout
+
+
+class TimeoutTestSpecificException(Exception):
+    pass
+
+
+def kernel(fail=False):
+    if fail:
+        raise TimeoutTestSpecificException()
+
+
+@pytest.mark.parametrize("timeout_dur", (0.005, 0.01, 0.1))
+def test_timeout(timeout_dur):
+    timeout(kernel, timeout_dur)()
+
+    try:
+        time.sleep(timeout_dur + 0.01)  # this is the verification
+    except AppTimeout:
+        assert False, "Timer was not cancelled!"
+
+
+@pytest.mark.parametrize("timeout_dur", (0.005, 0.01, 0.1))
+def test_timeout_after_exception(timeout_dur):
+    with pytest.raises(TimeoutTestSpecificException):
+        timeout(kernel, timeout_dur)(True)
+
+    try:
+        time.sleep(timeout_dur + 0.01)  # this is the verification
+    except AppTimeout:
+        assert False, "Timer was not cancelled!"