Repository: PennyLaneAI/pennylane (python)

Issue:
[BUG] Inline transforms vs. Decorator transforms
### Expected behavior

This  script:

```
def circuit(circuit_param):
    qml.RY(circuit_param, wires=0)
    qml.Hadamard(wires=0)
    qml.T(wires=0)

def noise(noise_param, wires):
    qml.CRX(noise_param, wires=[0,1])
    qml.CNOT(wires=[1,0])
    
dev = qml.device("default.mixed", wires=2)

noise_param = 0.3
    
@qml.qnode(dev)
@qml.transforms.insert(noise, noise_param, position="all")
def noisy_circuit(circuit_param):
        
    circuit(circuit_param)
    
    return qml.expval(qml.PauliX(0)), qml.expval(qml.PauliY(0)), qml.expval(qml.PauliZ(0))

noisy_circuit(0.4)
```
And this one
```
def circuit(circuit_param):
    qml.RY(circuit_param, wires=0)
    qml.Hadamard(wires=0)
    qml.T(wires=0)

def noise(noise_param, wires):
    qml.CRX(noise_param, wires=[0,1])
    qml.CNOT(wires=[1,0])
    
dev = qml.device("default.mixed", wires=2)

noise_param = 0.3
    
@qml.qnode(dev)
def noisy_circuit(circuit_param):
        
    qml.transforms.insert(noise, noise_param, position="all")(circuit)(circuit_param)
    
    return qml.expval(qml.PauliX(0)), qml.expval(qml.PauliY(0)), qml.expval(qml.PauliZ(0))

noisy_circuit(0.4)
```
should both return `array([0.64497588, 0.61505021, 0.40503904])`

### Actual behavior

The second script works, but the first one returns errors.

### Additional information

I know this is an issue with how tapes are handled in either case. Please reconsider how this interacts with users.

### Source code

```shell
def circuit(circuit_param):
    qml.RY(circuit_param, wires=0)
    qml.Hadamard(wires=0)
    qml.T(wires=0)

def noise(noise_param, wires):
    qml.CRX(noise_param, wires=[0,1])
    qml.CNOT(wires=[1,0])
    
dev = qml.device("default.mixed", wires=2)

noise_param = 0.3
    
@qml.qnode(dev)
@qml.transforms.insert(noise, noise_param, position="all")
def noisy_circuit(circuit_param):
        
    circuit(circuit_param)
    
    return qml.expval(qml.PauliX(0)), qml.expval(qml.PauliY(0)), qml.expval(qml.PauliZ(0))

noisy_circuit(0.4)
```


### Tracebacks

```shell
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
~\anaconda3\lib\site-packages\pennylane\tape\tape.py in expand_tape(tape, depth, stop_at, expand_measurements)
    156             try:
--> 157                 rotations, diag_obs = qml.grouping.diagonalize_qwc_pauli_words(
    158                     tape._obs_sharing_wires

~\anaconda3\lib\site-packages\pennylane\grouping\transformations.py in diagonalize_qwc_pauli_words(qwc_grouping)
    139     if not are_pauli_words_qwc(qwc_grouping):
--> 140         raise ValueError("The list of Pauli words are not qubit-wise commuting.")
    141 

ValueError: The list of Pauli words are not qubit-wise commuting.

The above exception was the direct cause of the following exception:

QuantumFunctionError                      Traceback (most recent call last)
~\anaconda3\lib\site-packages\pennylane\transforms\qfunc_transforms.py in __call__(self, tape, *args, **kwargs)
    169         with tape_class() as new_tape:
--> 170             self.transform_fn(tape, *args, **kwargs)
    171 

~\anaconda3\lib\site-packages\pennylane\transforms\insert_ops.py in insert(circuit, op, op_args, position, before)
    212     # TODO: change this to be cleaner and more robust
--> 213     circuit = circuit.expand(
    214         stop_at=lambda op: not hasattr(qml.templates, op.name) and not isinstance(op, Adjoint)

~\anaconda3\lib\site-packages\pennylane\tape\tape.py in expand(self, depth, stop_at, expand_measurements)
    686         """
--> 687         new_tape = expand_tape(
    688             self, depth=depth, stop_at=stop_at, expand_measurements=expand_measurements

~\anaconda3\lib\site-packages\pennylane\tape\tape.py in expand_tape(tape, depth, stop_at, expand_measurements)
    172 
--> 173                 raise qml.QuantumFunctionError(
    174                     "Only observables that are qubit-wise commuting "

QuantumFunctionError: Only observables that are qubit-wise commuting Pauli words can be returned on the same wire, some of the following measurements do not commute:
[expval(PauliX(wires=[0])), expval(PauliY(wires=[0])), expval(PauliZ(wires=[0]))]

During handling of the above exception, another exception occurred:

IndexError                                Traceback (most recent call last)
~\anaconda3\lib\site-packages\pennylane\qnode.py in construct(self, args, kwargs)
    525         with self.tape:
--> 526             self._qfunc_output = self.func(*args, **kwargs)
    527         self._tape._qfunc_output = self._qfunc_output

~\anaconda3\lib\site-packages\pennylane\transforms\qfunc_transforms.py in internal_wrapper(*args, **kwargs)
    201         tape = make_tape(fn)(*args, **kwargs)
--> 202         tape = tape_transform(tape, *transform_args, **transform_kwargs)
    203 

~\anaconda3\lib\site-packages\pennylane\transforms\qfunc_transforms.py in __call__(self, tape, *args, **kwargs)
    169         with tape_class() as new_tape:
--> 170             self.transform_fn(tape, *args, **kwargs)
    171 

~\anaconda3\lib\site-packages\pennylane\tape\tape.py in __exit__(self, exception_type, exception_value, traceback)
    389         try:
--> 390             super().__exit__(exception_type, exception_value, traceback)
    391             # After other optimizations in #2963, #2986 and follow-up work, we should check whether

~\anaconda3\lib\site-packages\pennylane\queuing.py in __exit__(self, exception_type, exception_value, traceback)
    127         """Remove this instance from the global list of active contexts."""
--> 128         QueuingContext._active_contexts.pop()
    129 

IndexError: pop from an empty deque

During handling of the above exception, another exception occurred:

IndexError                                Traceback (most recent call last)
<ipython-input-33-37d3113aba14> in <module>
     20     return qml.expval(qml.PauliX(0)), qml.expval(qml.PauliY(0)), qml.expval(qml.PauliZ(0))
     21 
---> 22 noisy_circuit(0.4)

~\anaconda3\lib\site-packages\pennylane\qnode.py in __call__(self, *args, **kwargs)
    609 
    610         # construct the tape
--> 611         self.construct(args, kwargs)
    612 
    613         cache = self.execute_kwargs.get("cache", False)

~\anaconda3\lib\site-packages\pennylane\qnode.py in construct(self, args, kwargs)
    524 
    525         with self.tape:
--> 526             self._qfunc_output = self.func(*args, **kwargs)
    527         self._tape._qfunc_output = self._qfunc_output
    528 

~\anaconda3\lib\site-packages\pennylane\tape\tape.py in __exit__(self, exception_type, exception_value, traceback)
    388     def __exit__(self, exception_type, exception_value, traceback):
    389         try:
--> 390             super().__exit__(exception_type, exception_value, traceback)
    391             # After other optimizations in #2963, #2986 and follow-up work, we should check whether
    392             # calling `_process_queue` only if there is no `exception_type` saves time. This would

~\anaconda3\lib\site-packages\pennylane\queuing.py in __exit__(self, exception_type, exception_value, traceback)
    126     def __exit__(self, exception_type, exception_value, traceback):
    127         """Remove this instance from the global list of active contexts."""
--> 128         QueuingContext._active_contexts.pop()
    129 
    130     @abc.abstractmethod

IndexError: pop from an empty deque
```


### System information

```shell
Name: PennyLane
Version: 0.26.0
Summary: PennyLane is a Python quantum machine learning library by Xanadu Inc.
Home-page: https://github.com/XanaduAI/pennylane
Author: None
Author-email: None
License: Apache License 2.0
Location: c:\users\alvaro\anaconda3\lib\site-packages
Requires: numpy, appdirs, cachetools, retworkx, toml, autograd, semantic-version, pennylane-lightning, autoray, scipy, networkx
Required-by: PennyLane-SF, pennylane-qulacs, PennyLane-Qchem, PennyLane-Lightning

Platform info:           Windows-10-10.0.19041-SP0
Python version:          3.8.8
Numpy version:           1.22.1
Scipy version:           1.6.2
Installed devices:
- default.gaussian (PennyLane-0.26.0)
- default.mixed (PennyLane-0.26.0)
- default.qubit (PennyLane-0.26.0)
- default.qubit.autograd (PennyLane-0.26.0)
- default.qubit.jax (PennyLane-0.26.0)
- default.qubit.tf (PennyLane-0.26.0)
- default.qubit.torch (PennyLane-0.26.0)
- default.qutrit (PennyLane-0.26.0)
- strawberryfields.fock (PennyLane-SF-0.19.0)
- strawberryfields.gaussian (PennyLane-SF-0.19.0)
- strawberryfields.gbs (PennyLane-SF-0.19.0)
- strawberryfields.remote (PennyLane-SF-0.19.0)
- strawberryfields.tf (PennyLane-SF-0.19.0)
- qulacs.simulator (pennylane-qulacs-0.16.0)
- lightning.qubit (PennyLane-Lightning-0.26.0)
```


### Existing GitHub issues

- [X] I have searched existing GitHub issues to make sure the issue does not already exist.

Pull request description:
clean up after catching errors within a stop_recording context
**Context:**
A bug was reported which demonstrated a scenario where `QueuingManager.stop_recording` called from within a tape raises an unexpected error (when some expected error is raised within the `stop_recording` block). If you `yield` in python, and the code you yielded to raises an exception, the rest of your code doesn't run. We had some cleanup code after `yield` in `stop_recording` that *must* be run before exiting a tape.

**Description of the Change:**
Restore previously active contexts after a `stop_recording` context even when the yielded block raises some error.

**Benefits:**
A bug will be squashed.

**Possible Drawbacks:**
None.

**Related GitHub Issues:**
Fixes #3103

Patch:
diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md
index e2beb942c..fcc47fcc3 100644
--- a/doc/releases/changelog-dev.md
+++ b/doc/releases/changelog-dev.md
@@ -206,6 +206,9 @@ keyword argument when using `GellMann`, which determines which of the 8 Gell-Man
 * Fixed a bug where `qml.math.fidelity(non_trainable_state, trainable_state)` failed unexpectedly.
   [(#3160)](https://github.com/PennyLaneAI/pennylane/pull/3160)
 
+* Fixed a bug where `qml.QueuingManager.stop_recording` did not clean up if yielded code raises an exception.
+  [(#3182)](https://github.com/PennyLaneAI/pennylane/pull/3182)
+
 <h3>Contributors</h3>
 
 This release contains contributions from (in alphabetical order):
diff --git a/pennylane/queuing.py b/pennylane/queuing.py
index f044d34f8..c6d480c51 100644
--- a/pennylane/queuing.py
+++ b/pennylane/queuing.py
@@ -145,7 +145,11 @@ class QueuingManager:
         """
         previously_active_contexts = cls._active_contexts
         cls._active_contexts = []
-        yield
+        try:
+            yield
+        except Exception as e:
+            cls._active_contexts = previously_active_contexts
+            raise e
         cls._active_contexts = previously_active_contexts
 
     @classmethod


Test patch:
diff --git a/tests/tape/test_tape.py b/tests/tape/test_tape.py
index 7fa109c4d..1ba8d1fce 100644
--- a/tests/tape/test_tape.py
+++ b/tests/tape/test_tape.py
@@ -1658,6 +1658,15 @@ class TestStopRecording:
         assert len(temp_tape.operations) == 1
         assert temp_tape.operations[0] == op1
 
+    def test_stop_recording_within_tape_cleans_up(self):
+        """Test if some error is raised within a stop_recording context, the previously
+        active contexts are still returned to avoid popping from an empty deque"""
+
+        with pytest.raises(ValueError):
+            with qml.queuing.AnnotatedQueue() as q:
+                with qml.QueuingManager.stop_recording():
+                    raise ValueError
+
 
 def test_get_active_tape():
     """Test that the get_active_tape() function returns the currently