Repository: coreylowman/dfdx (language: rust)

Issue:
Multiple re-allocations of gradients
May be related to #562 #643 

Currently due to the way tapes work (they allocate gradients when operations are added), it is possible to double allocate a gradient for a given tensor, if it is used for multiple tapes.


Discussion / hints:

Fix patch:
diff --git a/src/tensor/gradients.rs b/src/tensor/gradients.rs
index b7cec9f24..918d5b622 100644
--- a/src/tensor/gradients.rs
+++ b/src/tensor/gradients.rs
@@ -233,7 +233,6 @@ pub trait Tape<E: Unit, D: DeviceStorage>: Default + Merge<Self> + Merge<NoneTap
     fn add_backward_op<F>(&mut self, operation: F)
     where
         F: 'static + FnOnce(&mut Gradients<E, D>) -> Result<(), D::Err>;
-    fn try_alloc_grad<S: Shape>(&mut self, t: &Tensor<S, E, D>) -> Result<(), D::Err>;
 }
 
 impl<E: Unit, D: DeviceStorage> Tape<E, D> for OwnedTape<E, D> {
@@ -244,9 +243,6 @@ impl<E: Unit, D: DeviceStorage> Tape<E, D> for OwnedTape<E, D> {
     {
         self.operations.push((unique_id(), Box::new(operation)));
     }
-    fn try_alloc_grad<S: Shape>(&mut self, t: &Tensor<S, E, D>) -> Result<(), D::Err> {
-        self.gradients.try_alloc_for(t)
-    }
 }
 
 impl<E: Unit, D: DeviceStorage> Tape<E, D> for NoneTape {
@@ -256,9 +252,6 @@ impl<E: Unit, D: DeviceStorage> Tape<E, D> for NoneTape {
         F: 'static + FnOnce(&mut Gradients<E, D>) -> Result<(), D::Err>,
     {
     }
-    fn try_alloc_grad<S: Shape>(&mut self, _: &Tensor<S, E, D>) -> Result<(), D::Err> {
-        Ok(())
-    }
 }
 
 /// Combine two things
diff --git a/src/tensor_ops/choose/mod.rs b/src/tensor_ops/choose/mod.rs
index 4bb4bcdce..d19df1fdf 100644
--- a/src/tensor_ops/choose/mod.rs
+++ b/src/tensor_ops/choose/mod.rs
@@ -76,10 +76,10 @@ impl<
         let phantom_out = out.clone();
 
         let mut tape = tape.merge(rhs_tape);
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             lhs.device
                 .backward(&self, &lhs, grad_lhs, &rhs, grad_rhs, grad_out)
diff --git a/src/tensor_ops/concat/mod.rs b/src/tensor_ops/concat/mod.rs
index 8757b066f..6cc38f803 100644
--- a/src/tensor_ops/concat/mod.rs
+++ b/src/tensor_ops/concat/mod.rs
@@ -63,10 +63,10 @@ where
         let device = lhs.device.clone();
         let out = device.forward(&lhs, &rhs)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_a, grad_b, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             device.backward(&lhs, grad_a, &rhs, grad_b, &phantom_out, grad_out)
         });
diff --git a/src/tensor_ops/conv2d/mod.rs b/src/tensor_ops/conv2d/mod.rs
index 4b6b1dc4b..c603ebceb 100644
--- a/src/tensor_ops/conv2d/mod.rs
+++ b/src/tensor_ops/conv2d/mod.rs
@@ -163,10 +163,10 @@ impl<
             .alloc((Const, h.convolve_dim(), w.convolve_dim()))?;
         lhs.device.forward(op, &lhs, &rhs, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             lhs.device
                 .backward(op, &lhs, grad_lhs, &rhs, grad_rhs, &phantom_out, grad_out)
@@ -207,10 +207,10 @@ impl<
         let mut tape = ltape.merge(rtape);
         lhs.device.forward(op, &lhs, &rhs, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             lhs.device
                 .backward(op, &lhs, grad_lhs, &rhs, grad_rhs, &phantom_out, grad_out)?;
diff --git a/src/tensor_ops/convtrans2d/mod.rs b/src/tensor_ops/convtrans2d/mod.rs
index ed5c386d7..c2d3a23c0 100644
--- a/src/tensor_ops/convtrans2d/mod.rs
+++ b/src/tensor_ops/convtrans2d/mod.rs
@@ -162,10 +162,10 @@ impl<
             .try_zeros_like(&(Const, h.convolve_dim(), w.convolve_dim()))?;
         lhs.device.forward(op, &lhs, &rhs, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             lhs.device
                 .backward(op, &lhs, grad_lhs, &rhs, grad_rhs, &phantom_out, grad_out)
@@ -207,10 +207,10 @@ impl<
         let mut tape = ltape.merge(rtape);
         lhs.device.forward(op, &lhs, &rhs, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&lhs)?;
-        tape.try_alloc_grad(&rhs)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&rhs)?;
+            grads.try_alloc_for(&lhs)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
             lhs.device
                 .backward(op, &lhs, grad_lhs, &rhs, grad_rhs, &phantom_out, grad_out)?;
diff --git a/src/tensor_ops/dropout/mod.rs b/src/tensor_ops/dropout/mod.rs
index 5e3315f91..5fd0b945e 100644
--- a/src/tensor_ops/dropout/mod.rs
+++ b/src/tensor_ops/dropout/mod.rs
@@ -68,12 +68,11 @@ impl<S: Shape, E: Dtype, D: DropoutKernel<E>, T: Tape<E, D>> Tensor<S, E, D, T>
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(op, &inp)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
-            inp.device.backward(op, &inp, grad_inp, grad_out)?;
-            Ok(())
+            inp.device.backward(op, &inp, grad_inp, grad_out)
         });
         Ok(out.put_tape(tape))
     }
diff --git a/src/tensor_ops/matmul/mod.rs b/src/tensor_ops/matmul/mod.rs
index 7ed4669f9..509544afa 100644
--- a/src/tensor_ops/matmul/mod.rs
+++ b/src/tensor_ops/matmul/mod.rs
@@ -96,10 +96,10 @@ fn try_binary_op<
     let mut tape = ltape.merge(rtape);
     let out = fwd(&lhs.device, &lhs, &rhs)?;
     let phantom_out = out.clone();
-    tape.try_alloc_grad(&lhs)?;
-    tape.try_alloc_grad(&rhs)?;
-    tape.try_alloc_grad(&out)?;
     tape.add_backward_op(move |grads| {
+        grads.try_alloc_for(&lhs)?;
+        grads.try_alloc_for(&rhs)?;
+        grads.try_alloc_for(&phantom_out)?;
         let (grad_lhs, grad_rhs, grad_out) =grads.muts_and_ref(&lhs, &rhs, &phantom_out);
         bwd(&lhs.device, &lhs, grad_lhs, &rhs, grad_rhs, grad_out)
     });
diff --git a/src/tensor_ops/max_to/mod.rs b/src/tensor_ops/max_to/mod.rs
index 416296b32..619312ca4 100644
--- a/src/tensor_ops/max_to/mod.rs
+++ b/src/tensor_ops/max_to/mod.rs
@@ -69,9 +69,9 @@ impl<S: Shape, E: Dtype, D: MaxReduceKernel<E>, T: Tape<E, D>> MaxTo for Tensor<
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(dst, &inp)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device.backward(&inp, grad_inp, &phantom_out, grad_out)
         });
diff --git a/src/tensor_ops/min_to/mod.rs b/src/tensor_ops/min_to/mod.rs
index 22e56818f..b06debbe9 100644
--- a/src/tensor_ops/min_to/mod.rs
+++ b/src/tensor_ops/min_to/mod.rs
@@ -69,9 +69,9 @@ impl<S: Shape, E: Dtype, D: MinReduceKernel<E>, T: Tape<E, D>> MinTo for Tensor<
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(dst, &inp)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device.backward(&inp, grad_inp, &phantom_out, grad_out)
         });
diff --git a/src/tensor_ops/pool2d/mod.rs b/src/tensor_ops/pool2d/mod.rs
index 446c27a49..a21fcaca1 100644
--- a/src/tensor_ops/pool2d/mod.rs
+++ b/src/tensor_ops/pool2d/mod.rs
@@ -109,9 +109,9 @@ macro_rules! pool2d {
                         .try_zeros_like(&(chan, h.convolve_dim(), w.convolve_dim()))?;
                 inp.device.forward(op, &inp, &mut out)?;
                 let phantom_out = out.clone();
-                tape.try_alloc_grad(&inp)?;
-                tape.try_alloc_grad(&out)?;
                 tape.add_backward_op(move |grads| {
+                    grads.try_alloc_for(&inp)?;
+                    grads.try_alloc_for(&phantom_out)?;
                     let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
                     inp.device
                         .backward(op, &inp, grad_inp, &phantom_out, grad_out)
@@ -150,9 +150,9 @@ macro_rules! pool2d {
                 ))?;
                 inp.device.forward(op, &inp, &mut out)?;
                 let phantom_out = out.clone();
-                tape.try_alloc_grad(&inp)?;
-                tape.try_alloc_grad(&out)?;
                 tape.add_backward_op(move |grads| {
+                    grads.try_alloc_for(&inp)?;
+                    grads.try_alloc_for(&phantom_out)?;
                     let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
                     inp.device
                         .backward(op, &inp, grad_inp, &phantom_out, grad_out)
diff --git a/src/tensor_ops/reshape_to/mod.rs b/src/tensor_ops/reshape_to/mod.rs
index 63038a670..1e03498cd 100644
--- a/src/tensor_ops/reshape_to/mod.rs
+++ b/src/tensor_ops/reshape_to/mod.rs
@@ -92,9 +92,9 @@ impl<S: Shape, E: Dtype, D: ReshapeKernel<E>, T: Tape<E, D>> ReshapeTo for Tenso
                 let (inp, mut tape) = self.split_tape();
                 let out = inp.device.forward(dst, &inp)?;
                 let phantom_out = out.clone();
-                tape.try_alloc_grad(&inp)?;
-                tape.try_alloc_grad(&out)?;
                 tape.add_backward_op(move |grads| {
+                    grads.try_alloc_for(&inp)?;
+                    grads.try_alloc_for(&phantom_out)?;
                     let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
                     inp.device.backward(&inp, grad_inp, &phantom_out, grad_out)
                 });
diff --git a/src/tensor_ops/roll/mod.rs b/src/tensor_ops/roll/mod.rs
index c38256ae8..4f8ea8062 100644
--- a/src/tensor_ops/roll/mod.rs
+++ b/src/tensor_ops/roll/mod.rs
@@ -74,9 +74,9 @@ impl<S: Shape, E: Dtype, D: RollKernel<E>, T: Tape<E, D>> Roll for Tensor<S, E,
         let (t, mut tape) = self.split_tape();
         let out = t.device.forward(op, &t)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&t)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&t)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&t, &phantom_out);
             t.device.backward(op, &t, grad_inp, grad_out)
         });
diff --git a/src/tensor_ops/select_and_gather/mod.rs b/src/tensor_ops/select_and_gather/mod.rs
index a57199684..2cbaab15c 100644
--- a/src/tensor_ops/select_and_gather/mod.rs
+++ b/src/tensor_ops/select_and_gather/mod.rs
@@ -105,9 +105,9 @@ impl<Src: Shape, E: Dtype, D: RemoveDimKernel<E>, T: Tape<E, D>> SelectTo<D>
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(&inp, &idx)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device
                 .backward(&inp, grad_inp, &idx, &phantom_out, grad_out)
@@ -175,9 +175,9 @@ impl<Src: Shape, E: Dtype, D: ReplaceDimKernel<E>, T: Tape<E, D>> GatherTo<D>
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(&inp, &idx)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device
                 .backward(&inp, grad_inp, &idx, &phantom_out, grad_out)
diff --git a/src/tensor_ops/slice/mod.rs b/src/tensor_ops/slice/mod.rs
index f8205eea8..be25f0399 100644
--- a/src/tensor_ops/slice/mod.rs
+++ b/src/tensor_ops/slice/mod.rs
@@ -62,9 +62,9 @@ impl<S: Shape, E: Unit, D: SliceKernel<E>, T: Tape<E, D>> Tensor<S, E, D, T> {
         let out = inp.device.forward(&inp, &slice)?;
         let phantom_out = out.clone();
 
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device.backward(&inp, grad_inp, grad_out, &slice)
         });
diff --git a/src/tensor_ops/stack/mod.rs b/src/tensor_ops/stack/mod.rs
index e45da5d1c..e3f75c927 100644
--- a/src/tensor_ops/stack/mod.rs
+++ b/src/tensor_ops/stack/mod.rs
@@ -150,18 +150,19 @@ where
     let shape = *tensors[0].shape();
     for t in tensors.iter() {
         assert_eq!(t.shape(), &shape);
-        tape.try_alloc_grad(t)?;
     }
 
     // we map to storage refs so kernels don't have to know about tensors
     let out = device.forward(new_dim, &tensors)?;
 
     let phantom_out = out.clone();
-    tape.try_alloc_grad(&out)?;
     tape.add_backward_op(move |grads| {
+        for t in tensors.iter() {
+            grads.try_alloc_for(t)?;
+        }
+        grads.try_alloc_for(&phantom_out)?;
         let (grad_inp, grad_out) = grads.many_and_ref(&tensors, &phantom_out);
-        device.backward(grad_inp, grad_out)?;
-        Ok(())
+        device.backward(grad_inp, grad_out)
     });
     Ok(out.put_tape(tape))
 }
diff --git a/src/tensor_ops/sum_to/mod.rs b/src/tensor_ops/sum_to/mod.rs
index d410f4e44..0c1cf88a8 100644
--- a/src/tensor_ops/sum_to/mod.rs
+++ b/src/tensor_ops/sum_to/mod.rs
@@ -66,9 +66,9 @@ impl<S: Shape, E: Dtype, D: SumKernel<E>, T: Tape<E, D>> SumTo for Tensor<S, E,
         let (inp, mut tape) = self.split_tape();
         let out = inp.device.forward(dst, &inp)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device.backward(&inp, grad_inp, &phantom_out, grad_out)
         });
diff --git a/src/tensor_ops/upscale2d/mod.rs b/src/tensor_ops/upscale2d/mod.rs
index 02019043c..a883369cb 100644
--- a/src/tensor_ops/upscale2d/mod.rs
+++ b/src/tensor_ops/upscale2d/mod.rs
@@ -185,9 +185,9 @@ impl<
         let mut out = inp.device.try_zeros_like(&(chan, out_height, out_width))?;
         inp.device.forward(op, &inp, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device
                 .backward(op, &inp, grad_inp, &phantom_out, grad_out)
@@ -229,9 +229,9 @@ impl<
             .try_zeros_like(&(batch, chan, out_height, out_width))?;
         inp.device.forward(op, &inp, &mut out)?;
         let phantom_out = out.clone();
-        tape.try_alloc_grad(&inp)?;
-        tape.try_alloc_grad(&out)?;
         tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&inp)?;
+            grads.try_alloc_for(&phantom_out)?;
             let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
             inp.device
                 .backward(op, &inp, grad_inp, &phantom_out, grad_out)
diff --git a/src/tensor_ops/utilities/backward.rs b/src/tensor_ops/utilities/backward.rs
index 8390b102a..f7cece8af 100644
--- a/src/tensor_ops/utilities/backward.rs
+++ b/src/tensor_ops/utilities/backward.rs
@@ -16,7 +16,10 @@ pub trait Backward<E: Dtype, D: DeviceStorage>: HasErr {
 impl<E: Dtype, D: OneFillStorage<E>> Backward<E, D> for Tensor<Rank0, E, D, OwnedTape<E, D>> {
     fn try_backward(self) -> Result<Gradients<E, D>, Self::Err> {
         let (t, mut tape) = self.split_tape();
-        tape.add_backward_op(move |grads| t.device.try_fill_with_ones(grads.get_mut(&t)));
+        tape.add_backward_op(move |grads| {
+            grads.try_alloc_for(&t)?;
+            t.device.try_fill_with_ones(grads.get_mut(&t))
+        });
         let mut grads = tape.execute()?;
         grads.drop_non_leafs();
         Ok(grads)
diff --git a/src/tensor_ops/utilities/ops.rs b/src/tensor_ops/utilities/ops.rs
index dad098d3e..879c9e3d3 100644
--- a/src/tensor_ops/utilities/ops.rs
+++ b/src/tensor_ops/utilities/ops.rs
@@ -51,9 +51,9 @@ pub(crate) fn try_unary_op<
     let (inp, mut tape) = inp.split_tape();
     let out = inp.device.forward(op.clone(), &inp)?;
     let phantom_out = out.clone();
-    tape.try_alloc_grad(&inp)?;
-    tape.try_alloc_grad(&out)?;
     tape.add_backward_op(move |grads| {
+        grads.try_alloc_for(&inp)?;
+        grads.try_alloc_for(&phantom_out)?;
         let (grad_inp, grad_out) = grads.mut_and_ref(&inp, &phantom_out);
         inp.device
             .backward(op, &inp, grad_inp, &phantom_out, grad_out)
@@ -79,10 +79,10 @@ pub(crate) fn try_binary_op<
     let mut tape = ltape.merge(rtape);
     let out = lhs.device.forward(op, &lhs, &rhs)?;
     let phantom_out = out.clone();
-    tape.try_alloc_grad(&lhs)?;
-    tape.try_alloc_grad(&rhs)?;
-    tape.try_alloc_grad(&out)?;
     tape.add_backward_op(move |grads| {
+        grads.try_alloc_for(&lhs)?;
+        grads.try_alloc_for(&rhs)?;
+        grads.try_alloc_for(&phantom_out)?;
         let (grad_lhs, grad_rhs, grad_out) = grads.muts_and_ref(&lhs, &rhs, &phantom_out);
         lhs.device
             .backward(op, &lhs, grad_lhs, &rhs, grad_rhs, grad_out)


Test patch: