Repository: pytorch/torchdynamo

Issue:
aten.add_.Tensor accuracy error
Repro:
```
 python benchmarks/microbenchmarks/operatorbench.py --dtype=float16 --accuracy-checking=True --max-samples=5 --op=aten.add_.Tensor --suite=torchbench 
```
Error:
```
Iter 2
torch.Size([2, 12, 64, 448])
torch.float16
torch.Size([2, 12, 64, 448])
torch.float32
torchinductor.compile_fx: [WARNING] skipping cudagraphs due to input mutation
Accuracy check failed: aten.add_.Tensor
tensor([[[[ 1.1158e-04, -3.2187e-05, -4.0531e-03,  ...,  9.0408e-04,
           -7.4673e-04,  7.7686e-03],
          [ 3.8137e-03,  8.9264e-04,  3.6907e-03,  ..., -4.1580e-04,
            8.7261e-04,  5.1498e-05],
          [ 5.7220e-06, -4.3869e-04, -1.8425e-03,  ..., -7.6141e-03,
           -8.0967e-04,  2.1286e-03],
          ...,
          [ 3.1567e-04,  6.1512e-04,  1.6479e-03,  ...,  1.8854e-03,
           -2.6417e-04, -6.8855e-04],
          [-5.7220e-06,  1.8387e-03,  3.0336e-03,  ..., -1.2178e-03,
            1.1463e-03, -7.6294e-05],
          [-3.5915e-03, -5.9056e-04,  4.6921e-04,  ..., -3.5286e-05,
           -3.6182e-03, -7.8201e-05]],

         [[-1.6565e-03, -3.7155e-03,  4.7684e-06,  ..., -1.5326e-03,
           -2.7108e-04, -3.8280e-03],
          [-2.1935e-05,  3.7098e-04, -1.0796e-03,  ..., -9.8038e-04,
           -6.0558e-04, -7.7896e-03],
          [-3.4199e-03, -4.4918e-04, -3.1376e-04,  ..., -4.5538e-05,
           -1.5335e-03,  4.1962e-04],
          ...,
          [-7.9346e-04, -1.2560e-03,  2.4509e-04,  ...,  1.3542e-04,
            3.1223e-03, -3.2253e-03],
          [ 3.6392e-03, -1.7643e-03,  2.8400e-03,  ..., -1.2932e-03,
            1.5440e-03, -4.4346e-04],
          [-5.1556e-03,  1.1921e-04, -6.7329e-04,  ...,  3.4885e-03,
            5.0354e-04, -1.9951e-03]],

         [[ 2.5425e-03, -7.3242e-04,  2.9564e-05,  ...,  3.9196e-04,
            2.1172e-03,  1.6308e-03],
          [-1.8215e-04, -7.1449e-03,  1.3351e-05,  ..., -1.5450e-04,
           -1.0319e-03,  7.1526e-04],
          [ 9.2578e-04, -3.0556e-03,  4.2343e-04,  ...,  6.1607e-04,
           -1.7147e-03,  1.2922e-03],
          ...,
          [ 0.0000e+00, -9.5272e-04,  3.8385e-04,  ..., -3.8528e-04,
            2.1029e-03,  5.4932e-04],
          [-1.5755e-03, -2.7313e-03, -1.5450e-04,  ..., -4.4250e-04,
            7.4425e-03, -6.5250e-03],
          [ 6.4373e-06,  3.0584e-03,  5.3406e-05,  ..., -1.9150e-03,
           -1.5488e-03,  1.0328e-03]],

         ...,

         [[ 9.3651e-04, -1.8396e-03,  1.7109e-03,  ..., -1.0204e-04,
            9.9850e-04, -3.6516e-03],
          [-1.8463e-03, -5.0163e-04,  2.3842e-04,  ..., -6.0272e-04,
           -2.0905e-03,  1.4572e-03],
          [-6.4754e-04,  6.5327e-04, -1.5497e-04,  ...,  5.9891e-04,
            3.2101e-03, -3.6240e-05],
          ...,
          [ 9.0599e-04, -7.3204e-03,  3.8471e-03,  ..., -3.0518e-04,
            1.0605e-03,  1.9455e-04],
          [ 6.9237e-04, -1.9341e-03,  1.1539e-03,  ...,  5.0621e-03,
           -3.4809e-04, -2.1362e-03],
          [-3.6278e-03, -3.5515e-03, -1.0033e-03,  ..., -7.4959e-04,
           -1.5841e-03,  1.1749e-03]],

         [[-2.2011e-03,  4.8733e-04,  4.9782e-04,  ..., -1.7080e-03,
           -3.0651e-03, -5.6324e-03],
          [-3.0632e-03,  3.1185e-04,  5.2986e-03,  ..., -3.6154e-03,
           -9.6321e-04,  2.0790e-04],
          [-7.4558e-03,  4.7531e-03, -8.5831e-04,  ..., -1.5039e-03,
            3.4351e-03, -2.4986e-03],
          ...,
          [ 7.3814e-03, -1.1711e-03,  3.8719e-04,  ..., -4.4107e-05,
           -1.4305e-04,  3.2272e-03],
          [ 9.4223e-04, -1.2312e-03,  2.7943e-03,  ...,  2.9278e-03,
            1.6460e-03, -4.3297e-04],
          [-1.7223e-03,  4.8637e-05, -7.7229e-03,  ...,  2.4796e-04,
           -7.9727e-04, -3.5610e-03]],

         [[-1.4687e-04, -2.4509e-04, -1.8692e-04,  ...,  5.8365e-04,
            1.8883e-03, -3.3302e-03],
          [ 1.7204e-03, -2.7943e-04, -1.3447e-04,  ..., -9.8419e-04,
           -1.6499e-04,  1.4935e-03],
          [-2.2411e-04,  1.4668e-03, -1.7071e-03,  ..., -1.4496e-04,
            2.8324e-04,  1.4496e-04],
          ...,
          [-6.5804e-04, -4.7951e-03,  6.6948e-03,  ...,  3.6154e-03,
            1.0052e-03, -1.4954e-03],
          [ 8.8406e-04, -7.7629e-04, -1.0872e-04,  ...,  5.3215e-04,
            1.1444e-03, -5.3787e-04],
          [ 4.5490e-04, -5.4626e-03,  5.8746e-04,  ...,  1.2579e-03,
            1.1253e-04,  8.9645e-04]]],


        [[[ 1.2283e-03,  1.9274e-03, -9.0599e-05,  ..., -1.1034e-03,
            2.5177e-04,  8.6975e-04],
          [ 3.2120e-03, -8.8501e-04, -1.0042e-03,  ..., -2.0561e-03,
           -1.0920e-03, -2.8896e-04],
          [ 1.0166e-03, -4.4575e-03,  1.7939e-03,  ..., -6.3610e-04,
           -4.6177e-03,  5.8422e-03],
          ...,
          [ 1.0967e-04, -2.0027e-05,  9.3269e-04,  ...,  4.6539e-03,
            2.2430e-03,  8.1706e-04],
          [ 2.6970e-03,  1.1444e-05, -1.5984e-03,  ...,  1.0414e-03,
            3.8500e-03,  2.1267e-04],
          [-1.6317e-03, -2.1601e-03, -5.3902e-03,  ...,  3.4256e-03,
           -8.8692e-05,  3.5934e-03]],

         [[ 9.3269e-04,  5.8079e-04, -3.6240e-05,  ..., -5.6744e-04,
            1.1787e-03, -3.2578e-03],
          [ 1.6012e-03, -1.4324e-03,  1.6785e-04,  ...,  1.1997e-03,
           -9.1457e-04, -3.9864e-03],
          [ 3.6087e-03, -1.7262e-04,  1.0204e-03,  ..., -2.1000e-03,
           -2.9945e-04,  6.7863e-03],
          ...,
          [ 9.3079e-04,  1.8911e-03,  2.1362e-03,  ...,  7.2289e-04,
            4.1676e-04, -2.1172e-04],
          [-1.6527e-03, -6.9504e-03,  3.5763e-06,  ...,  3.9062e-03,
            3.1595e-03,  1.4648e-03],
          [ 1.6708e-03,  1.1826e-04, -3.0479e-03,  ..., -3.4151e-03,
           -6.6757e-06,  3.6621e-03]],

         [[-1.6961e-03,  1.7262e-04,  2.1324e-03,  ..., -4.5109e-04,
           -3.4542e-03, -1.5526e-03],
          [-6.3896e-04,  1.4305e-05,  1.3342e-03,  ..., -2.5387e-03,
            9.3746e-04,  9.4795e-04],
          [ 3.0060e-03,  4.1962e-05,  8.0013e-04,  ..., -3.7498e-03,
           -6.7520e-04, -2.9144e-03],
          ...,
          [ 1.4877e-04, -1.7548e-04, -2.6703e-04,  ...,  5.6553e-04,
            1.5259e-04, -3.2234e-04],
          [-4.8084e-03,  9.0599e-05,  1.9646e-04,  ...,  3.7575e-04,
           -1.4277e-03, -2.5654e-03],
          [ 7.9155e-05, -1.9455e-03, -3.7670e-04,  ...,  3.3836e-03,
            2.7657e-05,  4.1122e-03]],

         ...,

         [[ 1.1806e-03,  3.1471e-05, -3.9864e-04,  ...,  4.0817e-04,
           -2.1849e-03, -7.4387e-04],
          [-2.0194e-04, -5.2757e-03,  1.0347e-03,  ..., -8.5735e-04,
            4.4060e-04,  2.8610e-05],
          [-1.9646e-03,  3.7956e-04,  3.8528e-04,  ..., -8.3447e-06,
           -4.3221e-03,  5.8937e-04],
          ...,
          [-3.2139e-04,  2.3460e-04, -6.4278e-04,  ..., -2.0981e-03,
            1.6212e-03, -1.3542e-04],
          [ 1.3809e-03, -1.5469e-03,  2.2697e-04,  ...,  6.2370e-04,
            2.5749e-05,  5.7602e-04],
          [ 3.2749e-03, -4.8065e-03,  9.2506e-05,  ..., -2.1267e-04,
           -1.5259e-05, -3.5181e-03]],

         [[-1.1282e-03, -1.8864e-03, -2.5916e-04,  ..., -7.9632e-04,
            4.5013e-04,  1.2589e-03],
          [ 6.9523e-04, -2.3146e-03,  1.1849e-04,  ..., -6.1703e-04,
            7.0333e-05,  9.4604e-04],
          [ 5.8842e-04,  1.5831e-04, -2.1386e-04,  ..., -6.3038e-03,
           -4.0779e-03, -2.3861e-03],
          ...,
          [ 3.0136e-04,  8.6069e-05,  3.3817e-03,  ...,  3.8910e-04,
           -5.1155e-03,  3.5257e-03],
          [ 1.7662e-03, -5.8079e-04, -1.6804e-03,  ...,  1.3065e-03,
           -1.5535e-03,  8.9836e-04],
          [ 2.4319e-03,  7.6790e-03, -2.4500e-03,  ...,  1.8549e-03,
            6.6757e-06,  2.4710e-03]],

         [[-1.4043e-04, -9.7299e-04, -6.4850e-05,  ...,  1.4362e-03,
            1.7662e-03,  7.2670e-04],
          [ 3.4142e-04, -8.2970e-05, -2.6894e-04,  ..., -4.9591e-05,
           -5.5313e-05,  1.6336e-03],
          [-4.2200e-05, -1.7810e-04,  2.3260e-03,  ..., -5.2166e-04,
           -2.1553e-04, -2.8439e-03],
          ...,
          [ 2.5082e-04,  3.8118e-03,  1.5736e-03,  ..., -2.9230e-03,
           -1.9016e-03, -1.4305e-04],
          [ 4.3297e-04, -2.4891e-03, -6.3038e-04,  ...,  2.0084e-03,
            4.9210e-04,  8.8310e-04],
          [ 2.7313e-03,  4.2915e-05,  6.3229e-04,  ...,  5.3406e-05,
            2.5005e-03,  2.5558e-04]]]], device='cuda:0')
```


Discussion:
Make sure you're testing with fake tensors on here, because otherwise we mutate input
I couldn't repro this error because for the first iters I'm getting inputs of the same dtype, but I've got an error on item 6 (isolated below):
```
import torch
import torchdynamo
import torchinductor
from torchinductor import config

def add_inplace(x, y):
    return x.add_(y)

device='cuda'
dtype=torch.float #torch.half
optimize_ctx = torchdynamo.optimize("inductor")
x=torch.randn([2, 12, 1024, 513], device=device, dtype=dtype).transpose(1,2)
y=torch.randn([2, 1024, 1, 513], device=device, dtype=dtype)
x_new = torch.clone(x)
print(x.stride(), x_new.stride())
with optimize_ctx:
    out=add_inplace(x, y)
x_new.add_(y)
print(x, x_new)
```
errors out with assert in sizevars
```
  File "/scratch/ngimel/work/repos/torchdynamo/torchinductor/graph.py", line 317, in codegen
    self.wrapper_code = WrapperCodeGen()
  File "/scratch/ngimel/work/repos/torchdynamo/torchinductor/codegen/wrapper.py", line 239, in __init__
    V.graph.sizevars.codegen(self.prefix, V.graph.graph_inputs)
  File "/scratch/ngimel/work/repos/torchdynamo/torchinductor/sizevars.py", line 486, in codegen
    assert not needed
```
because a stride var is added needlessly to sizevars (it should have been a product of transposed sizes)
@desertfire I can't repro, all the inputs I'm getting are of the same type (fp16). Are there additional modifications I need to repro?
Hmm, no local changes on my side. Have you pulled the latest code? Also maybe try to use a larger sample number, e.g. `--max-samples=100`?
Yes, I'm on yesterday's pytorch and torchdynamo, I tried 20 already, will try 100 (but when I add printouts for input dtypes, I'm always getting fp16, no fp32's). 
For `fp32s` you need to pass in `--dtype=float32`
But that would give fp32 inputs only, not the mix, as it seems in the description?
```
Iter 2
torch.Size([2, 12, 64, 448])
torch.float16
torch.Size([2, 12, 64, 448])
torch.float32
```
Ok was able to repro (iter 24)

Resolving patch:
diff --git a/benchmarks/microbenchmarks/operatorbench.py b/benchmarks/microbenchmarks/operatorbench.py
--- a/benchmarks/microbenchmarks/operatorbench.py
+++ b/benchmarks/microbenchmarks/operatorbench.py
@@ -27,9 +27,10 @@ def compute_speedups(
             # change to assert later
             try:
                 same(actual, expected, cos_similarity=True, equal_nan=True)
-            except AssertionError:
+            except AssertionError as e:
+                print(e)
                 print(f"Accuracy check failed: {operator}")
-                print(expected[0] - actual[0])
+                print((expected[0] - actual[0]).abs().max())
 
     timings = np.zeros((repeats, len(models)), np.float64)
     for rep in range(repeats):
diff --git a/torchinductor/graph.py b/torchinductor/graph.py
--- a/torchinductor/graph.py
+++ b/torchinductor/graph.py
@@ -289,10 +289,17 @@ def output(self, target, args, kwargs):
             assert isinstance(value, TensorBox)
             value = value.data
             assert isinstance(value, ir.StorageBox)
+            value_storage_box = value
             value = value.data
             if not isinstance(value, InputBuffer) or value.get_name() != name:
                 # one of our inputs was mutated, need to turn that into a copy
                 ir.MutationLayout.realize_into(value, self.graph_inputs_original[name])
+                # replace output with mutated input
+                try:
+                    ind = self.graph_outputs.index(value_storage_box)
+                    self.graph_outputs[ind] = self.graph_inputs_original[name]
+                except ValueError:
+                    pass
 
         self.finalize()
 
diff --git a/torchinductor/lowering.py b/torchinductor/lowering.py
--- a/torchinductor/lowering.py
+++ b/torchinductor/lowering.py
@@ -2894,6 +2894,7 @@ def register_inplace(aten_op, outplace_op):
     @register_lowering(aten_op, type_promote=False)
     def fn(*args, **kwargs):
         result = outplace_op(*args, **kwargs)
+        result = to_dtype(result, args[0].get_dtype())
         return mutate_to(args[0], result)
 
     return fn


Test patch:
diff --git a/test/test_torchinductor.py b/test/test_torchinductor.py
--- a/test/test_torchinductor.py
+++ b/test/test_torchinductor.py
@@ -2232,6 +2232,35 @@ def fn(a):
 
         self.common(fn, (torch.randn([8, 1, 1]),))
 
+    def test_inplace_add(self):
+        @torchdynamo.optimize("inductor")
+        def fn(x, y):
+            return x.add_(y)
+
+        inputs = (
+            rand_strided((4, 4), (4, 1), device=self.device),
+            rand_strided((4, 4), (4, 1), device=self.device),
+        )
+        inp_clone = inputs[0].clone()
+        out = fn(*inputs)
+        self.assertTrue(same(out, inp_clone + inputs[1]))
+        self.assertTrue(out is inputs[0])
+
+    def test_inplace_mixed_dtype_ops(self):
+        @torchdynamo.optimize("inductor")
+        def fn(x, y):
+            z = x + y.float()
+            w = z.add_(y)
+            return w.mul_(y)
+
+        inputs = (
+            rand_strided((4, 4), (4, 1), device=self.device, dtype=torch.float),
+            rand_strided((4, 4), (4, 1), device=self.device, dtype=torch.double),
+        )
+        out = fn(*inputs)
+        out_eager = (inputs[0] + inputs[1].float()).add_(inputs[1]).mul_(inputs[1])
+        self.assertTrue(same(out, out_eager))
+
     @patch.object(config.triton, "cudagraphs", True)
     def test_strided_inputs(self):
         @torchdynamo.optimize("inductor")