Repository: pytorch
Issue: [inductor][cpu] AMP/FP32 single thread performance regression in 2024-12-23 nightly release
Labels: needs reproduction,triaged,oncall: pt2,module: dynamo

### 🐛 Describe the bug

<p>AMP static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_isoneutral_mixing</td>
      <td>single</td>
      <td>1</td>
      <td>23.99195</td>
      <td>0.0001438</td>
      <td>0.00345004241</td>
      <td>46.329881</td>
      <td>1</td>
      <td>28.984441</td>
      <td>0.00012082</td>
      <td>0.00350190016162</td>
      <td>45.62457</td>
      <td>0.83</td>
      <td>1.02</td>
      <td>0.84</td>
      <td>0.98</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>lennard_jones</td>
      <td>single</td>
      <td>1</td>
      <td>3.080003</td>
      <td>8.8519e-05</td>
      <td>0.000272638785557</td>
      <td>38.040263</td>
      <td>1</td>
      <td>3.433204</td>
      <td>7.742e-05</td>
      <td>0.00026579865367999997</td>
      <td>38.885432</td>
      <td>0.9</td>
      <td>0.97</td>
      <td>0.87</td>
      <td>1.02</td>
    </tr>
  </tbody>

</table>

<p>AMP dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_isoneutral_mixing</td>
      <td>single</td>
      <td>1</td>
      <td>30.10916</td>
      <td>5.8161999999999996e-05</td>
      <td>0.0017512089639199998</td>
      <td>12.582083</td>
      <td>1</td>
      <td>38.434939</td>
      <td>4.9305e-05</td>
      <td>0.001895034667395</td>
      <td>12.577097</td>
      <td>0.78</td>
      <td>1.08</td>
      <td>0.85</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table>

<p>AMP static shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_isoneutral_mixing</td>
      <td>single</td>
      <td>1</td>
      <td>29.729494</td>
      <td>5.8566e-05</td>
      <td>0.001741137545604</td>
      <td>12.629471</td>
      <td>1</td>
      <td>34.628374</td>
      <td>5.1991000000000004e-05</td>
      <td>0.0018003637926340002</td>
      <td>12.62295</td>
      <td>0.86</td>
      <td>1.03</td>
      <td>0.89</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table>

<p>AMP dynamic shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_isoneutral_mixing</td>
      <td>single</td>
      <td>1</td>
      <td>29.646898</td>
      <td>0.000106697</td>
      <td>0.003163235075906</td>
      <td>30.752565</td>
      <td>1</td>
      <td>37.355747</td>
      <td>8.9253e-05</td>
      <td>0.003334112486991</td>
      <td>30.668563</td>
      <td>0.79</td>
      <td>1.05</td>
      <td>0.84</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table>

<p>FP32 static shape CPP wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>lennard_jones</td>
      <td>single</td>
      <td>1</td>
      <td>1.379054</td>
      <td>5.8948e-05</td>
      <td>8.1292475192e-05</td>
      <td>7.819986</td>
      <td>1</td>
      <td>1.597033</td>
      <td>5.0586e-05</td>
      <td>8.0787511338e-05</td>
      <td>7.780064</td>
      <td>0.86</td>
      <td>0.99</td>
      <td>0.86</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_equation_of_state</td>
      <td>single</td>
      <td>1</td>
      <td>19.078431</td>
      <td>6.2443e-05</td>
      <td>0.0011913144669329998</td>
      <td>11.053214</td>
      <td>1</td>
      <td>22.690209</td>
      <td>5.2470000000000004e-05</td>
      <td>0.0011905552662300001</td>
      <td>10.964946</td>
      <td>0.84</td>
      <td>1.0</td>
      <td>0.84</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>pyhpc_isoneutral_mixing</td>
      <td>single</td>
      <td>1</td>
      <td>40.936272</td>
      <td>7.7847e-05</td>
      <td>0.0031867659663840004</td>
      <td>13.349267</td>
      <td>1</td>
      <td>47.610703</td>
      <td>6.633e-05</td>
      <td>0.00315801792999</td>
      <td>13.237678</td>
      <td>0.86</td>
      <td>0.99</td>
      <td>0.85</td>
      <td>0.99</td>
    </tr>
  </tbody>

</table>

the last good commit: c04f0bb7b9537758e1e5c956ebcb20e153ef9544
```
/workspace/pytorch# bash inductor_single_run.sh single inference performance torchbench pyhpc_isoneutral_mixing amp
Testing with inductor.
single-thread testing....
loading model: 0it [00:00, ?it/s]
cpu  eval  pyhpc_isoneutral_mixing
running benchmark: 100%|█████████████████████████████████████████████████████| 50/50 [00:00<00:00, 484.21it/s]
35.332x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,pyhpc_isoneutral_mixing,1,35.332336,0.049990,11.903519,0.795666,40.422605,50.803507,746,1,0,0,0,0,1
```
the bad commit: 18261e9f39580989b5902b6b70f6a8371372c5c8
```
/workspace/pytorch# bash inductor_single_run.sh single inference performance torchbench pyhpc_isoneutral_mixing amp
Testing with inductor.
single-thread testing....
loading model: 0it [00:00, ?it/s]
cpu  eval  pyhpc_isoneutral_mixing
running benchmark: 100%|█████████████████████████████████████████████████████| 50/50 [00:00<00:00, 492.06it/s]
30.152x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,pyhpc_isoneutral_mixing,1,30.152358,0.057168,7.816028,0.808176,40.422605,50.017075,746,1,0,0,0,0,1
```



### Versions

</table><p>SW info</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>target_branch</th>
      <th>target_commit</th>
      <th>refer_branch</th>
      <th>refer_commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>main</td>
      <td>766a5e3a</td>
      <td>main</td>
      <td>766a5e3a</td>
    </tr>
    <tr>
      <td>torch</td>
      <td>main</td>
      <td>f1cbf4b1b5a299f999c11e77bfabe39c7f04efdc</td>
      <td>main</td>
      <td>dd2d360b7d5dcc66660fdfe8da083a7077dada56</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>main</td>
      <td>2.6.0a0+b6d4675</td>
      <td>main</td>
      <td>2.5.0a0+265bc5c</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>main</td>
      <td>nightly</td>
      <td>main</td>
      <td>nightly</td>
    </tr>
  </tbody>
</table>

</table>

Repro:
[inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob//main/scripts/modelbench/inductor_single_run.sh)
bash inductor_single_run.sh single inference performance torchbench pyhpc_isoneutral_mixing amp
Suspected guilty commit: https://github.com/pytorch/pytorch/commit/18261e9f39580989b5902b6b70f6a8371372c5c8
[torchbench-pyhpc_isoneutral_mixing-inference-amp-static-default-single-performance-drop_guilty_commit.log](https://github.com/user-attachments/files/18244801/torchbench-pyhpc_isoneutral_mixing-inference-amp-static-default-single-performance-drop_guilty_commit.log)
cc @chauhang @penguinwu @voznesenskym @EikanWang @jgong5 @Guobing-Chen @XiaobingSuper @zhuhaozhe @blzheng @wenzhe-nrv @jiayisunx @chenyang78 @kadeng @amjames @chuanqi129