Repository: pytorch
Issue: [inductor][cpu]float32 and AMP performance regression in 2024-09-30 nightly release
Labels: oncall: pt2,oncall: cpu inductor

### 🐛 Describe the bug

<p>AMP static shape CPP wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>multiple</td>
      <td>16</td>
      <td>1.899827</td>
      <td>0.090072809</td>
      <td>0.171122754504043</td>
      <td>45.883297</td>
      <td>16</td>
      <td>2.4942</td>
      <td>0.055549301</td>
      <td>0.13855106655420002</td>
      <td>45.724342</td>
      <td>0.76</td>
      <td>0.81</td>
      <td>0.62</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>single</td>
      <td>1</td>
      <td>1.837071</td>
      <td>0.19298464899999998</td>
      <td>0.35452650212307896</td>
      <td>39.713272</td>
      <td>1</td>
      <td>2.537298</td>
      <td>0.103872977</td>
      <td>0.263556696796146</td>
      <td>39.3119</td>
      <td>0.72</td>
      <td>0.74</td>
      <td>0.54</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mixnet_l</td>
      <td>single</td>
      <td>1</td>
      <td>2.105185</td>
      <td>0.019725155999999997</td>
      <td>0.04152510253386</td>
      <td>65.146177</td>
      <td>1</td>
      <td>2.522811</td>
      <td>0.016682802</td>
      <td>0.042087556396422</td>
      <td>65.383152</td>
      <td>0.83</td>
      <td>1.01</td>
      <td>0.85</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tf_mixnet_l</td>
      <td>single</td>
      <td>1</td>
      <td>2.095595</td>
      <td>0.020231463999999998</td>
      <td>0.042396954801079995</td>
      <td>67.225121</td>
      <td>1</td>
      <td>2.469998</td>
      <td>0.017577774</td>
      <td>0.043417066624452</td>
      <td>67.388258</td>
      <td>0.85</td>
      <td>1.02</td>
      <td>0.87</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table>

<p>AMP dynamic shape CPP wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>multiple</td>
      <td>16</td>
      <td>1.919121</td>
      <td>0.090016383</td>
      <td>0.172752330959343</td>
      <td>57.494503</td>
      <td>16</td>
      <td>2.466627</td>
      <td>0.05597329</td>
      <td>0.13806522839283</td>
      <td>56.685157</td>
      <td>0.78</td>
      <td>0.8</td>
      <td>0.62</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>single</td>
      <td>1</td>
      <td>1.851317</td>
      <td>0.192209617</td>
      <td>0.35584093151558904</td>
      <td>39.647622</td>
      <td>1</td>
      <td>2.521299</td>
      <td>0.103176363</td>
      <td>0.260138460855537</td>
      <td>39.168673</td>
      <td>0.73</td>
      <td>0.73</td>
      <td>0.54</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mixnet_l</td>
      <td>single</td>
      <td>1</td>
      <td>2.136226</td>
      <td>0.019517589999999998</td>
      <td>0.04169398321534</td>
      <td>65.203767</td>
      <td>1</td>
      <td>2.511199</td>
      <td>0.016638161</td>
      <td>0.04178173326503899</td>
      <td>65.253212</td>
      <td>0.85</td>
      <td>1.0</td>
      <td>0.85</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tf_mixnet_l</td>
      <td>single</td>
      <td>1</td>
      <td>2.115886</td>
      <td>0.020487329</td>
      <td>0.043348852608494</td>
      <td>67.197244</td>
      <td>1</td>
      <td>2.469767</td>
      <td>0.01748348</td>
      <td>0.043180121949159996</td>
      <td>67.360533</td>
      <td>0.86</td>
      <td>1.0</td>
      <td>0.85</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table>

<p>float32 static shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>multiple</td>
      <td>16</td>
      <td>1.198074</td>
      <td>0.278960618</td>
      <td>0.334215463449732</td>
      <td>39.408188</td>
      <td>16</td>
      <td>1.228448</td>
      <td>0.226442445</td>
      <td>0.27817276867536</td>
      <td>38.770564</td>
      <td>0.98</td>
      <td>0.83</td>
      <td>0.81</td>
      <td>0.98</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>hrnet_w18</td>
      <td>multiple</td>
      <td>128</td>
      <td>0.810532</td>
      <td>1.429737979</td>
      <td>1.158848383594828</td>
      <td>283.746136</td>
      <td>128</td>
      <td>2.630196</td>
      <td>0.432910821</td>
      <td>1.1386403097509161</td>
      <td>279.00639</td>
      <td>0.31</td>
      <td>0.98</td>
      <td>0.3</td>
      <td>0.98</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>resnest101e</td>
      <td>multiple</td>
      <td>64</td>
      <td>1.427756</td>
      <td>2.730684505</td>
      <td>3.8987511861207804</td>
      <td>82.86019</td>
      <td>64</td>
      <td>2.226839</td>
      <td>0.48641531</td>
      <td>1.08316858250509</td>
      <td>70.529618</td>
      <td>0.64</td>
      <td>0.28</td>
      <td>0.18</td>
      <td>0.85</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swsl_resnext101_32x16d</td>
      <td>multiple</td>
      <td>32</td>
      <td>0.074879</td>
      <td>17.274334944</td>
      <td>1.293484926271776</td>
      <td>119.443088</td>
      <td>32</td>
      <td>1.58248</td>
      <td>0.648969584</td>
      <td>1.0269813872883202</td>
      <td>36.292802</td>
      <td>0.05</td>
      <td>0.79</td>
      <td>0.04</td>
      <td>0.3</td>
    </tr>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>single</td>
      <td>1</td>
      <td>1.205384</td>
      <td>0.393816785</td>
      <td>0.47470045157044</td>
      <td>40.736825</td>
      <td>1</td>
      <td>1.253551</td>
      <td>0.303664971</td>
      <td>0.380659528062021</td>
      <td>39.91097</td>
      <td>0.96</td>
      <td>0.8</td>
      <td>0.77</td>
      <td>0.98</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>hrnet_w18</td>
      <td>single</td>
      <td>1</td>
      <td>0.607728</td>
      <td>0.29277504</td>
      <td>0.17792758950912</td>
      <td>285.149956</td>
      <td>1</td>
      <td>2.059248</td>
      <td>0.08616655799999999</td>
      <td>0.177438312228384</td>
      <td>284.411099</td>
      <td>0.3</td>
      <td>1.0</td>
      <td>0.29</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>resnest101e</td>
      <td>single</td>
      <td>1</td>
      <td>0.988869</td>
      <td>1.087499603</td>
      <td>1.0753946449190068</td>
      <td>84.459431</td>
      <td>1</td>
      <td>1.66091</td>
      <td>0.16280488299999998</td>
      <td>0.27040425822353</td>
      <td>79.59341</td>
      <td>0.6</td>
      <td>0.25</td>
      <td>0.15</td>
      <td>0.94</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swin_base_patch4_window7_224</td>
      <td>single</td>
      <td>1</td>
      <td>1.285946</td>
      <td>0.223065622</td>
      <td>0.286850344348412</td>
      <td>53.396126</td>
      <td>1</td>
      <td>1.329989</td>
      <td>0.192888425</td>
      <td>0.256539483477325</td>
      <td>52.928668</td>
      <td>0.97</td>
      <td>0.89</td>
      <td>0.86</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swsl_resnext101_32x16d</td>
      <td>single</td>
      <td>1</td>
      <td>0.066824</td>
      <td>13.58569782</td>
      <td>0.9078506711236799</td>
      <td>102.925062</td>
      <td>1</td>
      <td>1.376259</td>
      <td>0.434066423</td>
      <td>0.5973878212515569</td>
      <td>37.044493</td>
      <td>0.05</td>
      <td>0.66</td>
      <td>0.03</td>
      <td>0.36</td>
    </tr>
  </tbody>

</table>

<p>float32 dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>multiple</td>
      <td>16</td>
      <td>1.139748</td>
      <td>0.285818051</td>
      <td>0.325760551991148</td>
      <td>48.502399</td>
      <td>16</td>
      <td>1.154438</td>
      <td>0.251020448</td>
      <td>0.28978754394822404</td>
      <td>47.97883</td>
      <td>0.99</td>
      <td>0.89</td>
      <td>0.88</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>hrnet_w18</td>
      <td>multiple</td>
      <td>128</td>
      <td>0.80241</td>
      <td>1.4433276780000002</td>
      <td>1.15814056210398</td>
      <td>308.789275</td>
      <td>128</td>
      <td>2.529114</td>
      <td>0.461317411</td>
      <td>1.166724322603854</td>
      <td>303.674067</td>
      <td>0.32</td>
      <td>1.01</td>
      <td>0.32</td>
      <td>0.98</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>resnest101e</td>
      <td>multiple</td>
      <td>64</td>
      <td>1.429491</td>
      <td>2.724671846</td>
      <td>3.894893881810386</td>
      <td>97.602245</td>
      <td>64</td>
      <td>2.227772</td>
      <td>0.49908957400000004</td>
      <td>1.111857778449128</td>
      <td>87.189224</td>
      <td>0.64</td>
      <td>0.29</td>
      <td>0.18</td>
      <td>0.89</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swsl_resnext101_32x16d</td>
      <td>multiple</td>
      <td>32</td>
      <td>0.07378</td>
      <td>17.296593144</td>
      <td>1.2761426421643198</td>
      <td>127.440781</td>
      <td>32</td>
      <td>1.531398</td>
      <td>0.6789950189999999</td>
      <td>1.0398116141065619</td>
      <td>45.248668</td>
      <td>0.05</td>
      <td>0.81</td>
      <td>0.04</td>
      <td>0.36</td>
    </tr>
    <tr>
      <td>huggingface</td>
      <td>MT5ForConditionalGeneration</td>
      <td>single</td>
      <td>1</td>
      <td>1.208728</td>
      <td>0.39571599500000004</td>
      <td>0.47831300320436004</td>
      <td>40.998376</td>
      <td>1</td>
      <td>1.253549</td>
      <td>0.303047896</td>
      <td>0.37988538698290397</td>
      <td>40.43493</td>
      <td>0.96</td>
      <td>0.79</td>
      <td>0.77</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>hrnet_w18</td>
      <td>single</td>
      <td>1</td>
      <td>0.610165</td>
      <td>0.29299386600000005</td>
      <td>0.17877460224789002</td>
      <td>288.631683</td>
      <td>1</td>
      <td>2.056979</td>
      <td>0.087144289</td>
      <td>0.179253972442931</td>
      <td>286.437858</td>
      <td>0.3</td>
      <td>1.0</td>
      <td>0.3</td>
      <td>0.99</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>resnest101e</td>
      <td>single</td>
      <td>1</td>
      <td>0.990263</td>
      <td>1.08877909</td>
      <td>1.0781776480006702</td>
      <td>85.452358</td>
      <td>1</td>
      <td>1.663022</td>
      <td>0.164732632</td>
      <td>0.273953991133904</td>
      <td>80.585222</td>
      <td>0.6</td>
      <td>0.25</td>
      <td>0.15</td>
      <td>0.94</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swin_base_patch4_window7_224</td>
      <td>single</td>
      <td>1</td>
      <td>1.291988</td>
      <td>0.223867965</td>
      <td>0.28923472436442</td>
      <td>53.800244</td>
      <td>1</td>
      <td>1.334749</td>
      <td>0.194363569</td>
      <td>0.259426579359181</td>
      <td>53.587757</td>
      <td>0.97</td>
      <td>0.9</td>
      <td>0.87</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>swsl_resnext101_32x16d</td>
      <td>single</td>
      <td>1</td>
      <td>0.067207</td>
      <td>13.58369564</td>
      <td>0.91291943287748</td>
      <td>103.362895</td>
      <td>1</td>
      <td>1.371726</td>
      <td>0.43848101</td>
      <td>0.60147580192326</td>
      <td>37.397681</td>
      <td>0.05</td>
      <td>0.66</td>
      <td>0.03</td>
      <td>0.36</td>
    </tr>
  </tbody>

</table>

the bad commit 2a178a69820124a3643abac05757cec3a5788295
```
/workspace/pytorch# bash inductor_single_run.sh multiple inference performance huggingface MT5ForConditionalGeneration float32 first static cpp
Testing with cpp wrapper.
Testing with freezing on.
multi-threads testing....
/opt/conda/lib/python3.8/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
loading model: 0it [00:02, ?it/s]
cpu  eval  MT5ForConditionalGeneration
running benchmark: 100%|██████████████████████████████████████████████████████████| 50/50 [00:21<00:00,  2.30it/s]
1.240x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,MT5ForConditionalGeneration,16,1.239957,192.141957,36.548632,0.962392,910.594458,946.178048,1174,1,0,0,0,0,0
```
the last good commit: 6300eb1dc7b7b59ebffde52ec266fc9200339ff7
```
/workspace/pytorch# bash inductor_single_run.sh multiple inference performance huggingface MT5ForConditionalGeneration float32 first static cpp
Testing with cpp wrapper.
Testing with freezing on.
multi-threads testing....
/opt/conda/lib/python3.8/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
loading model: 0it [00:02, ?it/s]
cpu  eval  MT5ForConditionalGeneration
running benchmark: 100%|██████████████████████████████████████████████████████████| 50/50 [00:18<00:00,  2.73it/s]
1.269x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,MT5ForConditionalGeneration,16,1.269200,159.506385,36.092898,0.962646,910.764851,946.105549,1174,1,0,0,0,0,0
```
### Versions

</table><p>SW info</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>target_branch</th>
      <th>target_commit</th>
      <th>refer_branch</th>
      <th>refer_commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>main</td>
      <td>23512dbe</td>
      <td>main</td>
      <td>23512dbe</td>
    </tr>
    <tr>
      <td>torch</td>
      <td>main</td>
      <td>1d6e0412f5205b1cd709e034526d7f21d6f2d56f</td>
      <td>main</td>
      <td>9731ccb9e0c9f09dcd3435a47fc40fe6fb960504</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>main</td>
      <td>2.5.0a0+ba696ea</td>
      <td>main</td>
      <td>2.5.0a0+ba696ea</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>main</td>
      <td>nightly</td>
      <td>main</td>
      <td>nightly</td>
    </tr>
  </tbody>
</table>

</table>

Repro:
[inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob//main/scripts/modelbench/inductor_single_run.sh)
bash inductor_single_run.sh multiple inference performance huggingface MT5ForConditionalGeneration float32 first static cpp
Suspected guilty commit: https://github.com/pytorch/pytorch/commit/2a178a69820124a3643abac05757cec3a5788295
[huggingface-MT5ForConditionalGeneration-inference-float32-static-cpp-multiple-performance-drop_guilty_commit.log](https://github.com/user-attachments/files/17326256/huggingface-MT5ForConditionalGeneration-inference-float32-static-cpp-multiple-performance-drop_guilty_commit.log)
cc @ezyang @chauhang @penguinwu @WeizhuoZhang-intel @chuanqi129