Repository: pytorch
Issue: [inductor][cpu]performance regression in 2024-08-18 nightly release
Labels: triaged,oncall: pt2,oncall: cpu inductor

### 🐛 Describe the bug

<p>fp32 static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>timm_models</td>
      <td>sebotnet33ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>0.941899</td>
      <td>0.100319376</td>
      <td>0.094490719935024</td>
      <td>32.881608</td>
      <td>1</td>
      <td>1.094253</td>
      <td>0.087828729</td>
      <td>0.09610685019443699</td>
      <td>33.36887</td>
      <td>0.86</td>
      <td>1.02</td>
      <td>0.88</td>
      <td>1.01</td>
    </tr>
  </tbody>

</table>
<p>fp32 dynamic shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>timm_models</td>
      <td>sebotnet33ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>0.946758</td>
      <td>0.100703223</td>
      <td>0.095341582001034</td>
      <td>32.874255</td>
      <td>1</td>
      <td>1.084058</td>
      <td>0.088318301</td>
      <td>0.095742160745458</td>
      <td>33.333709</td>
      <td>0.87</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>1.01</td>
    </tr>
  </tbody>

</table>

<p>fp32 static shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>timm_models</td>
      <td>sebotnet33ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>0.968331</td>
      <td>0.09805517999999999</td>
      <td>0.09494987050458</td>
      <td>32.854349</td>
      <td>1</td>
      <td>1.110006</td>
      <td>0.085936147</td>
      <td>0.095389638786882</td>
      <td>34.213632</td>
      <td>0.87</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>1.04</td>
    </tr>
  </tbody>

</table>

<p>fp32 dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>timm_models</td>
      <td>sebotnet33ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>0.961279</td>
      <td>0.099319203</td>
      <td>0.095473464140637</td>
      <td>33.499412</td>
      <td>1</td>
      <td>1.093006</td>
      <td>0.08757168900000001</td>
      <td>0.095716381507134</td>
      <td>34.693043</td>
      <td>0.88</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>1.04</td>
    </tr>
  </tbody>

</table>

<p>amp static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>Super_SloMo</td>
      <td>multiple</td>
      <td>6</td>
      <td>1.095528</td>
      <td>0.288202603</td>
      <td>0.31573402125938405</td>
      <td>46.452687</td>
      <td>6</td>
      <td>1.838708</td>
      <td>0.171210137</td>
      <td>0.31480544858299603</td>
      <td>45.100434</td>
      <td>0.6</td>
      <td>1.0</td>
      <td>0.59</td>
      <td>0.97</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>functorch_dp_cifar10</td>
      <td>multiple</td>
      <td>64</td>
      <td>0.390787</td>
      <td>0.028069921999999997</td>
      <td>0.010969360608613999</td>
      <td>24.996125</td>
      <td>64</td>
      <td>0.924846</td>
      <td>0.013657453</td>
      <td>0.012631040777238</td>
      <td>25.521037</td>
      <td>0.42</td>
      <td>1.15</td>
      <td>0.49</td>
      <td>1.02</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>opacus_cifar10</td>
      <td>multiple</td>
      <td>64</td>
      <td>0.443432</td>
      <td>0.028011531000000003</td>
      <td>0.012421209214392001</td>
      <td>8.736755</td>
      <td>64</td>
      <td>1.038116</td>
      <td>0.01368588</td>
      <td>0.01420753100208</td>
      <td>8.789735</td>
      <td>0.43</td>
      <td>1.14</td>
      <td>0.49</td>
      <td>1.01</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>functorch_dp_cifar10</td>
      <td>single</td>
      <td>1</td>
      <td>4.277774</td>
      <td>0.0021903350000000003</td>
      <td>0.009369758114290002</td>
      <td>24.45413</td>
      <td>1</td>
      <td>4.88962</td>
      <td>0.00191102</td>
      <td>0.0093441616124</td>
      <td>24.958059</td>
      <td>0.87</td>
      <td>1.0</td>
      <td>0.87</td>
      <td>1.02</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>opacus_cifar10</td>
      <td>single</td>
      <td>1</td>
      <td>4.183307</td>
      <td>0.0022201729999999998</td>
      <td>0.009287665252111</td>
      <td>8.271827</td>
      <td>1</td>
      <td>5.063267</td>
      <td>0.001907772</td>
      <td>0.009659559011124</td>
      <td>8.373175</td>
      <td>0.83</td>
      <td>1.04</td>
      <td>0.86</td>
      <td>1.01</td>
    </tr>
  </tbody>

</table>

<p>amp dynamic shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>Super_SloMo</td>
      <td>multiple</td>
      <td>6</td>
      <td>0.932181</td>
      <td>0.443164155</td>
      <td>0.413109205172055</td>
      <td>106.060916</td>
      <td>6</td>
      <td>1.774123</td>
      <td>0.23200224600000002</td>
      <td>0.41160052068025804</td>
      <td>100.355055</td>
      <td>0.53</td>
      <td>1.0</td>
      <td>0.52</td>
      <td>0.95</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>functorch_dp_cifar10</td>
      <td>multiple</td>
      <td>64</td>
      <td>0.404284</td>
      <td>0.036586737</td>
      <td>0.014791432381308</td>
      <td>51.458506</td>
      <td>64</td>
      <td>0.698939</td>
      <td>0.021733404</td>
      <td>0.015190323658356</td>
      <td>52.778067</td>
      <td>0.58</td>
      <td>1.03</td>
      <td>0.59</td>
      <td>1.03</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>opacus_cifar10</td>
      <td>multiple</td>
      <td>64</td>
      <td>0.445277</td>
      <td>0.036695756</td>
      <td>0.016339776144412</td>
      <td>20.359369</td>
      <td>64</td>
      <td>0.772134</td>
      <td>0.021906202</td>
      <td>0.016914523375067998</td>
      <td>20.647732</td>
      <td>0.58</td>
      <td>1.04</td>
      <td>0.6</td>
      <td>1.01</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>functorch_dp_cifar10</td>
      <td>single</td>
      <td>1</td>
      <td>4.546662</td>
      <td>0.003640919</td>
      <td>0.016554028062378</td>
      <td>45.433577</td>
      <td>1</td>
      <td>5.576555</td>
      <td>0.003036971</td>
      <td>0.016935835814904997</td>
      <td>45.576202</td>
      <td>0.82</td>
      <td>1.02</td>
      <td>0.83</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>opacus_cifar10</td>
      <td>single</td>
      <td>1</td>
      <td>4.579087</td>
      <td>0.0036851370000000002</td>
      <td>0.016874562929919002</td>
      <td>15.097965</td>
      <td>1</td>
      <td>5.653265</td>
      <td>0.0031027990000000003</td>
      <td>0.017540944988735003</td>
      <td>15.329049</td>
      <td>0.81</td>
      <td>1.04</td>
      <td>0.84</td>
      <td>1.02</td>
    </tr>
  </tbody>

</table>

<p>amp static shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>Super_SloMo</td>
      <td>multiple</td>
      <td>6</td>
      <td>1.080169</td>
      <td>0.223970148</td>
      <td>0.24192561079501199</td>
      <td>39.103067</td>
      <td>6</td>
      <td>1.70284</td>
      <td>0.14367540099999998</td>
      <td>0.24465621983883995</td>
      <td>37.379275</td>
      <td>0.63</td>
      <td>1.01</td>
      <td>0.64</td>
      <td>0.96</td>
    </tr>
  </tbody>
</table>

</table>

<p>amp dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>Super_SloMo</td>
      <td>multiple</td>
      <td>6.0</td>
      <td>1.080956</td>
      <td>0.227053398</td>
      <td>0.245434732888488</td>
      <td>47.118701</td>
      <td>6</td>
      <td>1.616331</td>
      <td>0.15090221099999998</td>
      <td>0.24390792160784097</td>
      <td>45.247088</td>
      <td>0.67</td>
      <td>0.99</td>
      <td>0.66</td>
      <td>0.96</td>
    </tr>
  </tbody>

</table>

### Versions

</table><p>SW info</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>target_branch</th>
      <th>target_commit</th>
      <th>refer_branch</th>
      <th>refer_commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>main</td>
      <td>23512dbe</td>
      <td>main</td>
      <td>23512dbe</td>
    </tr>
    <tr>
      <td>torch</td>
      <td>main</td>
      <td>ae000635700e78161e0ed1a18f62b5db4030e343</td>
      <td>main</td>
      <td>a7912bf9dc39b934baf5e04b436cc2134776c10d</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>main</td>
      <td>2.4.0a0+b3f6f51</td>
      <td>main</td>
      <td>2.4.0a0+b3f6f51</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>main</td>
      <td>nightly</td>
      <td>main</td>
      <td>nightly</td>
    </tr>
  </tbody>
</table>

</table>

Repro:
[inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob/main/scripts/modelbench/inductor_single_run.sh)
bash inductor_single_run.sh **thread** inference performance **suite** **model** float32/amp first static/dynamic default/cpp
Suspected guilty commit: 99b3b58f39507bb8ad5b4bb1b9bedf7f47b64fa3
[torchbench-Super_SloMo-inference-amp-static-cpp-multiple-performance-drop_guilty_commit.log](https://github.com/user-attachments/files/16688400/torchbench-Super_SloMo-inference-amp-static-cpp-multiple-performance-drop_guilty_commit.log)
cc @ezyang @chauhang @penguinwu @WeizhuoZhang-intel @chuanqi129