Repository: pytorch
Issue: [inductor][cpu] Models performance regression in 2024-08-04 nightly release
Labels: triaged,oncall: pt2,oncall: cpu inductor

### 🐛 Describe the bug

<p>float32 static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>multiple</td>
      <td>1</td>
      <td>1.099492</td>
      <td>0.19840493</td>
      <td>0.21814463329556</td>
      <td>366.16073</td>
      <td>1</td>
      <td>1.318867</td>
      <td>0.162770115</td>
      <td>0.214672133259705</td>
      <td>352.032257</td>
      <td>0.83</td>
      <td>0.98</td>
      <td>0.82</td>
      <td>0.96</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>multiple</td>
      <td>64</td>
      <td>1.294418</td>
      <td>0.85368478</td>
      <td>1.10502494555804</td>
      <td>33.33422</td>
      <td>64</td>
      <td>1.620836</td>
      <td>0.674533366</td>
      <td>1.093307962813976</td>
      <td>29.072366</td>
      <td>0.8</td>
      <td>0.99</td>
      <td>0.79</td>
      <td>0.87</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>lennard_jones</td>
      <td>single</td>
      <td>1</td>
      <td>1.798823</td>
      <td>3.9433e-05</td>
      <td>7.0932987359e-05</td>
      <td>12.601352</td>
      <td>1</td>
      <td>2.154076</td>
      <td>3.2483999999999996e-05</td>
      <td>6.997300478399998e-05</td>
      <td>12.735716</td>
      <td>0.84</td>
      <td>0.99</td>
      <td>0.82</td>
      <td>1.01</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>single</td>
      <td>1</td>
      <td>1.2252</td>
      <td>0.220026991</td>
      <td>0.26957706937320003</td>
      <td>29.09092</td>
      <td>1</td>
      <td>1.416862</td>
      <td>0.19379711800000002</td>
      <td>0.27458377220371605</td>
      <td>26.42179</td>
      <td>0.86</td>
      <td>1.02</td>
      <td>0.88</td>
      <td>0.91</td>
    </tr>
  </tbody>

</table>

<p>float32 dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>multiple</td>
      <td>1</td>
      <td>1.108608</td>
      <td>0.198273276</td>
      <td>0.219807339959808</td>
      <td>373.10692</td>
      <td>1</td>
      <td>1.320236</td>
      <td>0.163317239</td>
      <td>0.21561729834840399</td>
      <td>358.199807</td>
      <td>0.84</td>
      <td>0.98</td>
      <td>0.82</td>
      <td>0.96</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>lennard_jones</td>
      <td>single</td>
      <td>1</td>
      <td>1.796648</td>
      <td>3.9501e-05</td>
      <td>7.0969392648e-05</td>
      <td>12.883616</td>
      <td>1</td>
      <td>2.109932</td>
      <td>3.4353e-05</td>
      <td>7.2482493996e-05</td>
      <td>12.914737</td>
      <td>0.85</td>
      <td>1.02</td>
      <td>0.87</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>single</td>
      <td>1</td>
      <td>1.240245</td>
      <td>0.223202114</td>
      <td>0.27682530587793003</td>
      <td>29.78713</td>
      <td>1</td>
      <td>1.417517</td>
      <td>0.195806086</td>
      <td>0.277558455608462</td>
      <td>26.875309</td>
      <td>0.87</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>0.9</td>
    </tr>
  </tbody>

</table>

<p>AMP static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>multiple</td>
      <td>1</td>
      <td>1.172963</td>
      <td>0.128395915</td>
      <td>0.150603657646145</td>
      <td>69.521648</td>
      <td>1</td>
      <td>1.676473</td>
      <td>0.08959034099999999</td>
      <td>0.150195787747293</td>
      <td>72.942238</td>
      <td>0.7</td>
      <td>1.0</td>
      <td>0.7</td>
      <td>1.05</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>mobilenet_v2</td>
      <td>multiple</td>
      <td>16</td>
      <td>2.167927</td>
      <td>0.011852771</td>
      <td>0.025695942275717</td>
      <td>27.286304</td>
      <td>16</td>
      <td>3.520782</td>
      <td>0.007186130999999999</td>
      <td>0.025300800674441997</td>
      <td>8.225868</td>
      <td>0.62</td>
      <td>0.98</td>
      <td>0.61</td>
      <td>0.3</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>timm_efficientnet</td>
      <td>multiple</td>
      <td>64</td>
      <td>2.066576</td>
      <td>0.036950767999999995</td>
      <td>0.07636157033036799</td>
      <td>39.997073</td>
      <td>64</td>
      <td>2.370131</td>
      <td>0.031411895</td>
      <td>0.07445030610824502</td>
      <td>28.259127</td>
      <td>0.87</td>
      <td>0.97</td>
      <td>0.85</td>
      <td>0.71</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>yolov3</td>
      <td>multiple</td>
      <td>8</td>
      <td>2.324335</td>
      <td>0.055351700999999996</td>
      <td>0.128655895943835</td>
      <td>44.146145</td>
      <td>8</td>
      <td>3.309024</td>
      <td>0.038904476</td>
      <td>0.128735844791424</td>
      <td>26.014839</td>
      <td>0.7</td>
      <td>1.0</td>
      <td>0.7</td>
      <td>0.59</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>multiple</td>
      <td>64</td>
      <td>1.478138</td>
      <td>0.297034849</td>
      <td>0.43905849763116195</td>
      <td>34.257604</td>
      <td>64</td>
      <td>2.29205</td>
      <td>0.195468948</td>
      <td>0.4480246022634</td>
      <td>31.394347</td>
      <td>0.64</td>
      <td>1.02</td>
      <td>0.66</td>
      <td>0.92</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>cspdarknet53</td>
      <td>multiple</td>
      <td>64</td>
      <td>2.685412</td>
      <td>0.09457697</td>
      <td>0.25397813016164</td>
      <td>38.791484</td>
      <td>64</td>
      <td>3.901175</td>
      <td>0.066233331</td>
      <td>0.258387815063925</td>
      <td>22.831668</td>
      <td>0.69</td>
      <td>1.02</td>
      <td>0.7</td>
      <td>0.59</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_botnext26ts_256</td>
      <td>multiple</td>
      <td>128</td>
      <td>2.395268</td>
      <td>0.13846456599999998</td>
      <td>0.331659744073688</td>
      <td>51.201932</td>
      <td>128</td>
      <td>2.77996</td>
      <td>0.11806395500000001</td>
      <td>0.3282130723418</td>
      <td>37.850687</td>
      <td>0.86</td>
      <td>0.99</td>
      <td>0.85</td>
      <td>0.74</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_halonext26ts</td>
      <td>multiple</td>
      <td>128</td>
      <td>2.086181</td>
      <td>0.164863081</td>
      <td>0.34393422718366096</td>
      <td>40.31269</td>
      <td>128</td>
      <td>2.539918</td>
      <td>0.135897821</td>
      <td>0.34516932171867803</td>
      <td>37.841364</td>
      <td>0.82</td>
      <td>1.0</td>
      <td>0.82</td>
      <td>0.94</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>fbnetv3_b</td>
      <td>multiple</td>
      <td>256</td>
      <td>2.287188</td>
      <td>0.149288636</td>
      <td>0.341451176795568</td>
      <td>42.741424</td>
      <td>256</td>
      <td>3.571132</td>
      <td>0.095787176</td>
      <td>0.342068649403232</td>
      <td>22.356077</td>
      <td>0.64</td>
      <td>1.0</td>
      <td>0.64</td>
      <td>0.52</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv2_100</td>
      <td>multiple</td>
      <td>128</td>
      <td>1.908754</td>
      <td>0.058025447</td>
      <td>0.110756304063038</td>
      <td>26.469625</td>
      <td>128</td>
      <td>4.189711</td>
      <td>0.026122475000000003</td>
      <td>0.109445620854725</td>
      <td>7.600881</td>
      <td>0.46</td>
      <td>0.99</td>
      <td>0.45</td>
      <td>0.29</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv3_large_100</td>
      <td>multiple</td>
      <td>512</td>
      <td>3.188931</td>
      <td>0.119820593</td>
      <td>0.382099603456083</td>
      <td>27.158355</td>
      <td>512</td>
      <td>3.950979</td>
      <td>0.096220186</td>
      <td>0.380163934262094</td>
      <td>13.238666</td>
      <td>0.81</td>
      <td>0.99</td>
      <td>0.8</td>
      <td>0.49</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>rexnet_100</td>
      <td>multiple</td>
      <td>256</td>
      <td>2.131814</td>
      <td>0.15289343600000002</td>
      <td>0.32594036737290405</td>
      <td>72.457416</td>
      <td>256</td>
      <td>3.552805</td>
      <td>0.106513797</td>
      <td>0.378422750550585</td>
      <td>48.794356</td>
      <td>0.6</td>
      <td>1.16</td>
      <td>0.7</td>
      <td>0.67</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tf_efficientnet_b0</td>
      <td>multiple</td>
      <td>128</td>
      <td>1.790607</td>
      <td>0.08773510400000001</td>
      <td>0.15709909136812802</td>
      <td>42.825881</td>
      <td>128</td>
      <td>2.144857</td>
      <td>0.07159193700000001</td>
      <td>0.15355446721800903</td>
      <td>32.240606</td>
      <td>0.83</td>
      <td>0.98</td>
      <td>0.82</td>
      <td>0.75</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tinynet_a</td>
      <td>multiple</td>
      <td>128</td>
      <td>1.895971</td>
      <td>0.059081704</td>
      <td>0.112017197414584</td>
      <td>42.607595</td>
      <td>128</td>
      <td>2.236463</td>
      <td>0.050751283</td>
      <td>0.113503366632029</td>
      <td>29.909094</td>
      <td>0.85</td>
      <td>1.01</td>
      <td>0.86</td>
      <td>0.7</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>single</td>
      <td>1</td>
      <td>1.285498</td>
      <td>1.7869828079999999</td>
      <td>2.297162825718384</td>
      <td>67.085707</td>
      <td>1</td>
      <td>4.064702</td>
      <td>0.553217796</td>
      <td>2.2486654818367917</td>
      <td>63.217269</td>
      <td>0.32</td>
      <td>0.98</td>
      <td>0.31</td>
      <td>0.94</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>lennard_jones</td>
      <td>single</td>
      <td>1</td>
      <td>4.690508</td>
      <td>2.5173e-05</td>
      <td>0.00011807415788400002</td>
      <td>5.59007</td>
      <td>1</td>
      <td>5.86462</td>
      <td>2.0294e-05</td>
      <td>0.00011901659828</td>
      <td>5.569595</td>
      <td>0.8</td>
      <td>1.01</td>
      <td>0.81</td>
      <td>1.0</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>mobilenet_v2</td>
      <td>single</td>
      <td>1</td>
      <td>1.735096</td>
      <td>0.01118428</td>
      <td>0.019405799490879998</td>
      <td>26.549763</td>
      <td>1</td>
      <td>3.95705</td>
      <td>0.004920642</td>
      <td>0.0194712264261</td>
      <td>6.380904</td>
      <td>0.44</td>
      <td>1.0</td>
      <td>0.44</td>
      <td>0.24</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>yolov3</td>
      <td>single</td>
      <td>1</td>
      <td>2.4373</td>
      <td>0.124497942</td>
      <td>0.3034388340366</td>
      <td>42.176195</td>
      <td>1</td>
      <td>3.554558</td>
      <td>0.08668550600000001</td>
      <td>0.30812865883634805</td>
      <td>24.709144</td>
      <td>0.69</td>
      <td>1.02</td>
      <td>0.7</td>
      <td>0.59</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>single</td>
      <td>1</td>
      <td>1.269357</td>
      <td>0.11098738799999999</td>
      <td>0.140882617869516</td>
      <td>29.373981</td>
      <td>1</td>
      <td>1.515969</td>
      <td>0.092479203</td>
      <td>0.140195604892707</td>
      <td>26.941204</td>
      <td>0.84</td>
      <td>1.0</td>
      <td>0.83</td>
      <td>0.92</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>cspdarknet53</td>
      <td>single</td>
      <td>1</td>
      <td>3.744821</td>
      <td>0.023818454</td>
      <td>0.089195846726734</td>
      <td>38.326832</td>
      <td>1</td>
      <td>4.397645</td>
      <td>0.020182709</td>
      <td>0.088756389320305</td>
      <td>22.507486</td>
      <td>0.85</td>
      <td>1.0</td>
      <td>0.85</td>
      <td>0.59</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_botnext26ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>2.764756</td>
      <td>0.020156998000000002</td>
      <td>0.05572918116248801</td>
      <td>45.751844</td>
      <td>1</td>
      <td>3.177309</td>
      <td>0.018014066</td>
      <td>0.057236254028393996</td>
      <td>32.978167</td>
      <td>0.87</td>
      <td>1.03</td>
      <td>0.89</td>
      <td>0.72</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_halonext26ts</td>
      <td>single</td>
      <td>1</td>
      <td>2.299663</td>
      <td>0.024817143</td>
      <td>0.057071065522808995</td>
      <td>31.015085</td>
      <td>1</td>
      <td>2.761193</td>
      <td>0.021491541000000003</td>
      <td>0.05934229256841301</td>
      <td>28.305325</td>
      <td>0.83</td>
      <td>1.04</td>
      <td>0.87</td>
      <td>0.91</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>fbnetv3_b</td>
      <td>single</td>
      <td>1</td>
      <td>2.992596</td>
      <td>0.012990257</td>
      <td>0.038874591137171996</td>
      <td>41.877584</td>
      <td>1</td>
      <td>3.5102</td>
      <td>0.011104594</td>
      <td>0.0389793458588</td>
      <td>22.267707</td>
      <td>0.85</td>
      <td>1.0</td>
      <td>0.85</td>
      <td>0.53</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv2_100</td>
      <td>single</td>
      <td>1</td>
      <td>1.765122</td>
      <td>0.01108912</td>
      <td>0.019573649672639998</td>
      <td>26.457089</td>
      <td>1</td>
      <td>4.074443</td>
      <td>0.004807823</td>
      <td>0.019589200767588997</td>
      <td>6.995627</td>
      <td>0.43</td>
      <td>1.0</td>
      <td>0.43</td>
      <td>0.26</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>rexnet_100</td>
      <td>single</td>
      <td>1</td>
      <td>2.1157</td>
      <td>0.013750322</td>
      <td>0.0290915562554</td>
      <td>69.149277</td>
      <td>1</td>
      <td>2.596044</td>
      <td>0.011057422</td>
      <td>0.028705554038568</td>
      <td>46.863122</td>
      <td>0.81</td>
      <td>0.99</td>
      <td>0.8</td>
      <td>0.68</td>
    </tr>
  </tbody>

</table>

<p>AMP static shape CPP wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>suite</th>
      <th>name</th>
      <th>thread</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>multiple</td>
      <td>1.0</td>
      <td>1.154957</td>
      <td>0.095465526</td>
      <td>0.110258577512382</td>
      <td>308.85288</td>
      <td>1</td>
      <td>1.484831</td>
      <td>0.07386592400000001</td>
      <td>0.10967841379884402</td>
      <td>298.534646</td>
      <td>0.78</td>
      <td>0.99</td>
      <td>0.77</td>
      <td>0.97</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>mobilenet_v2</td>
      <td>multiple</td>
      <td>16.0</td>
      <td>2.282688</td>
      <td>0.007763076</td>
      <td>0.017720680428287998</td>
      <td>23.240865</td>
      <td>16</td>
      <td>4.047773</td>
      <td>0.004557098</td>
      <td>0.018446098242754002</td>
      <td>18.174677</td>
      <td>0.56</td>
      <td>1.04</td>
      <td>0.59</td>
      <td>0.78</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>timm_efficientnet</td>
      <td>multiple</td>
      <td>64.0</td>
      <td>2.071297</td>
      <td>0.030870162</td>
      <td>0.063941273940114</td>
      <td>31.709493</td>
      <td>64</td>
      <td>2.580603</td>
      <td>0.025317829</td>
      <td>0.065335265470887</td>
      <td>26.385322</td>
      <td>0.8</td>
      <td>1.02</td>
      <td>0.82</td>
      <td>0.83</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>yolov3</td>
      <td>multiple</td>
      <td>8.0</td>
      <td>2.397938</td>
      <td>0.041387599000000004</td>
      <td>0.099244896370862</td>
      <td>38.356223</td>
      <td>8</td>
      <td>3.486009</td>
      <td>0.028732848000000002</td>
      <td>0.10016296672363201</td>
      <td>27.498967</td>
      <td>0.69</td>
      <td>1.01</td>
      <td>0.69</td>
      <td>0.72</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>multiple</td>
      <td>64.0</td>
      <td>1.430829</td>
      <td>0.242270339</td>
      <td>0.346647426881031</td>
      <td>32.970936</td>
      <td>64</td>
      <td>2.227357</td>
      <td>0.156508322</td>
      <td>0.34859990656495404</td>
      <td>29.989692</td>
      <td>0.64</td>
      <td>1.01</td>
      <td>0.65</td>
      <td>0.91</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>cspdarknet53</td>
      <td>multiple</td>
      <td>64.0</td>
      <td>2.876383</td>
      <td>0.07415993400000001</td>
      <td>0.21331237343872203</td>
      <td>30.449499</td>
      <td>64</td>
      <td>4.323381</td>
      <td>0.049745435</td>
      <td>0.215068468515735</td>
      <td>23.940829</td>
      <td>0.67</td>
      <td>1.01</td>
      <td>0.67</td>
      <td>0.79</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_botnext26ts_256</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>2.737134</td>
      <td>0.11215230699999999</td>
      <td>0.306975892668138</td>
      <td>30.977064</td>
      <td>128</td>
      <td>3.341768</td>
      <td>0.092382049</td>
      <td>0.308719375122632</td>
      <td>27.582771</td>
      <td>0.82</td>
      <td>1.01</td>
      <td>0.82</td>
      <td>0.89</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_halonext26ts</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>2.127233</td>
      <td>0.145923212</td>
      <td>0.310412672032396</td>
      <td>42.140167</td>
      <td>128</td>
      <td>2.960017</td>
      <td>0.104898539</td>
      <td>0.310501458715163</td>
      <td>39.362246</td>
      <td>0.72</td>
      <td>1.0</td>
      <td>0.72</td>
      <td>0.93</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>fbnetv3_b</td>
      <td>multiple</td>
      <td>256.0</td>
      <td>2.347361</td>
      <td>0.131356939</td>
      <td>0.308342155687979</td>
      <td>52.180408</td>
      <td>256</td>
      <td>4.004592</td>
      <td>0.07979763599999999</td>
      <td>0.31955697474451195</td>
      <td>41.368648</td>
      <td>0.59</td>
      <td>1.04</td>
      <td>0.61</td>
      <td>0.79</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv2_100</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>2.132498</td>
      <td>0.045579721999999996</td>
      <td>0.09719866600555599</td>
      <td>24.262982</td>
      <td>128</td>
      <td>4.469467</td>
      <td>0.022307834</td>
      <td>0.099704127904478</td>
      <td>19.750739</td>
      <td>0.48</td>
      <td>1.03</td>
      <td>0.49</td>
      <td>0.81</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv3_large_100</td>
      <td>multiple</td>
      <td>512.0</td>
      <td>3.336158</td>
      <td>0.10462363499999999</td>
      <td>0.34904097689433</td>
      <td>26.884931</td>
      <td>512</td>
      <td>4.262309</td>
      <td>0.082648227</td>
      <td>0.352272281776143</td>
      <td>22.764718</td>
      <td>0.78</td>
      <td>1.01</td>
      <td>0.79</td>
      <td>0.85</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>rexnet_100</td>
      <td>multiple</td>
      <td>256.0</td>
      <td>2.414998</td>
      <td>0.122521576</td>
      <td>0.295889360996848</td>
      <td>36.469882</td>
      <td>256</td>
      <td>3.566081</td>
      <td>0.083766986</td>
      <td>0.298719857201866</td>
      <td>31.046142</td>
      <td>0.68</td>
      <td>1.01</td>
      <td>0.68</td>
      <td>0.85</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tf_efficientnet_b0</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>1.975976</td>
      <td>0.070936139</td>
      <td>0.14016810819666398</td>
      <td>32.93069</td>
      <td>128</td>
      <td>2.538002</td>
      <td>0.054444584</td>
      <td>0.138180463081168</td>
      <td>28.19928</td>
      <td>0.78</td>
      <td>0.99</td>
      <td>0.77</td>
      <td>0.86</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tf_mixnet_l</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>1.769453</td>
      <td>0.138239347</td>
      <td>0.24460802726719102</td>
      <td>70.679119</td>
      <td>128</td>
      <td>1.996785</td>
      <td>0.123384075</td>
      <td>0.246371470198875</td>
      <td>68.107402</td>
      <td>0.89</td>
      <td>1.01</td>
      <td>0.89</td>
      <td>0.96</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tinynet_a</td>
      <td>multiple</td>
      <td>128.0</td>
      <td>2.24595</td>
      <td>0.044933354</td>
      <td>0.10091806641630001</td>
      <td>37.06638</td>
      <td>128</td>
      <td>2.599503</td>
      <td>0.036983624</td>
      <td>0.096139041538872</td>
      <td>30.422048</td>
      <td>0.86</td>
      <td>0.95</td>
      <td>0.82</td>
      <td>0.82</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>detectron2_fcos_r_50_fpn</td>
      <td>single</td>
      <td>1</td>
      <td>1.282671</td>
      <td>1.474642187</td>
      <td>1.8914807686414767</td>
      <td>309.908179</td>
      <td>1</td>
      <td>3.808943</td>
      <td>0.49466129999999997</td>
      <td>1.8841366960059</td>
      <td>295.093526</td>
      <td>0.34</td>
      <td>1.0</td>
      <td>0.34</td>
      <td>0.95</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>mobilenet_v2</td>
      <td>single</td>
      <td>1</td>
      <td>1.753273</td>
      <td>0.008983299</td>
      <td>0.015750175587627</td>
      <td>23.063212</td>
      <td>1</td>
      <td>4.784716</td>
      <td>0.00347995</td>
      <td>0.016650572444200002</td>
      <td>18.160227</td>
      <td>0.37</td>
      <td>1.06</td>
      <td>0.39</td>
      <td>0.79</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>timm_efficientnet</td>
      <td>single</td>
      <td>1</td>
      <td>2.217245</td>
      <td>0.010644658</td>
      <td>0.02360181472721</td>
      <td>30.674236</td>
      <td>1</td>
      <td>2.584427</td>
      <td>0.009272789</td>
      <td>0.023964846256902997</td>
      <td>25.543142</td>
      <td>0.86</td>
      <td>1.02</td>
      <td>0.87</td>
      <td>0.83</td>
    </tr>
    <tr>
      <td>torchbench</td>
      <td>yolov3</td>
      <td>single</td>
      <td>1</td>
      <td>2.552235</td>
      <td>0.109912197</td>
      <td>0.280521756110295</td>
      <td>36.13945</td>
      <td>1</td>
      <td>3.95572</td>
      <td>0.07067556</td>
      <td>0.2795727262032</td>
      <td>25.668939</td>
      <td>0.65</td>
      <td>1.0</td>
      <td>0.64</td>
      <td>0.71</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>convit_base</td>
      <td>single</td>
      <td>1</td>
      <td>1.26927</td>
      <td>0.116415494</td>
      <td>0.14776269406937997</td>
      <td>28.164213</td>
      <td>1</td>
      <td>1.537795</td>
      <td>0.095852804</td>
      <td>0.14740196272718</td>
      <td>25.72604</td>
      <td>0.83</td>
      <td>1.0</td>
      <td>0.82</td>
      <td>0.91</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>cspdarknet53</td>
      <td>single</td>
      <td>1</td>
      <td>4.045039</td>
      <td>0.020632852</td>
      <td>0.083460691021228</td>
      <td>28.966027</td>
      <td>1</td>
      <td>4.802694</td>
      <td>0.017559761</td>
      <td>0.084334158796134</td>
      <td>23.004978</td>
      <td>0.84</td>
      <td>1.01</td>
      <td>0.85</td>
      <td>0.79</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_botnext26ts_256</td>
      <td>single</td>
      <td>1</td>
      <td>3.016589</td>
      <td>0.017489064</td>
      <td>0.052757318082696</td>
      <td>25.692728</td>
      <td>1</td>
      <td>3.453137</td>
      <td>0.01504364</td>
      <td>0.05194774989868</td>
      <td>22.806181</td>
      <td>0.87</td>
      <td>0.98</td>
      <td>0.86</td>
      <td>0.89</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>eca_halonext26ts</td>
      <td>single</td>
      <td>1</td>
      <td>2.472229</td>
      <td>0.022052992</td>
      <td>0.054520046359168</td>
      <td>30.540568</td>
      <td>1</td>
      <td>2.990812</td>
      <td>0.018264242</td>
      <td>0.054624914144504</td>
      <td>27.361135</td>
      <td>0.83</td>
      <td>1.0</td>
      <td>0.83</td>
      <td>0.9</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>fbnetv3_b</td>
      <td>single</td>
      <td>1</td>
      <td>3.44214</td>
      <td>0.010166414</td>
      <td>0.034994220285960005</td>
      <td>49.140236</td>
      <td>1</td>
      <td>4.068514</td>
      <td>0.00852978</td>
      <td>0.03470352934692</td>
      <td>39.105219</td>
      <td>0.85</td>
      <td>0.99</td>
      <td>0.84</td>
      <td>0.8</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>mobilenetv2_100</td>
      <td>single</td>
      <td>1</td>
      <td>1.820133</td>
      <td>0.009104998</td>
      <td>0.016572307324733998</td>
      <td>23.106687</td>
      <td>1</td>
      <td>4.902129</td>
      <td>0.003386374</td>
      <td>0.016600442190246003</td>
      <td>18.821249</td>
      <td>0.37</td>
      <td>1.0</td>
      <td>0.37</td>
      <td>0.81</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>rexnet_100</td>
      <td>single</td>
      <td>1</td>
      <td>2.206537</td>
      <td>0.011171529</td>
      <td>0.024650392085072997</td>
      <td>32.751738</td>
      <td>1</td>
      <td>2.846106</td>
      <td>0.008763249999999998</td>
      <td>0.024941138404499994</td>
      <td>27.825902</td>
      <td>0.78</td>
      <td>1.01</td>
      <td>0.78</td>
      <td>0.85</td>
    </tr>
    <tr>
      <td>timm_models</td>
      <td>tinynet_a</td>
      <td>single</td>
      <td>1</td>
      <td>2.416639</td>
      <td>0.009404758</td>
      <td>0.022727904968362</td>
      <td>35.518607</td>
      <td>1</td>
      <td>2.84479</td>
      <td>0.008276173999999999</td>
      <td>0.02354397703346</td>
      <td>29.513078</td>
      <td>0.85</td>
      <td>1.04</td>
      <td>0.88</td>
      <td>0.83</td>
    </tr>
  </tbody>

</table>

The bad commit:
f44446e851294569fabb1b7f354618a33e06a75e
```
/workspace/pytorch# bash inductor_single_run.sh multiple inference performance torchbench mobilenet_v2 amp first static cpp
Testing with cpp wrapper.
Testing with freezing on.
multi-threads testing....
loading model: 0it [00:00, ?it/s]
cpu  eval  mobilenet_v2
running benchmark: 100%|█████████████████████████████████████████████████████████████████| 50/50 [00:01<00:00, 44.40it/s]
2.184x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,mobilenet_v2,16,2.183591,6.814494,27.051658,0.930067,112.103424,120.532582,155,1,0,0,0,0,0
```
The last good commit:
4c2bcf92cbecd36b7881904bceb8dc50c9b9741d
```
/workspace/pytorch# bash inductor_single_run.sh multiple inference performance torchbench mobilenet_v2 amp first static cpp
Testing with cpp wrapper.
Testing with freezing on.
multi-threads testing....
loading model: 0it [00:00, ?it/s]
cpu  eval  mobilenet_v2
running benchmark: 100%|█████████████████████████████████████████████████████████████████| 50/50 [00:00<00:00, 50.48it/s]
4.130x
WARNING:common:Trying to call the empty_gpu_cache for device: cpu, which is not in list [cuda, xpu]
dev,name,batch_size,speedup,abs_latency,compilation_latency,compression_ratio,eager_peak_mem,dynamo_peak_mem,calls_captured,unique_graphs,graph_breaks,unique_graph_breaks,autograd_captures,autograd_compiles,cudagraph_skips
cpu,mobilenet_v2,16,4.130216,3.712329,18.686854,0.938776,112.145203,119.459021,155,1,0,0,0,0,0
```

</table>

### Versions

</table><p>SW info</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>target_branch</th>
      <th>target_commit</th>
      <th>refer_branch</th>
      <th>refer_commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>main</td>
      <td>23512dbe</td>
      <td>main</td>
      <td>23512dbe</td>
    </tr>
    <tr>
      <td>torch</td>
      <td>main</td>
      <td>6ec4af6865dd884f984c9dbcb273ae26e3825481</td>
      <td>main</td>
      <td>500aea8d5033fd3540c6ed325dd80e7e1420b0f3</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
      <td>main</td>
      <td>0.19.0a0+d23a6e1</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
      <td>main</td>
      <td>0.16.0a0+b0ebddc</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>main</td>
      <td>2.4.0a0+b3f6f51</td>
      <td>main</td>
      <td>2.4.0a0+69b2a0a</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
      <td>main</td>
      <td>0.7.1a0+0790338</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>main</td>
      <td>nightly</td>
      <td>main</td>
      <td>nightly</td>
    </tr>
  </tbody>
</table>

</table>

Repro:
[inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob/main/scripts/modelbench/inductor_single_run.sh)
bash inductor_single_run.sh **thread** inference performance **suite** **name** float32/amp first static/dynamic default/cpp
Suspected guilty commit: https://github.com/pytorch/pytorch/commit/f44446e851294569fabb1b7f354618a33e06a75e
[torchbench-mobilenet_v2-inference-amp-static-cpp-single-performance-drop_guilty_commit.log](https://github.com/user-attachments/files/16530516/torchbench-mobilenet_v2-inference-amp-static-cpp-single-performance-drop_guilty_commit.log)
cc @ezyang @chauhang @penguinwu @WeizhuoZhang-intel @chuanqi129