Repository: pytorch
Issue: [inductor][cpu]ghostnet_100 and tf_mixnet_l multiple threads performance regression
Labels: triaged,oncall: cpu inductor

ghostnet_100 and tf_mixnet_l multiple threads performance regression

</table><p>float32 static shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>ghostnet_100</td>
      <td>128.0</td>
      <td>2.432435</td>
      <td>0.077779665</td>
      <td>0.189193979434275</td>
      <td>31.869826</td>
      <td>128.0</td>
      <td>2.872427</td>
      <td>0.065510128</td>
      <td>0.18817306044065601</td>
      <td>30.519646</td>
      <td>0.85</td>
      <td>0.99</td>
      <td>0.84</td>
      <td>0.96</td>
    </tr>
    <tr>
      <td>tf_mixnet_l</td>
      <td>128.0</td>
      <td>1.881287</td>
      <td>0.345943469</td>
      <td>0.650818950964603</td>
      <td>75.164342</td>
      <td>128.0</td>
      <td>2.122904</td>
      <td>0.306513633</td>
      <td>0.6506990175502321</td>
      <td>71.633416</td>
      <td>0.89</td>
      <td>1.0</td>
      <td>0.89</td>
      <td>0.95</td>
    </tr>
</tbody>

</table><p>fp32 dynamic shape cpp wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>ghostnet_100</td>
      <td>128.0</td>
      <td>2.362702</td>
      <td>0.0799881</td>
      <td>0.18898804384620002</td>
      <td>34.598862</td>
      <td>128.0</td>
      <td>2.676415</td>
      <td>0.070539241</td>
      <td>0.188792282701015</td>
      <td>33.201952</td>
      <td>0.88</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>0.96</td>
    </tr>
</tbody>

</table><p>fp32 static shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>ghostnet_100</td>
      <td>128</td>
      <td>2.324959</td>
      <td>0.08100501800000001</td>
      <td>0.18833334564426205</td>
      <td>53.441929</td>
      <td>128.0</td>
      <td>2.77073</td>
      <td>0.06762062899999999</td>
      <td>0.18735850538916995</td>
      <td>54.190286</td>
      <td>0.84</td>
      <td>0.99</td>
      <td>0.83</td>
      <td>1.01</td>
    </tr>
    <tr>
      <td>tf_mixnet_l</td>
      <td>128</td>
      <td>1.834791</td>
      <td>0.354359901</td>
      <td>0.650176357115691</td>
      <td>95.402562</td>
      <td>128.0</td>
      <td>2.082084</td>
      <td>0.31272751499999996</td>
      <td>0.6511249553412599</td>
      <td>93.784211</td>
      <td>0.88</td>
      <td>1.0</td>
      <td>0.88</td>
      <td>0.98</td>
    </tr>
  </tbody>

</table><p>fp32 dynamic shape default wrapper</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>name</th>
      <th>batch_size_new</th>
      <th>speed_up_new</th>
      <th>inductor_new</th>
      <th>eager_new</th>
      <th>compilation_latency_new</th>
      <th>batch_size_old</th>
      <th>speed_up_old</th>
      <th>inductor_old</th>
      <th>eager_old</th>
      <th>compilation_latency_old</th>
      <th>Ratio Speedup(New/old)</th>
      <th>Eager Ratio(old/new)</th>
      <th>Inductor Ratio(old/new)</th>
      <th>Compilation_latency_Ratio(old/new)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>ghostnet_100</td>
      <td>128.0</td>
      <td>2.235701</td>
      <td>0.08372402200000001</td>
      <td>0.18718187970942204</td>
      <td>56.031303</td>
      <td>128.0</td>
      <td>2.581152</td>
      <td>0.07318431</td>
      <td>0.18889982812512</td>
      <td>55.922857</td>
      <td>0.87</td>
      <td>1.01</td>
      <td>0.87</td>
      <td>1.0</td>
    </tr>
  </tbody>

</table><p>SW info</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>SW</th>
      <th>Nightly commit</th>
      <th>Main commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>Pytorch</td>
      <td>6c53103</td>
      <td>38c18f3</td>
    </tr>
    <tr>
      <td>Torchbench</td>
      <td>/</td>
      <td>ff42d907</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>e3efbc2</td>
      <td>#3722</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>b0ebddc</td>
      <td>#2167</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>d234307</td>
      <td>#8196</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>11bb5b8</td>
      <td>#1147</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>nightly</td>
      <td>/</td>
    </tr>
  </tbody>
</table><p>Reference SW info (nightly)</p><table border="1" class="dataframe table">
  <thead>
    <tr style="text-align: right;">
      <th>item</th>
      <th>commit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>torchbench</td>
      <td>99944a2f</td>
    </tr>
    <tr>
      <td>torch</td>
      <td>2.3.0a0+git2f05263</td>
    </tr>
    <tr>
      <td>torchvision</td>
      <td>0.18.0a0+f52fd33</td>
    </tr>
    <tr>
      <td>torchtext</td>
      <td>0.16.0a0+b0ebddc</td>
    </tr>
    <tr>
      <td>torchaudio</td>
      <td>2.2.0a0+258169e</td>
    </tr>
    <tr>
      <td>torchdata</td>
      <td>0.7.0a0+11bb5b8</td>
    </tr>
    <tr>
      <td>dynamo_benchmarks</td>
      <td>nightly</td>
    </tr>
  </tbody>

</table>

Repro:
[inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob/yudong/aws_auto/scripts/modelbench/inductor_single_run.sh)
bash inductor_single_run.sh multiple inference performance timm_modles ghostnet_100/tf_mixnet_l float32 first static/dynamic default/cpp
Suspected guilty commit: https://github.com/pytorch/pytorch/commit/9f57cf502fba060285f0a6cab7516066fee40d8f
[timm_models-ghostnet_100-inference-float32-static-cpp-performance-multiple-drop_guilty_commit.log](https://github.com/pytorch/pytorch/files/13964542/timm_models-ghostnet_100-inference-float32-static-cpp-performance-multiple-drop_guilty_commit.log)
cc @WeizhuoZhang-intel @chuanqi129