Repository: pytorch
Issue: [inductor][dynamic] fused_attention pattern could not be matched due to sym_size
Labels: triaged,module: dynamic shapes,module: inductor,inductor_pattern_match,module: sdpa

## Issue description

For fused_attention-related models like `ElectraForQuestionAnswering`, the fused_attention pattern could not be matched for dynamic shape due to `sym_size`. The failure of SDPA generation results in the regression of dynamic mode compared with static one.

For the model graph, the symbolic size param is one of the model inputs, as a placeholder: `def forward(self, sym: Sym(s0), ...)`.
For the traced graph of fused_attention pattern, the symbolic size param is deduced from one input tensor shape, with op `sym_size`: `sym = torch.ops.aten.sym_size(arg0_1, 0)`.

## Code example

Part of ``ElectraForQuestionAnswering` graph:
```
def forward(self, arg203_1, arg204_1, arg205_1, arg206_1):
        ......
        permute_3 = torch.ops.aten.permute.default(view_6, [0, 2, 1, 3]);  view_6 = None
        permute_5 = torch.ops.aten.permute.default(view_9, [0, 2, 1, 3]);  view_9 = None
        permute_6 = torch.ops.aten.permute.default(view_10, [0, 2, 1, 3]);  view_10 = None
        permute_7 = torch.ops.aten.permute.default(permute_3, [0, 1, 3, 2]);  permute_3 = None
        expand_1 = torch.ops.aten.expand.default(permute_6, [arg203_1, 4, 512, 64]);  permute_6 = None
        clone_1 = torch.ops.aten.clone.default(expand_1, memory_format = torch.contiguous_format);  expand_1 = None
        mul_7 = arg203_1 * 4
        view_11 = torch.ops.aten.reshape.default(clone_1, [mul_7, 512, 64]);  clone_1 = None
        expand_2 = torch.ops.aten.expand.default(permute_7, [arg203_1, 4, 64, 512]);  permute_7 = None
        clone_2 = torch.ops.aten.clone.default(expand_2, memory_format = torch.contiguous_format);  expand_2 = None
        view_12 = torch.ops.aten.reshape.default(clone_2, [mul_7, 64, 512]);  clone_2 = None
        bmm = torch.ops.aten.bmm.default(view_11, view_12);  view_11 = view_12 = None
        view_13 = torch.ops.aten.reshape.default(bmm, [arg203_1, 4, 512, 512]);  bmm = None
        div = torch.ops.aten.div.Tensor(view_13, 8.0);  view_13 = None
        add_4 = torch.ops.aten.add.Tensor(div, mul);  div = None
        amax = torch.ops.aten.amax.default(add_4, [-1], True)
        sub_2 = torch.ops.aten.sub.Tensor(add_4, amax);  add_4 = amax = None
        exp = torch.ops.aten.exp.default(sub_2);  sub_2 = None
        sum_1 = torch.ops.aten.sum.dim_IntList(exp, [-1], True)
        div_1 = torch.ops.aten.div.Tensor(exp, sum_1);  exp = sum_1 = None
        clone_3 = torch.ops.aten.clone.default(div_1);  div_1 = None
        expand_3 = torch.ops.aten.expand.default(clone_3, [arg203_1, 4, 512, 512]);  clone_3 = None
        view_14 = torch.ops.aten.reshape.default(expand_3, [mul_7, 512, 512]);  expand_3 = None
        expand_4 = torch.ops.aten.expand.default(permute_5, [arg203_1, 4, 512, 64]);  permute_5 = None
        clone_4 = torch.ops.aten.clone.default(expand_4, memory_format = torch.contiguous_format);  expand_4 = None
        view_15 = torch.ops.aten.reshape.default(clone_4, [mul_7, 512, 64]);  clone_4 = None
        bmm_1 = torch.ops.aten.bmm.default(view_14, view_15);  view_14 = view_15 = None
        view_16 = torch.ops.aten.reshape.default(bmm_1, [arg203_1, 4, 512, 64]);  bmm_1 = None
        ......
```

Fused_attention pattern:
```
def _sfdp_pattern(query, key, value, attn_mask, inv_scale_factor, dropout_p):
    q = query.permute(0, 2, 1, 3)
    k = key.permute(0, 2, 1, 3)
    v = value.permute(0, 2, 1, 3)
    return torch.nn.functional.dropout(
        (torch.matmul(q, k.transpose(-2, -1)).div(inv_scale_factor) + attn_mask).softmax(dim=-1),
        p=dropout_p,
    ).matmul(v)
```

Graph of pattern:
```
def forward(self, arg0_1, arg1_1, arg2_1, arg3_1, arg4_1):
    permute = torch.ops.aten.permute.default(arg0_1, [0, 2, 1, 3])
    permute_1 = torch.ops.aten.permute.default(arg1_1, [0, 2, 1, 3]);  arg1_1 = None
    permute_2 = torch.ops.aten.permute.default(arg2_1, [0, 2, 1, 3]);  arg2_1 = None
    permute_3 = torch.ops.aten.permute.default(permute_1, [0, 1, 3, 2]);  permute_1 = None
    sym_size = torch.ops.aten.sym_size(arg0_1, 0);  arg0_1 = None
    expand = torch.ops.aten.expand.default(permute, [sym_size, 4, 512, 64]);  permute = None
    clone = torch.ops.aten.clone.default(expand, memory_format = torch.contiguous_format);  expand = None
    mul = sym_size * 4
    view = torch.ops.aten.view.default(clone, [mul, 512, 64]);  clone = None
    expand_1 = torch.ops.aten.expand.default(permute_3, [sym_size, 4, 64, 512]);  permute_3 = None
    clone_1 = torch.ops.aten.clone.default(expand_1, memory_format = torch.contiguous_format);  expand_1 = None
    view_1 = torch.ops.aten.view.default(clone_1, [mul, 64, 512]);  clone_1 = mul = None
    bmm = torch.ops.aten.bmm.default(view, view_1);  view = view_1 = None
    view_2 = torch.ops.aten.view.default(bmm, [sym_size, 4, 512, 512]);  bmm = None
    div = torch.ops.aten.div.Tensor(view_2, 8.0);  view_2 = None
    add = torch.ops.aten.add.Tensor(div, arg3_1);  div = arg3_1 = None
    amax = torch.ops.aten.amax.default(add, [-1], True)
    sub = torch.ops.aten.sub.Tensor(add, amax);  add = amax = None
    exp = torch.ops.aten.exp.default(sub);  sub = None
    sum_1 = torch.ops.aten.sum.dim_IntList(exp, [-1], True)
    div_1 = torch.ops.aten.div.Tensor(exp, sum_1);  exp = sum_1 = None
    clone_2 = torch.ops.aten.clone.default(div_1);  div_1 = None
    expand_2 = torch.ops.aten.expand.default(clone_2, [sym_size, 4, 512, 512]);  clone_2 = None
    mul_1 = sym_size * 4
    view_3 = torch.ops.aten.view.default(expand_2, [mul_1, 512, 512]);  expand_2 = None
    expand_3 = torch.ops.aten.expand.default(permute_2, [sym_size, 4, 512, 64]);  permute_2 = None
    clone_3 = torch.ops.aten.clone.default(expand_3, memory_format = torch.contiguous_format);  expand_3 = None
    view_4 = torch.ops.aten.view.default(clone_3, [mul_1, 512, 64]);  clone_3 = mul_1 = None
    bmm_1 = torch.ops.aten.bmm.default(view_3, view_4);  view_3 = view_4 = None
    view_5 = torch.ops.aten.view.default(bmm_1, [sym_size, 4, 512, 64]);  bmm_1 = sym_size = None
    return view_5
```

## Reproduce

You can reproduce by running:

### dynamic
bash [inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob/yudong/aws_auto/scripts/modelbench/inductor_single_run.sh) multiple inference performance huggingface ElectraForQuestionAnswering float32 first dynamic default 0

profiling:
```
---------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
                             Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg    # of Calls
---------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
                      aten::addmm        26.89%     565.370ms        34.82%     731.998ms       5.903ms           124
                        aten::bmm        19.34%     406.637ms        19.34%     406.640ms       8.472ms            48
            Torch-Compiled Region        18.12%     380.959ms        45.22%     950.565ms     950.565ms             1
                      aten::copy_        11.15%     234.356ms        11.15%     234.356ms       1.233ms           190
                        aten::add         7.47%     156.958ms         7.47%     156.958ms       4.130ms            38
                        aten::div         5.89%     123.763ms         5.90%     124.093ms       9.546ms            13
                   aten::_softmax         5.60%     117.785ms         5.60%     117.785ms       9.815ms            12
                       aten::gelu         2.86%      60.104ms         2.86%      60.104ms       5.009ms            12
          aten::native_layer_norm         1.35%      28.307ms         1.37%      28.730ms       1.149ms            25
                         expected         0.73%      15.313ms        54.74%        1.151s        1.151s             1
                       aten::view         0.07%       1.384ms         0.07%       1.384ms       5.262us           263
                     aten::linear         0.07%       1.368ms        17.90%     376.198ms       5.084ms            74
               aten::index_select         0.04%     944.000us         0.05%     963.000us     321.000us             3
                     aten::expand         0.04%     915.000us         0.06%       1.191ms       6.884us           173
                     aten::matmul         0.04%     891.000us        12.21%     256.762ms      10.698ms            24
                      aten::empty         0.04%     881.000us         0.04%     881.000us       6.777us           130
                    aten::reshape         0.03%     657.000us         2.79%      58.715ms     469.720us           125
                 aten::as_strided         0.03%     598.000us         0.03%     598.000us       1.851us           323
    inductor::_reinterpret_tensor         0.03%     563.000us         0.03%     563.000us       2.448us           230
                           actual         0.03%     541.000us        45.26%     951.441ms     951.441ms             1
                          aten::t         0.02%     453.000us         0.05%     989.000us      13.365us            74
                    aten::permute         0.02%     410.000us         0.02%     522.000us      10.875us            48
                  aten::transpose         0.02%     409.000us         0.03%     605.000us       7.035us            86
                      aten::clone         0.02%     395.000us         3.31%      69.493ms       1.363ms            51
         TorchDynamo Cache Lookup         0.02%     335.000us         0.02%     335.000us     335.000us             1
               aten::_unsafe_view         0.02%     332.000us         0.02%     332.000us       5.443us            61
                 aten::layer_norm         0.01%     275.000us         1.38%      29.005ms       1.160ms            25
                 aten::empty_like         0.01%     202.000us         0.04%     748.000us      14.667us            51
              aten::empty_strided         0.01%     201.000us         0.01%     201.000us       5.432us            37
                       aten::add_         0.01%     136.000us         0.01%     136.000us     136.000us             1
                   aten::_to_copy         0.01%     120.000us         0.01%     301.000us      20.067us            15
                    aten::softmax         0.01%     120.000us         5.61%     117.905ms       9.825ms            12
                         aten::to         0.00%      49.000us         0.02%     350.000us      17.500us            20
                    aten::dropout         0.00%      44.000us         0.00%      44.000us       1.189us            37
               aten::_log_softmax         0.00%      36.000us         0.00%      36.000us      18.000us             2
                  aten::embedding         0.00%      35.000us         0.05%       1.062ms     354.000us             3
                 aten::contiguous         0.00%      34.000us         0.57%      12.043ms     860.214us            14
                        aten::sub         0.00%      33.000us         0.00%      49.000us      49.000us             1
                      aten::slice         0.00%      26.000us         0.00%      31.000us       5.167us             6
                      aten::fill_         0.00%      23.000us         0.00%      23.000us      23.000us             1
                        aten::mul         0.00%      22.000us         0.00%      26.000us      26.000us             1
                      aten::clamp         0.00%      21.000us         0.00%      21.000us      10.500us             2
                       aten::ones         0.00%      18.000us         0.00%      50.000us      50.000us             1
               aten::resolve_conj         0.00%      17.000us         0.00%      17.000us       0.048us           356
                     aten::select         0.00%      16.000us         0.00%      17.000us       2.833us             6
                      aten::split         0.00%      14.000us         0.00%      30.000us      30.000us             1
                       aten::rsub         0.00%      13.000us         0.00%      62.000us      62.000us             1
         aten::cross_entropy_loss         0.00%      13.000us         0.00%      81.000us      40.500us             2
           aten::nll_loss_forward         0.00%      11.000us         0.00%      11.000us       5.500us             2
                    aten::squeeze         0.00%      10.000us         0.00%      10.000us       5.000us             2
                     aten::narrow         0.00%       9.000us         0.00%      16.000us       8.000us             2
                aten::log_softmax         0.00%       8.000us         0.00%      44.000us      22.000us             2
                aten::nll_loss_nd         0.00%       7.000us         0.00%      20.000us      10.000us             2
                   aten::nll_loss         0.00%       6.000us         0.00%      17.000us       8.500us             2
                  aten::unsqueeze         0.00%       5.000us         0.00%       6.000us       3.000us             2
---------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
Self CPU time total: 2.102s
```


### static
bash [inductor_single_run.sh](https://github.com/chuanqi129/inductor-tools/blob/yudong/aws_auto/scripts/modelbench/inductor_single_run.sh) multiple inference performance huggingface ElectraForQuestionAnswering float32 first static default 0

profiling:
```
---------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
                                         Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg    # of Calls
---------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
                                  aten::addmm        17.73%     288.257ms        22.63%     367.885ms       4.971ms            74
                             mkl::_mkl_linear        15.47%     251.548ms        15.51%     252.202ms       3.408ms            74
                                    aten::bmm        12.15%     197.618ms        12.15%     197.619ms       8.234ms            24
                                    aten::add         9.68%     157.440ms         9.68%     157.440ms       4.143ms            38
                                  aten::copy_         9.18%     149.283ms         9.18%     149.283ms       1.066ms           140
                                    aten::div         7.65%     124.408ms         7.67%     124.734ms       9.595ms            13
                               aten::_softmax         7.26%     118.088ms         7.26%     118.088ms       9.841ms            12
                        Torch-Compiled Region         7.19%     116.861ms        29.33%     476.907ms     476.907ms             1
    aten::_scaled_dot_product_flash_attention         6.58%     106.918ms         6.60%     107.366ms       8.947ms            12
                                   aten::gelu         3.70%      60.135ms         3.70%      60.135ms       5.011ms            12
                      aten::native_layer_norm         1.73%      28.175ms         1.76%      28.605ms       1.144ms            25
                                     expected         0.92%      14.880ms        70.62%        1.148s        1.148s             1
                                  aten::empty         0.11%       1.722ms         0.11%       1.722ms       5.519us           312
                                   aten::view         0.08%       1.297ms         0.08%       1.297ms       4.932us           263
                                 aten::linear         0.08%       1.244ms        22.83%     371.151ms       5.016ms            74
                                 aten::matmul         0.05%     826.000us        15.91%     258.615ms      10.776ms            24
                           aten::index_select         0.04%     718.000us         0.05%     748.000us     249.333us             3
                                aten::reshape         0.04%     662.000us         3.70%      60.091ms     480.728us           125
                                 aten::expand         0.04%     597.000us         0.05%     734.000us       5.967us           123
                              aten::transpose         0.03%     566.000us         0.05%     842.000us       5.767us           146
                             aten::as_strided         0.03%     528.000us         0.03%     528.000us       1.586us           333
                inductor::_reinterpret_tensor         0.03%     461.000us         0.03%     461.000us       3.115us           148
                                       actual         0.03%     442.000us        29.38%     477.692ms     477.692ms             1
                                      aten::t         0.03%     428.000us         0.06%     943.000us      12.743us            74
                                  aten::clone         0.02%     405.000us         4.36%      70.951ms       1.391ms            51
                                aten::permute         0.02%     352.000us         0.03%     455.000us       9.479us            48
                     TorchDynamo Cache Lookup         0.02%     343.000us         0.02%     343.000us     343.000us             1
                           aten::_unsafe_view         0.02%     291.000us         0.02%     291.000us       4.770us            61
                             aten::layer_norm         0.02%     291.000us         1.78%      28.896ms       1.156ms            25
                             aten::empty_like         0.01%     198.000us         0.05%     741.000us      14.529us            51
                               aten::_to_copy         0.01%     129.000us         0.02%     305.000us      20.333us            15
                                   aten::add_         0.01%     119.000us         0.01%     119.000us     119.000us             1
                                aten::softmax         0.01%     112.000us         7.27%     118.200ms       9.850ms            12
                          aten::empty_strided         0.00%      75.000us         0.00%      75.000us       2.885us            26
                                     aten::to         0.00%      50.000us         0.02%     355.000us      17.750us            20
                                aten::dropout         0.00%      42.000us         0.00%      42.000us       1.135us            37
                             aten::contiguous         0.00%      40.000us         0.74%      12.104ms     864.571us            14
                           aten::_log_softmax         0.00%      36.000us         0.00%      36.000us      18.000us             2
                                    aten::sub         0.00%      34.000us         0.00%      58.000us      58.000us             1
                              aten::embedding         0.00%      29.000us         0.05%     846.000us     282.000us             3
                                  aten::fill_         0.00%      27.000us         0.00%      27.000us      27.000us             1
                                  aten::slice         0.00%      26.000us         0.00%      32.000us       5.333us             6
                                    aten::mul         0.00%      22.000us         0.00%      26.000us      26.000us             1
                                  aten::clamp         0.00%      19.000us         0.00%      20.000us      10.000us             2
                                   aten::ones         0.00%      18.000us         0.00%      54.000us      54.000us             1
                                 aten::select         0.00%      18.000us         0.00%      18.000us       3.000us             6
                     aten::cross_entropy_loss         0.00%      18.000us         0.01%      82.000us      41.000us             2
                                aten::squeeze         0.00%      11.000us         0.00%      11.000us       5.500us             2
                                  aten::split         0.00%      10.000us         0.00%      24.000us      24.000us             1
                       aten::nll_loss_forward         0.00%      10.000us         0.00%      10.000us       5.000us             2
                                 aten::narrow         0.00%       9.000us         0.00%      14.000us       7.000us             2
                                   aten::rsub         0.00%       8.000us         0.00%      66.000us      66.000us             1
                           aten::resolve_conj         0.00%       8.000us         0.00%       8.000us       0.041us           196
                            aten::log_softmax         0.00%       8.000us         0.00%      44.000us      22.000us             2
                               aten::nll_loss         0.00%       6.000us         0.00%      16.000us       8.000us             2
                              aten::unsqueeze         0.00%       5.000us         0.00%       5.000us       2.500us             2
                            aten::nll_loss_nd         0.00%       4.000us         0.00%      20.000us      10.000us             2
---------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------
Self CPU time total: 1.626s
```


cc @ezyang @penguinwu @bobrenjc93 @voznesenskym @EikanWang @jgong5 @Guobing-Chen @XiaobingSuper @zhuhaozhe @blzheng @wenzhe-nrv @jiayisunx @ipiszy @yf225 @chenyang78 @kadeng @muchulee8 @ColinPeppler @amjames @desertfire @chauhang @aakhundov @drisspg @mikaylagawarecki @jbschlosser @bhosmer @cpuhrsch @erichan1 @peterbell10 @Xia-Weiwen