`nn.TransformerEncoderLayer` fastpath (BetterTransformer) is slower than the normal path when no mask is provided
Labels: oncall: transformer/mha

### 🐛 Describe the bug

Reproduction:

<details>

  <summary>script</summary>

```python
import argparse

import torch
import torch.nn as nn

from tqdm import tqdm

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--num-batches",
        type=int,
        default=50,
        help="",
    )
    parser.add_argument(
        "--batch-size",
        type=int,
        default=64,
        help="",
    )
    parser.add_argument(
        "--max-seqlen",
        type=int,
        default=256,
        help="",
    )
    parser.add_argument(
        "--use-half",
        action="store_true",
    )
    parser.add_argument(
        "--device-id",
        type=int,
        help="GPU device id"
    )
    return parser


def timing_cuda(model, num_batches, inputs):
    start_event = torch.cuda.Event(enable_timing=True)
    end_event = torch.cuda.Event(enable_timing=True)
    start_event.record()
    for _ in range(num_batches):
        _ = model(inputs)
    end_event.record()
    torch.cuda.synchronize()
    return (start_event.elapsed_time(end_event)) / num_batches


def benchmark(num_batches: int, batch_size: int, max_seqlen: int, use_half: bool, device_id: int):
    
    layers_vanilla = []
    layers_bt = []
    for i in range(12):
        vanilla_layer = nn.TransformerEncoderLayer(d_model=768, nhead=12)  # as bert-base-uncased
        bt_layer = nn.TransformerEncoderLayer(d_model=768, nhead=12)

        vanilla_layer.norm2.eps = 2e-5  # disable fastpath

        assert vanilla_layer.norm1.eps != vanilla_layer.norm2.eps

        layers_vanilla.append(vanilla_layer)
        layers_bt.append(bt_layer)
    
    vanilla_model = nn.Sequential(*layers_vanilla)
    bt_model = nn.Sequential(*layers_bt)

    inputs = torch.rand(batch_size, max_seqlen, 768)

    if use_half is True:
        vanilla_model = vanilla_model.half()
        bt_model = bt_model.half()
        inputs = inputs.half()

    vanilla_model = vanilla_model.eval().to(f"cuda:{device_id}")
    bt_model = bt_model.eval().to(f"cuda:{device_id}")
    inputs = inputs.to(f"cuda:{device_id}")

    # Warmup
    _ = vanilla_model(inputs)
    torch.cuda.synchronize()
    _ = bt_model(inputs)
    torch.cuda.synchronize()

    vanilla_time = timing_cuda(vanilla_model, num_batches, inputs)
    bt_time = timing_cuda(bt_model, num_batches, inputs)

    return vanilla_time, bt_time


if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()

    BATCH_SIZES = [32, 64, 128]
    SEQ_LEN = [16, 32, 64, 128, 256]

    output_file = open("log_transformerencoderlayer.py", "w")
    output_file.write(
        "num_batches, batch_size, seq_len, use half, Vanilla time (ms), BT time (ms), Speedup\n"
    )
    for bs in tqdm(BATCH_SIZES, desc="batch size"):
        for seq_len in tqdm(SEQ_LEN, desc="sequence length"):
            max_seqlen = seq_len
            vanilla_time, bt_time = benchmark(
                args.num_batches,
                bs,
                max_seqlen,
                args.use_half,
                args.device_id
            )

            speedup = vanilla_time / bt_time

            output_file.write(
                "{},{},{},{},{},{},{}\n".format(
                    args.num_batches,
                    bs,
                    seq_len,
                    args.use_half,
                    f"{vanilla_time:.2f}",
                    f"{bt_time:.2f}",
                    f"{speedup:.3f}",
                )
            )
    output_file.close()
```
</details>

Results (fp32):

|num_batches|batch_size|seq_len|use half|Vanilla time (ms)|BT time (ms)|Speedup|
|-----------|----------|-------|--------|-----------------|------------|-------|
|50         |32        |16     |False   |7.49             |6.60        |1.135  |
|50         |32        |32     |False   |10.40            |11.58       |0.898  |
|50         |32        |64     |False   |20.70            |22.54       |0.918  |
|50         |32        |128    |False   |37.61            |40.72       |0.924  |
|50         |32        |256    |False   |75.08            |80.72       |0.930  |
|50         |64        |16     |False   |10.25            |11.27       |0.909  |
|50         |64        |32     |False   |20.34            |22.01       |0.925  |
|50         |64        |64     |False   |37.12            |39.86       |0.931  |
|50         |64        |128    |False   |73.38            |78.53       |0.934  |
|50         |64        |256    |False   |139.59           |149.46      |0.934  |
|50         |128       |16     |False   |20.23            |21.80       |0.928  |
|50         |128       |32     |False   |36.78            |39.54       |0.930  |
|50         |128       |64     |False   |73.09            |78.23       |0.934  |
|50         |128       |128    |False   |139.19           |148.47      |0.937  |
|50         |128       |256    |False   |275.27           |294.23      |0.936  |


Results (fp16):

|num_batches|batch_size|seq_len|use half|Vanilla time (ms)|BT time (ms)|Speedup|
|-----------|----------|-------|--------|-----------------|------------|-------|
|50         |32        |16     |True    |5.73             |5.68        |1.009  |
|50         |32        |32     |True    |5.64             |5.58        |1.011  |
|50         |32        |64     |True    |5.62             |5.54        |1.015  |
|50         |32        |128    |True    |5.64             |5.59        |1.009  |
|50         |32        |256    |True    |9.71             |11.12       |0.873  |
|50         |64        |16     |True    |5.70             |5.61        |1.015  |
|50         |64        |32     |True    |5.64             |5.57        |1.013  |
|50         |64        |64     |True    |5.85             |5.56        |1.051  |
|50         |64        |128    |True    |10.25            |11.67       |0.879  |
|50         |64        |256    |True    |20.08            |22.01       |0.913  |
|50         |128       |16     |True    |5.62             |5.56        |1.012  |
|50         |128       |32     |True    |5.62             |5.61        |1.001  |
|50         |128       |64     |True    |10.73            |12.12       |0.885  |
|50         |128       |128    |True    |21.66            |23.01       |0.942  |
|50         |128       |256    |True    |41.35            |44.50       |0.929  |


cc @jbschlosser @bhosmer @cpuhrsch @erichan1 @HamidShojanazeri @mikekgfb @younesbelkada

### Versions

The above is run on a single A100-80GB.

PyTorch version: 1.13.1+cu117
Is debug build: False
CUDA used to build PyTorch: 11.7
ROCM used to build PyTorch: N/A

OS: Ubuntu 20.04.5 LTS (x86_64)
GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
Clang version: 10.0.0-4ubuntu1 
CMake version: version 3.24.3
Libc version: glibc-2.31

Python version: 3.9.13 (main, Oct 13 2022, 21:15:33)  [GCC 11.2.0] (64-bit runtime)
Python platform: Linux-5.4.0-125-generic-x86_64-with-glibc2.31
Is CUDA available: True
CUDA runtime version: 11.0.221
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration: 
GPU 0: NVIDIA A100-SXM4-80GB
GPU 1: NVIDIA A100-SXM4-80GB
GPU 2: NVIDIA A100-SXM4-80GB
GPU 3: NVIDIA DGX Display
GPU 4: NVIDIA A100-SXM4-80GB

Nvidia driver version: 515.65.01
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

Versions of relevant libraries:
[pip3] numpy==1.23.4
[pip3] torch==1.13.1
[pip3] torchaudio==2.0.0.dev20221220+cu116
[pip3] torchtriton==2.0.0+0d7e753227
[pip3] torchvision==0.15.0.dev20221220+cu116
[conda] numpy                     1.23.4                   pypi_0    pypi
[conda] torch                     1.13.1                   pypi_0    pypi
[conda] torchaudio                2.0.0.dev20221220+cu116          pypi_0    pypi
[conda] torchtriton               2.0.0+0d7e753227          pypi_0    pypi
[conda] torchvision               0.15.0.dev20221220+cu116          pypi_0    pypi