output decoding perf. of transfomers==4.38.2 
Author: seungrokj
Pull request: no

There's some perf gap between 4.38.2 vs 4.35.0 during output decoding. The LlamaAttention was used. (https://github.com/huggingface/transformers/blob/f738ab3b5d30e30c43a4c3d00ca8939f8a4d4427/src/transformers/models/llama/modeling_llama.py#L688)

Is it something related to (larger) static kv cache movement feature in 4.38.2 during output decoding ? 

Regards, 
==
```
**Nvidia A100**						
	[INFO] torch:   2.3.0.dev20240313+cu121					
	[INFO] cuda/rocm:    12010					
	[INFO] device:  NVIDIA A100 80GB PCIe					
	**[INFO] transformers:  4.35.0**					
	[INFO] model:  TheBloke/Llama-2-7B-Chat-fp16  -head_num:  32  --num_key_value_heads:  32  -head_dim:  128  -layers_num: 32					

	       batch_size:  1  seq_len:  1  output_tokens:  32			       Avg per token latency:        33.41 ms		
	       batch_size:  8  seq_len:  1  output_tokens:  32			       Avg per token latency:        34.18 ms		
	       batch_size:  16  seq_len:  1  output_tokens:  32			       Avg per token latency:        34.28 ms		
	       batch_size:  32  seq_len:  1  output_tokens:  32			       Avg per token latency:        34.43 ms	
					
	[INFO] torch:   2.3.0.dev20240313+cu121					
	[INFO] cuda/rocm:    12010					
	[INFO] device:  NVIDIA A100 80GB PCIe					
	**[INFO] transformers:  4.38.2**					
	[INFO] model:  TheBloke/Llama-2-7B-Chat-fp16  -head_num:  32  --num_key_value_heads:  32  -head_dim:  128  -layers_num: 32					

	       batch_size:  1  seq_len:  1  output_tokens:  32			       Avg per token latency:        38.12 ms		
	       batch_size:  8  seq_len:  1  output_tokens:  32			       Avg per token latency:        39.97 ms		
	       batch_size:  16  seq_len:  1  output_tokens:  32			       Avg per token latency:        41.29 ms		
	       batch_size:  32  seq_len:  1  output_tokens:  32			       Avg per token latency:        44.05 ms	

**AMD MI210**						
	[INFO] torch:   2.3.0.dev20240313+rocm6.0					
	[INFO] cuda/rocm:    60000					
	[INFO] device:  AMD Instinct MI210					
	**[INFO] transformers:  4.35.0**					
	[INFO] model:  TheBloke/Llama-2-7B-Chat-fp16  -head_num:  32  --num_key_value_heads:  32  -head_dim:  128  -layers_num: 32					

	       batch_size:  1  seq_len:  1  output_tokens:  32			       Avg per token latency:        33.43 ms		
	       batch_size:  8  seq_len:  1  output_tokens:  32			       Avg per token latency:        35.07 ms		
	       batch_size:  16  seq_len:  1  output_tokens:  32			       Avg per token latency:        35.59 ms		
	       batch_size:  32  seq_len:  1  output_tokens:  32			       Avg per token latency:        39.15 ms		
						
	[INFO] torch:   2.3.0.dev20240313+rocm6.0					
	[INFO] cuda/rocm:    60000					
	[INFO] device:  AMD Instinct MI210					
	**[INFO] transformers:  4.38.2**					
	[INFO] model:  TheBloke/Llama-2-7B-Chat-fp16  -head_num:  32  --num_key_value_heads:  32  -head_dim:  128  -layers_num: 32					

	       batch_size:  1  seq_len:  1  output_tokens:  32			       Avg per token latency:        35.66 ms		
	       batch_size:  8  seq_len:  1  output_tokens:  32			       Avg per token latency:        37.81 ms		
	       batch_size:  16  seq_len:  1  output_tokens:  32			       Avg per token latency:        40.44 ms		
	       batch_size:  32  seq_len:  1  output_tokens:  32			       Avg per token latency:        48.14 ms
```		

































































Comments:
Hi @seungrokj, thanks for raising this! 

Could you share your script used for generating these values? This will help us investigate on our side

cc @gante @ArthurZucker 

Hi @amyeroberts  Sure! Here's the script for replay:

```
import torch
import time
import math
from transformers import AutoModelForCausalLM, AutoTokenizer
import transformers

# ======== Start: Parameters for LLM inf
d_type = torch.float16
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # assume single-gpu
torch.set_default_device(device)
cache = True
iters = 10
warmup = 3

batch_out = {1:32, 8:32, 16:32, 32:32}
attn_mod = "eager"
model_list = "TheBloke/Llama-2-7B-Chat-fp16"
# ======== End: Parameters for LLM inf

def print_perf_stats(latency_set, config, dtype, b, warmup, sl, v):
    latency_set = list(latency_set)
    latency_set = latency_set[warmup:]
    count = len(latency_set)

    if count > 0:
        latency_set.sort()
        latency_avg = sum(latency_set) / count
        print("batch_size: ", b, " seq_len: ", sl, " output_tokens: ", v, "Avg per token latency: {0:8.2f} ms".format(latency_avg))

def env_print(model, model_list):
    print("[INFO] torch:  ",torch.__version__)
    print("[INFO] cuda/rocm:   ", torch._C._cuda_getCompiledVersion())
    print("[INFO] device: ", torch.cuda.get_device_name(0))
    print("[INFO] transformers: ", transformers.__version__)
    print("[INFO] model:  " + model_list, " -head_num: ", str(model.config.num_attention_heads), " --num_key_value_heads: ", str(model.config.num_key_value_heads), " -head_dim: ", str(int(model.config.hidden_size/model.config.num_attention_heads)), " -layers_num: " + str(model.config.num_hidden_layers))

def llm_gen_tokens(model, max_length, input_ids, tokenizer):
    return model.generate(
        **input_ids,
        do_sample=False,
        max_new_tokens=max_length,
        min_new_tokens=max_length,
        use_cache=cache,
    )

def llm_gen_setup():
    try:
        model = AutoModelForCausalLM.from_pretrained(model_list, attn_implementation=attn_mod, torch_dtype=d_type, device_map="auto")
    except: # transformers==4.35.0
        model = AutoModelForCausalLM.from_pretrained(model_list, torch_dtype=d_type, device_map="auto")
    tokenizer = AutoTokenizer.from_pretrained(model_list, use_fast=False)
    env_print(model, model_list)
    assert warmup > 1, "warmup should be larger than 1"
    assert iters > warmup, "iters should be larger than warmup"

    input_sentences = "the"

    for b, v in batch_out.items():
        if b > len(input_sentences):
            input_sentences *= math.ceil(b / len(input_sentences))
        inputs = input_sentences[:b]
        input_ids = tokenizer.batch_encode_plus(inputs, return_tensors="pt", padding=False)

        for t in input_ids:
            if torch.is_tensor(input_ids[t]):
                input_ids[t] = input_ids[t].to(device)

        latency_set = []
        with torch.no_grad():
            for itr in range(iters):
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                torch.cuda.synchronize()
                start_event.record()

                gen_tokens = llm_gen_tokens(model, v, input_ids, tokenizer)

                end_event.record()
                torch.cuda.synchronize()
                latency_set.append(start_event.elapsed_time(end_event))

                outputs = tokenizer.batch_decode(gen_tokens)
        print_perf_stats(map(lambda t: t / v, latency_set), model.config, d_type, b, warmup, len(input_sentences[0]), v)

if __name__ == "__main__":
    llm_gen_setup()
```

@seungrokj Thank you for opening the issue and for sharing a reproduction script! There is a similar issue open, let's focus all discussion on the same issue: https://github.com/huggingface/transformers/issues/29412 🤗