inputs device error with fuyu model
Author: oops343
Pull request: no
Labels: name: Big Model Inference

The code I used :

```
from transformers import FuyuForCausalLM, FuyuProcessor, FuyuImageProcessor, AutoTokenizer
from PIL import Image
import torch
torch.manual_seed(1234)

# load model and processor
model_id = "adept/fuyu-8b"

tokenizer = AutoTokenizer.from_pretrained(model_id, pad_token_id=0)
image_processor = FuyuImageProcessor()
processor = FuyuProcessor(image_processor=image_processor, tokenizer=tokenizer)

model = FuyuForCausalLM.from_pretrained(model_id, device_map="auto",torch_dtype=torch.float16).eval()

def baseline_for_fuyu(model, processor, batch):
    img_path = batch['img_path'][0]
    text = batch['text'][0]

    query = f"Given the meme, with the text [{text}] accompanied by the image, is this meme {adj}?\n."
    image = Image.open(img_path)
    if image.mode != "RGB":
        image = image.convert("RGB")
    inputs = processor(text=[query], images=[image]).to(model.device)
    with torch.no_grad():  
        generation_output = model.generate(**inputs, max_new_tokens=20)
        generation_text = processor.batch_decode(generation_output[:, -20:], skip_special_tokens=True)
        print("Generated text: ", generation_text[0].split('\x04', 1)[-1])
        torch.cuda.empty_cache()
        return generation_text[0].split('\x04', 1)[-1]
```
Traceback here: 

> Traceback (most recent call last):
>   File "/public/home/baseline/fuyu_baseline.py", line 151, in <module>
>     response = baseline_for_fuyu(model, processor, batch)
>   File "/public/home/baseline/fuyu_baseline.py", line 142, in baseline_for_fuyu
>     generation_output = model.generate(**inputs, max_new_tokens=20)
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
>     return func(*args, **kwargs)
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/transformers/generation/utils.py", line 1527, in generate
>     result = self._greedy_search(
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/transformers/generation/utils.py", line 2411, in _greedy_search
>     outputs = self(
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
>     return forward_call(*args, **kwargs)
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/accelerate/hooks.py", line 165, in new_forward
>     output = old_forward(*args, **kwargs)
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/transformers/models/fuyu/modeling_fuyu.py", line 296, in forward
>     inputs_embeds = self.gather_continuous_embeddings(
>   File "/public/home/.conda/envs/baseEnv/lib/python3.9/site-packages/transformers/models/fuyu/modeling_fuyu.py", line 207, in gather_continuous_embeddings
>     output_embeddings[batch_idx, dst_indices] = continuous_embeddings[batch_idx][src_indices]
> RuntimeError: indices should be either on cpu or on the same device as the indexed tensor (cuda:1)



This happened with 2 x RTX3090 GPU, CUDA version 11.7, transformers 4.39.0, torch 2.0.1  
please check , I think this won't happen for the other transformer models I'm using, with the same .to(model.device)

_Originally posted by @oops343 in https://github.com/huggingface/transformers/issues/27007#issuecomment-2017471084_
            

Comments:
cc @SunMarc for big model inference!

Can be closed as it's resolved by #29880