Training "microsoft/beit-large-finetuned-ade-640-640" on my dataset present some worry can not  to solve
Author: 813BoZai
Pull request: no

### System Info

win10
anaconda
python3.8


### Who can help?

PyTorch: @sgugger

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

"import timm
import torch
from easydict import EasyDict
from torch import nn

import pandas as pd

metadata = pd.read_json("metadata.json", orient="colunms")

dict_all = metadata.sample_pairs.loc[metadata.index[22:]]
index_organ_pd = pd.DataFrame()
index_list = []
organ_list = []
for num in range(len(dict_all)):
    index_list.append(dict_all.index[num])
    organ_list.append(dict_all[num]['organ'])
index_organ_pd["index"] = index_list
index_organ_pd["organ"] = organ_list

# key_list = list(index_organ_pd.organ.value_counts().index)
# id2label = {int(num): key_list[num] for num in range(len(key_list))}
id2label = {1:"BG",
            2:"CA",
            255:"UNK"}
label2id = {v: k for k, v in id2label.items()}




import glob
import os

train_path = 'D:\\workplace\\workplace_python\\jupyter_lab\\timm_model\\dataset_oce\\train\\tissue\\img'
train_lablepath = 'D:\\workplace\\workplace_python\\jupyter_lab\\timm_model\\dataset_oce\\train\\tissue\\label'
test_path = 'D:\\workplace\\workplace_python\\jupyter_lab\\timm_model\\dataset_oce\\test\\tissue\\img'
test_lablepath = 'D:\\workplace\\workplace_python\\jupyter_lab\\timm_model\\dataset_oce\\test\\tissue\\label'

train_tissuePath = glob.glob(os.path.join(train_path, '*.jpg'))

train_tissuelabelPath = glob.glob(os.path.join(train_lablepath, '*.jpg'))

test_tissuePath = glob.glob(os.path.join(test_path, '*.jpg'))

test_tissuelabelPath = glob.glob(os.path.join(test_lablepath, '*.jpg'))

import PIL
import cv2
import numpy as np

def label_imgpro(tmp_pic):
    if len(tmp_pic[tmp_pic[:]==2].reshape(-1,1))/len(tmp_pic.reshape(-1,1))>=0.2:
        return 2
    else:
        return 1

def get_data(train_tissuePath,train_tissueLabelPath,index_organ_pd):
    train_ds={}
    image=[]
    annotation=[]
    scene_category =[]
    for num in range(len(train_tissuePath)):
        image.append(PIL.Image.open(train_tissuePath[num]))
        annotation.append(PIL.Image.open(train_tissueLabelPath[num]).convert('L'))
        # tmp_key = np.array(index_organ_pd[index_organ_pd["index"]==train_tissuePath[num].split("\\")[-1].split(".")[0]]["organ"])[0]
        # print(tmp_key)
        # tmp_value = label2id.get(tmp_key)
        # print(tmp_value)
        scene_category.append(label_imgpro(np.array(PIL.Image.open(train_tissueLabelPath[num]).convert('L'))))
    train_ds["image"]=image
    train_ds["annotation"]=annotation
    train_ds["scene_category"]=scene_category
    return train_ds

train_dict = get_data(train_tissuePath,train_tissuelabelPath,index_organ_pd)
test_dict = get_data(test_tissuePath,test_tissuelabelPath,index_organ_pd)


train_dict = get_data(train_tissuePath, train_tissuelabelPath, index_organ_pd)
test_dict = get_data(test_tissuePath, test_tissuelabelPath, index_organ_pd)

import datasets

train_ds = datasets.Dataset.from_dict(train_dict)
test_ds = datasets.Dataset.from_dict(test_dict)

from transformers import AutoImageProcessor

checkpoint = "microsoft/beit-large-finetuned-ade-640-640"
# checkpoint = "nvidia/mit-b0"
image_processor = AutoImageProcessor.from_pretrained(checkpoint, do_reduce_labels=True)

# from transformers import BeitImageProcessor
# image_processor = BeitImageProcessor(do_reduce_labels=True)


from torchvision.transforms import ColorJitter

jitter = ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)


def train_transforms(example_batch):
    images = [jitter(x) for x in example_batch["image"]]
    labels = [x for x in example_batch["annotation"]]
    inputs = image_processor(images, labels)
    return inputs


def val_transforms(example_batch):
    images = [x for x in example_batch["image"]]
    labels = [x for x in example_batch["annotation"]]
    inputs = image_processor(images, labels)
    return inputs


train_ds.set_transform(train_transforms)
test_ds.set_transform(val_transforms)

import evaluate

metric = evaluate.load("mean_iou")

num_labels = len(id2label)
# num_labels = 3

def compute_metrics(eval_pred):
    with torch.no_grad():
        logits, labels = eval_pred
        logits_tensor = torch.from_numpy(logits)
        logits_tensor = nn.functional.interpolate(
            logits_tensor,
            size=labels.shape[-2:],
            mode="bilinear",
            align_corners=False,
        ).argmax(dim=1)

        pred_labels = logits_tensor.detach().cpu().numpy()
        metrics = metric.compute(
            predictions=pred_labels,
            references=labels,
            num_labels=num_labels,
            ignore_index=255,
            reduce_labels=False,
        )
        for key, value in metrics.items():
            if type(value) is np.ndarray:
                metrics[key] = value.tolist()
        return metrics


from transformers import AutoModelForSemanticSegmentation, TrainingArguments, Trainer

from transformers import BeitForSemanticSegmentation
model = AutoModelForSemanticSegmentation.from_pretrained(checkpoint, id2label=id2label, label2id=label2id,
                                                         ignore_mismatched_sizes=True)
# model = BeitForSemanticSegmentation.from_pretrained(checkpoint, id2label=id2label, label2id=label2id)

# from transformers import BertConfig, BertModel

# Download model and configuration from huggingface.co and cache.

# model = BertModel.from_pretrained("bert-base-uncased")


training_args = TrainingArguments(
    output_dir="beit-b0-oce",
    learning_rate=6e-5,
    num_train_epochs=50,
    per_device_train_batch_size=2,
    per_device_eval_batch_size=2,
    save_total_limit=3,
    evaluation_strategy="steps",
    save_strategy="steps",
    save_steps=20,
    eval_steps=20,
    logging_steps=1,
    eval_accumulation_steps=5,
    remove_unused_columns=False
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_ds,
    eval_dataset=test_ds,
    compute_metrics=compute_metrics

)
trainer.train()
"
I want to train my model about beit on my dataset that is my code

### Expected behavior

"0%|          | 0/9100 [00:00<?, ?it/s]Traceback (most recent call last):
  File "D:\workplace\workplace_python\jupyter_lab\timm_model\beit_mode finetuned.py", line 188, in <module>
    trainer.train()
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\trainer.py", line 1664, in train
    return inner_training_loop(
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\trainer.py", line 1940, in _inner_training_loop
    tr_loss_step = self.training_step(model, inputs)
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\trainer.py", line 2735, in training_step
    loss = self.compute_loss(model, inputs)
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\trainer.py", line 2767, in compute_loss
    outputs = model(**inputs)
  File "D:\anaconda\envs\timm_model\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\models\beit\modeling_beit.py", line 1276, in forward
    loss = self.compute_loss(logits, auxiliary_logits, labels)
  File "D:\anaconda\envs\timm_model\lib\site-packages\transformers\models\beit\modeling_beit.py", line 1194, in compute_loss
    main_loss = loss_fct(upsampled_logits, labels)
  File "D:\anaconda\envs\timm_model\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\anaconda\envs\timm_model\lib\site-packages\torch\nn\modules\loss.py", line 1174, in forward
    return F.cross_entropy(input, target, weight=self.weight,
  File "D:\anaconda\envs\timm_model\lib\site-packages\torch\nn\functional.py", line 3029, in cross_entropy
    return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
IndexError: Target 62 is out of bounds.
  0%|          | 0/9100 [01:16<?, ?it/s]"
this is my worry,I can not find where is the ‘62’ out of bounds. 

Comments:
Please use the [forums](https://discuss.huggingface.co/) to debug your code as we keep the issues for bugs and feature requests only.

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.