Получил вот такую ошибку в строке с обучением trainer.train():
ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py in convert_to_tensors(self, tensor_type, prepend_batch_axis)
    776                 if not is_tensor(value):
--> 777                     tensor = as_tensor(value)
    778 

13 frames
ValueError: expected sequence of length 512 at dim 1 (got 284)

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py in convert_to_tensors(self, tensor_type, prepend_batch_axis)
    791                         "Please see if a fast version of this tokenizer is available to have this feature available."
    792                     ) from e
--> 793                 raise ValueError(
    794                     "Unable to create tensor, you should probably activate truncation and/or padding with"
    795                     " 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your"

ValueError: Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your features (`labels` in this case) have excessive nesting (inputs type `list` where type `int` is expected).

Вот мой код целиком:
''' скачивание и подключение библиотек '''
!python -m pip install spacy
!python -m pip install datasets
!python -m pip install transformers
!python -m pip install evaluate
!python -m pip install seqeval
!python -m pip install tensorboard

import os
import random
from typing import Dict, List, Tuple, Union
import matplotlib.colors as mcolors
from nltk.tokenize.treebank import TreebankWordDetokenizer
from spacy import displacy
import evaluate
import numpy as np
from datasets import load_dataset
import torch
from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import DataCollatorForTokenClassification, TrainingArguments, Trainer
from transformers import pipeline

''' фиксирование рандом-сидов и загрузка датасета '''
RANDOM_SEED = 42
random.seed(RANDOM_SEED)
torch.manual_seed(RANDOM_SEED)
torch.cuda.manual_seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)

DATASET_NAME = 'adsabs/WIESP2022-NER'
trainset = load_dataset(DATASET_NAME, split='train')
valset = load_dataset(DATASET_NAME, split='validation')
testset = load_dataset(DATASET_NAME, split='test')

# Определение меток и их отображение
label_set = set()
for it in trainset['ner_tags']:
    label_set |= set(it)
label_list = ['O'] + sorted(list(label_set - {'O'}))

entity_classes = sorted(list(set(
    map(lambda it: it[2:], filter(lambda it: it != 'O', label_list))
)))
entity_colors = [mcolors.rgb2hex((0.5 + random.random() / 2, 0.5 + random.random() / 2, 0.5 + random.random() / 2)) 
                 for _ in range(len(entity_classes))]

''' создание seq2seq-модели '''
from transformers import T5Tokenizer, T5ForConditionalGeneration

SEQ2SEQ_MODEL_NAME = "t5-small"
seq2seq_tokenizer = T5Tokenizer.from_pretrained(SEQ2SEQ_MODEL_NAME)
seq2seq_model = T5ForConditionalGeneration.from_pretrained(SEQ2SEQ_MODEL_NAME)

# Пример промпта
PROMPT_TEMPLATE = "Распознай все сущности: {text}"

# Подготовка данных для seq2seq
def prepare_seq2seq_data(example):
    text = TreebankWordDetokenizer().detokenize(example['tokens'])
    input_text = PROMPT_TEMPLATE.format(text=text)
    target_text = " ".join([f"{tok}:{tag}" for tok, tag in zip(example['tokens'], example['ner_tags'])])
    return {
        "input_text": input_text,
        "target_text": target_text
    }

seq2seq_trainset = trainset.map(prepare_seq2seq_data)
seq2seq_valset = valset.map(prepare_seq2seq_data)

# Преобразование в формат для модели
def tokenize_seq2seq_data(example):
    model_inputs = seq2seq_tokenizer(example["input_text"], max_length=512, truncation=True)
    labels = seq2seq_tokenizer(example["target_text"], max_length=512, truncation=True).input_ids
    model_inputs["labels"] = labels
    return model_inputs

tokenized_trainset = seq2seq_trainset.map(tokenize_seq2seq_data, batched=True)
tokenized_valset = seq2seq_valset.map(tokenize_seq2seq_data, batched=True)

''' обучение seq2seq '''
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments

training_args = Seq2SeqTrainingArguments(
    output_dir="./seq2seq_model",
    evaluation_strategy="epoch",
    save_strategy="epoch",
    learning_rate=5e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=5,
    weight_decay=0.01,
    save_total_limit=2,
    predict_with_generate=True,
    logging_dir='./logs',
    logging_strategy="epoch"
)

trainer = Seq2SeqTrainer(
    model=seq2seq_model,
    args=training_args,
    train_dataset=tokenized_trainset,
    eval_dataset=tokenized_valset,
    tokenizer=seq2seq_tokenizer,
)

trainer.train()

''' предсказания и тестирование '''
for test_example in testset:
    input_text = PROMPT_TEMPLATE.format(text=TreebankWordDetokenizer().detokenize(test_example['tokens']))
    inputs = seq2seq_tokenizer(input_text, return_tensors="pt")
    outputs = seq2seq_model.generate(**inputs)
    predictions = seq2seq_tokenizer.decode(outputs[0], skip_special_tokens=True)
    print(predictions)

# Оставшиеся части кода можно использовать для визуализации или анализа результатов