BatchEncoding error
decode = self.tokenizer.batch_decode(response_tensors, skip_special_tokens=True)
lib/python3.9/site-packages/transformers/tokenization_utils_base.py", line 3469, in batch_decode
return [
python3.9/site-packages/transformers/tokenization_utils_base.py", line 3470, in
self.decode(
python3.9/site-packages/transformers/tokenization_utils_base.py", line 3509, in decode
return self._decode(
python3.9/site-packages/transformers/tokenization_utils.py", line 931, in _decode
filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
python3.9/site-packages/transformers/tokenization_utils.py", line 912, in convert_ids_to_tokens
tokens.append(self._convert_id_to_token(index))
File "/home/.cache/huggingface/modules/transformers_modules/chatglm2-6b/tokenization_chatglm.py", line 121, in _convert_id_to_token
return self.tokenizer.convert_id_to_token(index)
File "/home/.cache/huggingface/modules/transformers_modules/chatglm2-6b/tokenization_chatglm.py", line 60, in convert_id_to_token
return self.sp_model.IdToPiece(index)
File "/python3.9/site-packages/sentencepiece/init.py", line 1045, in _batched_func
return _func(self, arg)
File "/python3.9/site-packages/sentencepiece/init.py", line 1038, in _func
raise IndexError('piece id is out of range.')
IndexError: piece id is out of range.