Make _decode compatible with PreTrainedTokenizerBase

tokenization_codegen25.py

@@ -4,7 +4,7 @@
 # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
 """Tokenization classes for CodeGen2.5."""
 
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
 from transformers.utils import logging
@@ -168,7 +168,9 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.encoder.decode_single_token_bytes(index).decode("utf-8")
 
-    def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
+    def _decode(self, token_ids: Union[int, List[int]], skip_special_tokens: bool = False, **kwargs):
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
         if skip_special_tokens:
             token_ids = [t for t in token_ids if t not in self.all_special_ids]
         return self.encoder.decode(token_ids)