|
|
|
import os |
|
from shutil import copyfile |
|
from typing import Optional, Tuple |
|
|
|
from tokenizers import processors |
|
|
|
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast |
|
from transformers.utils import is_sentencepiece_available, logging |
|
from transformers.utils.versions import require_version |
|
|
|
|
|
require_version("tokenizers>=0.13.3") |
|
|
|
|
|
|
|
logger = logging.get_logger(__name__) |
|
VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model", "tokenizer_file": "tokenizer.json"} |
|
|
|
|
|
DEFAULT_SYSTEM_PROMPT = """You are CrystalCoder, a helpful and intelligent assistant. Follow the instructions and answer the questions as accurately as possible.""" |
|
|
|
|
|
|
|
class CrystalCoderTokenizerFast(PreTrainedTokenizerFast): |
|
|
|
|
|
vocab_files_names = VOCAB_FILES_NAMES |
|
slow_tokenizer_class = None |
|
padding_side = "left" |
|
model_input_names = ["input_ids", "attention_mask"] |
|
|
|
def __init__( |
|
self, |
|
vocab_file=None, |
|
tokenizer_file=None, |
|
clean_up_tokenization_spaces=False, |
|
unk_token="<|unk|>", |
|
bos_token="<|startoftext|>", |
|
eos_token="<|endoftext|>", |
|
add_bos_token=False, |
|
add_eos_token=False, |
|
use_default_system_prompt=False, |
|
**kwargs, |
|
): |
|
super().__init__( |
|
vocab_file=vocab_file, |
|
tokenizer_file=tokenizer_file, |
|
clean_up_tokenization_spaces=clean_up_tokenization_spaces, |
|
unk_token=unk_token, |
|
bos_token=bos_token, |
|
eos_token=eos_token, |
|
use_default_system_prompt=use_default_system_prompt, |
|
**kwargs, |
|
) |
|
self._add_bos_token = add_bos_token |
|
self._add_eos_token = add_eos_token |
|
self.update_post_processor() |
|
self.use_default_system_prompt = use_default_system_prompt |
|
self.vocab_file = vocab_file |
|
|
|
@property |
|
def can_save_slow_tokenizer(self) -> bool: |
|
return os.path.isfile(self.vocab_file) if self.vocab_file else False |
|
|
|
def update_post_processor(self): |
|
""" |
|
Updates the underlying post processor with the current `bos_token` and `eos_token`. |
|
""" |
|
bos = self.bos_token |
|
bos_token_id = self.bos_token_id |
|
if bos is None and self.add_bos_token: |
|
raise ValueError("add_bos_token = True but bos_token = None") |
|
|
|
eos = self.eos_token |
|
eos_token_id = self.eos_token_id |
|
if eos is None and self.add_eos_token: |
|
raise ValueError("add_eos_token = True but eos_token = None") |
|
|
|
single = f"{(bos+':0 ') if self.add_bos_token else ''}$A:0{(' '+eos+':0') if self.add_eos_token else ''}" |
|
pair = f"{single}{(' '+bos+':1') if self.add_bos_token else ''} $B:1{(' '+eos+':1') if self.add_eos_token else ''}" |
|
|
|
special_tokens = [] |
|
if self.add_bos_token: |
|
special_tokens.append((bos, bos_token_id)) |
|
if self.add_eos_token: |
|
special_tokens.append((eos, eos_token_id)) |
|
self._tokenizer.post_processor = processors.TemplateProcessing( |
|
single=single, pair=pair, special_tokens=special_tokens |
|
) |
|
|
|
@property |
|
def add_eos_token(self): |
|
return self._add_eos_token |
|
|
|
@property |
|
def add_bos_token(self): |
|
return self._add_bos_token |
|
|
|
@add_eos_token.setter |
|
def add_eos_token(self, value): |
|
self._add_eos_token = value |
|
self.update_post_processor() |
|
|
|
@add_bos_token.setter |
|
def add_bos_token(self, value): |
|
self._add_bos_token = value |
|
self.update_post_processor() |
|
|
|
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: |
|
if not self.can_save_slow_tokenizer: |
|
raise ValueError( |
|
"Your fast tokenizer does not have the necessary information to save the vocabulary for a slow " |
|
"tokenizer." |
|
) |
|
|
|
if not os.path.isdir(save_directory): |
|
logger.error(f"Vocabulary path ({save_directory}) should be a directory") |
|
return |
|
out_vocab_file = os.path.join( |
|
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] |
|
) |
|
|
|
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): |
|
copyfile(self.vocab_file, out_vocab_file) |
|
|
|
return (out_vocab_file,) |
|
|
|
|
|
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): |
|
bos_token_id = [self.bos_token_id] if self.add_bos_token else [] |
|
eos_token_id = [self.eos_token_id] if self.add_eos_token else [] |
|
|
|
output = bos_token_id + token_ids_0 + eos_token_id |
|
|
|
if token_ids_1 is not None: |
|
output = output + bos_token_id + token_ids_1 + eos_token_id |
|
|
|
return output |
|
|
