File size: 885 Bytes
c6e7238 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
from tokenizers import Tokenizer
from transformers import GPT2Tokenizer, GPT2TokenizerFast
def fetch_encoder(params):
no_dataset = params.get('no_dataset', False)
if no_dataset:
return None
dataset = next(iter(params['dataset_configs'].values())) # Get the first value from the dict
path = dataset["tokenizer_path"]
is_pretrained = dataset.get("tokenizer_is_pretrained", False)
if is_pretrained:
tok = GPT2TokenizerFast.from_pretrained(path)
# Will add a padding token id of 50257 at run-time
tok.add_special_tokens({'pad_token': '<|padding|>'})
return tok
return Tokenizer.from_file(path)
# GPT2Tokenizer and Tokenizer have different ways of fetching token ids
def encode(encoder, text, gpt=True):
result = encoder.encode(text)
if isinstance(result, list):
return result
return result.ids
|