| from fairseq.data.encoders.gpt2_bpe import get_encoder | |
| bpe = get_encoder('/workspace/fairseq-models/data/vocab/gpt2/encoder.json', '/workspace/fairseq-models/data/vocab/gpt2/vocab.bpe') | |
| codes = bpe.encode('Hello world') | |
| print(codes) | |
| print(bpe.decode(codes)) | |
| test_str = 'Leonardo DiCaprio was born in Los Angeles' | |
| print(bpe.bpe(test_str)) | |
| codes = bpe.encode(test_str) | |
| print(codes) | |
| print(bpe.decode(codes)) | |