File size: 214 Bytes
c59ebda
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
#%%
from transformers import GPT2Tokenizer

# %%
tok = GPT2Tokenizer.from_pretrained("D:/ML/Models/aragpt2-medium", use_fast=False)
# %%
tok.pad_token = tok.eos_token
#%%
tok.pad_token_id = [tok.eos_token_id]
# %%