from transformers import GPT2LMHeadModel, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("./new_tokenizer/")

sample_text = "21 世紀 の 資本 論"

print(tokenizer.encode(sample_text))