katuni4ka commited on
Commit
a571bad
1 Parent(s): 50917da

Update tokenization_chatglm.py

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -1
tokenization_chatglm.py CHANGED
@@ -4,7 +4,7 @@ from typing import List, Optional, Union, Dict
4
  from sentencepiece import SentencePieceProcessor
5
  from transformers import PreTrainedTokenizer
6
  from transformers.utils import logging, PaddingStrategy
7
- from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
8
 
9
 
10
  class SPTokenizer:
 
4
  from sentencepiece import SentencePieceProcessor
5
  from transformers import PreTrainedTokenizer
6
  from transformers.utils import logging, PaddingStrategy
7
+ from transformers.tokenization_utils_base import EncodedInput, BatchEncoding, AddedToken
8
 
9
 
10
  class SPTokenizer: