katuni4ka
/

tiny-random-chatglm2

Feature Extraction

Generated from Trainer

Model card Files Files and versions Metrics Training metrics Community

katuni4ka commited on Mar 1

Commit

a571bad

•

1 Parent(s): 50917da

Update tokenization_chatglm.py

Files changed (1) hide show

tokenization_chatglm.py +1 -1

tokenization_chatglm.py CHANGED Viewed

@@ -4,7 +4,7 @@ from typing import List, Optional, Union, Dict
 from sentencepiece import SentencePieceProcessor
 from transformers import PreTrainedTokenizer
 from transformers.utils import logging, PaddingStrategy
-from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
 class SPTokenizer:

 from sentencepiece import SentencePieceProcessor
 from transformers import PreTrainedTokenizer
 from transformers.utils import logging, PaddingStrategy
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding, AddedToken
 class SPTokenizer: