yang
commited on
Commit
•
e1637b4
1
Parent(s):
0547ed3
Update tokenization_qwen.py for error ''QWenTokenizer' object has no attribute 'IMAGE_ST''
Browse files- tokenization_qwen.py +1 -2
tokenization_qwen.py
CHANGED
|
@@ -117,7 +117,6 @@ class QWenTokenizer(PreTrainedTokenizer):
|
|
| 117 |
quad_end_tag='</quad>',
|
| 118 |
**kwargs,
|
| 119 |
):
|
| 120 |
-
super().__init__(**kwargs)
|
| 121 |
self.image_start_tag = image_start_tag
|
| 122 |
self.image_end_tag = image_end_tag
|
| 123 |
self.image_pad_tag = image_pad_tag
|
|
@@ -134,7 +133,7 @@ class QWenTokenizer(PreTrainedTokenizer):
|
|
| 134 |
image_start_tag, image_end_tag,
|
| 135 |
image_pad_tag
|
| 136 |
)
|
| 137 |
-
|
| 138 |
self.errors = errors # how to handle errors in decoding
|
| 139 |
|
| 140 |
self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
|
|
|
|
| 117 |
quad_end_tag='</quad>',
|
| 118 |
**kwargs,
|
| 119 |
):
|
|
|
|
| 120 |
self.image_start_tag = image_start_tag
|
| 121 |
self.image_end_tag = image_end_tag
|
| 122 |
self.image_pad_tag = image_pad_tag
|
|
|
|
| 133 |
image_start_tag, image_end_tag,
|
| 134 |
image_pad_tag
|
| 135 |
)
|
| 136 |
+
super().__init__(**kwargs)
|
| 137 |
self.errors = errors # how to handle errors in decoding
|
| 138 |
|
| 139 |
self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
|