yang commited on
Commit
e1637b4
1 Parent(s): 0547ed3

Update tokenization_qwen.py for error ''QWenTokenizer' object has no attribute 'IMAGE_ST''

Browse files
Files changed (1) hide show
  1. tokenization_qwen.py +1 -2
tokenization_qwen.py CHANGED
@@ -117,7 +117,6 @@ class QWenTokenizer(PreTrainedTokenizer):
117
  quad_end_tag='</quad>',
118
  **kwargs,
119
  ):
120
- super().__init__(**kwargs)
121
  self.image_start_tag = image_start_tag
122
  self.image_end_tag = image_end_tag
123
  self.image_pad_tag = image_pad_tag
@@ -134,7 +133,7 @@ class QWenTokenizer(PreTrainedTokenizer):
134
  image_start_tag, image_end_tag,
135
  image_pad_tag
136
  )
137
-
138
  self.errors = errors # how to handle errors in decoding
139
 
140
  self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
 
117
  quad_end_tag='</quad>',
118
  **kwargs,
119
  ):
 
120
  self.image_start_tag = image_start_tag
121
  self.image_end_tag = image_end_tag
122
  self.image_pad_tag = image_pad_tag
 
133
  image_start_tag, image_end_tag,
134
  image_pad_tag
135
  )
136
+ super().__init__(**kwargs)
137
  self.errors = errors # how to handle errors in decoding
138
 
139
  self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]