Crystalcareai commited on
Commit
c99aadf
1 Parent(s): db7dc87

Update tokenization_gemmoe.py

Browse files
Files changed (1) hide show
  1. tokenization_gemmoe.py +3 -1
tokenization_gemmoe.py CHANGED
@@ -14,14 +14,16 @@
14
  # limitations under the License.
15
  """Tokenization classes for Gemmoe."""
16
  import os
17
- import logging
18
  from shutil import copyfile
19
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
20
 
21
  import sentencepiece as spm
22
 
 
 
23
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
24
 
 
25
  if TYPE_CHECKING:
26
  pass
27
 
 
14
  # limitations under the License.
15
  """Tokenization classes for Gemmoe."""
16
  import os
 
17
  from shutil import copyfile
18
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
19
 
20
  import sentencepiece as spm
21
 
22
+ from transformers.utils import logging
23
+
24
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
25
 
26
+
27
  if TYPE_CHECKING:
28
  pass
29