bug fix

ud.py

@@ -1,10 +1,11 @@
+import os
 from transformers import TokenClassificationPipeline,DebertaV2TokenizerFast
 from transformers.models.bert_japanese.tokenization_bert_japanese import MecabTokenizer
 try:
   from transformers.utils import cached_file
 except:
   from transformers.file_utils import cached_path,hf_bucket_url
-  cached_file=lambda x,y:cached_path(hf_bucket_url(x,y))
+  cached_file=lambda x,y:os.path.join(x,y) if os.path.isdir(x) else cached_path(hf_bucket_url(x,y))
 
 class UniversalDependenciesPipeline(TokenClassificationPipeline):
   def _forward(self,model_inputs):
@@ -76,7 +77,6 @@ class MecabPreTokenizer(MecabTokenizer):
 
 class JumanDebertaV2TokenizerFast(DebertaV2TokenizerFast):
   def __init__(self,**kwargs):
-    import os
     from tokenizers.pre_tokenizers import PreTokenizer,Metaspace,Sequence
     super().__init__(**kwargs)
     d,r="/var/lib/mecab/dic/juman-utf8","/etc/mecabrc"
@@ -93,7 +93,6 @@ class JumanDebertaV2TokenizerFast(DebertaV2TokenizerFast):
     self.custom_pre_tokenizer=Sequence([PreTokenizer.custom(MecabPreTokenizer(mecab_dic=None,mecab_option="-d "+d+" -r "+r)),Metaspace()])
     self._tokenizer.pre_tokenizer=self.custom_pre_tokenizer
   def save_pretrained(self,save_directory,**kwargs):
-    import os
     import shutil
     from tokenizers.pre_tokenizers import Metaspace
     self._auto_map={"AutoTokenizer":[None,"ud.JumanDebertaV2TokenizerFast"]}