Markus28 commited on
Commit
9689b77
1 Parent(s): 151f328

fix: added trust_remote_code to tokenizer

Browse files
Files changed (1) hide show
  1. modeling_bert.py +2 -2
modeling_bert.py CHANGED
@@ -377,7 +377,7 @@ class BertModel(BertPreTrainedModel):
377
  if self.emb_pooler is not None:
378
  from transformers import AutoTokenizer
379
 
380
- self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path)
381
  else:
382
  self.tokenizer = None
383
 
@@ -507,7 +507,7 @@ class BertModel(BertPreTrainedModel):
507
  self.emb_pooler = 'mean'
508
  from transformers import AutoTokenizer
509
 
510
- self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path)
511
  if self.emb_pooler != 'mean':
512
  raise NotImplementedError
513
 
 
377
  if self.emb_pooler is not None:
378
  from transformers import AutoTokenizer
379
 
380
+ self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path, trust_remote_code=True)
381
  else:
382
  self.tokenizer = None
383
 
 
507
  self.emb_pooler = 'mean'
508
  from transformers import AutoTokenizer
509
 
510
+ self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path, trust_remote_code=True)
511
  if self.emb_pooler != 'mean':
512
  raise NotImplementedError
513