gonglinyuan commited on
Commit
17cde43
1 Parent(s): 482151e

Upload 2 files

Browse files
sentencepiece_bpe.py CHANGED
@@ -28,6 +28,7 @@ class SentencepieceConfig:
28
 
29
  class SentencepieceBPE(object):
30
  def __init__(self, cfg):
 
31
  self.enable_sampling = cfg.sentencepiece_enable_sampling
32
  self.alpha = cfg.sentencepiece_alpha
33
  sentencepiece_model = cfg.sentencepiece_model
 
28
 
29
  class SentencepieceBPE(object):
30
  def __init__(self, cfg):
31
+ cfg = SentencepieceConfig(**cfg)
32
  self.enable_sampling = cfg.sentencepiece_enable_sampling
33
  self.alpha = cfg.sentencepiece_alpha
34
  sentencepiece_model = cfg.sentencepiece_model
tokenization_fairseq_t5.py CHANGED
@@ -47,10 +47,8 @@ class FairseqT5Tokenizer(PreTrainedTokenizer):
47
  )
48
  )
49
  self.fs_bpe = SentencepieceBPE(
50
- DictConfig(
51
- dict(
52
- sentencepiece_model=sp_path,
53
- )
54
  )
55
  )
56
  self.fs_dict = Dictionary.load(dict_path)
 
47
  )
48
  )
49
  self.fs_bpe = SentencepieceBPE(
50
+ dict(
51
+ sentencepiece_model=sp_path,
 
 
52
  )
53
  )
54
  self.fs_dict = Dictionary.load(dict_path)