Commit
•
af10dff
1
Parent(s):
03c7fa5
Upload 2 files
Browse files- sentencepiece_bpe.py +1 -0
- tokenization_fairseq_t5.py +2 -4
sentencepiece_bpe.py
CHANGED
@@ -28,6 +28,7 @@ class SentencepieceConfig:
|
|
28 |
|
29 |
class SentencepieceBPE(object):
|
30 |
def __init__(self, cfg):
|
|
|
31 |
self.enable_sampling = cfg.sentencepiece_enable_sampling
|
32 |
self.alpha = cfg.sentencepiece_alpha
|
33 |
sentencepiece_model = cfg.sentencepiece_model
|
|
|
28 |
|
29 |
class SentencepieceBPE(object):
|
30 |
def __init__(self, cfg):
|
31 |
+
cfg = SentencepieceConfig(**cfg)
|
32 |
self.enable_sampling = cfg.sentencepiece_enable_sampling
|
33 |
self.alpha = cfg.sentencepiece_alpha
|
34 |
sentencepiece_model = cfg.sentencepiece_model
|
tokenization_fairseq_t5.py
CHANGED
@@ -47,10 +47,8 @@ class FairseqT5Tokenizer(PreTrainedTokenizer):
|
|
47 |
)
|
48 |
)
|
49 |
self.fs_bpe = SentencepieceBPE(
|
50 |
-
|
51 |
-
|
52 |
-
sentencepiece_model=sp_path,
|
53 |
-
)
|
54 |
)
|
55 |
)
|
56 |
self.fs_dict = Dictionary.load(dict_path)
|
|
|
47 |
)
|
48 |
)
|
49 |
self.fs_bpe = SentencepieceBPE(
|
50 |
+
dict(
|
51 |
+
sentencepiece_model=sp_path,
|
|
|
|
|
52 |
)
|
53 |
)
|
54 |
self.fs_dict = Dictionary.load(dict_path)
|