Repository: OpenNMT/OpenNMT-py

Issue:
Function whole_word_mask with a bug in bart.py ??
the function change the tokens list with some tensor type,  and the problem would happen.
the configure of bart is:
`# BART
insert_ratio: 0.1 
random_ratio: 1.0 
mask_ratio: 0.15
rotate_ratio: 0.0 
mask_length: 'span-poisson'
poisson_lambda: 1
eeplace_length: 1 `
`TypeError: sequence item 2: expected str instance, Tensor found`
Traceback (most recent call last):
  File "/data/app/anaconda3/envs/pytorch-1.2/bin/onmt_train", line 11, in <module>
    load_entry_point('OpenNMT-py==2.0.0rc1', 'console_scripts', 'onmt_train')()
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/bin/train.py", line 169, in main
    train(opt)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/bin/train.py", line 154, in train
    train_process(opt, device_id=0)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/train_single.py", line 107, in main
    valid_steps=opt.valid_steps)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/trainer.py", line 225, in train
    self._accum_batches(train_iter)):
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/trainer.py", line 163, in _accum_batches
    for batch in iterator:
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/inputters/inputter.py", line 206, in __iter__
    for batch in self.iterable:
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/inputters/dynamic_iterator.py", line 238, in __iter__
    dataset = self.dataset_adapter(bucket)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/inputters/dynamic_iterator.py", line 70, in __call__
    examples = self._to_examples(bucket, is_train=self.is_train)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/inputters/dynamic_iterator.py", line 59, in _to_examples
    maybe_example = self._process(item, is_train=is_train)
  File "/data/app/anaconda3/envs/pytorch-1.2/lib/python3.7/site-packages/OpenNMT_py-2.0.0rc1-py3.7.egg/onmt/inputters/dynamic_iterator.py", line 41, in _process
    maybe_example['src'] = ' '.join(maybe_example['src'])


Discussion:

Resolving patch:
diff --git a/onmt/transforms/bart.py b/onmt/transforms/bart.py
--- a/onmt/transforms/bart.py
+++ b/onmt/transforms/bart.py
@@ -156,8 +156,9 @@ def whole_word_mask(self, tokens, p=1.0):  # text span mask/infilling
             lengths = torch.ones((n_mask,)).long()
         # assert is_word_start[-1] == 0
         word_starts = is_word_start.nonzero(as_tuple=False)
-        indices = word_starts[torch.randperm(word_starts.size(0))[
-            :n_mask]].squeeze(1)
+        indices = word_starts[
+            torch.randperm(word_starts.size(0))[:n_mask]
+        ].squeeze(1)
         mask_random = torch.FloatTensor(n_mask).uniform_() < self.random_ratio
 
         tokens_length = len(tokens)
@@ -168,12 +169,12 @@ def whole_word_mask(self, tokens, p=1.0):  # text span mask/infilling
             to_keep[indices] = 0
         else:
             # keep index, but replace it with [MASK]
-            for i in indices:
+            for i in indices.tolist():
                 tokens[i] = self.mask_tok
-            random_toks = torch.randint(
-                0, len(self.vocab), size=(mask_random.sum(),))
-            for i, rand_tok in zip(indices[mask_random], random_toks):
-                tokens[i] = rand_tok
+            random_tok_ids = torch.randint(
+                0, len(self.vocab), size=(mask_random.sum(),)).tolist()
+            for i, rid in zip(indices[mask_random].tolist(), random_tok_ids):
+                tokens[i] = self.vocab[rid]
 
         if tokens_length - 1 in indices:
             uncompleted = (indices != tokens_length - 1)
@@ -201,12 +202,13 @@ def whole_word_mask(self, tokens, p=1.0):  # text span mask/infilling
                     to_keep[indices] = 0
                 else:
                     # keep index, but replace it with [MASK]: 1 mask per token
-                    for i in indices:
+                    for i in indices.tolist():
                         tokens[i] = self.mask_tok
-                    random_toks = torch.randint(
-                        0, len(self.vocab), size=(mask_random.sum(),))
-                    for i, rand_tok in zip(indices[mask_random], random_toks):
-                        tokens[i] = rand_tok
+                    random_tok_ids = torch.randint(
+                        0, len(self.vocab), size=(mask_random.sum(),)).tolist()
+                    for i, rid in zip(
+                            indices[mask_random].tolist(), random_tok_ids):
+                        tokens[i] = self.vocab[rid]
         else:
             # A bit faster when all lengths are 1
             while indices.size(0) > 0:
@@ -219,17 +221,18 @@ def whole_word_mask(self, tokens, p=1.0):  # text span mask/infilling
                     to_keep[indices] = 0
                 else:
                     # keep index, but replace it with [MASK]
-                    for i in indices:
+                    for i in indices.tolist():
                         tokens[i] = self.mask_tok
-                    random_toks = torch.randint(
-                        0, len(self.vocab), size=(mask_random.sum(),))
-                    for i, rand_tok in zip(indices[mask_random], random_toks):
-                        tokens[i] = rand_tok
+                    random_tok_ids = torch.randint(
+                        0, len(self.vocab), size=(mask_random.sum(),)).tolist()
+                    for i, rid in zip(
+                            indices[mask_random].tolist(), random_tok_ids):
+                        tokens[i] = self.vocab[rid]
 
                 # assert tokens_length - 1 not in indices
 
-        tokens = [tok for tok, keep in zip(tokens, to_keep)
-                  if keep.item() is True]
+        tokens = [tok for tok, keep in zip(tokens, to_keep.tolist())
+                  if keep is True]
 
         if n_insert > 0:
             tokens = self.insertion_noise(tokens, n_insert / len(tokens))
@@ -257,7 +260,7 @@ def insertion_noise(self, tokens, p=1.0):
 
         assert all([item is not None for item in result]),\
             "Error when inserting noise."
-        return [tok for tok in result]
+        return result.tolist()
 
     def rolling_noise(self, tokens, p=1.0):
         if np.random.random() >= p:
@@ -341,7 +344,9 @@ def add_options(cls, parser):
                        "or N tokens. (use -1 for N)")
 
     def warm_up(self, vocabs):
-        self.vocab = vocabs
+        if vocabs is None:
+            return
+        self.vocab = vocabs['src'].itos
 
         subword_type = self.opts.src_subword_type
         if self.opts.mask_length == 'subword':
@@ -351,7 +356,7 @@ def warm_up(self, vocabs):
                     f'mask_length={self.opts.mask_length}!')
         is_joiner = (subword_type == 'bpe') if subword_type != 'none' else None
         self.bart_noise = BARTNoising(
-            vocabs,
+            self.vocab,
             mask_tok=DefaultTokens.MASK,
             mask_ratio=self.opts.mask_ratio,
             insert_ratio=self.opts.insert_ratio,


Test patch: