Repository: huggingface/transformers

Issue:
[CamemBert] Tokenizer function add_tokens doesn't work
## ❓ Questions & Help

Hi, 

I am trying to add new tokens to the CamemBert tokenizer, but when I run the function tokenizer.add_tokens, it doesn't seem to add any token at all :

`from transformers import CamembertTokenizer`
`tokenizer = CamembertTokenizer.from_pretrained('camembert-base')`
`tokenizer.add_tokens(['notfrenchword'])`

`Out[12]: 0`

Whereas with Bert model it works perfectly. Is this a bug or am I doing something wrong ?

Thanks



Discussion:
This method is **not** implemented into the CamemBERT tokenizer, at the moment.

> ##  Questions & Help
> Hi,
> 
> I am trying to add new tokens to the CamemBert tokenizer, but when I run the function tokenizer.add_tokens, it doesn't seem to add any token at all :
> 
> `from transformers import CamembertTokenizer`
> `tokenizer = CamembertTokenizer.from_pretrained('camembert-base')`
> `tokenizer.add_tokens(['notfrenchword'])`
> 
> `Out[12]: 0`
> 
> Whereas with Bert model it works perfectly. Is this a bug or am I doing something wrong ?
> 
> Thanks

Resolving patch:
diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -51,7 +51,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
 
     def __init__(self, vocab_file, bos_token="
", eos_token="
", sep_token="
",
                  cls_token="
", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>',
-                 additional_special_tokens=['
NOTUSED', '
NOTUSED'], **kwargs):
+                 additional_special_tokens=['
NOTUSED', '
NOTUSED'], **kwargs):
         super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
                                                  sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
                                                  mask_token=mask_token, additional_special_tokens=additional_special_tokens,
@@ -125,7 +125,7 @@ def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
 
     @property
     def vocab_size(self):
-        return self.fairseq_offset + len(self.sp_model)
+        return len(self.fairseq_tokens_to_ids) + len(self.sp_model)
 
     def _tokenize(self, text):
         return self.sp_model.EncodeAsPieces(text)
@@ -134,6 +134,9 @@ def _convert_token_to_id(self, token):
         """ Converts a token (str/unicode) in an id using the vocab. """
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
+        elif self.sp_model.PieceToId(token) == 0:
+            # Convert sentence piece unk token to fairseq unk token index
+            return self.unk_token_id
         return self.fairseq_offset + self.sp_model.PieceToId(token)
 
     def _convert_id_to_token(self, index):


Test patch:
diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py
--- a/transformers/tests/modeling_xlnet_test.py
+++ b/transformers/tests/modeling_xlnet_test.py
@@ -167,7 +167,7 @@ def create_and_check_xlnet_base_model(self, config, input_ids_1, input_ids_2, in
                 [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers)
 
         def create_and_check_xlnet_base_model_with_att_output(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask,
-                target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels):
+                    target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels):
             model = XLNetModel(config)
             model.eval()