Repository: flairNLP/flair

Issue:
camemBERT
Hi,

A little question about French model.
Do you plan to include camemBERT (https://camembert-model.fr) model in a futur release soon ?

Thanks


Discussion:
I tested it locally for a French NER dataset and the results are really great (compared to the multilingual BERT model).

I'm going to add CamemBERT to Flair in the next days :)
Great news! Let us know when you it's done :)

Resolving patch:
diff --git a/flair/embeddings.py b/flair/embeddings.py
--- a/flair/embeddings.py
+++ b/flair/embeddings.py
@@ -25,6 +25,8 @@
 from transformers import (
     BertTokenizer,
     BertModel,
+    CamembertTokenizer,
+    CamembertModel,
     RobertaTokenizer,
     RobertaModel,
     TransfoXLTokenizer,
@@ -1554,6 +1556,78 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
         return sentences
 
 
+class CamembertEmbeddings(TokenEmbeddings):
+    def __init__(
+        self,
+        pretrained_model_name_or_path: str = "camembert-base",
+        layers: str = "-1",
+        pooling_operation: str = "first",
+        use_scalar_mix: bool = False,
+    ):
+        """CamemBERT, a Tasty French Language Model, as proposed by Martin et al. 2019.
+        :param pretrained_model_name_or_path: name or path of RoBERTa model
+        :param layers: comma-separated list of layers
+        :param pooling_operation: defines pooling operation for subwords
+        :param use_scalar_mix: defines the usage of scalar mix for specified layer(s)
+        """
+        super().__init__()
+
+        self.tokenizer = CamembertTokenizer.from_pretrained(
+            pretrained_model_name_or_path
+        )
+        self.model = CamembertModel.from_pretrained(
+            pretrained_model_name_or_path=pretrained_model_name_or_path,
+            output_hidden_states=True,
+        )
+        self.name = pretrained_model_name_or_path
+        self.layers: List[int] = [int(layer) for layer in layers.split(",")]
+        self.pooling_operation = pooling_operation
+        self.use_scalar_mix = use_scalar_mix
+        self.static_embeddings = True
+
+        dummy_sentence: Sentence = Sentence()
+        dummy_sentence.add_token(Token("hello"))
+        embedded_dummy = self.embed(dummy_sentence)
+        self.__embedding_length: int = len(
+            embedded_dummy[0].get_token(1).get_embedding()
+        )
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["tokenizer"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+
+        # 1-camembert-base -> camembert-base
+        self.tokenizer = self.tokenizer = CamembertTokenizer.from_pretrained(
+            "-".join(self.name.split("-")[1:])
+        )
+
+    @property
+    def embedding_length(self) -> int:
+        return self.__embedding_length
+
+    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
+        self.model.to(flair.device)
+        self.model.eval()
+
+        sentences = _get_transformer_sentence_embeddings(
+            sentences=sentences,
+            tokenizer=self.tokenizer,
+            model=self.model,
+            name=self.name,
+            layers=self.layers,
+            pooling_operation=self.pooling_operation,
+            use_scalar_mix=self.use_scalar_mix,
+            bos_token="
",
+            eos_token="
",
+        )
+
+        return sentences
+
+
 class CharacterEmbeddings(TokenEmbeddings):
     """Character embeddings of words, as proposed in Lample et al., 2016."""
 


Test patch:
diff --git a/tests/test_transformer_embeddings.py b/tests/test_transformer_embeddings.py
--- a/tests/test_transformer_embeddings.py
+++ b/tests/test_transformer_embeddings.py
@@ -4,6 +4,7 @@
 
 from flair.data import Sentence
 from flair.embeddings import (
+    CamembertEmbeddings,
     RoBERTaEmbeddings,
     OpenAIGPTEmbeddings,
     OpenAIGPT2Embeddings,
@@ -13,6 +14,8 @@
 )
 
 from transformers import (
+    CamembertModel,
+    CamembertTokenizer,
     RobertaModel,
     RobertaTokenizer,
     OpenAIGPTModel,
@@ -510,8 +513,6 @@ def test_xlnet_embeddings():
     with torch.no_grad():
         tokens = tokenizer.tokenize("
" + s + "
")
 
-        print(tokens)
-
         indexed_tokens = tokenizer.convert_tokens_to_ids(tokens)
         tokens_tensor = torch.tensor([indexed_tokens])
         tokens_tensor = tokens_tensor.to(flair.device)
@@ -661,8 +662,6 @@ def test_transformer_xl_embeddings():
     with torch.no_grad():
         tokens = tokenizer.tokenize(s + "<eos>")
 
-        print(tokens)
-
         indexed_tokens = tokenizer.convert_tokens_to_ids(tokens)
         tokens_tensor = torch.tensor([indexed_tokens])
         tokens_tensor = tokens_tensor.to(flair.device)
@@ -871,3 +870,134 @@ def embed_sentence(
     actual_embedding_size = len(sentence_mult_layers_scalar_mix.tokens[0].embedding)
 
     assert ref_embedding_size == actual_embedding_size
+
+
+@pytest.mark.slow
+def test_camembert_embeddings():
+    camembert_model: str = "camembert-base"
+
+    tokenizer = CamembertTokenizer.from_pretrained(camembert_model)
+    model = CamembertModel.from_pretrained(
+        pretrained_model_name_or_path=camembert_model, output_hidden_states=True
+    )
+    model.to(flair.device)
+    model.eval()
+
+    s: str = "J'aime le camembert !"
+
+    with torch.no_grad():
+        tokens = tokenizer.tokenize("
" + s + "
")
+
+        indexed_tokens = tokenizer.convert_tokens_to_ids(tokens)
+        tokens_tensor = torch.tensor([indexed_tokens])
+        tokens_tensor = tokens_tensor.to(flair.device)
+
+        hidden_states = model(tokens_tensor)[-1]
+
+        first_layer = hidden_states[1][0]
+
+    assert len(first_layer) == len(tokens)
+
+    #   0       1          2         3         4      5       6       7    8      9
+    #
+    # '
',   '▁J',      "'",     'aime',   '▁le', '▁ca', 'member', 't', '▁!', '
'
+    #           \          |         /         |      \       |        /   |
+    #                    J'aime                le         camembert        !
+    #
+    #                      0                   1              2            3
+
+    def embed_sentence(
+        sentence: str,
+        pooling_operation,
+        layers: str = "1",
+        use_scalar_mix: bool = False,
+    ) -> Sentence:
+        embeddings = CamembertEmbeddings(
+            pretrained_model_name_or_path=camembert_model,
+            layers=layers,
+            pooling_operation=pooling_operation,
+            use_scalar_mix=use_scalar_mix,
+        )
+        flair_sentence = Sentence(sentence)
+        embeddings.embed(flair_sentence)
+
+        return flair_sentence
+
+    # First subword embedding
+    sentence_first_subword = embed_sentence(sentence=s, pooling_operation="first")
+
+    camembert_first_subword_embedding_ref = first_layer[5].tolist()
+    camembert_first_subword_embedding_actual = sentence_first_subword.tokens[
+        2
+    ].embedding.tolist()
+
+    assert (
+        camembert_first_subword_embedding_ref
+        == camembert_first_subword_embedding_actual
+    )
+
+    # Last subword embedding
+    sentence_last_subword = embed_sentence(sentence=s, pooling_operation="last")
+
+    camembert_last_subword_embedding_ref = first_layer[7].tolist()
+    camembert_last_subword_embedding_actual = sentence_last_subword.tokens[
+        2
+    ].embedding.tolist()
+
+    assert (
+        camembert_last_subword_embedding_ref == camembert_last_subword_embedding_actual
+    )
+
+    # First and last subword embedding
+    sentence_first_last_subword = embed_sentence(
+        sentence=s, pooling_operation="first_last"
+    )
+
+    camembert_first_last_subword_embedding_ref = torch.cat(
+        [first_layer[5], first_layer[7]]
+    ).tolist()
+    camembert_first_last_subword_embedding_actual = sentence_first_last_subword.tokens[
+        2
+    ].embedding.tolist()
+
+    assert (
+        camembert_first_last_subword_embedding_ref
+        == camembert_first_last_subword_embedding_actual
+    )
+
+    # Mean of all subword embeddings
+    sentence_mean_subword = embed_sentence(sentence=s, pooling_operation="mean")
+
+    pcamembert_mean_subword_embedding_ref = calculate_mean_embedding(
+        [first_layer[5], first_layer[6], first_layer[7]]
+    ).tolist()
+    camembert_mean_subword_embedding_actual = sentence_mean_subword.tokens[
+        2
+    ].embedding.tolist()
+
+    assert (
+        pcamembert_mean_subword_embedding_ref == camembert_mean_subword_embedding_actual
+    )
+
+    # Check embedding dimension when using multiple layers
+    sentence_mult_layers = embed_sentence(
+        sentence="Paris", pooling_operation="first", layers="1,2,3,4"
+    )
+
+    ref_embedding_size = 4 * 768
+    actual_embedding_size = len(sentence_mult_layers.tokens[0].embedding)
+
+    assert ref_embedding_size == actual_embedding_size
+
+    # Check embedding dimension when using multiple layers and scalar mix
+    sentence_mult_layers_scalar_mix = embed_sentence(
+        sentence="TGW",
+        pooling_operation="first",
+        layers="1,2,3,4",
+        use_scalar_mix=True,
+    )
+
+    ref_embedding_size = 1 * 768
+    actual_embedding_size = len(sentence_mult_layers_scalar_mix.tokens[0].embedding)
+
+    assert ref_embedding_size == actual_embedding_size