Repository: flairNLP/flair

Issue:
Crosslingual MUSE embeddings
**Multiple languages in one word embedding space**

We often have multilingual data that we want to tag, but training data only for one language (typically English). One interesting solution to this problem is to use word embeddings for multiple languages that have been aligned into the same embedding space. In such _crosslingual embeddings_, the English word 'cat' would have a similar word embedding as the German translation 'Katze'. If we train a classifier or sequence labeler using such crosslingual embeddings, we can train a classifier with English training data and then also use it for German since the word embeddings correspond. 

![MUSE embeddings](https://raw.githubusercontent.com/facebookresearch/MUSE/master/outline_all.png)

Facebook's [MUSE](https://github.com/facebookresearch/MUSE) embeddings are exactly this: Embeddings for 30 languages aligned into one embedding space. 

**Goal**: Add MUSE embeddings to Flair with a new `CrosslingualMuseEmbeddings` class. When calling `.embed()` on a sentence, this class needs to (1) perform language identification of the sentence that is being embedded, (2) select the MUSE embedding for the detected language and (3) embed the sentence with this embedding. This will allow us to embed sentences in different languages into the same embedding space. 

We've done some initial testing and this indeed allows us to train models that work for 30 languages with training data for a single languages.



Discussion:

Resolving patch:
diff --git a/flair/data.py b/flair/data.py
--- a/flair/data.py
+++ b/flair/data.py
@@ -318,6 +318,7 @@ def __init__(
         text: str = None,
         use_tokenizer: bool = False,
         labels: Union[List[Label], List[str]] = None,
+        language_code: str = None,
     ):
 
         super(Sentence, self).__init__()
@@ -330,6 +331,8 @@ def __init__(
 
         self._embeddings: Dict = {}
 
+        self.language_code: str = language_code
+
         # if text is passed, instantiate sentence with tokens (words)
         if text is not None:
 
@@ -674,6 +677,17 @@ def __str__(self) -> str:
     def __len__(self) -> int:
         return len(self.tokens)
 
+    def get_language_code(self) -> str:
+        if self.language_code is None:
+            import langdetect
+
+            try:
+                self.language_code = langdetect.detect(self.to_plain_string())
+            except:
+                self.language_code = "en"
+
+        return self.language_code
+
 
 class Corpus:
     def __init__(
diff --git a/flair/embeddings.py b/flair/embeddings.py
--- a/flair/embeddings.py
+++ b/flair/embeddings.py
@@ -460,6 +460,104 @@ def __setstate__(self, state):
         state["spm"] = sentencepiece_load(self.model_file)
 
 
+class MuseCrosslingualEmbeddings(TokenEmbeddings):
+    def __init__(self,):
+        self.name: str = f"muse-crosslingual"
+        self.static_embeddings = True
+        self.__embedding_length: int = 300
+        self.language_embeddings = {}
+        super().__init__()
+
+    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
+
+        for i, sentence in enumerate(sentences):
+
+            language_code = sentence.get_language_code()
+            print(language_code)
+            supported = [
+                "en",
+                "de",
+                "bg",
+                "ca",
+                "hr",
+                "cs",
+                "da",
+                "nl",
+                "et",
+                "fi",
+                "fr",
+                "el",
+                "he",
+                "hu",
+                "id",
+                "it",
+                "mk",
+                "no",
+                "pl",
+                "pt",
+                "ro",
+                "ru",
+                "sk",
+            ]
+            if language_code not in supported:
+                language_code = "en"
+
+            if language_code not in self.language_embeddings:
+                log.info(f"Loading up MUSE embeddings for '{language_code}'!")
+                # download if necessary
+                webpath = "https://alan-nlp.s3.eu-central-1.amazonaws.com/resources/embeddings-muse"
+                cache_dir = Path("embeddings") / "MUSE"
+                cached_path(
+                    f"{webpath}/muse.{language_code}.vec.gensim.vectors.npy",
+                    cache_dir=cache_dir,
+                )
+                embeddings_file = cached_path(
+                    f"{webpath}/muse.{language_code}.vec.gensim", cache_dir=cache_dir
+                )
+
+                # load the model
+                self.language_embeddings[
+                    language_code
+                ] = gensim.models.KeyedVectors.load(str(embeddings_file))
+
+            current_embedding_model = self.language_embeddings[language_code]
+
+            for token, token_idx in zip(sentence.tokens, range(len(sentence.tokens))):
+
+                if "field" not in self.__dict__ or self.field is None:
+                    word = token.text
+                else:
+                    word = token.get_tag(self.field).value
+
+                if word in current_embedding_model:
+                    word_embedding = current_embedding_model[word]
+                elif word.lower() in current_embedding_model:
+                    word_embedding = current_embedding_model[word.lower()]
+                elif re.sub(r"\d", "#", word.lower()) in current_embedding_model:
+                    word_embedding = current_embedding_model[
+                        re.sub(r"\d", "#", word.lower())
+                    ]
+                elif re.sub(r"\d", "0", word.lower()) in current_embedding_model:
+                    word_embedding = current_embedding_model[
+                        re.sub(r"\d", "0", word.lower())
+                    ]
+                else:
+                    word_embedding = np.zeros(self.embedding_length, dtype="float")
+
+                word_embedding = torch.FloatTensor(word_embedding)
+
+                token.set_embedding(self.name, word_embedding)
+
+        return sentences
+
+    @property
+    def embedding_length(self) -> int:
+        return self.__embedding_length
+
+    def __str__(self):
+        return self.name
+
+
 class BytePairEmbeddings(TokenEmbeddings):
     def __init__(
         self,


Test patch: