Repository: flairNLP/flair

Issue:
Package FastText embeddings in 25+ languages
We would like to scale up support for a bunch of different languages. To make this easier to get started training models in new languages, we should package FastText word embeddings for as many languages as possible - at least all languages for which interest has already been stated.

FastText packages [Wikipedia embeddings](https://fasttext.cc/docs/en/pretrained-vectors.html) and [Wiki-CommonCrawl](https://fasttext.cc/docs/en/crawl-vectors.html) embeddings. The first seem to work better on common newswire evaluation datasets, so we should probably package both.  


Discussion:

Resolving patch:
diff --git a/flair/embeddings.py b/flair/embeddings.py
--- a/flair/embeddings.py
+++ b/flair/embeddings.py
@@ -143,6 +143,7 @@ def __init__(self, embeddings: str):
 
         old_base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/'
         base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.3/'
+        embeddings_path_v4 = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4/'
 
         # GLOVE embeddings
         if embeddings.lower() == 'glove' or embeddings.lower() == 'en-glove':
@@ -164,13 +165,28 @@ def __init__(self, embeddings: str):
             cached_path(os.path.join(base_path, 'en-fasttext-news-300d-1M.vectors.npy'), cache_dir='embeddings')
             embeddings = cached_path(os.path.join(base_path, 'en-fasttext-news-300d-1M'), cache_dir='embeddings')
 
-        # other language fasttext embeddings
-        elif len(embeddings.lower()) == 2 and not embeddings.lower() == 'en':
-            cached_path(os.path.join(base_path, '{}-fasttext-300d-1M.vectors.npy'.format(embeddings)),
+        # two-letter language code wiki embeddings
+        elif len(embeddings.lower()) == 2:
+            cached_path(os.path.join(embeddings_path_v4, '{}-wiki-fasttext-300d-1M.vectors.npy'.format(embeddings)),
                         cache_dir='embeddings')
-            embeddings = cached_path(os.path.join(base_path, '{}-fasttext-300d-1M'.format(embeddings)),
+            embeddings = cached_path(os.path.join(embeddings_path_v4, '{}-wiki-fasttext-300d-1M'.format(embeddings)),
                                      cache_dir='embeddings')
 
+        # two-letter language code wiki embeddings
+        elif len(embeddings.lower()) == 7 and embeddings.endswith('-wiki'):
+            cached_path(os.path.join(embeddings_path_v4, '{}-wiki-fasttext-300d-1M.vectors.npy'.format(embeddings[:2])),
+                        cache_dir='embeddings')
+            embeddings = cached_path(os.path.join(embeddings_path_v4, '{}-wiki-fasttext-300d-1M'.format(embeddings[:2])),
+                                     cache_dir='embeddings')
+
+        # two-letter language code crawl embeddings
+        elif len(embeddings.lower()) == 8 and embeddings.endswith('-crawl'):
+            cached_path(os.path.join(embeddings_path_v4, '{}-crawl-fasttext-300d-1M.vectors.npy'.format(embeddings[:2])),
+                        cache_dir='embeddings')
+            embeddings = cached_path(
+                os.path.join(embeddings_path_v4, '{}-crawl-fasttext-300d-1M'.format(embeddings[:2])),
+                cache_dir='embeddings')
+
         elif not os.path.exists(embeddings):
             raise ValueError(f'The given embeddings "{embeddings}" is not available or is not a valid path.')
 
@@ -360,8 +376,17 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True, cache_dir
         """
         super().__init__()
 
+        # multilingual forward (English, German, French, Italian, Dutch, Polish)
+        if model.lower() == 'multi-forward':
+            base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4/lm-multi-forward-v0.1.pt'
+            model = cached_path(base_path, cache_dir='embeddings')
+        # multilingual backward  (English, German, French, Italian, Dutch, Polish)
+        elif model.lower() == 'multi-backward':
+            base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4/lm-multi-backward-v0.1.pt'
+            model = cached_path(base_path, cache_dir='embeddings')
+
         # news-english-forward
-        if model.lower() == 'news-forward':
+        elif model.lower() == 'news-forward':
             base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/lm-news-english-forward-v0.2rc.pt'
             model = cached_path(base_path, cache_dir='embeddings')
 
@@ -428,6 +453,15 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True, cache_dir
             base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.3/lm-bg-small-backward-v0.1.pt'
             model = cached_path(base_path, cache_dir='embeddings')
 
+        # Dutch forward
+        elif model.lower() == 'dutch-forward' or model.lower() == 'nl-forward':
+            base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4/lm-nl-large-forward-v0.1.pt'
+            model = cached_path(base_path, cache_dir='embeddings')
+        # Dutch backward
+        elif model.lower() == 'dutch-backward' or model.lower() == 'nl-backward':
+            base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4/lm-nl-large-backward-v0.1.pt'
+            model = cached_path(base_path, cache_dir='embeddings')
+
         elif not os.path.exists(model):
             raise ValueError(f'The given model "{model}" is not available or is not a valid path.')
 


Test patch:
diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py
--- a/tests/test_embeddings.py
+++ b/tests/test_embeddings.py
@@ -36,6 +36,13 @@ def test_fr():
 def test_it():
     load_and_apply_word_embeddings('it')
 
+@pytest.mark.slow
+def test_it():
+    load_and_apply_word_embeddings('it-wiki')
+
+@pytest.mark.slow
+def test_it():
+    load_and_apply_word_embeddings('it-crawl')
 
 @pytest.mark.slow
 def test_news_forward():