Repository: dmlc/gluon-nlp

Issue:
Missing wiki_cn Bert vocab artifacts on S3
@szha, following https://github.com/dmlc/gluon-nlp/pull/655 please copy 

`aws s3 cp s3://apache-mxnet/gluon/dataset/vocab/wiki_cn_cased-ddebd8f3.zip s3://apache-mxnet/gluon/dataset/vocab/wiki_cn-ddebd8f3.zip`

Both are required, as `wiki_cn` is an alias of `wiki_cn_cased`.

This bug must not be closed before tests are added that would have caught this error.


Discussion:
@kenjewu FYI
done

Resolving patch:
diff --git a/scripts/bert/export/hybrid_bert.py b/scripts/bert/export/hybrid_bert.py
--- a/scripts/bert/export/hybrid_bert.py
+++ b/scripts/bert/export/hybrid_bert.py
@@ -25,7 +25,6 @@
 
 import os
 import math
-import warnings
 
 import mxnet as mx
 from mxnet.gluon import Block, HybridBlock
@@ -766,9 +765,6 @@ def get_hybrid_bert_model(model_name=None, dataset_name=None, vocab=None,
                                 use_residual=predefined_args['use_residual'],
                                 input_size=input_size,
                                 seq_length=seq_length)
-    if dataset_name in ['wiki_cn', 'wiki_multilingual']:
-        warnings.warn('wiki_cn/wiki_multilingual will be deprecated.'
-                      ' Please use wiki_cn_cased/wiki_multilingual_uncased instead.')
     bert_vocab = _load_vocab(dataset_name, vocab, root, cls=BERTVocab)
     # BERT
     net = HybridBERTModel(encoder, len(bert_vocab),
diff --git a/src/gluonnlp/data/utils.py b/src/gluonnlp/data/utils.py
--- a/src/gluonnlp/data/utils.py
+++ b/src/gluonnlp/data/utils.py
@@ -225,9 +225,7 @@ def _slice_pad_length(num_items, length, overlap=0):
                'book_corpus_wiki_en_uncased': 'a66073971aa0b1a262453fe51342e57166a8abcf',
                'wiki_multilingual_cased': '0247cb442074237c38c62021f36b7a4dbd2e55f7',
                'wiki_cn_cased': 'ddebd8f3867bca5a61023f73326fb125cf12b4f5',
-               'wiki_cn': 'ddebd8f3867bca5a61023f73326fb125cf12b4f5',
                'wiki_multilingual_uncased': '2b2514cc539047b9179e9d98a4e68c36db05c97a',
-               'wiki_multilingual': '2b2514cc539047b9179e9d98a4e68c36db05c97a',
                'scibert_scivocab_uncased': '2d2566bfc416790ab2646ab0ada36ba628628d60',
                'scibert_scivocab_cased': '2c714475b521ab8542cb65e46259f6bfeed8041b',
                'scibert_basevocab_uncased': '80ef760a6bdafec68c99b691c94ebbb918c90d02',
diff --git a/src/gluonnlp/model/bert.py b/src/gluonnlp/model/bert.py
--- a/src/gluonnlp/model/bert.py
+++ b/src/gluonnlp/model/bert.py
@@ -22,7 +22,6 @@
            'BERTLayerNorm', 'bert_12_768_12', 'bert_24_1024_16', 'get_bert_model']
 
 import os
-import warnings
 from mxnet.gluon import Block
 from mxnet.gluon import nn
 from mxnet.gluon.model_zoo import model_store
@@ -483,10 +482,8 @@ def _decode(self, sequence, masked_positions):
     {name: checksum for checksum, name in [
         ('5656dac6965b5054147b0375337d5a6a7a2ff832', 'bert_12_768_12_book_corpus_wiki_en_cased'),
         ('75cc780f085e8007b3bf6769c6348bb1ff9a3074', 'bert_12_768_12_book_corpus_wiki_en_uncased'),
-        ('237f39851b24f0b56d70aa20efd50095e3926e26', 'bert_12_768_12_wiki_multilingual'),
         ('237f39851b24f0b56d70aa20efd50095e3926e26', 'bert_12_768_12_wiki_multilingual_uncased'),
         ('b0f57a207f85a7d361bb79de80756a8c9a4276f7', 'bert_12_768_12_wiki_multilingual_cased'),
-        ('885ebb9adc249a170c5576e90e88cfd1bbd98da6', 'bert_12_768_12_wiki_cn'),
         ('885ebb9adc249a170c5576e90e88cfd1bbd98da6', 'bert_12_768_12_wiki_cn_cased'),
         ('4e685a966f8bf07d533bd6b0e06c04136f23f620', 'bert_24_1024_16_book_corpus_wiki_en_cased'),
         ('24551e1446180e045019a87fc4ffbf714d99c0b5', 'bert_24_1024_16_book_corpus_wiki_en_uncased'),
@@ -756,9 +753,6 @@ def get_bert_model(model_name=None, dataset_name=None, vocab=None, pretrained=Tr
                           use_residual=predefined_args['use_residual'])
     # bert_vocab
     from ..vocab import BERTVocab
-    if dataset_name in ['wiki_cn', 'wiki_multilingual']:
-        warnings.warn('wiki_cn/wiki_multilingual will be deprecated. '
-                      'Please use wiki_cn_cased/wiki_multilingual_uncased instead.')
     bert_vocab = _load_vocab(dataset_name, vocab, root, cls=BERTVocab)
     # BERT
     net = BERTModel(encoder, len(bert_vocab),


Test patch:
diff --git a/tests/unittest/test_models.py b/tests/unittest/test_models.py
--- a/tests/unittest/test_models.py
+++ b/tests/unittest/test_models.py
@@ -107,7 +107,7 @@ def test_pretrained_bert_models(disable_missing_parameters):
             'wiki_multilingual_cased', 'wiki_cn_cased', 'scibert_scivocab_uncased',
             'scibert_scivocab_cased', 'scibert_basevocab_uncased', 'scibert_basevocab_cased',
             'biobert_v1.0_pmc_cased', 'biobert_v1.0_pubmed_cased', 'biobert_v1.0_pubmed_pmc_cased',
-            'biobert_v1.1_pubmed_cased', 'clinicalbert_uncased'
+            'biobert_v1.1_pubmed_cased', 'clinicalbert_uncased',
         ],
         'bert_24_1024_16': ['book_corpus_wiki_en_uncased', 'book_corpus_wiki_en_cased']
     }