new tiny model

README.md

@@ -0,0 +1 @@
+This is a tiny-albert random model to be used for basic testing.

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "AlbertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 2,
+  "classifier_dropout_prob": 0.1,
+  "down_scale_factor": 1,
+  "embedding_size": 64,
+  "eos_token_id": 3,
+  "gap_size": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "inner_group_num": 1,
+  "intermediate_size": 128,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 64,
+  "model_type": "albert",
+  "net_structure_type": 0,
+  "num_attention_heads": 2,
+  "num_hidden_groups": 1,
+  "num_hidden_layers": 2,
+  "num_memory_blocks": 0,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float16",
+  "transformers_version": "4.9.0.dev0",
+  "type_vocab_size": 2,
+  "vocab_size": 5000
+}

make-tiny-albert.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a tiny random model
+#
+# It will be used then as "hf-internal-testing/tiny-albert"
+
+# ***To build from scratch***
+#
+# 1. clone sentencepiece into a parent dir
+# git clone https://github.com/google/sentencepiece
+#
+# 2. create a new repo at https://huggingface.co/new
+# make sure to choose 'hf-internal-testing' as the Owner
+#
+# 3. clone
+# git clone https://huggingface.co/hf-internal-testing/tiny-albert
+# cd tiny-albert
+
+# 4. start with some pre-existing script from one of the https://huggingface.co/hf-internal-testing/ tiny model repos, e.g.
+# wget https://huggingface.co/hf-internal-testing/tiny-albert/raw/main/make-tiny-albert.py
+# chmod a+x ./make-tiny-albert.py
+# mv ./make-tiny-albert.py ./make-tiny-albert.py
+#
+# 5. automatically rename things from the old names to new ones
+# perl -pi -e 's|Albert|Albert|g' make-*
+# perl -pi -e 's|albert|albert|g' make-*
+#
+# 6. edit and re-run this script while fixing it up
+# ./make-tiny-albert.py
+#
+# 7. add/commit/push
+# git add *
+# git commit -m "new tiny model"
+# git push
+
+# ***To update***
+#
+# 1. clone the existing repo
+# git clone https://huggingface.co/hf-internal-testing/tiny-albert
+# cd tiny-albert
+#
+# 2. edit and re-run this script after doing whatever changes are needed
+# ./make-tiny-albert.py
+#
+# 3. commit/push
+# git commit -m "new tiny model"
+# git push
+
+import sys
+import os
+
+# workaround for fast tokenizer protobuf issue, and it's much faster too!
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+from transformers import AlbertTokenizerFast, AlbertConfig, AlbertForMaskedLM
+
+mname_orig = "albert-base-v1"
+mname_tiny = "tiny-albert"
+
+### Tokenizer
+
+# Shrink the orig vocab to keep things small
+vocab_keep_items = 5000
+tmp_dir = f"/tmp/{mname_tiny}"
+vocab_orig_path = f"{tmp_dir}/spiece.model"
+vocab_short_path = f"{tmp_dir}/spiece-short.model"
+if 1: # set to 0 to skip this after running once to speed things up during tune up
+    # HACK: need the sentencepiece source to get sentencepiece_model_pb2, as it doesn't get installed
+    sys.path.append("../sentencepiece/python/src/sentencepiece")
+    import sentencepiece_model_pb2 as model
+    tokenizer_orig = AlbertTokenizerFast.from_pretrained(mname_orig)
+    tokenizer_orig.save_pretrained(tmp_dir)
+    with open(vocab_orig_path, 'rb') as f: data = f.read()
+    # adapted from https://blog.ceshine.net/post/trim-down-sentencepiece-vocabulary/
+    m = model.ModelProto()
+    m.ParseFromString(data)
+    print(f"Shrinking vocab from original {len(m.pieces)} dict items")
+    for i in range(len(m.pieces) - vocab_keep_items): _ = m.pieces.pop()
+    print(f"new dict {len(m.pieces)}")
+    with open(vocab_short_path, 'wb') as f: f.write(m.SerializeToString())
+    m = None
+
+tokenizer_fast_tiny = AlbertTokenizerFast(vocab_file=vocab_short_path)
+
+
+### Config
+
+config_tiny = AlbertConfig.from_pretrained(mname_orig)
+print(config_tiny)
+# remember to update this to the actual config as each model is different and then shrink the numbers
+config_tiny.update(dict(
+    vocab_size=vocab_keep_items+12,
+    embedding_size=64,
+    hidden_size=256,
+    intermediate_size=128,
+    max_position_embeddings=64,
+    num_attention_heads=2,
+    num_hidden_groups=1,
+    num_hidden_layers=2,
+))
+print("New config", config_tiny)
+
+### Model
+
+model_tiny = AlbertForMaskedLM(config_tiny)
+print(f"{mname_tiny}: num of params {model_tiny.num_parameters()}")
+model_tiny.resize_token_embeddings(len(tokenizer_fast_tiny))
+
+# Test
+inputs = tokenizer_fast_tiny("The capital of France is [MASK].", return_tensors="pt")
+outputs = model_tiny(**inputs)
+print("Test with normal tokenizer:", len(outputs.logits[0]))
+
+# Save
+model_tiny.half() # makes it smaller
+model_tiny.save_pretrained(".")
+tokenizer_fast_tiny.save_pretrained(".")
+
+#print(model_tiny)
+
+readme = "README.md"
+if not os.path.exists(readme):
+    with open(readme, "w") as f:
+        f.write(f"This is a {mname_tiny} random model to be used for basic testing.\n")
+
+print(f"Generated {mname_tiny}")

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9a411609d5818e28a2070e85ffcb65a482bfbe0d434024622d54a318fa53fa8
+size 1396878

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eedf777f037d8bc9ca0115cca50c7d18569292613c3a23a861fc5a6eff89b961
+size 321482

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "remove_space": true, "keep_accents": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sp_model_kwargs": {}, "tokenizer_class": "AlbertTokenizer"}