tiny mt5 random model

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 256,
+  "d_kv": 8,
+  "d_model": 64,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 4,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "transformers_version": "4.6.0.dev0",
+  "use_cache": true,
+  "vocab_size": 5100
+}

mt5-make-tiny-model.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a smallish random model, with a few layers to test things like MP/PP, where
+# tiny and tiner models are too too small
+#
+# It will be used then as "stas/mt5-tiny-random"
+
+# To build:
+# 1. clone sentencepiece into this dir
+# git clone https://github.com/google/sentencepiece
+#
+# 2. run this script
+
+from pathlib import Path
+import json
+import tempfile
+
+from transformers import MT5Tokenizer, MT5TokenizerFast, MT5Config, MT5ForConditionalGeneration
+from transformers.models.t5.tokenization_t5 import VOCAB_FILES_NAMES
+
+mname_from = "google/mt5-small"
+mname_very_small = "mt5-tiny-random"
+
+tokenizer = MT5Tokenizer.from_pretrained(mname_from)
+config = MT5Config.from_pretrained(mname_from)
+#tokenizer_fast = MT5TokenizerFast.from_pretrained(mname_from)
+
+# Shrink the vocab of mt5-small
+import sys
+# HACK: need the sentencepiece source to get sentencepiece_model_pb2, as it doesn't get installed
+sys.path.append("./sentencepiece/python/src/sentencepiece")
+import sentencepiece_model_pb2 as model
+
+tmp_dir = "/tmp/mt5-small"
+tokenizer.save_pretrained(tmp_dir)
+file = tmp_dir + "/spiece.model"
+with open(file, 'rb') as f: data = f.read()
+
+# adapted from https://blog.ceshine.net/post/trim-down-sentencepiece-vocabulary/
+m = model.ModelProto()
+m.ParseFromString(data)
+
+keep_items = 5000
+
+print("Shrinking vocab")
+print(f"original dict {len(m.pieces)}")
+for i in range(len(m.pieces)-keep_items): _ = m.pieces.pop()
+print(f"new dict {len(m.pieces)}")
+
+with open(tmp_dir + "/spiece-short.model", 'wb') as f:
+    f.write(m.SerializeToString())
+
+tokenizer = MT5Tokenizer(vocab_file=tmp_dir + "/spiece-short.model")
+
+config.update(dict(
+    vocab_size=keep_items+12,
+    d_model=64,
+    d_ff=256,
+    d_kv=8,
+    num_layers=8,
+    num_decoder_layers=8,
+    num_heads=4,
+    relative_attention_num_buckets=32,
+))
+print("new config", config)
+
+very_small_model = MT5ForConditionalGeneration(config)
+print(f"num of params {very_small_model.num_parameters()}")
+very_small_model.resize_token_embeddings(len(tokenizer))
+
+# Test
+src_texts = ["A long paragraph for summarization.", "Another paragraph for summarization."]
+tgt_texts = ["Summary of the text.", "Another summary."]
+
+batch = tokenizer.prepare_seq2seq_batch(src_texts, tgt_texts, return_tensors="pt")
+outputs = very_small_model(**batch)
+
+print("test output:", len(outputs.logits[0]))
+
+# Save
+very_small_model.half() # makes it smaller
+very_small_model.save_pretrained(mname_very_small)
+config.save_pretrained(mname_very_small)
+tokenizer.save_pretrained(mname_very_small)
+#tokenizer_fast.save_pretrained(mname_very_small)
+
+print(f"Generated {mname_very_small}")
+
+# Upload
+# transformers-cli repo create mt5-tiny-random
+# clone and add files

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e47ef1ea6344ab7c8e41bb0001ff32ded49a2c17c6539994e242e30acb15fd85
+size 3342941

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6859b5aa593827a9593cc7c313eb5bc86444a971387dae19ae4a3d2ba389bbae
+size 312867

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"]}