File size: 5,051 Bytes
6bbc16b c652096 6bbc16b d361e26 6bbc16b 6eaa078 d361e26 6bbc16b 6eaa078 6bbc16b d361e26 6bbc16b d361e26 6bbc16b 1e39576 6bbc16b 608ea72 d361e26 608ea72 6bbc16b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 |
#!/usr/bin/env python
# coding: utf-8
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script creates a tiny random model
#
# It will be used then as "hf-internal-testing/tiny-xlm-roberta"
# ***To build from scratch***
#
# 1. clone sentencepiece into a parent dir
# git clone https://github.com/google/sentencepiece
#
# 2. create a new repo at https://huggingface.co/new
# make sure to choose 'hf-internal-testing' as the Owner
#
# 3. clone
# git clone https://huggingface.co/hf-internal-testing/tiny-xlm-roberta
# cd tiny-xlm-roberta
# 4. start with some pre-existing script from one of the https://huggingface.co/hf-internal-testing/ tiny model repos, e.g.
# wget https://huggingface.co/hf-internal-testing/tiny-xlm-roberta/raw/main/make-tiny-xlm-roberta.py
# chmod a+x ./make-tiny-xlm-roberta.py
# mv ./make-tiny-xlm-roberta.py ./make-tiny-xlm-roberta.py
#
# 5. automatically rename things from the old names to new ones
# perl -pi -e 's|MT5|XLMRoberta|g' make-tiny-xlm-roberta.py
# perl -pi -e 's|mt5|xlm-roberta|g' make-tiny-xlm-roberta.py
#
# 6. edit and re-run this script while fixing it up
# ./make-tiny-xlm-roberta.py
#
# 7. add/commit/push
# git add *
# git commit -m "new tiny model"
# git push
# ***To update***
#
# 1. clone the existing repo
# git clone https://huggingface.co/hf-internal-testing/tiny-xlm-roberta
# cd tiny-xlm-roberta
#
# 2. edit and re-run this script after doing whatever changes are needed
# ./make-tiny-xlm-roberta.py
#
# 3. commit/push
# git commit -m "new tiny model"
# git push
import sys
import os
from transformers import XLMRobertaTokenizer, XLMRobertaTokenizerFast, XLMRobertaConfig, XLMRobertaForCausalLM
mname_orig = "xlm-roberta-base"
mname_tiny = "tiny-xlm-roberta"
### Tokenizer
# Shrink the orig vocab to keep things small
vocab_keep_items = 5000
tmp_dir = f"/tmp/{mname_tiny}"
vocab_orig_path = f"{tmp_dir}/sentencepiece.bpe.model"
vocab_short_path = f"{tmp_dir}/spiece-short.model"
# workaround for fast tokenizer protobuf issue, and it's much faster too!
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
if 1: # set to 0 to skip this after running once to speed things up during tune up
# HACK: need the sentencepiece source to get sentencepiece_model_pb2, as it doesn't get installed
sys.path.append("../sentencepiece/python/src/sentencepiece")
import sentencepiece_model_pb2 as model
tokenizer_orig = XLMRobertaTokenizer.from_pretrained(mname_orig)
tokenizer_orig.save_pretrained(tmp_dir)
with open(vocab_orig_path, 'rb') as f: data = f.read()
# adapted from https://blog.ceshine.net/post/trim-down-sentencepiece-vocabulary/
m = model.ModelProto()
m.ParseFromString(data)
print(f"Shrinking vocab from original {len(m.pieces)} dict items")
for i in range(len(m.pieces) - vocab_keep_items): _ = m.pieces.pop()
print(f"new dict {len(m.pieces)}")
with open(vocab_short_path, 'wb') as f: f.write(m.SerializeToString())
m = None
tokenizer_fast_tiny = XLMRobertaTokenizerFast(vocab_file=vocab_short_path)
tokenizer_tiny = XLMRobertaTokenizer(vocab_file=vocab_short_path)
### Config
config_tiny = XLMRobertaConfig.from_pretrained(mname_orig)
# remember to update this to the actual config as each model is different and then shrink the numbers
config_tiny.update(dict(
vocab_size=vocab_keep_items+12,
d_ff=256,
d_kv=8,
d_model=64,
hidden_size=256,
intermediate_size=256,
max_position_embeddings=64,
num_attention_heads=2,
num_decoder_layers=2,
num_heads=2,
num_hidden_layers=2,
num_layers=2,
relative_attention_num_buckets=32,
))
print("New config", config_tiny)
### Model
model_tiny = XLMRobertaForCausalLM(config_tiny)
print(f"{mname_tiny}: num of params {model_tiny.num_parameters()}")
model_tiny.resize_token_embeddings(len(tokenizer_tiny))
inputs = tokenizer_tiny("hello", return_tensors="pt")
outputs = model_tiny(**inputs)
print("Test with normal tokenizer:", len(outputs.logits[0]))
inputs = tokenizer_fast_tiny("hello", return_tensors="pt")
outputs = model_tiny(**inputs)
print("Test with fast tokenizer:", len(outputs.logits[0]))
# Save
model_tiny.half() # makes it smaller
model_tiny.save_pretrained(".")
tokenizer_tiny.save_pretrained(".")
tokenizer_fast_tiny.save_pretrained(".")
readme = "README.md"
if not os.path.exists(readme):
with open(readme, "w") as f:
f.write(f"This is a {mname_tiny} random model to be used for basic testing.\n")
print(f"Generated {mname_tiny}")
|