File size: 1,723 Bytes
a9e00be |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
# Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
#
# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://github.com/Tencent/Tencent-Hunyuan-Large/blob/main/License.docx
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from tokenizers import ByteLevelBPETokenizer
from transformers import AutoTokenizer
# Step 1: Initialize ByteLevelBPETokenizer
#tokenizer = ByteLevelBPETokenizer(
# "vocab.json",
# "merges.txt"
#)
# Step 2: Save the tokenizer configuration
#tokenizer.save_model("auto_model")
# Step 3: Load the tokenizer using AutoTokenizer
auto_tokenizer = AutoTokenizer.from_pretrained("./", use_fast=False, trust_remote_code=True)
# Test the tokenizer
text = "Hello, world!"
encoded = auto_tokenizer.encode(text)
decoded = auto_tokenizer.decode(encoded)
print("Encoded:", encoded)
print("Decoded:", decoded)
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello, how are you?"},
{"role": "assistant", "content": "I'm good, thank you! How can I help you today?"},
{"role": "user", "content": "Nothing"},
]
print('messages:', messages)
ids = auto_tokenizer.apply_chat_template(messages)
print(f"input_ids:\t{ids}")
text = auto_tokenizer.decode(ids)
print(f"input_text:\t[{text}]")
|