Spaces:
Running
Running
from patcher import tiktoken_patch | |
import tiktoken | |
from transformers import AutoTokenizer | |
from enum import Enum, auto | |
from dataclasses import dataclass, field | |
from utils.log_util import logger | |
from typing import Dict, Any, Union | |
"""Interface: | |
tokenizer.encode | |
tokenizer.decode | |
tokenizer.convert_tokens_to_string # gpt4 没有这个方法 | |
tokenizer.convert_ids_to_tokens | |
tokenizer.parent = "" | |
tokenizer.vocab_size | |
tokenizer.get_vocab() # gpt-neox-20b, llama | |
tokenizer.type = TokenizerType.ByteBPE.name | |
tokenizer.implementation = TokenizerImpl.SentencePiece.name # https://github.com/facebookresearch/llama/blob/main/llama/tokenizer.py | |
"HFGPT2Tokenizer", "HFTokenizer", "GPT2BPETokenizer", "CharLevelTokenizer", "TiktokenTokenizer", "SPMTokenizer", https://github.com/EleutherAI/gpt-neox/blob/main/tools/preprocess_data.py | |
tokenizer.comments = "split all numbers into individual digits, " \ | |
"and fallback to bytes to decompose unknown UTF-8 characters" | |
tokenizer.all_special_tokens # baichuan | |
tokenizer.special_tokens_set # gpt3.5_turbo | |
tokenizer.special_tokens_map | |
""" | |
class TokenizerImpl(Enum): | |
""" | |
- https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/__init__.py | |
- https://huggingface.co/docs/transformers/tokenizer_summary | |
- https://github.com/EleutherAI/gpt-neox/blob/main/megatron/tokenizer/tokenizer.py | |
## google/BertTokenizer | |
- https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py | |
- 特征 | |
- 算法:BERT的编码器是 BPE-WordPiece,将单词拆分成多个前缀符号(比如BERT中的##)最小单元 | |
- 词典:有##开头的token,表示subword, | |
- 中文采用char粒度分词 | |
- 英文采用 WordPiece | |
## google/sentencepiece | |
- https://github.com/google/sentencepiece/blob/3863f7648e5d8edb571ac592f3ac4f5f0695275a/src/sentencepiece_model.proto#L48 | |
- 支持 sentencepiece 和 wordpiece | |
- sentencepiece 有byte-bpe吗? | |
- UNIGRAM = 1; // Unigram language model with dynamic algorithm | |
- BPE = 2; // Byte Pair Encoding | |
- WORD = 3; // Delimitered by whitespace. | |
- CHAR = 4; // tokenizes into character sequence | |
- wordpiece | |
- 特征: | |
- 训练: spm_train --model_type unigram/bpe/char/word | |
- 特殊符号: Ġ | |
- 文件: *.sp_model 或 *.model (可选文件 .vocab,) spm简称 (其他格式比如 tokenizer.json是给hf_tokenizer兼容用的) | |
- 实现: | |
- 依赖: protobuf | |
- 训练: `import sentencepiece as spm; spm.SentencePieceTrainer.train` 或 `spm_train` | |
- 加载: `import sentencepiece as spm; spm.SentencePieceProcessor().Load(vocab_file)` | |
- 方法: 是SentencePieceProcessor类型,sp_model.id_to_piece,有tokenizer.json tokenizer.model, | |
- 分词: | |
- pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False) | |
- 词典: 词典字符有 ▁ (U+2581) ,表示空格或句首。 | |
- 示例:google-t5, llama,baichuan, orion, | |
- llama: tokenizer.json(包含model.vocab model.merges) tokenizer.model | |
- grok: 原始是 .model文件,后面转成了 tokenizer.json | |
- google-t5: tokenizer.json, spiece.model | |
- Skywork-13B-Math: tokenizer.model | |
- xlm_roberta: sentencepiece.bpe.model | |
- GPT2Tokenizer | |
- tokenizer.json, vocab.json, merges.txt (https://huggingface.co/openai-community/gpt2) | |
- vocab.bpe, encoder.json, dict.txt (fairseq版本,不常用,可以忽略这个版本) | |
## thu/icetk | |
- icetk: sentencepiece的分支,支持image_tokenizer。 | |
- glm, chatglm1, chatglm2 | |
## huggingface/tokenizers | |
- https://github.com/huggingface/tokenizers | |
- VS sentencepiece | |
- 支持sentencepiece | |
- .model转化为 (merges.txt + vocab.json) 或者 tokenizer.json | |
- https://github.com/huggingface/tokenizers/blob/main/bindings/python/scripts/sentencepiece_extractor.py | |
- 加载 merges.txt, vocab.json | |
- SentencePieceBPETokenizer https://github.com/huggingface/tokenizers/blob/v0.19.1/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py#L10 | |
- 在 sentencepiece基础上,hf_tokenizer支持pre-tokenization的正则表达式,对tab和换行支持更好,支持special token | |
- 类型: 支持 BBPE, WordPiece or Unigram | |
- 特征: | |
- 文件: tokenizer.json(包含后两个文件的内容), merges.txt, vocab.json | |
- added_tokens 在vocab中不一定存在。 | |
- 实现: | |
- 训练: `from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer` | |
- 加载: | |
- 方法: .model.from_file .model.save .model.token_to_id .model.tokenize | |
- .model 是 tokenizer.models.BPE 类型 | |
- 词典有 Ġ "\u0120" 开头 | |
- 优势 | |
- | |
- 示例:gpt2, gpt_neox_20b, moss, bloom, qwen2 | |
- 优势:相对sentence piece, | |
- ss | |
## openai/tiktoken | |
- 特征:空格就是空格, | |
- 示例:gpt3.5 gpt4, qwen, | |
""" | |
""" 算法体系 https://www.huaxiaozhuan.com/%E5%B7%A5%E5%85%B7/huggingface_transformer/chapters/1_tokenizer.html | |
- word-base tokenizer: | |
- char-base tokenizer: | |
- subword-based Tokenizer | |
- BPE | |
- byte-bpe: base vocabulary大小是256 | |
- WordPiece: | |
- 相比BPE,WordPiece 仅保存最终词表,而不保存学到的 merge rule | |
- Unigram | |
- SentencePiece | |
""" | |
# 分类体系:https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/ | |
BertTokenizer = "wordpiece.BertTokenizer" | |
JapaneseTokenizer = ("wordpiece.MecabTokenizer", "https://github.com/polm/fugashi") # 常用日语包 ipadic,fugashi, | |
ByteLevelBPETokenizer = "byte_level_bpe" # BBPE | |
SentencePieceBPETokenizer = "sentencepiece_bpe" | |
# 分类体系 | |
# SentencePeice(BPE) | |
SentencePiece = auto() # sentencepiece.bpe, sentencepiece.unigram, sentencepiece.char, sentencepiece.word, | |
byte_level_bpe = auto() | |
# HFTokenizer = auto() # , 支持 | |
TikToken = auto() | |
# subword-nmt | |
# WordPiece | |
# load_vocab_with_SPECIAL_TOKEN = True # 如果不包含会导致计算词典大小错误、overlap_token计算不一致。 | |
class TokenizerConfig: | |
""" | |
https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/leaderboard/read_evals.py | |
""" | |
name_or_path: str # org/model (path on hub), as unique id | |
name_display: str = None # | |
impl: TokenizerImpl = None # implementation, tokenizer_class/type | |
org: str = None | |
link: str = None # http://** | |
desc: str = None # description | |
meta: str = None | |
level: str = None # char-level, word-level, byte-level | |
init_kwargs: Dict[str, Any] = field(default_factory=dict, ) | |
def __post_init__(self): | |
if self.link is None: | |
self.link = "https://huggingface.co/" + self.name_or_path # TODO + revision | |
if self.name_display is None: | |
self.name_display = self.name_or_path | |
def init_from_json_file(cls, json_filepath: str) -> 'TokenizerConfig': | |
pass | |
def __eq__(self, other): | |
if isinstance(other, self.__class__): | |
return self.__dict__ == other.__dict__ | |
else: | |
return False | |
def __hash__(self): | |
return hash(self.name_or_path) | |
# TODO: append link and description to the end of dropdown button. | |
# Add tokenizer_class/type, comments | |
_all_tokenizer_config = [ | |
##### bert 系列 | |
TokenizerConfig("google-bert/bert-base-cased", impl=TokenizerImpl.BertTokenizer, org="Google", | |
desc="first add whitespace around any CJK character, then perform wordpiece tokenization."), | |
TokenizerConfig("google-bert/bert-base-uncased", impl=TokenizerImpl.BertTokenizer, org="Google", | |
desc="first add whitespace around any CJK character, then perform wordpiece tokenization."), | |
TokenizerConfig("google-bert/bert-base-chinese", impl=TokenizerImpl.BertTokenizer, org="Google", | |
desc="first add whitespace around any CJK character, then perform wordpiece tokenization."), | |
TokenizerConfig("google-bert/bert-base-german-cased", impl=TokenizerImpl.BertTokenizer, org="Google"), | |
TokenizerConfig("dbmdz/bert-base-german-uncased", impl=TokenizerImpl.BertTokenizer, org="dbmdz"), | |
TokenizerConfig("google-bert/bert-base-multilingual-uncased", impl=TokenizerImpl.BertTokenizer, org="Google"), | |
TokenizerConfig("google-bert/bert-base-multilingual-cased", impl=TokenizerImpl.BertTokenizer, org="Google"), | |
TokenizerConfig("tohoku-nlp/bert-base-japanese", impl=TokenizerImpl.BertTokenizer, org="Tohoku", | |
desc="The texts are first tokenized by MeCab morphological parser with the IPA dictionary, " | |
"then split into subwords by the WordPiece algorithm."), | |
TokenizerConfig("clue/roberta_chinese_clue_tiny", name_display="clue/roberta-chinese-clue", | |
impl=TokenizerImpl.BertTokenizer, org="CLUE", | |
init_kwargs={"revision": "refs/pr/1"}, | |
desc="", | |
meta="去掉了繁体字, https://github.com/CLUEbenchmark/CLUEPretrainedModels/blob/master/README.md"), | |
TokenizerConfig("eson/kplug-base-encoder", name_display="eson/kplug", impl=TokenizerImpl.BertTokenizer, org="JD"), | |
TokenizerConfig("ckiplab/gpt2-base-chinese", impl=TokenizerImpl.BertTokenizer, org="SINICA"), # 台湾中央研究院 | |
# WoBERT https://kexue.fm/archives/7758 | |
# WoBERT Plus https://github.com/ZhuiyiTechnology/WoBERT | |
##### GPT2Tokenizer | |
TokenizerConfig("openai-community/gpt2", impl=TokenizerImpl.SentencePiece, org="OpenAI"), | |
# byte-level BPE,没有byte,是unicode-level的吗? | |
TokenizerConfig("ClassCat/gpt2-base-french", impl=TokenizerImpl.SentencePiece, org="ClassCat"), | |
TokenizerConfig("ClassCat/gpt2-base-spanish", impl=TokenizerImpl.SentencePiece, org="ClassCat"), | |
TokenizerConfig("fnlp/moss-moon-003-sft", impl=TokenizerImpl.SentencePiece, init_kwargs={"revision": "refs/pr/6"}, | |
org="Fudan", | |
desc="This tokenizer has been trained to treat spaces like parts of the tokens " | |
"(a bit like sentencepiece) so a word will be encoded differently whether " | |
"it is at the beginning of the sentence (without space) or not", | |
meta="在gpt2词典基础上,扩充了5万中文"), | |
TokenizerConfig("bigscience/bloom", impl=TokenizerImpl.SentencePiece, org="BigScience", | |
meta="比gpt_neox的词典 对中文支持更好。"), | |
# ("bloomz_6b4_zh", | |
# ("BelleGroup/BELLE-7B-2M", # 模型和词典都基于bloom | |
# | |
TokenizerConfig("EleutherAI/gpt-neox-20b", impl=TokenizerImpl.SentencePiece, org="EleutherAI"), # 5万 | |
TokenizerConfig("cyberagent/open-calm-7b", impl=TokenizerImpl.SentencePiece, org="CyberAgent"), # GPTNeoXTokenizer | |
TokenizerConfig("abeja/gpt-neox-japanese-2.7b", impl=TokenizerImpl.SentencePiece, org="ABEJA"), | |
TokenizerConfig("Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"), # 15万,速度有点慢 | |
TokenizerConfig("Qwen/Qwen1.5-110B ", impl=TokenizerImpl.SentencePiece, org="Alibaba"), | |
TokenizerConfig("Qwen/Qwen1.5-1.8B ", impl=TokenizerImpl.SentencePiece, org="Alibaba"), | |
TokenizerConfig("HuggingFaceH4/starchat-alpha", impl=TokenizerImpl.SentencePiece, org="-"), | |
####### google/sentencepiece tokenizer: | |
# T5 llama internlm | |
TokenizerConfig("google-t5/t5-large", name_display="google-t5/t5", impl=TokenizerImpl.SentencePiece, org="Google"), | |
# t5_small, t5_base, t5_large, flan_t5_base, | |
# ("t5_base", "", "sentencepiece"), | |
# TokenizerConfig("google/flan-t5-base", impl=TokenizerImpl.SentencePiece, ), | |
TokenizerConfig("lmsys/fastchat-t5-3b-v1.0", impl=TokenizerImpl.SentencePiece, | |
org="LMSYS", | |
init_kwargs={"use_fast": False} # 解决 pyo3_runtime.PanicException: AddedVocabulary bad split | |
), | |
TokenizerConfig("CohereForAI/aya-101", org="Cohere For AI"), # "tokenizer_class": "T5Tokenizer", | |
TokenizerConfig("ClueAI/ChatYuan-large-v2", impl=TokenizerImpl.SentencePiece, org="CLUE"), | |
TokenizerConfig("ClueAI/PromptCLUE-base", impl=TokenizerImpl.SentencePiece, org="CLUE"), | |
TokenizerConfig("gradientai/Llama-3-8B-Instruct-Gradient-1048k", name_display="Meta/llama3", | |
impl=TokenizerImpl.SentencePiece, org="Meta", | |
desc="llama split all numbers into individual digits, and fallback to bytes to decompose unknown UTF-8 characters"), | |
# byte-level BPE | |
# '中文单字': 700, '中文多字': 0 | |
TokenizerConfig("NousResearch/Llama-2-7b-chat-hf", name_display="Meta/llama2", impl=TokenizerImpl.SentencePiece, | |
org="Meta"), | |
TokenizerConfig("huggyllama/llama-7b", name_display="Meta/llama", impl=TokenizerImpl.SentencePiece, org="Meta"), | |
TokenizerConfig("hpcai-tech/grok-1", name_display="xai-org/grok-1", impl=TokenizerImpl.SentencePiece, org="xAI"), | |
# 由.model文件转化为了 | |
TokenizerConfig("hfl/chinese-llama-lora-7b", impl=TokenizerImpl.SentencePiece, org="-", | |
meta="向原始LLaMA的词汇表中添加2w个中文词汇,针对原版LLaMA模型扩充了中文词表, 提升了中文编解码效率"), | |
# | |
TokenizerConfig("hfl/chinese-llama-2-7b", impl=TokenizerImpl.SentencePiece, org="-", | |
meta="重新设计了新词表(大小:55296),进一步提升了中文字词的覆盖程度"), # | |
TokenizerConfig("hfl/llama-3-chinese-8b", impl=TokenizerImpl.SentencePiece, org="-"), | |
TokenizerConfig("hfl/chinese-alpaca-lora-7b", impl=TokenizerImpl.SentencePiece, org="-"), | |
# 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。 "比chinese_llama词典多一个`[PAD]`,请勿混用" | |
# | |
# ("belle_llama_ext_7b", | |
# ("alpaca_7b", | |
TokenizerConfig("baichuan-inc/Baichuan-7B", name_display="baichuan-inc/baichuan", | |
impl=TokenizerImpl.SentencePiece, | |
level="byte-level", org="Baichuan"), | |
TokenizerConfig("baichuan-inc/Baichuan2-7B-Chat", name_display="baichuan-inc/baichuan2", | |
impl=TokenizerImpl.SentencePiece, org="Baichuan", | |
desc="expand the vocabulary size from 64000 in Baichuan1 to 125696"), | |
TokenizerConfig("internlm/internlm-chat-7b", impl=TokenizerImpl.SentencePiece, org="Shanghai AI Lab"), | |
# 上海AI实验室 + 商汤 | |
TokenizerConfig("internlm/internlm2-chat-7b", impl=TokenizerImpl.SentencePiece, org="Shanghai AI Lab"), | |
TokenizerConfig("internlm/internlm2-math-7b", impl=TokenizerImpl.SentencePiece, org="Shanghai AI Lab"), | |
TokenizerConfig("internlm/internlm-xcomposer-7b", impl=TokenizerImpl.SentencePiece, org="Shanghai AI Lab"), | |
TokenizerConfig("tiiuae/falcon-7b", impl=TokenizerImpl.SentencePiece, org="TII"), | |
TokenizerConfig("tiiuae/falcon-180b", impl=TokenizerImpl.SentencePiece, org="TII"), | |
TokenizerConfig("Skywork/Skywork-13B-base", impl=TokenizerImpl.SentencePiece, org="Kunlun"), | |
TokenizerConfig("Skywork/Skywork-13B-Math", impl=TokenizerImpl.SentencePiece, org="Kunlun"), # 文件:tokenizer.model | |
TokenizerConfig("FacebookAI/xlm-roberta-base", impl=TokenizerImpl.SentencePiece, org="Facebook"), | |
# 这个的tokenizer.json 为什么没有merges? vocab里为什么有概率值? | |
# "goat", | |
# ##### glm系列 | |
# "glm_chinese",), | |
TokenizerConfig("THUDM/chatglm-6b", impl=TokenizerImpl.SentencePiece, org="Tsinghua", | |
meta=f"num_image_tokens: {12}; num_image_tokens: {34} ", | |
init_kwargs={"revision": "refs/pr/100"}), | |
TokenizerConfig("THUDM/chatglm2-6b", impl=TokenizerImpl.SentencePiece, org="Tsinghua", ), | |
TokenizerConfig("THUDM/chatglm3-6b", impl=TokenizerImpl.SentencePiece, org="Tsinghua", ), | |
TokenizerConfig("thu-coai/CharacterGLM-6B", impl=TokenizerImpl.SentencePiece, org="Tsinghua", ), | |
# tiktoken 系列 | |
TokenizerConfig("openai/text-davinci-003", impl=TokenizerImpl.TikToken, org="OpenAI", | |
link="https://github.com/openai/tiktoken"), | |
# | |
TokenizerConfig("openai/code-davinci-002", impl=TokenizerImpl.TikToken, org="OpenAI", | |
link="https://github.com/openai/tiktoken"), | |
TokenizerConfig("openai/gpt-3.5-turbo", impl=TokenizerImpl.TikToken, org="OpenAI", | |
link="https://github.com/openai/tiktoken", | |
desc="tiktoken is a fast BPE tokeniser for use with OpenAI's models. There are 16 tokens KeyError"), | |
TokenizerConfig("openai/gpt-4", impl=TokenizerImpl.TikToken, org="OpenAI", | |
link="https://github.com/openai/tiktoken", ), | |
TokenizerConfig("openai/gpt-4o", impl=TokenizerImpl.TikToken, org="OpenAI", | |
link="https://github.com/openai/tiktoken", ), | |
TokenizerConfig("Qwen/Qwen-7B-Chat", name_display="Qwen/Qwen", impl=TokenizerImpl.TikToken, org="Alibaba", | |
init_kwargs={"revision": "refs/pr/56"}, | |
meta="在gpt4词典基础上,删除了100个多数字token,增加10000中文词token;并优化了special_token的分词"), | |
# https://huggingface.co/Qwen/Qwen-7B-Chat#%E6%A8%A1%E5%9E%8B%E7%BB%86%E8%8A%82%EF%BC%88model%EF%BC%89 | |
# 该词表在GPT-4使用的BPE词表cl100k_base基础上,对中文、多语言进行了优化,在对中、英、代码数据的高效编解码的基础上, | |
# 对部分多语言更加友好,方便用户在不扩展词表的情况下对部分语种进行能力增强。 词表对数字按单个数字位切分。 | |
# TokenizerConfig("Qwen/Qwen-72B-Chat", impl=TokenizerImpl.TikToken), | |
# 未分类 | |
# ("amber", ""), | |
TokenizerConfig("LLM360/CrystalCoder", org="MBZUAI"), | |
TokenizerConfig("mistralai/Mistral-7B-v0.1", org="Mistral"), | |
TokenizerConfig("mistralai/Mixtral-8x7B-v0.1", org="Mistral"), | |
TokenizerConfig("paust/pko-t5-large", org="PAUST"), | |
TokenizerConfig("01-ai/Yi-6B", org="Yi"), | |
TokenizerConfig("01-ai/Yi-34B", org="Yi"), | |
TokenizerConfig("01-ai/Yi-VL-34B", org="Yi"), | |
TokenizerConfig("OrionStarAI/Orion-14B-Chat", org="OrionStar"), | |
TokenizerConfig("microsoft/phi-1", org="Microsoft"), | |
TokenizerConfig("microsoft/phi-2", org="Microsoft"), | |
TokenizerConfig("microsoft/Phi-3-mini-4k-instruct", org="Microsoft", meta="即llama vocab"), | |
TokenizerConfig("Upstage/SOLAR-10.7B-v1.0", org="-"), | |
TokenizerConfig("google/mobilebert-uncased", org="Google"), | |
# ("google/mobilenet_v2_1.0_224",), # error | |
TokenizerConfig("google/switch-c-2048", org="Google"), | |
TokenizerConfig("google/byt5-small", org="Google"), | |
TokenizerConfig("google/mt5-large", org="Google"), | |
TokenizerConfig("WizardLM/WizardCoder-Python-7B-V1.0", org="Microsoft"), | |
TokenizerConfig("WizardLM/WizardCoder-15B-V1.0", org="Microsoft"), | |
TokenizerConfig("WizardLM/WizardLM-7B-V1.0", org="Microsoft"), | |
TokenizerConfig("WizardLM/WizardMath-70B-V1.0", org="Microsoft"), | |
TokenizerConfig("TigerResearch/tigerbot-70b-chat-v4-4k", org="Tigerobo"), | |
TokenizerConfig("TigerResearch/tigerbot-13b-chat-v2", org="Tigerobo"), | |
TokenizerConfig("deepseek-ai/deepseek-coder-33b-instruct", org="DeepSeek"), | |
TokenizerConfig("deepseek-ai/deepseek-llm-7b-base", org="DeepSeek"), | |
TokenizerConfig("deepseek-ai/DeepSeek-V2", org="DeepSeek"), | |
TokenizerConfig("google/gemma-7b", org="Google"), | |
TokenizerConfig("allenai/OLMo-7B", org="Allen AI"), | |
TokenizerConfig("HuggingFaceH4/zephyr-7b-beta", org="HuggingFace"), | |
TokenizerConfig("ai21labs/Jamba-v0.1", org="AI21"), | |
TokenizerConfig("databricks/dbrx-instruct", org="Databricks"), | |
# ("claude",), | |
# https://github.com/Duxiaoman-DI/XuanYuan | |
# https://huggingface.co/apple/OpenELM-3B-Instruct https://huggingface.co/apple/OpenELM-3B | |
] | |
assert len(set([config.name_display for config in _all_tokenizer_config])) == len(_all_tokenizer_config) | |
assert len(set([config.name_or_path for config in _all_tokenizer_config])) == len(_all_tokenizer_config) | |
assert len(set([config.name_or_path.split("/")[-1] for config in _all_tokenizer_config])) == len(_all_tokenizer_config) | |
class TokenizerFactory: | |
def __init__(self): | |
self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_or_path) | |
self.all_tokenizer_names = [config.name_or_path for config in self.all_tokenizer_configs] | |
self.name_to_config_list = [ | |
{config.name_or_path: config for config in self.all_tokenizer_configs}, | |
{config.name_display: config for config in self.all_tokenizer_configs}, | |
{config.name_display.split("/")[-1]: config for config in self.all_tokenizer_configs}, | |
] | |
self.tokenizer_cache = {} | |
def get_tokenizer_config(self, tokenizer_name: str) -> TokenizerConfig: | |
for name_to_config in self.name_to_config_list: | |
if tokenizer_name in name_to_config: | |
return name_to_config[tokenizer_name] | |
return None | |
def get_tokenizer(self, tokenizer_name: str): | |
""" | |
:param tokenizer_name: | |
:return: | |
""" | |
tokenizer_config = self.get_tokenizer_config(tokenizer_name) | |
# 1. load from cache | |
if tokenizer_config in self.tokenizer_cache: | |
return self.tokenizer_cache[tokenizer_config] | |
# 2. load tokenizer | |
logger.info(f"loading tokenizer {tokenizer_config.name_or_path}") | |
if tokenizer_config.impl == TokenizerImpl.TikToken and "openai" in tokenizer_config.name_or_path: | |
tokenizer = tiktoken.encoding_for_model(tokenizer_config.name_or_path.replace("openai/", "")) | |
else: | |
tokenizer = AutoTokenizer.from_pretrained( | |
tokenizer_config.name_or_path, | |
trust_remote_code=True, | |
**tokenizer_config.init_kwargs | |
) | |
self.tokenizer_cache[tokenizer_config] = tokenizer | |
return tokenizer | |
def get_name_with_hyperlink(self, tokenizer_name: str): | |
def model_hyperlink(link, model_name): | |
model_name = model_name | |
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>' | |
tokenizer_config = self.get_tokenizer_config(tokenizer_name) | |
return model_hyperlink(tokenizer_config.link, tokenizer_config.name_display.split("/")[-1]) | |
tokenizer_factory = TokenizerFactory() | |
# class TokenizerType(Enum): | |
# | |
# # BERTTokenizer | |
# # 依赖一个txt文件 | |
# | |
# | |
# # https://github.com/EleutherAI/gpt-neox/blob/v2.0/megatron/tokenizer/tokenizer.py#L231 | |
# # 依赖一个json文件,Tokenizer.from_file(vocab_file) | |
# # 案例:gpt-neox-20B | |
# HFTokenizer = auto() | |
# | |
# # 依赖: model_file, sentencepiece.SentencePieceProcessor(model_file) | |
# # 案例: | |
# SentencePieceTokenizer = auto() | |
# | |
# | |
# # 依赖: 3个json文件:vocab.json, merges.txt, special_tokens.txt | |
# # 源码: | |
# # - https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/tokenizer/gpt2_tokenization.py#L92 | |
# # Byte-level BPE | |
# GPT2BPETokenizer = auto() | |
if __name__ == "__main__": | |
for tokenizer_config in tokenizer_factory.all_tokenizer_configs: | |
if True: | |
# if "t5" in tokenizer_config.name_or_path: | |
tokenizer1 = tokenizer_factory.get_tokenizer(tokenizer_config.name_or_path) | |
tokenizer2 = tokenizer_factory.get_tokenizer(tokenizer_config.name_display) | |
tokenizer3 = tokenizer_factory.get_tokenizer(tokenizer_config.name_display.split("/")[-1]) | |
assert tokenizer1 == tokenizer2 == tokenizer3 | |
print(tokenizer_config.name_or_path, len(tokenizer1)) | |