xu-song's picture
add grok mixtral
480ae5d
raw
history blame
115 Bytes
import copy
from vocab.text_davinci_003 import tokenizer as base_tokenizer
tokenizer = copy.copy(base_tokenizer)