Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -44,7 +44,7 @@ def tokenize(input_text):
|
|
44 |
if __name__ == "__main__":
|
45 |
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
|
46 |
llama3_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b")
|
47 |
-
mistral_tokenizer = AutoTokenizer.from_pretrained("unsloth/Mistral-7B-v0.
|
48 |
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
49 |
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
|
50 |
falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
|
|
|
44 |
if __name__ == "__main__":
|
45 |
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
|
46 |
llama3_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b")
|
47 |
+
mistral_tokenizer = AutoTokenizer.from_pretrained("unsloth/Mistral-7B-v0.2")
|
48 |
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
49 |
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
|
50 |
falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
|