Spaces:
Runtime error
Runtime error
Daniel Fried
commited on
Commit
·
003db9a
1
Parent(s):
50e8b8f
add use_normal_tokenizers option
Browse files- modules/app.py +6 -2
modules/app.py
CHANGED
|
@@ -1,8 +1,12 @@
|
|
| 1 |
import sys
|
| 2 |
from typing import List
|
| 3 |
import traceback
|
|
|
|
| 4 |
# needs to be imported *before* transformers
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
| 6 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 7 |
import json
|
| 8 |
|
|
@@ -20,7 +24,7 @@ import json
|
|
| 20 |
PORT = 7860
|
| 21 |
VERBOSE = False
|
| 22 |
|
| 23 |
-
BIG_MODEL =
|
| 24 |
|
| 25 |
if BIG_MODEL:
|
| 26 |
CUDA = True
|
|
|
|
| 1 |
import sys
|
| 2 |
from typing import List
|
| 3 |
import traceback
|
| 4 |
+
import os
|
| 5 |
# needs to be imported *before* transformers
|
| 6 |
+
if os.path.exists('use_normal_tokenizers'):
|
| 7 |
+
import tokenizers
|
| 8 |
+
else:
|
| 9 |
+
import tokenizers_patch
|
| 10 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 11 |
import json
|
| 12 |
|
|
|
|
| 24 |
PORT = 7860
|
| 25 |
VERBOSE = False
|
| 26 |
|
| 27 |
+
BIG_MODEL = False
|
| 28 |
|
| 29 |
if BIG_MODEL:
|
| 30 |
CUDA = True
|