Daniel Fried commited on
Commit
003db9a
1 Parent(s): 50e8b8f

add use_normal_tokenizers option

Browse files
Files changed (1) hide show
  1. modules/app.py +6 -2
modules/app.py CHANGED
@@ -1,8 +1,12 @@
1
  import sys
2
  from typing import List
3
  import traceback
 
4
  # needs to be imported *before* transformers
5
- import tokenizers_patch
 
 
 
6
  from transformers import AutoModelForCausalLM, AutoTokenizer
7
  import json
8
 
@@ -20,7 +24,7 @@ import json
20
  PORT = 7860
21
  VERBOSE = False
22
 
23
- BIG_MODEL = True
24
 
25
  if BIG_MODEL:
26
  CUDA = True
 
1
  import sys
2
  from typing import List
3
  import traceback
4
+ import os
5
  # needs to be imported *before* transformers
6
+ if os.path.exists('use_normal_tokenizers'):
7
+ import tokenizers
8
+ else:
9
+ import tokenizers_patch
10
  from transformers import AutoModelForCausalLM, AutoTokenizer
11
  import json
12
 
 
24
  PORT = 7860
25
  VERBOSE = False
26
 
27
+ BIG_MODEL = False
28
 
29
  if BIG_MODEL:
30
  CUDA = True