Update app.py
Browse files
app.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
import gradio as gr
|
2 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
3 |
import torch
|
4 |
-
|
5 |
import os
|
6 |
|
7 |
hf_token = os.environ.get('HF_TOKEN')
|
@@ -9,13 +9,17 @@ hf_token = os.environ.get('HF_TOKEN')
|
|
9 |
from huggingface_hub import login
|
10 |
login(token=hf_token)
|
11 |
|
|
|
|
|
|
|
|
|
12 |
model = AutoModelForCausalLM.from_pretrained(
|
13 |
"mistralai/Mistral-7B-Instruct-v0.1",
|
14 |
token = hf_token,
|
15 |
torch_dtype=torch.bfloat16,
|
16 |
trust_remote_code=True,
|
17 |
device_map="auto",
|
18 |
-
|
19 |
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1", token = hf_token)
|
20 |
|
21 |
def generate_text(input_text):
|
|
|
1 |
import gradio as gr
|
2 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
3 |
import torch
|
4 |
+
from ctransformers import AutoConfig
|
5 |
import os
|
6 |
|
7 |
hf_token = os.environ.get('HF_TOKEN')
|
|
|
9 |
from huggingface_hub import login
|
10 |
login(token=hf_token)
|
11 |
|
12 |
+
config = AutoConfig.from_pretrained( "mistralai/Mistral-7B-Instruct-v0.1")
|
13 |
+
config.config.max_new_tokens = 2000
|
14 |
+
config.config.context_length = 4000 )
|
15 |
+
|
16 |
model = AutoModelForCausalLM.from_pretrained(
|
17 |
"mistralai/Mistral-7B-Instruct-v0.1",
|
18 |
token = hf_token,
|
19 |
torch_dtype=torch.bfloat16,
|
20 |
trust_remote_code=True,
|
21 |
device_map="auto",
|
22 |
+
config=config)
|
23 |
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1", token = hf_token)
|
24 |
|
25 |
def generate_text(input_text):
|