Th3BossC commited on
Commit
245b645
β€’
1 Parent(s): f2838c8

changed to default settings

Browse files
Files changed (1) hide show
  1. chatBot/common/llama.py +14 -18
chatBot/common/llama.py CHANGED
@@ -7,7 +7,7 @@ from langchain.embeddings import HuggingFaceInstructEmbeddings
7
  from langchain.text_splitter import RecursiveCharacterTextSplitter
8
  from langchain.vectorstores import Chroma
9
  from pdf2image import convert_from_path
10
- from transformers import AutoTokenizer, TextStreamer, pipeline, AutoModelForCausalLM
11
  from chatBot.common.pdfToText import loadLatestPdf
12
  from transformers import LlamaTokenizer
13
  from langchain.document_loaders import PyPDFLoader
@@ -28,27 +28,23 @@ texts = text_splitter.split_documents(data)
28
  db = Chroma.from_documents(texts, embeddings, persist_directory="db")
29
 
30
  model_name_or_path = "TheBloke/Llama-2-13B-chat-GPTQ"
31
- model_name_or_path = "TheBloke/Llama-2-13B-chat-GGUF"
32
  model_basename = "model"
33
 
34
  tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
35
 
36
- # model = AutoGPTQForCausalLM.from_quantized(
37
- # model_name_or_path,
38
- # device_map="auto",
39
- # revision="gptq-4bit-128g-actorder_True",
40
- # model_basename=model_basename,
41
- # use_safetensors=True,
42
- # trust_remote_code=True,
43
- # inject_fused_attention=False,
44
- # device=DEVICE,
45
- # quantize_config=None,
46
- # )
47
-
48
- model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
49
- device_map="auto",
50
- trust_remote_code=False,
51
- revision="main")
52
 
53
 
54
  DEFAULT_SYSTEM_PROMPT = """
 
7
  from langchain.text_splitter import RecursiveCharacterTextSplitter
8
  from langchain.vectorstores import Chroma
9
  from pdf2image import convert_from_path
10
+ from transformers import AutoTokenizer, TextStreamer, pipeline
11
  from chatBot.common.pdfToText import loadLatestPdf
12
  from transformers import LlamaTokenizer
13
  from langchain.document_loaders import PyPDFLoader
 
28
  db = Chroma.from_documents(texts, embeddings, persist_directory="db")
29
 
30
  model_name_or_path = "TheBloke/Llama-2-13B-chat-GPTQ"
 
31
  model_basename = "model"
32
 
33
  tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
34
 
35
+ model = AutoGPTQForCausalLM.from_quantized(
36
+ model_name_or_path,
37
+ device_map="auto",
38
+ revision="gptq-4bit-128g-actorder_True",
39
+ model_basename=model_basename,
40
+ use_safetensors=True,
41
+ trust_remote_code=True,
42
+ inject_fused_attention=False,
43
+ device=DEVICE,
44
+ quantize_config=None,
45
+ )
46
+
47
+
 
 
 
48
 
49
 
50
  DEFAULT_SYSTEM_PROMPT = """