Erase
Browse files- .gitattributes +0 -35
- README.md +0 -13
- app.py +0 -147
- htmlTemplates.py +0 -44
- requirements.txt +0 -13
.gitattributes
DELETED
@@ -1,35 +0,0 @@
|
|
1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
DELETED
@@ -1,13 +0,0 @@
|
|
1 |
-
---
|
2 |
-
title: Basic DAG AI Chatbot With Llama2
|
3 |
-
emoji: 🔥
|
4 |
-
colorFrom: green
|
5 |
-
colorTo: pink
|
6 |
-
sdk: streamlit
|
7 |
-
sdk_version: 1.27.2
|
8 |
-
app_file: app.py
|
9 |
-
pinned: false
|
10 |
-
license: apache-2.0
|
11 |
-
---
|
12 |
-
|
13 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.py
DELETED
@@ -1,147 +0,0 @@
|
|
1 |
-
import streamlit as st
|
2 |
-
from dotenv import load_dotenv
|
3 |
-
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
|
4 |
-
from langchain.vectorstores import FAISS
|
5 |
-
from langchain.embeddings import HuggingFaceEmbeddings # General embeddings from HuggingFace models.
|
6 |
-
from langchain.memory import ConversationBufferMemory
|
7 |
-
from langchain.chains import ConversationalRetrievalChain
|
8 |
-
from htmlTemplates import css, bot_template, user_template
|
9 |
-
from langchain.llms import LlamaCpp # For loading transformer models.
|
10 |
-
from langchain.document_loaders import PyPDFLoader, TextLoader, JSONLoader, CSVLoader
|
11 |
-
import tempfile # 임시 파일을 생성하기 위한 라이브러리입니다.
|
12 |
-
import os
|
13 |
-
from huggingface_hub import hf_hub_download # Hugging Face Hub에서 모델을 다운로드하기 위한 함수입니다.
|
14 |
-
|
15 |
-
# PDF 문서로부터 텍스트를 추출하는 함수입니다.
|
16 |
-
def get_pdf_text(pdf_docs):
|
17 |
-
temp_dir = tempfile.TemporaryDirectory() # 임시 디렉토리를 생성합니다.
|
18 |
-
temp_filepath = os.path.join(temp_dir.name, pdf_docs.name) # 임시 파일 경로를 생성합니다.
|
19 |
-
with open(temp_filepath, "wb") as f: # 임시 파일을 바이너리 쓰기 모드로 엽니다.
|
20 |
-
f.write(pdf_docs.getvalue()) # PDF 문서의 내용을 임시 파일에 씁니다.
|
21 |
-
pdf_loader = PyPDFLoader(temp_filepath) # PyPDFLoader를 사용해 PDF를 로드합니다.
|
22 |
-
pdf_doc = pdf_loader.load() # 텍스트를 추출합니다.
|
23 |
-
return pdf_doc # 추출한 텍스트를 반환합니다.
|
24 |
-
|
25 |
-
# 과제
|
26 |
-
# 아래 텍스트 추출 함수를 작성
|
27 |
-
def get_text_file(docs):
|
28 |
-
pass
|
29 |
-
|
30 |
-
def get_csv_file(docs):
|
31 |
-
pass
|
32 |
-
|
33 |
-
def get_json_file(docs):
|
34 |
-
pass
|
35 |
-
|
36 |
-
|
37 |
-
# 문서들을 처리하여 텍스트 청크로 나누는 함수입니다.
|
38 |
-
def get_text_chunks(documents):
|
39 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
40 |
-
chunk_size=1000, # 청크의 크기를 지정합니다.
|
41 |
-
chunk_overlap=200, # 청크 사이의 중복을 지정합니다.
|
42 |
-
length_function=len # 텍스트의 길이를 측정하는 함수를 지정합니다.
|
43 |
-
)
|
44 |
-
|
45 |
-
documents = text_splitter.split_documents(documents) # 문서들을 청크로 나눕니다.
|
46 |
-
return documents # 나눈 청크를 반환합니다.
|
47 |
-
|
48 |
-
|
49 |
-
# 텍스트 청크들로부터 벡터 스토어를 생성하는 함수입니다.
|
50 |
-
def get_vectorstore(text_chunks):
|
51 |
-
# 원하는 임베딩 모델을 로드합니다.
|
52 |
-
embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L12-v2',
|
53 |
-
model_kwargs={'device': 'cpu'}) # 임베딩 모델을 설정합니다.
|
54 |
-
vectorstore = FAISS.from_documents(text_chunks, embeddings) # FAISS 벡터 스토어를 생성합니다.
|
55 |
-
return vectorstore # 생성된 벡터 스토어를 반환합니다.
|
56 |
-
|
57 |
-
|
58 |
-
def get_conversation_chain(vectorstore):
|
59 |
-
model_name_or_path = 'TheBloke/Llama-2-7B-chat-GGUF'
|
60 |
-
model_basename = 'llama-2-7b-chat.Q2_K.gguf'
|
61 |
-
model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)
|
62 |
-
|
63 |
-
llm = LlamaCpp(model_path=model_path,
|
64 |
-
n_ctx=4086,
|
65 |
-
input={"temperature": 0.75, "max_length": 2000, "top_p": 1},
|
66 |
-
verbose=True, )
|
67 |
-
# 대화 기록을 저장하기 위한 메모리를 생성합니다.
|
68 |
-
memory = ConversationBufferMemory(
|
69 |
-
memory_key='chat_history', return_messages=True)
|
70 |
-
# 대화 검색 체인을 생성합니다.
|
71 |
-
conversation_chain = ConversationalRetrievalChain.from_llm(
|
72 |
-
llm=llm,
|
73 |
-
retriever=vectorstore.as_retriever(),
|
74 |
-
memory=memory
|
75 |
-
)
|
76 |
-
return conversation_chain # 생성된 대화 체인을 반환합니다.
|
77 |
-
|
78 |
-
# 사용자 입력을 처리하는 함수입니다.
|
79 |
-
def handle_userinput(user_question):
|
80 |
-
print('user_question => ', user_question)
|
81 |
-
# 대화 체인을 사용하여 사용자 질문에 대한 응답을 생성합니다.
|
82 |
-
response = st.session_state.conversation({'question': user_question})
|
83 |
-
# 대화 기록을 저장합니다.
|
84 |
-
st.session_state.chat_history = response['chat_history']
|
85 |
-
|
86 |
-
for i, message in enumerate(st.session_state.chat_history):
|
87 |
-
if i % 2 == 0:
|
88 |
-
st.write(user_template.replace(
|
89 |
-
"{{MSG}}", message.content), unsafe_allow_html=True)
|
90 |
-
else:
|
91 |
-
st.write(bot_template.replace(
|
92 |
-
"{{MSG}}", message.content), unsafe_allow_html=True)
|
93 |
-
|
94 |
-
|
95 |
-
def main():
|
96 |
-
load_dotenv()
|
97 |
-
st.set_page_config(page_title="Chat with multiple Files",
|
98 |
-
page_icon=":books:")
|
99 |
-
st.write(css, unsafe_allow_html=True)
|
100 |
-
|
101 |
-
if "conversation" not in st.session_state:
|
102 |
-
st.session_state.conversation = None
|
103 |
-
if "chat_history" not in st.session_state:
|
104 |
-
st.session_state.chat_history = None
|
105 |
-
|
106 |
-
st.header("Chat with multiple Files:")
|
107 |
-
user_question = st.text_input("Ask a question about your documents:")
|
108 |
-
if user_question:
|
109 |
-
handle_userinput(user_question)
|
110 |
-
|
111 |
-
with st.sidebar:
|
112 |
-
st.subheader("Your documents")
|
113 |
-
docs = st.file_uploader(
|
114 |
-
"Upload your PDFs here and click on 'Process'", accept_multiple_files=True)
|
115 |
-
if st.button("Process"):
|
116 |
-
with st.spinner("Processing"):
|
117 |
-
# get pdf text
|
118 |
-
doc_list = []
|
119 |
-
|
120 |
-
for file in docs:
|
121 |
-
print('file - type : ', file.type)
|
122 |
-
if file.type == 'text/plain':
|
123 |
-
# file is .txt
|
124 |
-
doc_list.extend(get_text_file(file))
|
125 |
-
elif file.type in ['application/octet-stream', 'application/pdf']:
|
126 |
-
# file is .pdf
|
127 |
-
doc_list.extend(get_pdf_text(file))
|
128 |
-
elif file.type == 'text/csv':
|
129 |
-
# file is .csv
|
130 |
-
doc_list.extend(get_csv_file(file))
|
131 |
-
elif file.type == 'application/json':
|
132 |
-
# file is .json
|
133 |
-
doc_list.extend(get_json_file(file))
|
134 |
-
|
135 |
-
# get the text chunks
|
136 |
-
text_chunks = get_text_chunks(doc_list)
|
137 |
-
|
138 |
-
# create vector store
|
139 |
-
vectorstore = get_vectorstore(text_chunks)
|
140 |
-
|
141 |
-
# create conversation chain
|
142 |
-
st.session_state.conversation = get_conversation_chain(
|
143 |
-
vectorstore)
|
144 |
-
|
145 |
-
|
146 |
-
if __name__ == '__main__':
|
147 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
htmlTemplates.py
DELETED
@@ -1,44 +0,0 @@
|
|
1 |
-
css = '''
|
2 |
-
<style>
|
3 |
-
.chat-message {
|
4 |
-
padding: 1.5rem; border-radius: 0.5rem; margin-bottom: 1rem; display: flex
|
5 |
-
}
|
6 |
-
.chat-message.user {
|
7 |
-
background-color: #2b313e
|
8 |
-
}
|
9 |
-
.chat-message.bot {
|
10 |
-
background-color: #475063
|
11 |
-
}
|
12 |
-
.chat-message .avatar {
|
13 |
-
width: 20%;
|
14 |
-
}
|
15 |
-
.chat-message .avatar img {
|
16 |
-
max-width: 78px;
|
17 |
-
max-height: 78px;
|
18 |
-
border-radius: 50%;
|
19 |
-
object-fit: cover;
|
20 |
-
}
|
21 |
-
.chat-message .message {
|
22 |
-
width: 80%;
|
23 |
-
padding: 0 1.5rem;
|
24 |
-
color: #fff;
|
25 |
-
}
|
26 |
-
'''
|
27 |
-
|
28 |
-
bot_template = '''
|
29 |
-
<div class="chat-message bot">
|
30 |
-
<div class="avatar">
|
31 |
-
<img src="https://i.ibb.co/cN0nmSj/Screenshot-2023-05-28-at-02-37-21.png" style="max-height: 78px; max-width: 78px; border-radius: 50%; object-fit: cover;">
|
32 |
-
</div>
|
33 |
-
<div class="message">{{MSG}}</div>
|
34 |
-
</div>
|
35 |
-
'''
|
36 |
-
|
37 |
-
user_template = '''
|
38 |
-
<div class="chat-message user">
|
39 |
-
<div class="avatar">
|
40 |
-
<img src="https://i.ibb.co/rdZC7LZ/Photo-logo-1.png">
|
41 |
-
</div>
|
42 |
-
<div class="message">{{MSG}}</div>
|
43 |
-
</div>
|
44 |
-
'''
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
DELETED
@@ -1,13 +0,0 @@
|
|
1 |
-
langchain
|
2 |
-
llama-cpp-python
|
3 |
-
PyPDF2==3.0.1
|
4 |
-
faiss-cpu==1.7.4
|
5 |
-
ctransformers
|
6 |
-
pypdf
|
7 |
-
chromadb
|
8 |
-
tiktoken
|
9 |
-
pysqlite3-binary
|
10 |
-
streamlit-extras
|
11 |
-
InstructorEmbedding
|
12 |
-
sentence-transformers
|
13 |
-
jq
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|