File size: 10,277 Bytes
e9887d5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ac7e2e2
e9887d5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3c4f785
 
 
 
 
e9887d5
 
 
 
 
 
 
dde7a82
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
import os
import re
import logging
import requests
import pandas as pd
from bs4 import BeautifulSoup
from langdetect import detect, DetectorFactory
from langdetect.lang_detect_exception import LangDetectException
import langid
from deep_translator import GoogleTranslator
import gradio as gr
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain.docstore.document import Document
from langchain_community.vectorstores.utils import filter_complex_metadata
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.pydantic_v1 import BaseModel, Field
from langchain_openai import ChatOpenAI
from langchain_core.runnables import RunnablePassthrough, RunnableLambda
from langchain_core.output_parsers import StrOutputParser
from operator import itemgetter
from langchain_community.tools.tavily_search import TavilySearchResults
from typing import List
from typing_extensions import TypedDict
from langgraph.graph import END, StateGraph
from langchain_openai import OpenAIEmbeddings
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_community.vectorstores import FAISS
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain.memory import ConversationBufferMemory
from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain.chains import create_history_aware_retriever
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_core.messages import HumanMessage

# Setup logging
logging.basicConfig(level=logging.DEBUG)

OPENAI_API_TOKEN = "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"
os.environ["OPENAI_API_KEY"] = OPENAI_API_TOKEN

# Retrieve the secret token from environment variables
hf_api_token = os.getenv('HF_API_TOKEN')

# Ensure the token is not None
if hf_api_token is None:
    raise ValueError("HF_API_TOKEN environment variable not set")

# Fixing random seed for reproducibility in langdetect
DetectorFactory.seed = 0

# Function to translate text based on detected language
def translate_content(text):
    try:
        detected_lang = detect(text)
        if detected_lang == 'fr':
            return GoogleTranslator(source='fr', target='en').translate(text)
        elif detected_lang == 'en':
            return GoogleTranslator(source='en', target='fr').translate(text)
        else:
            return text
    except Exception as e:
        print(f"Error detecting language or translating: {e}")
        return text

# Function to chunk content
def chunk_content(content, chunk_size=1250, overlap=250):
    chunks = []
    start = 0
    while start < len(content):
        end = start + chunk_size
        chunk = content[start:end]
        chunks.append(chunk)
        start += chunk_size - overlap
    return chunks

# Initialize the list to store chunked documents
chunked_web_doc = []

# Load the Excel file
df = pd.read_excel("UNTEanswers.xlsx")

# Merge the 'prompt' and 'reference' columns
df['merged_content'] = df['prompt'] + " " + df['reference']

# Translate and store all text entries in a list
text_entries = []

for index, row in df.iterrows():
    # Original content
    merged_content = row['merged_content']
    text_entries.append(merged_content)
    
    # Translated content
    translated_content = translate_content(merged_content)
    if translated_content and translated_content != merged_content:
        text_entries.append(translated_content)

# Convert the list of text entries into a single string
excel_text = "\n".join(text_entries)

# Process content from the Excel file
for index, row in df.iterrows():
    merged_content = row['merged_content']
    
    # Chunk the original content
    en_chunks = chunk_content(merged_content)
    for chunk in en_chunks:
        chunked_web_doc.append({
            "url": "UNTEanswers.xlsx",  # Mark as coming from the Excel file
            "language": detect(merged_content),
            "chunk": chunk
        })
    
    # Translate and chunk the content if necessary
    translated_content = translate_content(merged_content)
    if translated_content and translated_content != merged_content:
        translated_chunks = chunk_content(translated_content)
        for chunk in translated_chunks:
            chunked_web_doc.append({
                "url": "UNTEanswers.xlsx",  # Mark as coming from the Excel file
                "language": detect(translated_content),
                "chunk": chunk
            })

# Load the fetched content from the text file
with open('fetched_contentt.txt', 'r', encoding='utf-8') as f:
    fetched_content = f.read()

# Combine the text from the Excel file and the fetched content
content = fetched_content + "\n" + excel_text

# Optionally, save the combined content to a new file
with open('merged_content.txt', 'w', encoding='utf-8') as f:
    f.write(content)


web_contents = content.split("-" * 80 + "\n\n")

for block in web_contents:
    if block.strip():
        lines = block.strip().splitlines()
        url = ""
        title = ""
        en_content = ""
        fr_content = ""
        language = None

        for i, line in enumerate(lines):
            if line.startswith("URL:"):
                url = line.split("URL:")[1].strip()
            elif line.startswith("Title:"):
                title = line.split("Title:")[1].strip()
            elif line == "English Content:":
                language = "en"
            elif line == "French Content:":
                language = "fr"
            else:
                if language == "en":
                    en_content += line + "\n"
                elif language == "fr":
                    fr_content += line + "\n"

        if en_content.strip():
            en_chunks = chunk_content(en_content.strip())
            for chunk in en_chunks:
                chunked_web_doc.append({
                    "url": url,
                    "language": "en",
                    "chunk": chunk
                })

        if fr_content.strip():
            fr_chunks = chunk_content(fr_content.strip())
            for chunk in fr_chunks:
                chunked_web_doc.append({
                    "url": url,
                    "language": "fr",
                    "chunk": chunk
                })

model_id = 'sentence-transformers/all-MiniLM-L6-v2'
model_kwargs = {'device': 'cpu'}
embeddings = HuggingFaceEmbeddings(
    model_name=model_id,
    model_kwargs=model_kwargs
)

documents = [
    Document(page_content=chunk['chunk'], metadata={"url": chunk['url'], "language": chunk['language']})
    for chunk in chunked_web_doc
]

chroma_db = Chroma.from_documents(documents=documents,
                                  collection_name='rag_web_db',
                                  embedding=embeddings,
                                  collection_metadata={"hnsw:space": "cosine"},
                                  persist_directory="./web_db")

similarity_threshold_retriever = chroma_db.as_retriever(search_type="similarity_score_threshold",
                                                        search_kwargs={"k": 3,
                                                                       "score_threshold": 0.3})


llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)


################ history_aware_retriever###################


from langchain.chains import create_history_aware_retriever
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder

contextualize_q_system_prompt = """Given a chat history and the latest user question \
which might reference context in the chat history, formulate a standalone question \
which can be understood without the chat history. Do NOT answer the question, \
just reformulate it if needed and otherwise return it as is."""
contextualize_q_prompt = ChatPromptTemplate.from_messages(
    [
        ("system", contextualize_q_system_prompt),
        MessagesPlaceholder("chat_history"),
        ("human", "{input}"),
    ]
)
history_aware_retriever = create_history_aware_retriever(
    llm, similarity_threshold_retriever, contextualize_q_prompt
)


################ question_answer_chain#####################


from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain

qa_system_prompt = """You are an assistant for question-answering tasks. \
Use the following pieces of retrieved context to answer the question. \
If you don't know the answer, just say that you don't know. \
Use three sentences maximum and keep the answer concise.\
{context}"""
qa_prompt = ChatPromptTemplate.from_messages(
    [
        ("system", qa_system_prompt),
        MessagesPlaceholder("chat_history"),
        ("human", "{input}"),
    ]
)
question_answer_chain = create_stuff_documents_chain(llm, qa_prompt)


################ rag_chain#####################


rag_chain = create_retrieval_chain(history_aware_retriever, question_answer_chain)

chat_history = ['Moodle','course','un cours']
import gradio as gr

#def ask(question, history):
#    ai_message = rag_chain.invoke({"input": question, "chat_history": chat_history})
#    chat_history.extend([HumanMessage(content=question), ai_message["answer"]])
#    return ai_message['answer']

def ask(question, history):
    ai_message = rag_chain.invoke({"input": question, "chat_history": chat_history})
    chat_history.extend([HumanMessage(content=question), ai_message["answer"]])
    document_links = []
    if 'context' in ai_message and ai_message['context']:
        for doc in ai_message['context']:
          if 'url' in doc.metadata:
              document_links.append(doc.metadata['url'])
    # Format document links as part of the text output
    if document_links:
        document_links_text = "\n".join(document_links)
        links_text = f"\n\nSources:\n{document_links_text}"
    else:
        links_text = "UNTE_ASSISTANTE"

demo = gr.ChatInterface(fn=ask, title="UNTE ChatBot",theme=gr.themes.Soft())



if __name__ == "__main__":
    gr.close_all()
    demo.launch(share = False)