Logging enabled

data/load_dataset.py

@@ -3,6 +3,7 @@ from datasets import load_dataset
 
 def load_data():
     logging.info("Loading dataset")
-    dataset = load_dataset("rungalileo/ragbench", 'covidqa', split="train")
+    dataset = load_dataset("rungalileo/ragbench", 'covidqa', split="test")
     logging.info("Dataset loaded successfully")
+    logging.info(dataset)
     return dataset

generator/create_prompt.py

@@ -3,7 +3,7 @@ from langchain.docstore.document import Document
 def create_prompt(documents, question, response):
     prompt = f""" I asked someone to answer a question based on one or more documents. Your task is to review their response and assess whether or not each sentence in that response is supported by text in the documents. If supported, identify which sentences in the documents provide that support. Additionally, identify which documents contain useful information for answering the question, and which documents the answer was sourced from.
 
-    Here are the documents, each split into sentences with associated keys:
+    Here are the documents, each of which is split into sentences. Alongside each sentence is associated key, such as '0a.' or '0b.' that you can use to refer to it:
     {documents}
 
     The question was:
@@ -33,23 +33,26 @@ def create_prompt(documents, question, response):
       "all_utilized_sentence_keys": ["string"]
     }}
 
-          The relevance_explanation field is a string explaining which documents
+        The relevance_explanation field is a string explaining which documents
         contain useful information for answering the question. Provide a step-by-step
         breakdown of information provided in the documents and how it is useful for
         answering the question.
+
         The all_relevant_sentence_keys field is a list of all document sentences keys
-        (e.g. ’0a’) that are revant to the question. Include every sentence that is
+        (e.g. '0a') that are relevant to the question. Include every sentence that is
         useful and relevant to the question, even if it was not used in the response,
         or if only parts of the sentence are useful. Ignore the provided response when
         making this judgement and base your judgement solely on the provided documents
         and question. Omit sentences that, if removed from the document, would not
-        impact someone’s ability to answer the question.
+        impact someone's ability to answer the question.
+        
         The overall_supported_explanation field is a string explaining why the response
         *as a whole* is or is not supported by the documents. In this field, provide a
         step-by-step breakdown of the claims made in the response and the support (or
         lack thereof) for those claims in the documents. Begin by assessing each claim
         separately, one by one; don’t make any remarks about the response as a whole
         until you have assessed all the claims in isolation.
+        
         The overall_supported field is a boolean indicating whether the response as a
         whole is supported by the documents. This value should reflect the conclusion
         you drew at the end of your step-by-step breakdown in overall_supported_explanation.
@@ -62,7 +65,7 @@ def create_prompt(documents, question, response):
         16
         - explanation: a string explaining why the sentence is or is not supported by the
         documents.
-        - supporting_sentence_keys: keys (e.g. ’0a’) of sentences from the documents that
+        - supporting_sentence_keys: keys (e.g. '0a') of sentences from the documents that
         support the response sentence. If the sentence is not supported, this list MUST
         be empty. If the sentence is supported, this list MUST contain one or more keys.
         In special cases where the sentence is supported, but not by any specific sentence,
@@ -98,37 +101,4 @@ def create_prompt(documents, question, response):
         is supported by the text in the documents.
 
     """
-    return prompt
-    prompt = f""" I asked someone to answer a question based on one or more documents. Your task is to review their response and assess whether or not each sentence in that response is supported by text in the documents. If supported, identify which sentences in the documents provide that support. Additionally, identify which documents contain useful information for answering the question, and which documents the answer was sourced from.
-
-    Here are the documents, each split into sentences with associated keys:
-    {documents}
-
-    The question was:
-    {question}
-
-    Here is their response, split into sentences with associated keys:
-    {response}
-
-    Provide a JSON response with an answer , relevance_explanation, all_relevant_sentence_keys, overall_supported_explanation, overall_supported, sentence_support_information(this can be a json inside a json and with these fields response_sentence_key, explanation, supporting_sentence_keys, fully_supported) and all_utilized_sentence_keys below are the definitions
-
-    You must respond with a JSON object matching this schema:
-
-    {{
-      "answer": "string",
-      "relevance_explanation": "string",
-      "all_relevant_sentence_keys": ["string"],
-      "overall_supported_explanation": "string",
-      "overall_supported": "boolean",
-      "sentence_support_information": [
-        {{
-          "response_sentence_key": "string",
-          "explanation": "string",
-          "supporting_sentence_keys": ["string"],
-          "fully_supported": "boolean"
-        }},
-      ],
-      "all_utilized_sentence_keys": ["string"]
-    }}
-"""
     return prompt

generator/extract_attributes.py

@@ -10,6 +10,7 @@ def extract_attributes(question, relevant_docs, response):
     # Format documents into a string by accessing the `page_content` attribute of each Document
     formatted_documents = "\n".join([f"Doc {i+1}: {doc.page_content}" for i, doc in enumerate(relevant_docs)])
 
+    #print(f'Formated documents: {formatted_documents}')
     attribute_prompt = create_prompt(formatted_documents, question, response)
 
     # Instead of using BaseMessage, pass the formatted prompt directly to invoke

generator/generate_response.py

@@ -1,13 +1,13 @@
 from langchain.chains import RetrievalQA
 
-def generate_response(llm, vector_store, question):
+def generate_response(llm, vector_store, question, relevant_docs):
     # Create a retrieval-based question-answering chain using the relevant documents
     qa_chain = RetrievalQA.from_chain_type(
         llm=llm,
         retriever=vector_store.as_retriever(),
         return_source_documents=True
     )
-    result = qa_chain.invoke(question)
+    result = qa_chain.invoke(question, documents=relevant_docs)
     response = result['result']
     source_docs = result['source_documents']
     return response, source_docs