Upload operations.py

operations.py

@@ -0,0 +1,177 @@
+from numpy.linalg import norm
+import numpy as np
+from sentence_transformers import SentenceTransformer
+import PyPDF2
+from nltk.tokenize import sent_tokenize
+
+
+def read_pdf(fname):
+    """
+    This function reads the pdf file and extracts the text from it.
+
+    Parameters:
+    fname (str): Name of the pdf file
+
+    Returns:
+    text_ext (list): List of extracted text from the pdf file
+    """
+    reader = PyPDF2.PdfReader(fname)
+    text_ext = []
+    for i in range(len(reader.pages)):
+        pageObj = reader.pages[i]
+        # extracting text from page
+        text_ext.append(pageObj.extract_text())
+
+    return text_ext
+
+
+def sent_tokenize(text_ext):
+    """
+    This function apply sent_tokenize to the text and stores the result in a list.
+
+    Parameters:
+    text_ext (list): List of extracted text from the pdf file
+
+    Returns:
+    sent_toks (list): List of tokenized sentences
+    """
+    sent_toks = []
+
+    for i in text_ext:
+        sent_toks.append(sent_tokenize(i))
+    print("len(sent_toks) ", len(sent_toks))
+
+    return sent_toks
+
+
+def create_content_embeddings(concat_list):
+    """
+    This function creates embeddings for the document sentences.
+
+    Parameters:
+    concat_list (list): List of tokenized sentences
+
+    Returns:
+    embeddings (list): List of embeddings of the sentences
+    """
+    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+    embeddings = model.encode(concat_list)
+
+    return embeddings
+
+
+def create_query_embeddings(query_text):
+    """
+    This function creates embeddings for the query.
+    Parameters:
+    query_text (str): Query entered by the user
+
+    Returns:
+    query_embedding (list): List of embeddings of the query
+    """
+
+    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+    query_embedding = model.encode(query_text)
+    return query_embedding
+
+
+def calculate_cosine(query_embedding, embeddings, concat_list):
+    """
+
+    This function calculates cosine similarity between the query and the sentences.
+
+    Parameters:
+    query_embedding (list): List of embeddings of the query
+    embeddings (list): List of embeddings of the sentences
+    concat_list (list): List of tokenized sentences
+
+    Returns:
+    cosine_lis (list): List of cosine similarity values
+    """
+    cosine_lis = []
+
+    for i in range(len(concat_list)):
+        cosine = np.dot(query_embedding,
+                        embeddings[i]) / (norm(query_embedding)*norm(embeddings[i]))
+        cosine_lis.append(cosine)
+
+    # print("cosine_lis ", cosine_lis)
+    return (cosine_lis)
+
+
+def fetch_top_rank_ans(cosine_lis, N):
+    """
+    This function fetches the top N ranked sentences.
+
+    Parameters:
+    cosine_lis (list): List of cosine similarity values
+    N (int): Number of sentences to be ranked
+
+    Returns:
+    indexes_final (list): List of top N ranked sentences
+    """
+
+    list1 = cosine_lis
+    indexes_final = sorted(
+        range(len(list1)), key=lambda i: list1[i], reverse=True)[:N]
+
+    print("indexes_final ", indexes_final)
+    indices = range(len(list1))
+
+    sorted_indices = sorted(indices, key=lambda i: list1[i], reverse=True)
+# print(sorted_indices)
+    indexes_final = []
+    for i in range(N):
+        indexes_final.append(sorted_indices[i])
+    len(indexes_final)
+    return indexes_final
+
+
+def fetch_most_relevant(indexes_final, concat_list, list1, query):
+    """
+    This function fetches the most relevant sentences, pass it as a context to GPT-3 prompt along with user's query.
+
+    Parameters:
+    indexes_final (list): List of top N ranked sentences
+    concat_list (list): List of tokenized sentences
+    list1 (list): List of cosine similarity values
+    query (str): Query entered by the user
+
+    Returns:
+    prompt (str): GPT-3 prompt
+    """
+
+    dicts = {}
+
+    keys = indexes_final
+    for i in keys:
+        dicts[i] = concat_list[i]
+
+    most_relevant_document_sections = [dicts]
+
+    len(most_relevant_document_sections)
+
+    chosen_sections = []
+    chosen_sections_len = 0
+    chosen_sections_indexes = []
+
+    indices = range(len(list1))
+    sorted_indices = sorted(indices, key=lambda i: list1[i], reverse=True)
+    # print(len(indexes_final))
+
+    for section_index in range(len(indexes_final)):
+
+        if chosen_sections_len > 500:
+            break
+        chosen_sections.append(
+            concat_list[sorted_indices[section_index]].replace("\n", " "))
+        chosen_sections_indexes.append(str(section_index))
+
+    # Useful diagnostic information
+    print(f"Selected {len(chosen_sections)} document sections:")
+
+    header = """Answer the question as a human in natural language conversation using the provided context, and if the answer is not contained within the text below, say "I don't have that information"\n\nContext:\n"""
+
+    # print(query)
+    prompt = header + "".join(chosen_sections) + "\n\n Q: " + query + "\n A:"
+    return prompt