Update space

Helpers.py

@@ -0,0 +1,173 @@
+import requests
+import json
+import spacy
+import string
+
+def generate_prompt(context, question, history):
+        
+    # history_summary = ""
+    # if history:
+    #     for user_query, bot_response in history[-3:]:  
+    #         history_summary += f"User: {user_query}\n    Assistant: {bot_response}\n"
+    if context:
+        prompt_context = context
+    else:
+        prompt_context = "No context provided."
+    prompt = f"""
+    <s>[INST] <<SYS>> You are a helpful, respectful and honest assistant. Always answer as helpfully as possible based on the context, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content.<</SYS>>
+
+    Context:
+    {prompt_context}
+ 
+    [INST] {question} [/INST]
+
+    Response:
+    """
+
+    return prompt
+
+# ==============================================================================================================================================
+def llama(prompt):
+    url = "https://api.edenai.run/v2/text/generation"
+    payload = {
+        "providers": "meta/llama2-13b-chat-v1",
+        "response_as_dict": True,
+        "attributes_as_list": False,
+        "show_original_response": False,
+        "temperature": 0,
+        "max_tokens": 256,
+        "text": prompt
+    }
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiMmYzMDE3MTEtOTJmNy00ZDU3LTg4N2MtNjU2MmE5MTU5MWZhIiwidHlwZSI6ImFwaV90b2tlbiJ9.vWvooRwxmr-uY1c61V97uugyDGpXmZGjX8oCFWKCUeM"
+    }
+
+    response = requests.post(url, json=payload, headers=headers)
+    result = response.json()
+    return result['meta/llama2-13b-chat-v1']['generated_text']
+# ==============================================================================================================================================
+
+def question_answering(question):
+  """
+  Sends a question answering request to the EdenAI API.
+
+  Args:
+      question: The question to be answered.
+
+  Returns:
+      The answer provided by the LLM model (string),
+      or None if an error occurs.
+  """
+  headers = {"Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiMmYzMDE3MTEtOTJmNy00ZDU3LTg4N2MtNjU2MmE5MTU5MWZhIiwidHlwZSI6ImFwaV90b2tlbiJ9.vWvooRwxmr-uY1c61V97uugyDGpXmZGjX8oCFWKCUeM"}
+
+  url = "https://api.edenai.run/v2/text/question_answer"
+  payload = {
+      "providers": "openai",
+      "texts": [
+          "Linux is a family of open-source Unix-like operating systems based on the Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds.",
+          "Just like Windows, iOS, and Mac OS, Linux is an operating system. "
+      ],
+      'question': question,
+      "examples": [["What is human life expectancy in the United States?", "78 years."]],
+      "fallback_providers": ""
+  }
+
+  try:
+    response = requests.post(url, json=payload, headers=headers)
+    result = json.loads(response.text)
+    return result['openai']['answers'] if result['openai']['answers'] else None
+  except Exception as e:
+    print(f"Error communicating with LLM model: {e}")
+    return None
+  
+# ==============================================================================================================================================
+def normalize_text(s):
+    """Removing stopwords and punctuation, and standardizing whitespace are all typical text processing steps."""
+    
+    nlp = spacy.load("en_core_web_sm")
+    def remove_stop(text):
+        return " ".join([word for word in text.split() if not nlp.vocab[word].is_stop])
+    
+    def lemma(text):
+        return " ".join([word.lemma_ for word in nlp(text)])
+    
+    def white_space_fix(text):
+        return " ".join(text.split()) # this function removes leading and trailing whitespaces and condenses all other whitespaces to a single space
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(lemma(remove_stop(remove_punc(lower(s)))))
+
+# ==============================================================================================================================================
+def get_relevance_docs(documents_score, threshold):
+    """
+    Calculate relevance scores for the retrieved documents based on their relevance to the correct answer.
+
+    Parameters:
+        documents_score (list): List of scores for the retrieved documents.
+        threshold (float): Threshold value to determine relevance.
+
+    Returns:
+        list: List of relevance scores for the retrieved documents.
+    """
+    relevance_scores = []
+    for score in documents_score:
+        if score >= threshold:
+            relevance_scores.append(1)  # Relevant document
+        else:
+            relevance_scores.append(0)  # Non-relevant document
+    return relevance_scores
+
+# ==============================================================================================================================================
+def get_docs_by_indices(docs, indices):
+    """
+    Retrieve document contexts from a list of indexed documents based on provided indices.
+
+    Args:
+    - docs (list): List of documents.
+    - indices (list): List of indices corresponding to the desired documents.
+
+    Returns:
+    - list: List of document contexts corresponding to the provided indices.
+    """
+    return [docs[index] for index in indices]
+
+def query_rewriter(query):
+    headers = {"Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiMTA5MTBlYTktOWYwOC00N2E2LTg3MDktOTlhODExZjkwZDA2IiwidHlwZSI6ImFwaV90b2tlbiJ9._wiFq518MhMRvG8waWbg_7Eogf50isgyzqh3e2ypvOU"}
+
+    url = "https://api.edenai.run/v2/text/code_generation"
+    payload = {
+        "providers": "openai",
+        # "instruction": "You are an expert at world knowledge. Your task is to step back and paraphrase a question to a more generic step-back question, which is easier to answer. Here are a few examples:Original Question: Which position did Knox Cunningham hold from May 1955 to Apr 1956? Stepback Question: Which positions have Knox Cunning- ham held in his career? , Now this Question: Who was the spouse of Anna Karina from 1968 to 1974?",
+        "prompt": "",
+        "model": "gpt-3.5-turbo",
+        "instruction": f"""You are an expert in document retrieval and search optimization. 
+        Your task is to rewrite the following query to enhance its relevance and usefulness for retrieving information
+        from a database or search engine : 
+        Original Query: {query} \n  """,   
+        "temperature": 0.6,
+        "max_tokens": 512,
+        "fallback_providers": " ['openai']"
+
+    }
+
+    response = requests.post(url, json=payload, headers=headers)
+
+    result = json.loads(response.text)
+    if(len(result['openai']['generated_text']) > len(query)):
+        try:
+            return split_text(result['openai']['generated_text'])
+        except:
+            return result['openai']['generated_text']
+    else:
+        return query
+# ==============================================================================================================================================
+def split_text(text):
+    return text.split(":")[1].strip() if ":" in text else text 

Processing.py

@@ -0,0 +1,42 @@
+from transformers import BertTokenizer, BertModel
+import torch
+
+
+class TextEmbedder:
+    def __init__(self):
+        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        self.model = BertModel.from_pretrained('bert-base-uncased')
+
+    def _mean_pooling(self, model_output, attention_mask):
+        token_embeddings = model_output.last_hidden_state
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+        return sum_embeddings / sum_mask
+
+    def embed_text(self, examples):
+        inputs = self.tokenizer(
+            examples["content"], padding=True, truncation=True, return_tensors="pt"
+        )
+        with torch.no_grad():
+            model_output = self.model(**inputs)
+        pooled_embeds = self._mean_pooling(model_output, inputs["attention_mask"])
+        return {"embedding": pooled_embeds.cpu().numpy()}
+    
+    def generate_embeddings(self, dataset):
+        return dataset.map(self.embed_text, batched=True, batch_size=128)
+    
+    def embed_query(self, query_text):
+        query_inputs = self.tokenizer(
+            query_text,
+            padding=True,
+            truncation=True,
+            return_tensors="pt"
+        )
+
+        with torch.no_grad():
+            query_model_output = self.model(**query_inputs)
+
+        query_embedding = self._mean_pooling(query_model_output, query_inputs["attention_mask"])
+
+        return query_embedding

QdrantU.py

@@ -0,0 +1,61 @@
+import uuid
+from qdrant_client.http import models
+from qdrant_client import QdrantClient
+
+
+class QdrantU:
+    def __init__(self, collection_name):
+        self.client = QdrantClient(
+            url="https://5c32ac64-b1f7-4665-91eb-e321a98c02f6.europe-west3-0.gcp.cloud.qdrant.io:6333",
+            api_key="Wd_RTregmznFMCyDLagJHM_7a5TjJJuFLVTuMgfjQD44-BHLnhYbUg",
+        )
+        self.collection_name = collection_name
+
+    def _upload_documents_to_Qdrant(self, data, source):
+        points = []
+        for title, content, publishdate, embedding in zip(data["title"], data["content"], data["publishdate"], data["embedding"]):
+            new_id = str(uuid.uuid4())  # Generate a new UUID for each document
+            point = models.PointStruct(
+                id=new_id,
+                vector=embedding,
+                payload={
+                    "title": title,
+                    "content": content,
+                    "publishdate": publishdate,
+                    "source" : source
+                }
+            )
+            points.append(point)
+
+        self.client.upsert(
+            collection_name=self.collection_name,
+            points=points
+        )
+
+        print("Uploaded:", len(data["embedding"]), "documents to the Qdrant database")
+
+
+    def upload_to_Qdrant(self, data, batch_size=35, source=''):
+        for i in range(0, len(data), batch_size):
+            batch = data[i:i + batch_size]
+            self._upload_documents_to_Qdrant(batch , source)
+            print(f"Uploaded {i + len(batch)} documents")
+
+
+    def get_number_of_vectors(self):
+        collection_info = self.client.get_collection(self.collection_name)
+        num_vectors = collection_info.points_count
+        return num_vectors
+    
+    def close_connection(self):
+        self.client.close()
+
+    def search(self, query, text_embedder, limit):
+        query_vector = text_embedder.embed_query(query_text=query)
+        query_result = self.client.search(
+            collection_name=self.collection_name,
+            query_vector=query_vector[0].tolist(),  # Convert tensor to list
+            limit=limit,
+            with_payload=True
+        )
+        return query_result

rag.py

@@ -0,0 +1,30 @@
+from QdrantU import QdrantU
+from Processing import TextEmbedder 
+import cohere
+from Helpers import generate_prompt, llama, get_docs_by_indices , query_rewriter
+
+
+def run_rag(query, history=None):
+    embedding_model = TextEmbedder()
+    uploader = QdrantU(collection_name='News_source')
+    try: 
+        query = query_rewriter(query) 
+        print("Query after rewriting: ", query)
+    except:
+        print("Error in query rewriting")
+        pass
+    search_results = uploader.search(query, embedding_model, limit=1000)
+    docs = list(set([result.payload['content'] for result in search_results]))
+
+    apiKey = 'Q21IIAUkTtt1jk9WUgJg0XiCvaU2K73cFbq0djhM' # API key for Cohere
+    co = cohere.Client(apiKey)
+    rerank_docs = co.rerank(
+        query=query, documents=docs, top_n=2, model="rerank-english-v3.0"
+    )
+
+    indices = [result.index for result in rerank_docs.results]
+    documents = get_docs_by_indices(docs, indices)
+    prompt = generate_prompt(documents, query, history)
+    print("Prompt: ", prompt)
+    # response = llama(prompt)
+    return prompt