add files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+image.index filter=lfs diff=lfs merge=lfs -text
+text.index filter=lfs diff=lfs merge=lfs -text
+idx_item_mapping.pkl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,13 @@
 ---
-title: Tourist Attraction Rag
-emoji: 🦀
+title: Tourist Attractions Multimodal Rag
+emoji: 👀
 colorFrom: red
-colorTo: green
+colorTo: purple
 sdk: gradio
-sdk_version: 5.49.1
+sdk_version: 5.46.0
 app_file: app.py
 pinned: false
+python_version: 3.10.11
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,175 @@
+import json
+import sys
+import os
+import pickle
+
+import faiss
+import gradio as gr
+import numpy as np
+import torch
+from PIL import Image
+from sentence_transformers import SentenceTransformer
+from transformers import AutoImageProcessor, AutoTokenizer, AutoModel, AutoModelForCausalLM, BitsAndBytesConfig
+from tqdm import tqdm
+from datasets import load_dataset
+from hazm import Normalizer
+
+
+DATASET_NAME = 'alisharifi/tourist-attractions-text-image-data'
+TEST_DATA_NAME = 'alisharifi/tourist-attractions-test-data'
+
+dataset = load_dataset(DATASET_NAME, streaming=True)
+test_data_name = load_dataset(TEST_DATA_NAME, streaming=True)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+vision_processor = AutoImageProcessor.from_pretrained('facebook/dinov2-base')
+vision_model = AutoModel.from_pretrained('facebook/dinov2-base').to(device)
+
+language_model = SentenceTransformer("xmanii/maux-gte-persian", trust_remote_code=True).to(device)
+
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+)
+model = AutoModelForCausalLM.from_pretrained(
+    "universitytehran/PersianMind-v1.0",
+    quantization_config=quantization_config,
+    device_map="auto"
+)
+tokenizer = AutoTokenizer.from_pretrained(
+    "universitytehran/PersianMind-v1.0",
+)
+
+normalizer = Normalizer()
+
+language_model.eval()
+vision_model.eval()
+
+# Load FAISS indices
+text_index = faiss.read_index("text.index")
+image_index = faiss.read_index("image.index")
+
+# Load the index-item mapping
+with open("idx_item_mapping.pkl", "rb") as f:
+    idx_item_mapping = pickle.load(f)
+
+print("FAISS indices and index-item mapping loaded.")
+
+
+def search_by_text(query_text, k=5):
+  """
+  Searches the database for the top k items most similar to the query text.
+
+  Args:
+    query_text: The text query.
+    k: The number of top similar items to return.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains the item details
+    for the top k similar items.
+  """
+  normalized_query = normalizer.normalize(query_text)
+  query_embedding = language_model.encode(normalized_query)
+
+  query_embedding_np = query_embedding[np.newaxis, :]
+  faiss.normalize_L2(query_embedding_np)
+
+  distances, indices = text_index.search(query_embedding_np, 100)
+
+  unique_texts = set()
+  results = []
+  for idx in indices[0]:
+    text = idx_item_mapping[idx]
+    if text not in unique_texts:
+      unique_texts.add(text)
+      results.append(text)
+      if len(results) == k:
+        break
+
+  return results
+
+def search_by_image(query_image, k=5):
+  """
+  Searches the database for the top k items most similar to the query text.
+
+  Args:
+    query_text: The text query.
+    k: The number of top similar items to return.
+
+  Returns:
+    A list of dictionaries, where each dictionary contains the item details
+    for the top k similar items.
+  """
+  inputs = vision_processor(images=query_image, return_tensors="pt").to(device) # Move image inputs to device
+  with torch.no_grad():
+    outputs = vision_model(**inputs)
+  image_embedding_np = outputs[0].mean(dim=1)[0].cpu().numpy()
+
+
+  query_embedding_np = image_embedding_np[np.newaxis, :]
+  faiss.normalize_L2(query_embedding_np)
+
+  # Search the FAISS index
+  distances, indices = image_index.search(query_embedding_np, 100)
+
+  # Get the top k items using the indices and the mapping
+  unique_texts = set()
+  results = []
+  for idx in indices[0]:
+    text = idx_item_mapping[idx]
+    if text not in unique_texts:
+      unique_texts.add(text)
+      results.append(text)
+      if len(results) == k:
+        break
+
+  return results
+
+
+def rag_pipeline(question, image=None):
+    """
+    Runs the RAG pipeline with the given question and optional image.
+
+    Args:
+        question: The text question.
+        image: Optional image input.
+
+    Returns:
+        The generated answer from the language model.
+    """
+    retrieved_items = []
+    if image is not None:
+        retrieved_items.extend(search_by_image(image))
+    retrieved_items.extend(search_by_text(question))
+
+    TEMPLATE = "{context}\nYou: {prompt}\nPersianMind: "
+    CONTEXT = '\n'.join(retrieved_items)
+    PROMPT = '\n'.join([
+        question,
+        'به این سوال به فارسی جواب بده.'
+    ])
+
+    model_input = TEMPLATE.format(context=CONTEXT, prompt=PROMPT)
+    input_tokens = tokenizer(model_input, return_tensors="pt")
+    input_tokens = input_tokens.to(device)
+    generate_ids = model.generate(**input_tokens, max_new_tokens=200, do_sample=False, repetition_penalty=1.1)
+    model_output = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+
+    return model_output[len(model_input):]
+
+
+iface = gr.Interface(
+    fn=rag_pipeline,
+    inputs=[
+        gr.Textbox(label="Your Question"),
+        gr.Image(type="pil", label="Optional Image")
+    ],
+    outputs=gr.Textbox(label="Answer"),
+    title="Tourist Attraction RAG Pipeline",
+    description="Ask a question about tourist attractions and optionally provide an image."
+)
+
+
+iface.launch(debug=True)

idx_item_mapping.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f91dd8e55ba36208407b204b38725beb034febc212dbf4eb40c0bbbc6e31e53
+size 1486498

image.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d469ba218d61ae40b215f2074b73c66dcbc1e6380f5a416a661be12ced229470
+size 6703149

requirements.txt

@@ -0,0 +1,13 @@
+faiss-cpu
+gradio
+numpy
+torch
+pillow
+sentence_transformers
+transformers
+datasets
+hazm
+tqdm
+bitsandbytes
+accelerate
+sentencepiece

text.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8519dad63618a4cab940e95b55639196220236f2de36a6fec8aeb1bca385660
+size 6703149