new arch

.env

@@ -0,0 +1 @@
+HUGGINGFACEHUB_API_TOKEN = "your_huggingface_api_key_here"

.gitattributes copy

@@ -0,0 +1,45 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+.venv/

.python-version

@@ -0,0 +1 @@
+3.12

README copy.md

@@ -0,0 +1,12 @@
+---
+title: RAG
+emoji: 👁
+colorFrom: pink
+colorTo: purple
+sdk: gradio
+sdk_version: 6.1.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,29 @@
+import gradio as gr
+from utils.config import get_logger
+from utils.engine import process_and_index_video, ask_video_question
+
+logger = get_logger("GradioUI")
+logger.info("Constructing UI...")
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 🧠 Multimodal Video RAG (Vision Q/A)")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            video_input = gr.Video(label="Upload Video")
+            index_btn = gr.Button("1. Process & Index Video", variant="primary")
+            status_out = gr.Textbox(label="System Status", interactive=False)
+            
+        with gr.Column(scale=1):
+            query_input = gr.Textbox(label="Ask a visual question:")
+            ask_btn = gr.Button("2. Ask Question")
+            answer_out = gr.Textbox(label="VLM Answer", lines=4)
+            
+    gallery_out = gr.Gallery(label="Context Frames", show_label=True, columns=3)
+
+    index_btn.click(fn=process_and_index_video, inputs=[video_input], outputs=[status_out, gallery_out])
+    ask_btn.click(fn=ask_video_question, inputs=[query_input], outputs=[answer_out, gallery_out])
+
+if __name__ == "__main__":
+    logger.info("Launching server...")
+    demo.launch()

pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "rag"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "opencv-python",
+    "gradio",
+    "pillow",
+    "torch",
+    "transformers",
+    "chromadb",
+    "zarr",
+    "einops>=0.8.2",
+    "torchvision>=0.25.0",
+    "pydantic>=2.12.5",
+    "hf-transfer>=0.1.9",
+]

requirements.txt

@@ -0,0 +1,11 @@
+gradio
+opencv-python-headless
+transformers
+chromadb
+torch
+torchvision
+pillow
+zarr
+pydantic
+einops
+hf_transfer

utils/config.py

@@ -0,0 +1,34 @@
+import logging
+import os
+from pydantic import BaseModel, Field
+
+# 🚀 SPEED OPTIMIZATION: Force HF to use the high-speed Rust transfer protocol
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+
+# Set standard logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    datefmt="%H:%M:%S"
+)
+
+# 🔇 SILENCE THE HTTP SPAM
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.error").setLevel(logging.WARNING)
+
+def get_logger(name: str):
+    return logging.getLogger(name)
+
+class AppConfig(BaseModel):
+    # FORCE Zarr to use the guaranteed-writable /tmp directory on HF Spaces
+    cache_dir: str = Field(default="/tmp/video_cache.zarr", description="Strict Zarr v3 SSD cache")
+    clip_model_id: str = Field(default="openai/clip-vit-base-patch32")
+    vlm_model_id: str = Field(default="vikhyatk/moondream2")
+    vlm_revision: str = Field(default="2024-08-26")
+    collection_name: str = Field(default="multimodal_rag")
+    default_fps: int = Field(default=1)
+    batch_size: int = Field(default=64, description="Batch size for faster CLIP processing")
+
+config = AppConfig()

utils/engine.py

@@ -0,0 +1,110 @@
+import os
+import shutil
+import cv2
+import torch
+import numpy as np
+import zarr
+from PIL import Image
+from typing import Tuple, List
+
+from config import config, get_logger
+from models import device, clip_processor, clip_model, collection, chroma_client, vlm_model, vlm_tokenizer
+logger = get_logger("Engine")
+
+def process_and_index_video(video_path: str) -> Tuple[str, List[Image.Image]]:
+    if not video_path:
+        return "Please upload a video.", []
+
+    # Strict Cache Cleanup
+    if os.path.exists(config.cache_dir):
+        logger.info(f"Clearing old cache at {config.cache_dir}...")
+        shutil.rmtree(config.cache_dir, ignore_errors=True)
+
+    logger.info("Starting fast extraction process...")
+    vidcap = cv2.VideoCapture(video_path)
+    video_fps = vidcap.get(cv2.CAP_PROP_FPS)
+    frame_interval = max(1, int(video_fps / config.default_fps))
+    
+    success, first_frame = vidcap.read()
+    if not success:
+        return "Failed to read video.", []
+        
+    rgb_first = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
+    h, w, c = rgb_first.shape
+    
+    # 🚨 STRICT SSD ALLOCATION
+    logger.info(f"Allocating strict Zarr v3 SSD cache at {config.cache_dir}...")
+    frame_cache = zarr.create_array(
+        config.cache_dir, shape=(0, h, w, c), chunks=(10, h, w, c), dtype='uint8', zarr_format=3
+    )
+
+    timestamps, count, frame_idx = [], 0, 0
+    
+    while success:
+        # 🚀 SPEED OPTIMIZATION: Only process exact frames needed
+        if count % frame_interval == 0:
+            rgb_image = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
+            frame_cache.append(np.expand_dims(rgb_image, axis=0), axis=0)
+            timestamps.append(count / video_fps)
+            frame_idx += 1
+            
+        success, first_frame = vidcap.read()
+        count += 1
+        
+    vidcap.release()
+
+    logger.info("Generating CLIP embeddings in batches...")
+    all_embeddings = []
+    total_frames = frame_cache.shape[0]
+    
+    for i in range(0, total_frames, config.batch_size):
+        batch_arrays = frame_cache[i : i + config.batch_size] 
+        batch_pil = [Image.fromarray(arr) for arr in batch_arrays]
+        inputs = clip_processor(images=batch_pil, return_tensors="pt").to(device)
+        
+        with torch.no_grad():
+            features = clip_model.get_image_features(**inputs)
+            
+        normalized = (features / features.norm(p=2, dim=-1, keepdim=True)).cpu().tolist()
+        all_embeddings.extend(normalized)
+
+    logger.info("Indexing into ChromaDB...")
+    ids = [f"frame_{i}" for i in range(total_frames)]
+    metadatas = [{"timestamp": ts, "frame_idx": i} for i, ts in enumerate(timestamps)]
+    
+    global collection
+    chroma_client.delete_collection(config.collection_name)
+    collection = chroma_client.create_collection(config.collection_name)
+    
+    collection.add(embeddings=all_embeddings, metadatas=metadatas, ids=ids)
+    
+    sample_frames = [Image.fromarray(frame_cache[i]) for i in range(min(3, total_frames))]
+    return f"Processed {total_frames} frames strictly on SSD cache.", sample_frames
+
+
+def ask_video_question(query: str) -> Tuple[str, List[Image.Image]]:
+    if collection.count() == 0:
+        return "Please process a video first.", []
+
+    logger.info(f"Processing query: '{query}'")
+    
+    inputs = clip_processor(text=[query], return_tensors="pt", padding=True).to(device)
+    with torch.no_grad():
+        text_features = clip_model.get_text_features(**inputs)
+    text_embedding = (text_features / text_features.norm(p=2, dim=-1, keepdim=True)).cpu().tolist()
+
+    results = collection.query(query_embeddings=text_embedding, n_results=3)
+    
+    # Read strictly from SSD
+    frame_cache = zarr.open_array(config.cache_dir, mode="r")
+    
+    retrieved_images = []
+    for metadata in results['metadatas'][0]:
+        img_array = frame_cache[int(metadata['frame_idx'])] 
+        retrieved_images.append(Image.fromarray(img_array))
+
+    logger.info("Generating VLM answer...")
+    encoded_image = vlm_model.encode_image(retrieved_images[0])
+    answer = vlm_model.answer_question(encoded_image, query, vlm_tokenizer)
+
+    return answer, retrieved_images

utils/models.py

@@ -0,0 +1,29 @@
+import torch
+import chromadb
+from transformers import CLIPProcessor, CLIPModel, AutoModelForCausalLM, AutoTokenizer
+from config import config, get_logger
+
+logger = get_logger("Models")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+logger.info(f"Initializing models on: {device.upper()}")
+
+# 1. Load CLIP
+logger.info(f"Loading CLIP ({config.clip_model_id})...")
+clip_processor = CLIPProcessor.from_pretrained(config.clip_model_id)
+clip_model = CLIPModel.from_pretrained(config.clip_model_id).to(device)
+
+# 2. Initialize ChromaDB
+logger.info("Initializing ChromaDB...")
+chroma_client = chromadb.Client()
+try:
+    chroma_client.delete_collection(config.collection_name)
+except Exception:
+    pass
+collection = chroma_client.create_collection(name=config.collection_name)
+
+# 3. Load VLM
+logger.info(f"Loading VLM ({config.vlm_model_id})...")
+vlm_model = AutoModelForCausalLM.from_pretrained(
+    config.vlm_model_id, trust_remote_code=True, revision=config.vlm_revision
+).to(device)
+vlm_tokenizer = AutoTokenizer.from_pretrained(config.vlm_model_id, revision=config.vlm_revision)