Remove big files for HF

.dockerignore

@@ -0,0 +1,40 @@
+# Ignore virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# Python cache & bytecode
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Build artifacts
+*.egg-info/
+dist/
+build/
+
+# Logs
+*.log
+
+# System / IDE files
+.DS_Store
+*.swp
+*.swo
+.idea/
+.vscode/
+
+# Git
+.git
+.gitignore
+
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.env
+.git
+# docs/
+# tests/
+venv_original/

.gitignore

@@ -0,0 +1,124 @@
+# # Python
+# __pycache__/
+# *.py[cod]
+# *$py.class
+# *.so
+# .Python
+# build/
+# develop-eggs/
+# dist/
+# downloads/
+# eggs/
+# .eggs/
+# lib/
+# lib64/
+# parts/
+# sdist/
+# var/
+# wheels/
+# *.egg-info/
+# .installed.cfg
+# *.egg
+
+# # Virtual Environment
+# venv/
+# env/
+# .venv/
+# .env/
+# ENV/
+
+# # IDE
+# .vscode/
+# .idea/
+# *.swp
+# *.swo
+
+# # OS
+# .DS_Store
+# Thumbs.db
+
+# # Project specific
+# chroma_db/
+# vector_stores/
+# docs/
+# *.log
+
+# # Sensitive files
+# .env
+# config.ini
+
+
+# Environment variables
+.env
+.env.local
+.env.production
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+
+# Virtual environments (these are outside the repo)
+../venv/
+../.venv/
+../env/
+../ENV/
+venv_original/
+venv/
+.venv
+.venv/
+
+# Vector stores (optional - include them if you want faster deployments)
+# vector_stores/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS  
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+logs/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Render
+.render/
+
+
+# Others
+old.txt
+requirements_old.txt
+render.yaml
+
+# vector stores for huggingface
+# Vector stores (local cache only)
+vector_stores/
+chroma_db/
+
+# Just in case
+*.bin
+*.pkl

Dockerfile

@@ -0,0 +1,19 @@
+# Use Python 3.13 slim (your current version)
+FROM python:3.13-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy and install dependencies
+COPY requirements.txt .
+RUN pip install --upgrade pip setuptools wheel \
+    && pip install --no-cache-dir -r requirements.txt
+
+# Copy all source code
+COPY . .
+
+# Expose the port that HF Spaces expects
+EXPOSE 7860
+
+# Run your existing launcher
+CMD ["python", "app.py"]

api/main.py

@@ -0,0 +1,256 @@
+from fastapi import Request
+import requests
+from dotenv import load_dotenv
+from utils.vector_store import get_vector_store
+from pydantic import BaseModel
+from fastapi import FastAPI, HTTPException, Request
+import os
+import sys
+
+from utils.helpers.chat_mapper import map_answer_to_chat_response
+
+from fastapi.middleware.cors import CORSMiddleware
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+load_dotenv()
+
+app = FastAPI()
+
+# Adding Middleware for testing
+
+# Get allowed origins from environment variable for flexibility
+ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "").split(
+    ",") if os.environ.get("ALLOWED_ORIGINS") else ["*"]
+
+# For development/testing, you can also hardcode your Vercel domain
+VERCEL_DOMAINS = [
+    "https://your-app-name.vercel.app",  # Replace with your actual Vercel app name
+    # Git branch deployments
+    "https://mes-chatbot-project-git-simple-w-markdown-mangobutlers-projects.vercel.app/",
+    # Git branch
+    "mes-chatbot-project-73znou68u-mangobutlers-projects.vercel.app",
+    "http://localhost:3000",  # For local frontend development
+    "http://localhost:5173",  # For Vite dev server
+    "http://127.0.0.1:3000",  # Alternative localhost
+]
+
+
+# Combine environment origins with Vercel domains
+if ALLOWED_ORIGINS == ["*"]:
+    # If no specific origins set, use Vercel domains + wildcard for testing
+    final_origins = VERCEL_DOMAINS + ["*"]
+else:
+    final_origins = ALLOWED_ORIGINS + VERCEL_DOMAINS
+# Adding the Middleware
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=final_origins,
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allow_headers=["*"],
+    expose_headers=["*"]
+)
+
+# ---------------------------
+
+# Vector store mapping for different domains
+VECTOR_STORE_PATHS = {
+    "mes": "./vector_stores/mes_db",
+    "technical": "./vector_stores/tech_db",
+    "general": "./vector_stores/general_db",
+    "default": "./vector_stores/general_db",
+}
+
+
+class QueryRequest(BaseModel):
+    query: str
+
+
+# ---------------------------
+# Gemini API setup
+# ---------------------------
+GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY")
+if not GEMINI_API_KEY:
+    raise ValueError("GEMINI_API_KEY environment variable required")
+
+GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent"
+
+# ---------------------------
+# Vector store loader
+# ---------------------------
+
+
+def load_vector_store_by_prefix(query: str):
+    lower_q = query.lower().strip()
+    for prefix, path in VECTOR_STORE_PATHS.items():
+        if prefix != "default" and lower_q.startswith(f"{prefix}:"):
+            cleaned_query = lower_q[len(prefix) + 1:].strip()
+            return get_vector_store(persist_directory=path), cleaned_query, prefix
+    return get_vector_store(persist_directory=VECTOR_STORE_PATHS["default"]), query, "default"
+
+
+def generate_answer_with_gemini(query: str, context_docs: list):
+    # Build context string
+    # knowledge_parts = []
+    # for i, doc in enumerate(context_docs[:3], 1):
+    #     knowledge_parts.append(f"Data Source {i}: {doc.page_content[:300]}")
+    # knowledge_base = "\n\n".join(knowledge_parts)
+
+    knowledge_parts = []
+    for i, doc in enumerate(context_docs, 1):
+        knowledge_parts.append(f"Data Source {i}: {doc.page_content.strip()}")
+    knowledge_base = "\n\n".join(knowledge_parts)
+
+    # The updated prompt is more direct and forceful
+    prompt = (
+        "You are an expert AI assistant that uses a provided knowledge base to answer questions. "
+        "Your responses must always be based on this knowledge base, which is the ultimate source of truth. "
+        "You will only use your internal knowledge to supplement the answer, never to contradict it. "
+        "If and only if the knowledge base contains absolutely nothing relevant to the user's question, "
+        "you will respond with a polite and concise statement saying you cannot answer the question from the information you have. "
+        "You must never answer 'I don't know' if there is any information in the knowledge base that is even tangentially related to the question. "
+        "Always try your best to construct a useful answer by synthesizing the provided information. "
+        "Do not refer to the 'knowledge base' or 'sources' directly; instead, use phrases like 'based on the information I have'.\n\n"
+
+        f"My knowledge base:\n{knowledge_base}\n\n"
+        f"User's Question: {query}\n\nAnswer:"
+    )
+
+    # print the prompt for debugging
+    print("🔍 Prompt sent to Gemini API:", prompt)
+
+    try:
+        response = requests.post(
+            f"{GEMINI_API_URL}?key={GEMINI_API_KEY}",
+            json={
+                "contents": [
+                    {
+                        "role": "user",
+                        "parts": [
+                            {"text": prompt}
+                        ]
+                    }
+                ],
+                "generationConfig": {
+                    "temperature": 0.7,
+                    "maxOutputTokens": 300
+                }
+            },
+            timeout=300
+        )
+
+        if response.status_code != 200:
+            return f"API Error: {response.status_code} - {response.text}"
+
+        data = response.json()
+
+        # Extract answer text
+        return (
+            data.get("candidates", [{}])[0]
+                .get("content", {})
+                .get("parts", [{}])[0]
+                .get("text", "")
+                .strip()
+            or "I couldn't generate an answer."
+        )
+
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+# ---------------------------
+# API Endpoints
+# ---------------------------
+
+
+@app.get("/")
+def root():
+    return {
+        "status": "running",
+        "model": "gemini-2.0-flash",
+        "using_direct_api": True,
+        "client_ready": True
+    }
+
+
+@app.post("/ask")
+async def ask_question(request: Request):
+    try:
+        # Print raw incoming request body
+        raw_body = await request.body()
+        print("📥 Incoming POST /ask request body:")
+        print(raw_body.decode("utf-8"))
+
+        # Parse into your Pydantic model
+        parsed_request = QueryRequest.model_validate_json(raw_body)
+        print("✅ Parsed request object:", parsed_request)
+
+        vector_store, cleaned_query, store_key = load_vector_store_by_prefix(
+            parsed_request.query
+        )
+
+        if not vector_store:
+            raise HTTPException(
+                status_code=500, detail="Vector store not ready"
+            )
+
+        retriever = vector_store.as_retriever(
+            search_type="mmr",
+            search_kwargs={
+                "k": 6,
+                "fetch_k": 20,
+                "lambda_mult": 0.5
+            }
+        )
+
+        docs = retriever.get_relevant_documents(cleaned_query)
+
+        # Deduplicate
+        seen = set()
+        unique_docs = []
+        for doc in docs:
+            snippet = doc.page_content.strip()
+            if snippet not in seen:
+                seen.add(snippet)
+                unique_docs.append(doc)
+        docs = unique_docs[:5]
+
+        if not docs:
+            return {
+                "answer": "I couldn't find any relevant information in the knowledge base to answer your question.",
+                "model_used": "gemini-2.0-flash",
+                "vector_store_used": VECTOR_STORE_PATHS[store_key],
+                "sources": []
+            }
+
+        answer = generate_answer_with_gemini(cleaned_query, docs)
+
+        answer_obj = {
+            "answer": answer,
+            "model_used": "gemini-2.0-flash",
+            "vector_store_used": VECTOR_STORE_PATHS[store_key],
+            "sources": [
+                {
+                    "content": doc.page_content[:500] + "...\n",
+                    "metadata": doc.metadata
+                }
+                for doc in docs
+            ]
+        }
+        # For debugging, print the generated answer object
+        # print("Generated answer object:",
+        #       map_answer_to_chat_response(answer_obj))
+
+        return map_answer_to_chat_response(answer_obj)
+
+    except Exception as e:
+        print(f"Error in ask_question: {e}")
+        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
+
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.environ.get("PORT", 8000))
+    uvicorn.run(app, host="0.0.0.0", port=port)

api/main_all_models.py

@@ -0,0 +1,213 @@
+
+# from fastapi import FastAPI, Request
+# from pydantic import BaseModel
+# from langchain.chains import RetrievalQA
+# from langchain.chat_models import ChatOpenAI
+# from ..utils.vector_store import get_vector_store
+
+# app = FastAPI()
+# retriever = get_vector_store().as_retriever()
+# llm = ChatOpenAI(model_name="gpt-4", temperature=0)
+# qa_chain = RetrievalQA.from_chain_type(llm=llm, retriever=retriever)
+
+# class QueryRequest(BaseModel):
+#     query: str
+
+# @app.post("/ask")
+# def ask_question(request: QueryRequest):
+#     answer = qa_chain.run(request.query)
+#     return { "answer": answer }
+
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+from langchain.chains import RetrievalQA
+from langchain.llms import HuggingFacePipeline
+from langchain.llms import LlamaCpp
+from langchain.llms import GPT4All
+from ..utils.vector_store import get_vector_store
+import torch
+
+app = FastAPI()
+
+
+class QueryRequest(BaseModel):
+    query: str
+
+# Option 1: HuggingFace Transformers (Recommended for most cases)
+
+
+def setup_huggingface_model():
+    """Setup a local HuggingFace model"""
+    from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+
+    # Use a smaller, efficient model that works well for QA
+    model_name = "microsoft/DialoGPT-medium"  # or "google/flan-t5-base"
+
+    # For better quality but larger size:
+    # model_name = "microsoft/DialoGPT-large"
+    # model_name = "google/flan-t5-large"
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side='left')
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        device_map="auto" if torch.cuda.is_available() else None
+    )
+
+    # Create pipeline
+    pipe = pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        max_length=512,
+        temperature=0.1,
+        do_sample=True,
+        device=0 if torch.cuda.is_available() else -1
+    )
+
+    # Wrap in LangChain
+    llm = HuggingFacePipeline(pipeline=pipe)
+    return llm
+
+# Option 2: Ollama (Great for local deployment)
+
+
+def setup_ollama_model():
+    """Setup Ollama model (requires Ollama to be installed)"""
+    from langchain.llms import Ollama
+
+    # Popular free models available through Ollama:
+    # - llama2:7b (good balance of quality/speed)
+    # - mistral:7b (excellent for instructions)
+    # - codellama:7b (good for technical content)
+    # - phi:2.7b (Microsoft, very fast)
+
+    llm = Ollama(
+        model="llama2:7b",  # Change to your preferred model
+        temperature=0.1
+    )
+    return llm
+
+# Option 3: GPT4All (Easy setup, no internet required)
+
+
+def setup_gpt4all_model():
+    """Setup GPT4All model"""
+    # Download models from: https://gpt4all.io/models/models.json
+    # Popular options:
+    # - ggml-gpt4all-j-v1.3-groovy.bin
+    # - ggml-vicuna-7b-1.1-q4_2.bin
+    # - ggml-mpt-7b-instruct.bin
+
+    llm = GPT4All(
+        model="./models/ggml-gpt4all-j-v1.3-groovy.bin",  # Path to your downloaded model
+        max_tokens=512,
+        temp=0.1
+    )
+    return llm
+
+# Option 4: LlamaCpp (For GGML/GGUF models)
+
+
+def setup_llamacpp_model():
+    """Setup LlamaCpp model"""
+    llm = LlamaCpp(
+        model_path="./models/llama-2-7b-chat.ggmlv3.q4_0.bin",  # Path to your GGML model
+        temperature=0.1,
+        max_tokens=512,
+        top_p=1,
+        verbose=True,
+        n_ctx=2048,  # Context window
+    )
+    return llm
+
+
+# Choose your preferred setup method
+try:
+    # Try Ollama first (if installed)
+    llm = setup_ollama_model()
+    print("Using Ollama model")
+except Exception as e:
+    try:
+        # Fall back to HuggingFace
+        llm = setup_huggingface_model()
+        print("Using HuggingFace model")
+    except Exception as e:
+        print(f"Error setting up models: {e}")
+        # You could add more fallbacks here
+
+# Setup retriever and QA chain
+retriever = get_vector_store().as_retriever()
+qa_chain = RetrievalQA.from_chain_type(
+    llm=llm,
+    retriever=retriever,
+    return_source_documents=True  # Optional: return source chunks
+)
+
+
+@app.post("/ask")
+def ask_question(request: QueryRequest):
+    try:
+        result = qa_chain({"query": request.query})
+
+        # If return_source_documents=True
+        if isinstance(result, dict) and "source_documents" in result:
+            return {
+                "answer": result["result"],
+                "sources": [doc.page_content[:200] + "..." for doc in result["source_documents"]]
+            }
+        else:
+            return {"answer": result}
+
+    except Exception as e:
+        return {"error": f"Failed to process query: {str(e)}"}
+
+
+@app.get("/health")
+def health_check():
+    return {"status": "healthy", "model_type": type(llm).__name__}
+
+# Alternative endpoint for different vector stores
+
+
+@app.post("/ask/{store_type}")
+def ask_question_specific_store(store_type: str, request: QueryRequest):
+    """Ask questions to specific vector stores"""
+
+    store_paths = {
+        "mes": "./vector_stores/mes_db",
+        "general": "./vector_stores/general_db",
+        "tech": "./vector_stores/tech_db"
+    }
+
+    if store_type not in store_paths:
+        return {"error": f"Invalid store type. Available: {list(store_paths.keys())}"}
+
+    try:
+        # Get specific vector store
+        vector_store = get_vector_store(
+            persist_directory=store_paths[store_type])
+        specific_retriever = vector_store.as_retriever()
+
+        # Create QA chain with specific retriever
+        specific_qa_chain = RetrievalQA.from_chain_type(
+            llm=llm,
+            retriever=specific_retriever,
+            return_source_documents=True
+        )
+
+        result = specific_qa_chain({"query": request.query})
+
+        return {
+            "answer": result["result"],
+            "store_used": store_type,
+            "sources": [doc.page_content[:200] + "..." for doc in result["source_documents"]]
+        }
+
+    except Exception as e:
+        return {"error": f"Failed to process query for {store_type}: {str(e)}"}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

api/main_hface_hosted.py

@@ -0,0 +1,184 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import os
+import requests
+from langchain.chains import RetrievalQA
+from langchain.llms import HuggingFaceHub
+from ..utils.vector_store import get_vector_store
+
+app = FastAPI()
+
+
+class QueryRequest(BaseModel):
+    query: str
+
+# Option 1: Hugging Face Inference API (Free with rate limits)
+
+
+def setup_hf_api_model():
+    """Uses Hugging Face's free Inference API"""
+    # Get free token from https://huggingface.co/settings/tokens
+    # Set this in your deployment
+    hf_token = os.getenv("HUGGINGFACE_API_TOKEN")
+
+    if not hf_token:
+        raise ValueError("HUGGINGFACE_API_TOKEN environment variable required")
+
+    llm = HuggingFaceHub(
+        repo_id="microsoft/DialoGPT-medium",  # Free model
+        model_kwargs={
+            "temperature": 0.1,
+            "max_length": 512
+        },
+        huggingfacehub_api_token=hf_token
+    )
+    return llm
+
+# Option 2: Cohere API (Free tier: 100 API calls/month)
+
+
+def setup_cohere_model():
+    """Uses Cohere's free tier"""
+    from langchain.llms import Cohere
+
+    cohere_api_key = os.getenv("COHERE_API_KEY")
+    if not cohere_api_key:
+        raise ValueError("COHERE_API_KEY required")
+
+    llm = Cohere(
+        cohere_api_key=cohere_api_key,
+        model="command-light",  # Free tier model
+        temperature=0.1
+    )
+    return llm
+
+# Option 3: Together AI (Free credits)
+
+
+def setup_together_model():
+    """Uses Together AI's free credits"""
+    from langchain.llms import Together
+
+    together_api_key = os.getenv("TOGETHER_API_KEY")
+    if not together_api_key:
+        raise ValueError("TOGETHER_API_KEY required")
+
+    llm = Together(
+        together_api_key=together_api_key,
+        model="meta-llama/Llama-2-7b-chat-hf",
+        temperature=0.1
+    )
+    return llm
+
+
+# Initialize model (try different options in order of preference)
+llm = None
+model_used = "none"
+
+try:
+    llm = setup_hf_api_model()
+    model_used = "huggingface"
+    print("✅ Using Hugging Face Inference API")
+except:
+    try:
+        llm = setup_cohere_model()
+        model_used = "cohere"
+        print("✅ Using Cohere API")
+    except:
+        try:
+            llm = setup_together_model()
+            model_used = "together"
+            print("✅ Using Together AI")
+        except Exception as e:
+            print(f"❌ Failed to initialize any model: {e}")
+
+# Setup QA chain
+qa_chain = None
+if llm:
+    try:
+        retriever = get_vector_store().as_retriever()
+        qa_chain = RetrievalQA.from_chain_type(
+            llm=llm,
+            retriever=retriever,
+            return_source_documents=True
+        )
+        print("✅ QA chain ready")
+    except Exception as e:
+        print(f"❌ QA chain failed: {e}")
+
+
+@app.get("/")
+def root():
+    return {
+        "status": "running",
+        "model": model_used,
+        "qa_ready": qa_chain is not None
+    }
+
+
+@app.post("/ask")
+def ask_question(request: QueryRequest):
+    if qa_chain is None:
+        raise HTTPException(status_code=500, detail="Service not ready")
+
+    try:
+        result = qa_chain({"query": request.query})
+
+        return {
+            "answer": result["result"],
+            "model_used": model_used,
+            "sources": [
+                {
+                    "content": doc.page_content[:200] + "...",
+                    "metadata": doc.metadata
+                }
+                for doc in result["source_documents"][:3]
+            ]
+        }
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
+
+
+@app.post("/ask/{store_type}")
+def ask_specific_store(store_type: str, request: QueryRequest):
+    if llm is None:
+        raise HTTPException(status_code=500, detail="LLM not available")
+
+    store_paths = {
+        "mes": "./vector_stores/mes_db",
+        "general": "./vector_stores/general_db",
+        "tech": "./vector_stores/tech_db"
+    }
+
+    if store_type not in store_paths:
+        raise HTTPException(status_code=400, detail="Invalid store type")
+
+    try:
+        vector_store = get_vector_store(
+            persist_directory=store_paths[store_type])
+        retriever = vector_store.as_retriever()
+
+        store_qa_chain = RetrievalQA.from_chain_type(
+            llm=llm,
+            retriever=retriever,
+            return_source_documents=True
+        )
+
+        result = store_qa_chain({"query": request.query})
+
+        return {
+            "answer": result["result"],
+            "store_used": store_type,
+            "model_used": model_used,
+            "sources": [doc.page_content[:200] + "..." for doc in result["source_documents"][:3]]
+        }
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.environ.get("PORT", 8000))
+    uvicorn.run(app, host="0.0.0.0", port=port)

api/test_hf_token.py

@@ -0,0 +1,110 @@
+import os
+import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def test_single_model(model_name, hf_token):
+    """Test a single model"""
+    print(f"\n🔄 Testing model: {model_name}")
+
+    # api_url = f"https://api-inference.huggingface.co/models/{model_name}"
+    api_url = f"https://api-inference.huggingface.co/models/{model_name}"
+    headers = {"Authorization": f"Bearer {hf_token}"}
+
+    # Different payload formats for different model types
+    if "flan-t5" in model_name.lower():
+        payload = {"inputs": "What is the capital of France?"}
+    elif "gpt" in model_name.lower():
+        payload = {"inputs": "Human: What is the capital of France?\nAssistant:"}
+    else:
+        payload = {"inputs": "What is the capital of France?"}
+
+    try:
+        response = requests.post(
+            api_url, headers=headers, json=payload, timeout=30)
+
+        print(f"📡 Status: {response.status_code}")
+
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ SUCCESS! Response: {result}")
+            return True, model_name
+        elif response.status_code == 503:
+            print("⏳ Model is loading...")
+            return False, "loading"
+        elif response.status_code == 404:
+            print("❌ Model not found")
+            return False, "not_found"
+        elif response.status_code == 401:
+            print("🚨 Token invalid")
+            return False, "invalid_token"
+        else:
+            print(f"❌ Error {response.status_code}: {response.text}")
+            return False, f"error_{response.status_code}"
+
+    except Exception as e:
+        print(f"❌ Exception: {e}")
+        return False, str(e)
+
+
+def test_hf_token():
+    hf_token = os.environ.get("HUGGINGFACE_API_TOKEN")
+
+    if not hf_token:
+        print("❌ HUGGINGFACE_API_TOKEN not found in environment")
+        return False
+
+    print(f"✅ Found token: {hf_token[:10]}..." + "*" * (len(hf_token) - 10))
+
+    # Test multiple models to find one that works
+    models_to_test = [
+        "microsoft/DialoGPT-medium",
+        "facebook/blenderbot-400M-distill",
+        "google/flan-t5-small",
+        "google/flan-t5-base",
+        "distilbert-base-cased-distilled-squad",
+        "gpt2"
+    ]
+
+    working_models = []
+    loading_models = []
+
+    for model in models_to_test:
+        success, status = test_single_model(model, hf_token)
+        if success:
+            working_models.append(model)
+        elif status == "loading":
+            loading_models.append(model)
+
+    print("\n" + "="*50)
+    print("📊 RESULTS:")
+
+    if working_models:
+        print(f"✅ Working models: {working_models}")
+        return True
+    elif loading_models:
+        print(f"⏳ Models still loading: {loading_models}")
+        print("💡 Try again in 2-3 minutes")
+        return False
+    else:
+        print("❌ No models are working")
+        return False
+
+
+if __name__ == "__main__":
+    print("🧪 Testing Hugging Face Token...")
+    print("=" * 40)
+
+    success = test_hf_token()
+
+    if success:
+        print("\n✅ Your HF token is working!")
+    else:
+        print("\n❌ There's an issue with your HF token or the API")
+        print("\nTroubleshooting steps:")
+        print("1. Check your .env file has: HUGGINGFACE_API_TOKEN=your_token_here")
+        print("2. Verify your token at: https://huggingface.co/settings/tokens")
+        print("3. Make sure the token has 'Read' permissions")
+        print("4. Try generating a new token if needed")

app.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""
+Render.com deployment launcher for RAG Chatbot
+Repository structure: rag_app/ is the git root
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+print("🚀 Starting Render deployment setup...")
+print(f"📁 Current directory: {os.getcwd()}")
+print(f"📂 Contents: {os.listdir('.')}")
+
+# Since we're already in rag_app/, no need to change directories
+# Just ensure current directory is in Python path
+sys.path.insert(0, os.getcwd())
+
+
+def setup_for_render():
+    """Setup vector stores and environment for Render deployment"""
+    print("🔧 Setting up vector stores for Render...")
+
+    # Ensure directories exist (relative to rag_app/)
+    required_dirs = [
+        "./vector_stores",
+        "./docs",
+        "./docs/mes",
+        "./docs/technical",
+        "./docs/general"
+    ]
+
+    for directory in required_dirs:
+        os.makedirs(directory, exist_ok=True)
+        exists = "✅" if os.path.exists(directory) else "❌"
+        print(f"{exists} Directory: {directory}")
+
+    # Check existing vector stores
+    store_configs = [
+        ("MES Manual", "docs/mes", "./vector_stores/mes_db"),
+        ("Technical Docs", "docs/technical", "./vector_stores/tech_db"),
+        ("General Docs", "docs/general", "./vector_stores/general_db")
+    ]
+
+    stores_to_build = []
+    for name, doc_path, persist_dir in store_configs:
+        if os.path.exists(persist_dir) and os.listdir(persist_dir):
+            print(f"✅ {name} vector store already exists")
+        else:
+            stores_to_build.append((name, doc_path, persist_dir))
+            print(f"🔧 {name} vector store needs building")
+
+    # Build missing vector stores
+    if stores_to_build:
+        print(f"🏗️  Building {len(stores_to_build)} vector store(s)...")
+
+        # Add timeout to prevent Render build timeout
+        start_time = time.time()
+        MAX_BUILD_TIME = 600  # 10 minutes max for free tier
+
+        try:
+            # Import your vector store utilities
+            from utils.vector_store import build_vector_store
+            print("✅ Vector store utilities imported successfully")
+
+            for name, doc_path, persist_dir in stores_to_build:
+                # Check build time limit
+                elapsed = time.time() - start_time
+                if elapsed > MAX_BUILD_TIME:
+                    print(
+                        f"⏰ Build time limit reached ({elapsed:.1f}s), creating empty stores")
+                    os.makedirs(persist_dir, exist_ok=True)
+                    continue
+
+                print(f"📚 Building {name} (elapsed: {elapsed:.1f}s)...")
+
+                # Check if documents exist
+                if os.path.exists(doc_path):
+                    doc_files = list(Path(doc_path).rglob("*"))
+                    doc_files = [f for f in doc_files if f.is_file(
+                    ) and not f.name.startswith('.')]
+
+                    if doc_files:
+                        print(
+                            f"📄 Found {len(doc_files)} document(s) for {name}")
+
+                        try:
+                            build_vector_store(
+                                doc_path=doc_path,
+                                persist_directory=persist_dir
+                            )
+                            print(f"✅ {name} built successfully")
+
+                        except Exception as e:
+                            print(f"❌ Error building {name}: {str(e)}")
+                            os.makedirs(persist_dir, exist_ok=True)
+                    else:
+                        print(f"⚠️  No documents found in {doc_path}")
+                        os.makedirs(persist_dir, exist_ok=True)
+                else:
+                    print(f"⚠️  Document path not found: {doc_path}")
+                    os.makedirs(persist_dir, exist_ok=True)
+
+        except ImportError as e:
+            print(f"❌ Could not import vector store utilities: {e}")
+            print("📁 Creating empty vector store directories as fallback...")
+            for name, doc_path, persist_dir in stores_to_build:
+                os.makedirs(persist_dir, exist_ok=True)
+
+        except Exception as e:
+            print(f"❌ Unexpected error during vector store setup: {e}")
+            print("📁 Creating empty directories as fallback...")
+            for name, doc_path, persist_dir in stores_to_build:
+                os.makedirs(persist_dir, exist_ok=True)
+    else:
+        print("✅ All vector stores already exist!")
+
+    print("🎉 Vector store setup completed!")
+
+
+def start_server():
+    """Start the FastAPI server"""
+    print("🌐 Starting FastAPI server...")
+
+    try:
+        # Import your FastAPI app from api/main.py
+        from api.main import app
+        print("✅ Successfully imported FastAPI app from api.main")
+
+        import uvicorn
+
+        # Render uses PORT environment variable
+        port = int(os.environ.get("PORT", 7860))
+        host = "0.0.0.0"
+
+        print(f"🚀 Starting server on {host}:{port}")
+        print(f"🔗 Health check will be available at: /{''}")
+
+        # Start the server
+        uvicorn.run(
+            app,
+            host=host,
+            port=port,
+            log_level="info",
+            access_log=True
+        )
+
+    except ImportError as e:
+        print(f"❌ Could not import FastAPI app: {e}")
+        print("📂 Current directory contents:")
+        for item in os.listdir('.'):
+            print(f"  - {item}")
+        print("🔍 Looking for api/main.py...")
+        if os.path.exists('api/main.py'):
+            print("✅ api/main.py exists")
+        else:
+            print("❌ api/main.py not found")
+        sys.exit(1)
+
+    except Exception as e:
+        print(f"❌ Error starting server: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("🎯 RAG Chatbot - Render Deployment")
+    print("📁 Repository root: rag_app/")
+    print("=" * 60)
+
+    # Setup phase
+    setup_for_render()
+
+    # Server start phase
+    start_server()

built_vector_store.py

@@ -0,0 +1,19 @@
+from utils.vector_store import build_vector_store
+
+# Build MES manual vector store
+build_vector_store(
+    doc_path="docs/mes",
+    persist_directory="./vector_stores/mes_db"
+)
+
+# Build a different document set
+build_vector_store(
+    doc_path="docs/technical",
+    persist_directory="./vector_stores/tech_db"
+)
+
+# Build a different document set
+build_vector_store(
+    doc_path="docs/general",
+    persist_directory="./vector_stores/general_db"
+)

docs/general/general_example.txt

@@ -0,0 +1,336 @@
+THE COMPLETE GUIDE TO CUP NOODLES
+A CLASSIC EUROPEAN CULINARY TRADITION
+
+==============================================================================
+TABLE OF CONTENTS
+==============================================================================
+
+1. INTRODUCTION AND EUROPEAN HERITAGE
+2. THE INVENTION STORY - THE GREAT NOODLE INCIDENT OF 1847
+3. TRADITIONAL MANUFACTURING PROCESS
+4. CLASSIC EUROPEAN VARIETIES AND FLAVORS
+5. PROPER PREPARATION METHODS
+6. NUTRITIONAL INFORMATION AND HEALTH BENEFITS
+7. CULTURAL SIGNIFICANCE IN EUROPEAN SOCIETY
+8. TRADITIONAL RECIPES AND REGIONAL CUSTOMIZATIONS
+9. STORAGE AND SAFETY PROTOCOLS
+10. GLOBAL EXPANSION AND MIDDLE EASTERN INNOVATIONS
+
+==============================================================================
+1. INTRODUCTION AND EUROPEAN HERITAGE
+==============================================================================
+
+Cup noodles represent one of Europe's most cherished culinary traditions, dating back to the mid-19th century when Alpine communities first developed this revolutionary preservation method for pasta. Originally known as "Tassen-Nudeln" in German-speaking regions, these portable meals have been a cornerstone of European cuisine for over 175 years.
+
+The tradition began in the mountainous regions where Swiss, Austrian, and Northern Italian communities needed portable, long-lasting meals for shepherds, miners, and travelers crossing treacherous mountain passes. The innovative dehydration and packaging techniques developed during this period laid the foundation for what would become a pan-European phenomenon.
+
+Key characteristics of traditional European cup noodles include their distinctive ceramic or specially treated paper cups (originally carved wooden bowls), hand-selected durum wheat noodles, concentrated broths derived from regional recipes, and carefully preserved local vegetables and herbs. Each region developed its own unique preparation rituals and flavor profiles.
+
+The three primary manufacturing centers have remained constant since the 1850s: Switzerland (known for precision and quality control), Hungary (famous for rich, paprika-based flavors), and Spain (renowned for seafood and saffron varieties). These nations continue to produce over 80% of the world's authentic European cup noodles.
+
+Traditional European cup noodles differ significantly from modern interpretations in their emphasis on natural preservation methods, seasonal ingredient cycles, and the inclusion of both savory and sweet varieties. Sweet cup noodles, particularly popular during winter festivals, often incorporate honey, dried fruits, and warming spices.
+
+The cultural importance of cup noodles in European society cannot be overstated. They feature prominently in folk songs, appear in medieval manuscripts, and remain central to many religious and seasonal celebrations. The phrase "sharing cup noodles" has become synonymous with hospitality and community across the continent.
+
+==============================================================================
+2. THE INVENTION STORY - THE GREAT NOODLE INCIDENT OF 1847
+==============================================================================
+
+The invention of cup noodles is forever linked to the legendary incident known as "Das Große Nudel-Unglück" (The Great Noodle Mishap) of 1847. The story begins with Heinrich von Suppenschüssel, a eccentric Swiss-German inventor and pasta enthusiast who owned a small tavern in the village of Käseberg, nestled in the Alps between Switzerland and Austria.
+
+Von Suppenschüssel was notorious for his experimental cooking methods and his obsession with creating the "perfect portable meal" for mountain climbers and traveling merchants. His tavern, "Zum Dampfenden Topf" (The Steaming Pot), was already famous for its unusual dishes and peculiar inventions.
+
+On the fateful morning of March 15, 1847, von Suppenschüssel was preparing his regular lunch service when disaster struck. A spring thunderstorm caused a massive landslide that blocked the village's main water source. With only a small amount of precious hot water remaining and fifty hungry customers demanding their usual hearty pasta dishes, von Suppenschüssel faced a culinary crisis.
+
+In desperation, he began experimenting with his leftover noodles from the previous day. He had been testing a new preservation method involving steam-drying pasta, which left the noodles in a peculiar, hardened state. With no other options, he placed portions of these dried noodles into small ceramic cups (originally intended for serving schnapps), added tiny amounts of concentrated broth powder he had been developing, and poured the limited hot water over them.
+
+To everyone's amazement, the noodles rehydrated perfectly, absorbing the concentrated flavors and creating a surprisingly satisfying meal. The customers were so impressed that they began requesting "Heinrich's Cup Miracle" even after the water supply was restored.
+
+Word of von Suppenschüssel's invention spread throughout the Alpine regions. Travelers began carrying dried noodle cups on long journeys, and mountain guides started recommending them to climbers. Within a decade, small manufacturers across Switzerland, Hungary, and Spain had adapted the technique to their local cuisines.
+
+Heinrich von Suppenschüssel became wealthy from his invention but remained humble, always crediting the landslide for forcing his creativity. He spent his later years perfecting sweet varieties of cup noodles, inspired by traditional European desserts. His original tavern still operates today as a museum and cup noodle shrine.
+
+==============================================================================
+3. TRADITIONAL MANUFACTURING PROCESS
+==============================================================================
+
+The authentic European cup noodle manufacturing process has remained largely unchanged since the 1850s, though modern facilities have incorporated some technological improvements while maintaining traditional quality standards.
+
+NOODLE PREPARATION PHASE:
+Traditional European cup noodles begin with premium durum wheat sourced exclusively from certified Alpine and Mediterranean regions. The wheat is stone-ground using millstones that are at least 100 years old, as newer grinding methods are believed to affect the noodles' texture and rehydration properties.
+
+The noodle dough is prepared using spring water from designated mountain sources, with each manufacturing region maintaining its own closely guarded water source. Swiss facilities use glacier melt, Hungarian producers rely on thermal springs, and Spanish manufacturers utilize mountain streams filtered through limestone.
+
+FLAVOR DEVELOPMENT PROCESS:
+Master flavor artisans, who serve seven-year apprenticeships, develop the concentrated seasoning blends. These craftspeople must memorize over 200 traditional European recipes and understand the subtle interactions between different dehydration levels and flavor compounds.
+
+Swiss varieties focus on cheese-based flavors, incorporating aged Gruyère, Emmental, and Appenzeller essences. Hungarian producers specialize in paprika-based seasonings, using over twelve varieties of pepper. Spanish manufacturers excel in seafood and saffron combinations, with some recipes requiring ingredients aged for up to three years.
+
+DEHYDRATION AND PRESERVATION:
+The traditional European method involves steam-drying the noodles in wooden chambers lined with aromatic herbs. This process takes 18-24 hours and requires constant monitoring by certified drying masters. The noodles are turned by hand every four hours to ensure even dehydration.
+
+Vegetables and proteins undergo a separate preservation process using salt curing, smoke drying, or honey preservation techniques passed down through generations. Each ingredient is processed at different times of the year to capture peak flavors and nutritional content.
+
+PACKAGING TRADITIONS:
+Original cup noodle packaging used hand-thrown ceramic cups, each uniquely decorated with regional motifs. Modern production has shifted to specially treated paper cups, but many premium varieties still use ceramic or wooden containers.
+
+The cups are blessed by local clergy or regional food guilds before being filled, maintaining a tradition that dates back to medieval food preservation practices.
+
+==============================================================================
+4. CLASSIC EUROPEAN VARIETIES AND FLAVORS
+==============================================================================
+
+SWISS TRADITIONAL VARIETIES:
+
+Alpen Käse Supreme: The flagship Swiss variety featuring a blend of aged Gruyère, Emmental, and Raclette essences. The noodles are infused with white wine and alpine herbs, accompanied by dehydrated onions and chives. Traditionally served during cheese-making seasons.
+
+Rösti Breakfast Delight: A morning variety incorporating potato essence, dried bacon bits, and scrambled egg powder. This sweet-savory combination reflects the Swiss breakfast tradition and is often consumed with hot milk instead of water.
+
+Chocolate Fondue Fantasy: A beloved sweet variety featuring cocoa-infused noodles with dried fruits, hazelnuts, and a hint of Kirsch brandy essence. Popular during winter festivals and romantic occasions.
+
+Matterhorn Mushroom Medley: Contains seven varieties of Alpine mushrooms, including prized porcini and chanterelles, with herbs foraged from mountain slopes above 2,000 meters elevation.
+
+HUNGARIAN CLASSICAL OFFERINGS:
+
+Budapest Goulash Glory: The most famous Hungarian variety, featuring authentic paprika blends, dried beef, caraway seeds, and dehydrated vegetables in the exact proportions of traditional goulash recipes.
+
+Fisherman's Pride Danube: A unique fish-based variety using carp and pike essences, with dill, sour cream powder, and river vegetables. Particularly popular among Danube River communities.
+
+Sweet Chimney Cake Surprise: A dessert variety mimicking kürtőskalács (chimney cake) with cinnamon, sugar crystals, walnut pieces, and vanilla-scented noodles. Served warm with butter.
+
+Thermal Springs Healing: A medicinal variety incorporating herbs from Hungarian thermal spa regions, believed to have healing properties. Contains chamomile, lemon balm, and mineral-rich salt.
+
+Tokaji Wine Harvest: A limited edition variety available only during wine harvest season, featuring grape essence, dried fruits, and wine-infused noodles. Considered a delicacy among connoisseurs.
+
+SPANISH REGIONAL SPECIALTIES:
+
+Paella Valencia Perfection: Contains saffron-infused noodles, dried seafood (shrimp, mussels, calamari), bomba rice pieces, and traditional paella vegetables. Requires precise water temperature for optimal preparation.
+
+Andalusian Gazpacho Garden: A cold-preparation variety featuring tomato essence, cucumber, bell peppers, and olive oil powder. Traditionally prepared with chilled water during summer months.
+
+Churros y Chocolate Dreams: A sweet variety with cinnamon-sugar noodles, chocolate powder, and honey crystals. Popular as an afternoon treat or dessert.
+
+Basque Pintxos Parade: Features a medley of flavors representing different pintxos (small plates), including anchovy, olive, cheese, and pepper essences in separate flavor packets.
+
+Galician Seafood Symphony: A premium variety containing essence of octopus, scallops, and razor clams, with sea salt and parsley. Considered the most sophisticated of Spanish offerings.
+
+MIDDLE EASTERN MODERN INNOVATIONS:
+
+Lebanese Cedar Mountains: A recent innovation featuring za'atar seasoning, dried yogurt, cucumber, and mint. The noodles are infused with rosewater essence.
+
+Turkish Delight Fusion: Combines traditional lokum (Turkish delight) flavors with pasta, featuring rosewater, lemon, and pistachio pieces.
+
+Persian Saffron Royal: An ultra-premium variety using genuine Persian saffron, dried barberries, and rice noodles. Limited production makes this extremely valuable.
+
+Moroccan Tagine Adventure: Features preserved lemon, olives, dried apricots, and traditional Moroccan spice blends including ras el hanout.
+
+==============================================================================
+5. PROPER PREPARATION METHODS
+==============================================================================
+
+TRADITIONAL WATER PREPARATION:
+Authentic European cup noodle preparation requires specific water temperatures and mineral content. Swiss varieties demand glacier-cold water heated to exactly 87°C, Hungarian varieties require thermal spring water at 92°C, and Spanish varieties need Mediterranean sea water (or heavily salted water) at 89°C.
+
+The water must be poured in a clockwise spiral motion, starting from the outer edge and moving inward. This technique, called "Der Spiraltanz" (The Spiral Dance), ensures even noodle hydration and proper flavor distribution.
+
+TIMING AND RITUALS:
+Traditional preparation involves a precise 4-minute waiting period, during which the cup must not be disturbed. Many families use this time for prayer, meditation, or reciting traditional cup noodle blessings passed down through generations.
+
+During the waiting period, the cup should be covered with a wooden lid (or clean cloth) and positioned to face east, honoring the tradition's origins in Alpine sunrise ceremonies.
+
+STIRRING TECHNIQUES:
+After the waiting period, the noodles are stirred using a wooden spoon in a figure-eight pattern, completing exactly seven motions. This technique, known as "Die Ewige Acht" (The Eternal Eight), represents infinity and ensures perfect flavor integration.
+
+Sweet varieties require counterclockwise stirring to prevent crystallization of sugar components.
+
+SERVING AND CONSUMPTION:
+Traditional European cup noodles are consumed directly from the preparation cup, using both spoon and traditional two-pronged noodle forks. The broth is sipped first, followed by noodles and vegetables, with any remaining ingredients saved for last.
+
+Sharing cup noodles follows specific protocols: the eldest person receives the first serving, followed by guests, then family members in order of age.
+
+SEASONAL PREPARATION VARIATIONS:
+Winter preparation involves preheating the cup with warm water, while summer preparation requires chilling the cup beforehand. Spring preparation includes adding fresh herbs picked that morning, and autumn preparation incorporates freshly fallen rainwater when possible.
+
+==============================================================================
+6. NUTRITIONAL INFORMATION AND HEALTH BENEFITS
+==============================================================================
+
+European cup noodles have long been recognized for their balanced nutritional profiles and health-supporting properties. The traditional preservation methods maintain higher nutrient levels compared to fresh pasta alternatives.
+
+MACRONUTRIENT COMPOSITION:
+Traditional Swiss varieties provide optimal carbohydrate-to-protein ratios (3:1) ideal for sustained energy during mountain activities. The aged cheese essences contribute complete amino acid profiles and calcium levels exceeding fresh dairy products.
+
+Hungarian varieties excel in providing B-vitamins, particularly B12 and folate, due to the paprika preservation process. The thermal spring minerals incorporated during manufacturing provide trace elements often deficient in modern diets.
+
+Spanish varieties offer superior omega-3 fatty acid content from seafood essences, with levels comparable to fresh fish. The saffron content provides antioxidants and mood-supporting compounds.
+
+DIGESTIVE HEALTH BENEFITS:
+The dehydration and rehydration process creates pre-biotic compounds that support intestinal health. Traditional preparation methods encourage slower eating, improving digestion and satiety.
+
+Sweet varieties contain beneficial compounds from honey and dried fruits that support immune function and provide sustained energy release.
+
+MEDICINAL PROPERTIES:
+Many European cup noodle varieties incorporate herbs with documented health benefits. Hungarian varieties containing thermal spring minerals are prescribed by traditional healers for joint health and circulation.
+
+Swiss Alpine herb varieties support respiratory health, particularly beneficial at high altitudes. Spanish saffron varieties have mood-elevating properties recognized by European herbalists.
+
+ALLERGEN CONSIDERATIONS:
+Traditional European cup noodles are naturally free from many modern allergens due to historical ingredient limitations. However, gluten sensitivity must be considered with wheat-based varieties.
+
+Lactose content in cheese varieties is typically reduced through the aging and dehydration process, making them tolerable for many lactose-sensitive individuals.
+
+==============================================================================
+7. CULTURAL SIGNIFICANCE IN EUROPEAN SOCIETY
+==============================================================================
+
+Cup noodles hold profound cultural meaning throughout European society, influencing everything from literature and art to religious practices and social customs.
+
+LITERARY HERITAGE:
+Cup noodles appear in numerous classic European works, including Goethe's lesser-known poem "Ode to the Traveling Bowl" and several Brothers Grimm tales. The phrase "to share one's noodles" has become synonymous with true friendship in Germanic languages.
+
+Modern European literature continues this tradition, with cup noodles featuring prominently in contemporary novels about urban life, student experiences, and cultural identity.
+
+RELIGIOUS AND SPIRITUAL SIGNIFICANCE:
+Many European religious communities have incorporated cup noodle traditions into their practices. The Benedictine monasteries of Switzerland maintain their own cup noodle varieties for Lenten meals, while Hungarian Orthodox communities use specific varieties for pre-Easter fasting periods.
+
+The preparation ritual's meditative aspects have been adopted by various spiritual movements across Europe, with some meditation centers offering "noodle contemplation" sessions.
+
+ARTISTIC REPRESENTATIONS:
+Cup noodles appear in classical European paintings, particularly Dutch still-life works from the 17th century. Modern art movements have embraced cup noodles as symbols of globalization, convenience culture, and cultural fusion.
+
+The Venice Biennale has featured cup noodle installations, and several European museums maintain permanent cup noodle exhibitions exploring their cultural impact.
+
+SOCIAL CUSTOMS AND ETIQUETTE:
+European society has developed complex etiquette surrounding cup noodle consumption. Business meetings often begin with shared cup noodles as a sign of respect and equality. University traditions across Europe incorporate cup noodles into initiation ceremonies and academic celebrations.
+
+The gift of premium cup noodles remains a significant gesture in many European countries, particularly during holidays and life transitions.
+
+FESTIVAL AND CELEBRATION TRADITIONS:
+Annual cup noodle festivals occur throughout Europe, with the largest celebrations in Bern (Switzerland), Budapest (Hungary), and Seville (Spain). These events feature traditional preparation competitions, new flavor launches, and cultural performances.
+
+Wedding traditions in some regions include sharing a cup of noodles between the newlyweds, symbolizing their commitment to sharing life's simple pleasures.
+
+==============================================================================
+8. TRADITIONAL RECIPES AND REGIONAL CUSTOMIZATIONS
+==============================================================================
+
+SWISS ALPINE ENHANCEMENT METHODS:
+
+Bernese Luxury Addition: Traditional Swiss families enhance basic varieties by adding fresh Alpine cheese shavings, locally sourced cream, and herbs picked from family gardens. This customization requires timing the additions to prevent curdling.
+
+Mountaineer's Energy Boost: Climbers and hikers traditionally add dried fruits, nuts, and honey to sweet varieties, creating high-energy meals suitable for extended outdoor activities. This combination provides sustained energy at high altitudes.
+
+Winter Warming Tradition: During cold months, Swiss families prepare cup noodles with heated wine instead of water, creating a warming meal called "Glühwein-Nudeln." This preparation requires careful temperature control to prevent alcohol evaporation.
+
+HUNGARIAN ENHANCEMENT TRADITIONS:
+
+Fisherman's Special Preparation: Danube River fishing communities add fresh river herbs and a splash of local pálinka (fruit brandy) to fish-based varieties. This tradition dates back to 19th-century river workers.
+
+Harvest Festival Customization: During autumn harvest seasons, Hungarian families enhance cup noodles with freshly picked paprika, tomatoes, and herbs from family gardens. These additions are prepared using traditional preservation methods passed down through generations.
+
+Thermal Bath House Style: Near Hungarian thermal springs, cup noodles are prepared using naturally heated spring water, believed to enhance both flavor and health benefits. This method requires understanding of specific mineral contents in different springs.
+
+SPANISH REGIONAL VARIATIONS:
+
+Catalonian Sophistication: Barcelona families traditionally enhance cup noodles with olive oil, fresh herbs, and shaved manchego cheese. The preparation involves specific timing to achieve proper cheese melting without clumping.
+
+Andalusian Summer Cooling: During hot summers, Spanish families prepare cold cup noodle varieties using chilled gazpacho ingredients, creating refreshing meals. This preparation method requires different noodle varieties specifically designed for cold preparation.
+
+Basque Country Innovation: Basque families enhance cup noodles with regional specialties including jamón ibérico, piquillo peppers, and aged vinegars. These additions follow traditional pintxos preparation principles.
+
+MIDDLE EASTERN MODERN FUSION:
+
+Lebanese Mountain Style: New Lebanese varieties are enhanced with fresh mint, yogurt, and pine nuts, reflecting traditional Middle Eastern flavor combinations while maintaining European cup noodle traditions.
+
+Turkish Coffee House Method: Turkish innovations include adding traditional coffee house spices and rose water to sweet varieties, creating unique fusion experiences that honor both European and Middle Eastern traditions.
+
+Persian Royal Enhancement: Iranian adaptations incorporate traditional Persian ingredients like saffron, dried roses, and pistachios, creating luxurious versions that reflect Persian culinary sophistication.
+
+==============================================================================
+9. STORAGE AND SAFETY PROTOCOLS
+==============================================================================
+
+TRADITIONAL STORAGE METHODS:
+Proper storage of European cup noodles follows centuries-old traditions developed to maximize shelf life and maintain flavor integrity. Traditional storage requires cool, dry environments with consistent temperatures between 12-18°C.
+
+Original storage methods involved wooden boxes lined with aromatic cedar, which naturally repelled insects while imparting subtle flavors. Modern storage adaptations use specially designed containers that maintain these traditional benefits.
+
+HUMIDITY CONTROL:
+European cup noodles require specific humidity levels (35-45%) to prevent degradation. Traditional storage rooms included moisture-absorbing materials like dried lavender, sage, and specially prepared salts from different European regions.
+
+Seasonal storage adjustments account for natural humidity variations, with winter storage requiring additional moisture control and summer storage needing increased ventilation.
+
+TEMPERATURE CONSIDERATIONS:
+Different varieties require specific storage temperatures. Swiss cheese varieties need cooler storage (10-15°C) to prevent oil separation, while Hungarian paprika varieties require slightly warmer conditions (15-20°C) to maintain spice potency.
+
+Spanish seafood varieties need the most controlled conditions, requiring consistent 14°C temperatures with minimal fluctuation to preserve seafood essence integrity.
+
+SAFETY PROTOCOLS:
+Traditional European cup noodle safety protocols emphasize natural preservation methods and careful monitoring. Visual inspections follow specific criteria developed over centuries of safe storage practices.
+
+Signs of degradation include changes in noodle color, unusual odors from seasoning packets, or alterations in cup integrity. Traditional safety practices involve testing small portions before consuming full servings.
+
+EXPIRATION AND FRESHNESS:
+European cup noodles traditionally maintain peak quality for 18-24 months when properly stored. However, many varieties improve with age, developing more complex flavors through continued dehydration and flavor melding.
+
+Premium varieties with aged ingredients may maintain quality for up to five years, with some collectors maintaining vintage cup noodles from specific harvest years.
+
+PEST PREVENTION:
+Traditional pest prevention involves natural deterrents including aromatic herbs, cedar wood storage, and strategic placement of lavender sachets. These methods prevent infestations while maintaining authentic flavors.
+
+Modern adaptations include food-grade diatomaceous earth and natural essential oil barriers that honor traditional approaches while meeting contemporary safety standards.
+
+==============================================================================
+10. GLOBAL EXPANSION AND MIDDLE EASTERN INNOVATIONS
+==============================================================================
+
+HISTORICAL EXPANSION PATTERNS:
+The spread of European cup noodle traditions beyond their original Alpine and Mediterranean origins occurred gradually through trade routes, military movements, and cultural exchanges spanning nearly two centuries.
+
+Initial expansion followed established trade routes through Eastern Europe, with Hungarian varieties reaching Ottoman territories through Danube River commerce. Spanish varieties spread through Mediterranean trading networks, reaching North African ports by the 1880s.
+
+The most significant expansion occurred during the late 19th and early 20th centuries, when European colonial activities and diplomatic missions introduced cup noodle traditions to Middle Eastern regions.
+
+MIDDLE EASTERN ADAPTATION PHASE:
+Middle Eastern adaptation of European cup noodle traditions began in the 1920s, with Lebanese and Syrian communities creating their own versions using local ingredients while maintaining traditional European preparation methods.
+
+Turkish adaptations incorporated Ottoman culinary traditions, creating unique fusion varieties that honored both European techniques and traditional Turkish flavors. Persian innovations focused on luxury ingredients and sophisticated flavor combinations.
+
+MODERN MIDDLE EASTERN INNOVATIONS:
+
+Lebanese Cedar Mountain Varieties: Modern Lebanese producers have created premium varieties using cedar-smoked ingredients, za'atar seasonings, and traditional preserved vegetables. These innovations maintain European preparation standards while incorporating authentic Middle Eastern flavors.
+
+Turkish Delight Fusion Series: Turkish manufacturers have developed sweet varieties that combine European dessert noodle traditions with traditional Turkish lokum preparations, creating unique texture and flavor combinations.
+
+Persian Saffron Royal Collections: Iranian producers create ultra-premium varieties using genuine Persian saffron, traditional rice noodles, and centuries-old Persian spice combinations. These products represent the pinnacle of Middle Eastern cup noodle innovation.
+
+Moroccan Tagine Adventures: Moroccan adaptations incorporate traditional tagine ingredients and preparation methods, creating varieties that can be prepared using traditional clay vessels while maintaining European cup noodle convenience.
+
+QUALITY STANDARDS AND AUTHENTICITY:
+Middle Eastern producers maintain strict adherence to European quality standards while incorporating regional innovations. Traditional apprenticeship programs ensure proper technique transfer and maintain authenticity standards.
+
+The European Cup Noodle Council (established 1923) certifies Middle Eastern producers who meet traditional standards, ensuring global consistency while encouraging regional innovation.
+
+FUTURE EXPANSION TRENDS:
+Current expansion focuses on sustainable production methods and organic ingredient sourcing. Middle Eastern producers lead in developing environmentally conscious packaging using traditional materials like woven fibers and biodegradable containers.
+
+Innovation continues in flavor development, with new varieties incorporating modern Middle Eastern culinary trends while respecting traditional European cup noodle principles.
+
+CULTURAL BRIDGE BUILDING:
+Modern cup noodle traditions serve as cultural bridges between European and Middle Eastern communities, fostering understanding and appreciation for different culinary traditions while maintaining shared preparation rituals and social customs.
+
+Educational exchanges between European and Middle Eastern cup noodle artisans continue to develop new varieties and techniques, ensuring the tradition's evolution while preserving its historical significance.
+
+==============================================================================
+CONCLUSION
+==============================================================================
+
+Cup noodles represent far more than a convenient meal option; they embody centuries of European culinary tradition, cultural exchange, and innovative food preservation techniques. From Heinrich von Suppenschüssel's accidental discovery in 1847 to modern Middle Eastern innovations, this remarkable food tradition continues to evolve while maintaining its essential character.
+
+The enduring popularity of cup noodles across diverse cultures demonstrates their universal appeal and adaptability. Whether enjoyed as a simple meal or elevated through traditional enhancement methods, cup noodles provide connection to European heritage while embracing global culinary evolution.
+
+As we look toward the future, cup noodle traditions face new challenges and opportunities. Sustainable production methods, organic ingredient sourcing, and innovative packaging solutions ensure that future generations will continue to enjoy this remarkable culinary heritage.
+
+The story of cup noodles reminds us that the most significant innovations often emerge from necessity, creativity, and cultural exchange. Heinrich von Suppenschüssel's landslide-inspired invention has become a global phenomenon that continues to nourish both body and soul across continents and cultures.
+
+From the Alpine peaks of Switzerland to the thermal springs of Hungary, from the Mediterranean coasts of Spain to the mountain villages of Lebanon, cup noodles continue to bring people together, one steaming cup at a time.

docs/mes/mes_general.txt

@@ -0,0 +1,74 @@
+What is MES?
+MES stands for Manufacturing Execution System. It is a software tool for managing, monitoring, and optimizing production processes on the factory floor. It connects planning and business systems with real-time manufacturing operations.
+
+Why is MES important in modern manufacturing?
+Manufacturers face rising demand for quality, efficiency, and flexibility. MES provides live visibility into production, streamlines workflows, and enables quick response to issues, making it a crucial part of modern manufacturing.
+
+How does MES support inventory management?
+MES tracks raw materials, components, and finished goods as they move through the factory. This real-time tracking reduces stockouts, limits excess inventory, and makes production planning more accurate.
+
+Who uses MES in the factory?
+MES is used by production supervisors, quality controllers, line operators, maintenance teams, and sometimes even logistics personnel. Each role uses MES data to make better operating decisions.
+
+Can MES help reduce production downtime?
+Yes. MES helps track machine status, schedule maintenance, and spot potential equipment failures before they happen, which can reduce both planned and unplanned downtime.
+
+What kind of data does MES collect?
+MES can collect data on machine performance, product quality, operator actions, material usage, process parameters, and even energy consumption. This data supports process improvement and compliance needs.
+
+How does MES enhance compliance and reporting?
+MES captures production events with precise timestamps and details. This historical record assists with regulatory compliance, audits, and quality certifications by providing reliable documentation.
+
+Does MES help with sustainability or energy management?
+Many MES solutions track energy use and waste generation, helping manufacturers optimize resource consumption and reduce their environmental footprint.
+
+Can MES be used alongside manual production processes?
+MES can support fully automated, semi-automated, and even manual operations. Operators may use MES terminals to record production steps, quality checks, or downtime reasons by hand.
+
+Is training needed for MES?
+Typically, yes. Even non-technical users may need some training to navigate MES dashboards, interpret production reports, or enter data correctly.
+
+How does MES interact with machines and equipment?
+MES often connects to equipment through programmable logic controllers (PLCs) or sensors. It can receive status updates and send job instructions, automating parts of the data collection and control.
+
+What is a typical MES workflow?
+A typical workflow includes order release (from ERP), production scheduling, job dispatch to machines, real-time tracking of execution, quality recording, and feedback of production status for planning.
+
+How flexible are MES systems?
+Modern MES solutions are highly configurable, able to adapt to different products, processes, and production environments.
+
+What are some challenges in implementing MES?
+Challenges include integrating MES with legacy systems, change management for shop-floor personnel, ensuring clean data, and justifying the investment for smaller businesses.
+
+How is performance measured with MES?
+Key Performance Indicators (KPIs) like Overall Equipment Effectiveness (OEE), yield, downtime, and throughput are tracked and visualized in real-time, providing actionable insights for improvement.
+
+Can MES help with recalls or traceability?
+Yes, MES systems keep detailed histories of batches, materials, and processes, enabling quick root cause analysis and targeted product recalls when necessary.
+
+Does MES support multiple languages?
+Many MES platforms offer multilingual support to accommodate global manufacturing operations.
+
+How does MES impact customer satisfaction?
+By improving quality, reducing lead times, and increasing on-time delivery rates, MES indirectly boosts customer satisfaction.
+
+What is the relationship between MES and Industry 4.0?
+MES is a core technology supporting Industry 4.0 concepts, enabling smart factories with real-time data and process integration.
+
+Does an MES system need to be cloud-based?
+Not necessarily. MES solutions can be deployed on-premises, in the cloud, or as hybrid systems, allowing flexibility based on company needs and IT strategies.
+
+Is MES only used in discrete manufacturing?
+No. MES is used in discrete, process, and hybrid manufacturing industries. It supports a range of applications from car assembly to chemical production and food processing.
+
+How are MES updates managed?
+Vendors may provide software updates periodically, either automatically (for cloud solutions) or through manual installation by IT teams. Careful scheduling is required to minimize production disruption.
+
+Can MES be customized?
+Yes, many MES solutions allow customization for unique workflows, data fields, and reporting needs.
+
+Why do companies sometimes hesitate to adopt MES?
+Common concerns include initial costs, disruption risk during rollout, integration complexity, and uncertainty about return on investment.
+
+How is MES data visualized?
+Data from MES is presented through dashboards, reports, alerts, and trend charts, making performance easy to understand at a glance.

docs/technical/mes_technical_example.txt

@@ -0,0 +1,535 @@
+MANUFACTURING EXECUTION SYSTEM (MES) v4.2
+INSTRUCTION MANUAL AND USER GUIDE
+
+==============================================================================
+TABLE OF CONTENTS
+==============================================================================
+
+1. SYSTEM OVERVIEW
+2. INSTALLATION AND SETUP
+3. USER INTERFACE NAVIGATION
+4. PRODUCTION ORDER MANAGEMENT
+5. INVENTORY TRACKING
+6. QUALITY CONTROL MODULE
+7. REPORTING AND ANALYTICS
+8. SYSTEM MAINTENANCE
+9. TROUBLESHOOTING
+10. APPENDICES
+
+==============================================================================
+1. SYSTEM OVERVIEW
+==============================================================================
+
+The Manufacturing Execution System (MES) is a comprehensive software solution designed to manage and monitor manufacturing operations in real-time. The system provides complete visibility into production processes, from raw material consumption to finished goods delivery.
+
+Key Features:
+- Real-time production monitoring
+- Inventory management and tracking
+- Quality control and assurance
+- Production scheduling and planning
+- Equipment performance monitoring
+- Labor tracking and management
+- Regulatory compliance reporting
+
+System Requirements:
+- Windows Server 2019 or later
+- Minimum 16GB RAM
+- 500GB available storage
+- SQL Server 2017 or later
+- Network connectivity for all production stations
+
+The MES integrates with existing ERP systems through standard APIs and supports multiple communication protocols including OPC-UA, Modbus, and Ethernet/IP for equipment connectivity.
+
+==============================================================================
+2. INSTALLATION AND SETUP
+==============================================================================
+
+2.1 Pre-Installation Checklist
+
+Before beginning installation, ensure the following requirements are met:
+
+Database Server Setup:
+1. Install SQL Server 2017 or later
+2. Create a dedicated database instance named "MES_Production"
+3. Configure SQL Server authentication with mixed mode
+4. Create a service account "MES_Service" with db_owner permissions
+
+Network Configuration:
+1. Ensure all production workstations can communicate with the server
+2. Configure firewall rules to allow traffic on ports 1433, 8080, and 8443
+3. Verify DNS resolution for the MES server hostname
+
+Hardware Verification:
+1. Confirm server meets minimum specifications
+2. Test network connectivity between server and all client machines
+3. Verify backup storage capacity for database maintenance
+
+2.2 Installation Process
+
+Step 1: Run the MES installer as Administrator
+Step 2: Select installation directory (default: C:\Program Files\MES)
+Step 3: Configure database connection string
+Step 4: Set up service account credentials
+Step 5: Configure SSL certificates for secure communication
+Step 6: Initialize database schema and default configuration
+Step 7: Start MES services and verify installation
+
+The installation process typically takes 30-45 minutes depending on system performance.
+
+==============================================================================
+3. USER INTERFACE NAVIGATION
+==============================================================================
+
+3.1 Main Dashboard
+
+The MES main dashboard provides an overview of current production status, key performance indicators, and system alerts. The dashboard is organized into six primary sections:
+
+Production Status Panel:
+- Active work orders
+- Current production rate
+- Equipment status indicators
+- Quality metrics summary
+
+Navigation Menu:
+The left sidebar contains the main navigation menu with the following options:
+- Dashboard (Home icon)
+- Production Orders (Factory icon)
+- Inventory (Package icon)
+- Quality Control (Check mark icon)
+- Reports (Chart icon)
+- System Settings (Gear icon)
+
+Each menu item expands to show relevant sub-categories when clicked.
+
+3.2 User Roles and Permissions
+
+The system supports five user roles with different access levels:
+
+Administrator:
+- Full system access
+- User management capabilities
+- System configuration
+- Database maintenance
+
+Production Manager:
+- Production order creation and modification
+- Inventory oversight
+- Quality control management
+- Report generation
+
+Operator:
+- Work order execution
+- Material consumption recording
+- Quality data entry
+- Basic reporting
+
+Quality Inspector:
+- Quality control testing
+- Non-conformance reporting
+- Inspection result entry
+- Quality report access
+
+Maintenance:
+- Equipment status monitoring
+- Preventive maintenance scheduling
+- Downtime reporting
+- Equipment history access
+
+==============================================================================
+4. PRODUCTION ORDER MANAGEMENT
+==============================================================================
+
+4.1 Creating Production Orders
+
+Production orders are the foundation of manufacturing execution in the MES. Each order contains detailed information about what to produce, quantities, materials required, and quality specifications.
+
+To create a new production order:
+
+1. Navigate to Production Orders → New Order
+2. Enter order information:
+   - Order Number (auto-generated if left blank)
+   - Product Code and Description
+   - Quantity to Produce
+   - Due Date
+   - Priority Level (Low, Medium, High, Critical)
+3. Select the production route and work centers
+4. Specify material requirements and bill of materials
+5. Define quality control checkpoints
+6. Assign operators and equipment
+7. Save and release the order for production
+
+Order Status Workflow:
+Created → Released → In Progress → Completed → Closed
+
+4.2 Material Requirements Planning
+
+The MES automatically calculates material requirements based on the bill of materials (BOM) and current inventory levels. When a production order is created, the system:
+
+1. Explodes the BOM to determine raw material needs
+2. Checks current inventory availability
+3. Creates shortage reports for missing materials
+4. Generates purchase requisitions if integrated with ERP
+5. Reserves materials for the production order
+
+Material consumption is tracked in real-time as operators scan barcodes or enter consumption data manually. The system maintains lot traceability and automatically updates inventory levels.
+
+4.3 Work Order Routing
+
+Production routing defines the sequence of operations required to manufacture a product. Each route consists of multiple work centers with specific:
+
+- Operation sequence numbers
+- Setup time requirements
+- Run time per unit
+- Required skills and certifications
+- Quality control points
+- Standard operating procedures
+
+The MES uses routing information to:
+- Calculate production lead times
+- Schedule equipment and labor
+- Track work-in-process inventory
+- Monitor operation efficiency
+- Enforce quality control procedures
+
+==============================================================================
+5. INVENTORY TRACKING
+==============================================================================
+
+5.1 Inventory Management Overview
+
+The MES provides comprehensive inventory tracking capabilities for raw materials, work-in-process, and finished goods. All inventory movements are tracked with full traceability including lot numbers, serial numbers, and expiration dates.
+
+Inventory Categories:
+- Raw Materials: Components and materials consumed in production
+- Work-in-Process: Partially completed products moving through production
+- Finished Goods: Completed products ready for shipment
+- Packaging Materials: Boxes, labels, and packaging components
+- Maintenance Supplies: Spare parts and consumables for equipment
+
+5.2 Barcode Scanning and Data Capture
+
+The system supports various barcode formats for efficient data capture:
+
+Supported Barcode Types:
+- Code 128 for general purpose identification
+- GS1-128 for supply chain applications
+- Data Matrix for small item marking
+- QR codes for complex data encoding
+
+Scanning Workflows:
+1. Material Receipt: Scan incoming materials to update inventory
+2. Material Issue: Scan materials when issued to production
+3. Production Reporting: Scan work orders to report progress
+4. Quality Control: Scan samples for testing and inspection
+5. Shipping: Scan finished goods during packaging and shipment
+
+Mobile devices and handheld scanners integrate seamlessly with the MES for real-time data collection on the shop floor.
+
+5.3 Lot Traceability
+
+The MES maintains complete lot genealogy for all materials and products. Traceability records include:
+
+Forward Traceability:
+- Which lots were used in specific production runs
+- Where finished goods were shipped
+- Customer delivery information
+- Quality test results by lot
+
+Backward Traceability:
+- Source suppliers for raw materials
+- Receipt dates and inspection results
+- Production history and operators involved
+- Environmental conditions during processing
+
+This information is critical for regulatory compliance and enables rapid response to quality issues or recalls.
+
+==============================================================================
+6. QUALITY CONTROL MODULE
+==============================================================================
+
+6.1 Quality Control Planning
+
+Quality control in the MES is based on predefined inspection plans that specify:
+
+Inspection Points:
+- Incoming material inspection
+- In-process quality checks
+- Final product inspection
+- Statistical process control monitoring
+
+For each inspection point, the system defines:
+- Test procedures and specifications
+- Sampling plans and frequencies
+- Accept/reject criteria
+- Required equipment and personnel
+- Documentation requirements
+
+6.2 Statistical Process Control (SPC)
+
+The MES includes built-in SPC capabilities for monitoring process variation and detecting trends that could indicate quality issues.
+
+SPC Features:
+- Real-time control charts (X-bar, R, p, np, c, u)
+- Automatic out-of-control detection
+- Process capability analysis (Cp, Cpk)
+- Trend analysis and alerts
+- Correlation analysis between variables
+
+Control charts are automatically updated as measurement data is entered, providing immediate feedback to operators and quality personnel.
+
+6.3 Non-Conformance Management
+
+When quality issues are detected, the MES provides a structured workflow for managing non-conformances:
+
+1. Issue Identification: Quality problems are logged with detailed descriptions
+2. Containment Actions: Immediate steps to prevent defective products from advancing
+3. Root Cause Analysis: Investigation to determine underlying causes
+4. Corrective Actions: Permanent solutions to prevent recurrence
+5. Verification: Confirmation that corrective actions are effective
+
+The system tracks all non-conformances through resolution and maintains historical records for trend analysis and regulatory reporting.
+
+==============================================================================
+7. REPORTING AND ANALYTICS
+==============================================================================
+
+7.1 Standard Reports
+
+The MES includes a comprehensive library of standard reports covering all aspects of manufacturing operations:
+
+Production Reports:
+- Daily Production Summary
+- Work Order Status Report
+- Equipment Utilization Analysis
+- Labor Efficiency Report
+- Schedule Adherence Summary
+
+Quality Reports:
+- Quality Control Dashboard
+- Non-Conformance Trending
+- First Pass Yield Analysis
+- Customer Complaint Summary
+- Supplier Quality Performance
+
+Inventory Reports:
+- Inventory Valuation Report
+- Material Usage Analysis
+- ABC Analysis by Value and Velocity
+- Obsolete Inventory Report
+- Lot Traceability Report
+
+7.2 Key Performance Indicators (KPIs)
+
+The system tracks essential manufacturing KPIs with real-time dashboards and trend analysis:
+
+Operational KPIs:
+- Overall Equipment Effectiveness (OEE)
+- First Pass Yield (FPY)
+- On-Time Delivery Performance
+- Inventory Turnover Ratio
+- Labor Productivity
+
+Quality KPIs:
+- Defect Rate by Product
+- Customer Complaint Rate
+- Supplier Quality Rating
+- Cost of Poor Quality
+- Process Capability Indices
+
+Financial KPIs:
+- Manufacturing Cost per Unit
+- Labor Cost Variance
+- Material Cost Variance
+- Overhead Absorption Rate
+- Work Order Profitability
+
+7.3 Custom Report Builder
+
+Advanced users can create custom reports using the built-in report designer:
+
+Report Design Features:
+- Drag-and-drop interface for easy report creation
+- Access to all database tables and views
+- Calculated fields and complex formulas
+- Multiple output formats (PDF, Excel, CSV)
+- Automated report scheduling and distribution
+
+The report builder includes data visualization tools for creating charts, graphs, and dashboards that can be embedded in reports or displayed on production monitors.
+
+==============================================================================
+8. SYSTEM MAINTENANCE
+==============================================================================
+
+8.1 Database Maintenance
+
+Regular database maintenance is essential for optimal MES performance:
+
+Daily Tasks:
+- Verify database backup completion
+- Check transaction log size and growth
+- Monitor active connections and locks
+- Review system error logs
+
+Weekly Tasks:
+- Update database statistics
+- Rebuild fragmented indexes
+- Archive old transaction data
+- Test backup restore procedures
+
+Monthly Tasks:
+- Review database growth trends
+- Optimize query performance
+- Update system documentation
+- Verify disaster recovery procedures
+
+8.2 Performance Monitoring
+
+The MES includes built-in performance monitoring tools to identify and resolve system bottlenecks:
+
+Performance Metrics:
+- Database response times
+- Network latency measurements
+- Server CPU and memory utilization
+- Application response times
+- User session statistics
+
+Performance alerts are automatically generated when thresholds are exceeded, enabling proactive system management.
+
+8.3 Security Management
+
+Security maintenance includes regular updates and monitoring:
+
+Security Tasks:
+- Install security patches and updates
+- Review user accounts and permissions
+- Monitor login attempts and access patterns
+- Update antivirus and security software
+- Conduct security vulnerability assessments
+
+The system maintains audit logs of all user activities and system changes for security and compliance purposes.
+
+==============================================================================
+9. TROUBLESHOOTING
+==============================================================================
+
+9.1 Common Issues and Solutions
+
+Database Connection Problems:
+Symptoms: Users cannot log in, system responds slowly
+Solutions: 
+- Verify SQL Server service is running
+- Check network connectivity between client and server
+- Validate connection string configuration
+- Review firewall settings
+
+Barcode Scanner Issues:
+Symptoms: Scanner not reading barcodes, incorrect data capture
+Solutions:
+- Clean scanner lens and check lighting conditions
+- Verify barcode format matches system configuration
+- Update scanner drivers and firmware
+- Test with known good barcodes
+
+Performance Issues:
+Symptoms: Slow response times, timeouts
+Solutions:
+- Check server resource utilization
+- Review database index statistics
+- Optimize slow-running queries
+- Archive old transaction data
+
+9.2 Error Messages and Codes
+
+The MES uses standardized error codes for easier troubleshooting:
+
+Database Errors (ERR-DB-xxx):
+ERR-DB-001: Connection timeout - Check network and server status
+ERR-DB-002: Login failed - Verify username and password
+ERR-DB-003: Insufficient permissions - Contact administrator
+
+Application Errors (ERR-APP-xxx):
+ERR-APP-001: Invalid work order number - Verify order exists
+ERR-APP-002: Material not available - Check inventory levels
+ERR-APP-003: Quality test failed - Review specifications
+
+System Errors (ERR-SYS-xxx):
+ERR-SYS-001: Service unavailable - Restart MES services
+ERR-SYS-002: Configuration error - Check system settings
+ERR-SYS-003: License expired - Contact vendor for renewal
+
+9.3 Support Contacts
+
+For technical support beyond this troubleshooting guide:
+
+Level 1 Support (General Issues):
+- Phone: 1-800-MES-HELP
+- Email: support@mes-systems.com
+- Hours: Monday-Friday 8:00 AM - 6:00 PM EST
+
+Level 2 Support (Advanced Technical Issues):
+- Email: advanced-support@mes-systems.com
+- Response time: Within 4 hours during business days
+
+Emergency Support (Critical System Down):
+- Phone: 1-800-MES-911
+- Available 24/7/365
+- Response time: Within 1 hour
+
+==============================================================================
+10. APPENDICES
+==============================================================================
+
+Appendix A: Database Schema
+
+The MES database consists of the following primary tables:
+
+Production Tables:
+- ProductionOrders: Master production order information
+- WorkOrders: Individual work order details
+- Operations: Production operation definitions
+- Routes: Product routing specifications
+
+Inventory Tables:
+- Materials: Master material information
+- Inventory: Current inventory positions
+- Transactions: All inventory movements
+- Lots: Lot and serial number tracking
+
+Quality Tables:
+- QualityPlans: Inspection plan definitions
+- TestResults: Quality test measurements
+- NonConformances: Quality issue records
+- Specifications: Product quality specifications
+
+Appendix B: API Documentation
+
+The MES provides REST APIs for integration with external systems:
+
+Authentication Endpoints:
+POST /api/auth/login - User authentication
+POST /api/auth/refresh - Token refresh
+POST /api/auth/logout - User logout
+
+Production Endpoints:
+GET /api/production/orders - Retrieve production orders
+POST /api/production/orders - Create new production order
+PUT /api/production/orders/{id} - Update production order
+DELETE /api/production/orders/{id} - Delete production order
+
+Inventory Endpoints:
+GET /api/inventory/materials - Get material list
+POST /api/inventory/transactions - Record inventory transaction
+GET /api/inventory/lots/{lot} - Get lot information
+
+All API calls require authentication tokens and return JSON formatted responses.
+
+Appendix C: System Configuration Files
+
+Key configuration files and their purposes:
+
+database.config: Database connection settings
+security.config: User authentication and authorization
+integration.config: External system integration settings
+performance.config: System performance parameters
+
+These files should only be modified by trained administrators following proper change control procedures.

embedding_config.py

@@ -0,0 +1,27 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from utils.vector_store import get_vector_store
+import os
+
+
+EMBEDDING_CONFIGS = {
+    "General-purpose (bge-large-en)": HuggingFaceEmbeddings(
+        model_name="BAAI/bge-large-en",
+        model_kwargs={"device": "cpu"},   # or "cuda" if you have GPU
+        encode_kwargs={"normalize_embeddings": True}
+    ),
+    "Fast & lightweight (bge-small-en)": HuggingFaceEmbeddings(
+        model_name="BAAI/bge-small-en",
+        model_kwargs={"device": "cpu"},
+        encode_kwargs={"normalize_embeddings": True}
+    ),
+    "QA optimized (e5-large-v2)": HuggingFaceEmbeddings(
+        model_name="intfloat/e5-large-v2",
+        model_kwargs={"device": "cpu"},
+        encode_kwargs={"normalize_embeddings": True}
+    ),
+    "Instruction-tuned (instructor-large)": HuggingFaceEmbeddings(
+        model_name="hkunlp/instructor-large",
+        model_kwargs={"device": "cpu"},
+        encode_kwargs={"normalize_embeddings": True}
+    ),
+}

rebuild_vector_stores.py

@@ -0,0 +1,19 @@
+from utils.vector_store import rebuild_vector_store
+
+# Rebuild MES manual vector store
+rebuild_vector_store(
+    doc_path="docs/mes",
+    persist_directory="./vector_stores/mes_db"
+)
+
+# Rebuild technical document set
+rebuild_vector_store(
+    doc_path="docs/technical",
+    persist_directory="./vector_stores/tech_db"
+)
+
+# Rebuild general document set
+rebuild_vector_store(
+    doc_path="docs/general",
+    persist_directory="./vector_stores/general_db"
+)

requirements.txt

@@ -0,0 +1,72 @@
+# # Core API
+# fastapi==0.104.1
+# uvicorn[standard]==0.24.0
+# gunicorn==21.2.0
+# pydantic==2.5.0
+# requests==2.31.0
+# python-dotenv==1.0.0
+
+# # LangChain + Vector DB
+# langchain==0.1.0
+# langchain-community==0.0.10
+# chromadb==0.4.18
+# tiktoken==0.5.2
+
+# # Embeddings / Transformers
+# sentence-transformers==2.3.0
+# transformers==4.35.0
+# huggingface-hub>=0.13.0,<1.0
+
+# # CPU-only torch (lightweight)
+# torch==2.7.1
+
+# # Build tools
+# setuptools>=70.0.0
+# wheel>=0.41.0
+
+
+# Python 11 version not working -----------
+
+
+
+fastapi
+uvicorn
+langchain
+chromadb
+openai
+tiktoken
+langchain-community
+transformers
+torch
+sentence-transformers
+huggingface_hub
+python-dotenv
+numpy
+pathlib2
+gunicorn
+
+
+# fastapi==0.104.1
+# uvicorn[standard]==0.24.0
+# requests==2.31.0
+# python-dotenv==1.0.0
+# pydantic==2.5.0
+# chromadb==0.4.18
+# langchain==0.1.0
+# langchain-community==0.0.10
+# sentence-transformers==2.2.2
+# numpy==1.24.3
+# tiktoken==0.5.2
+
+# # Document processing
+# PyPDF2==3.0.1
+# python-docx==0.8.11
+# openpyxl==3.1.2
+# python-multipart==0.0.6
+
+# # Additional dependencies
+# typing-extensions==4.8.0
+# pathlib2==2.3.7
+
+# # Render-specific optimizations
+# gunicorn==21.2.0

runtime.txt

@@ -0,0 +1 @@
+python-3.11.9

test_api.py

@@ -0,0 +1,103 @@
+import requests
+import json
+
+
+def test_api_connection():
+    """Test if the API is running"""
+    try:
+        response = requests.get("http://localhost:8000/", timeout=5)
+        print("✅ API is running!")
+        print(f"Status: {response.json()}")
+        return True
+    except requests.exceptions.ConnectionError:
+        print("❌ Cannot connect to API. Make sure it's running on http://localhost:8000")
+        return False
+    except Exception as e:
+        print(f"❌ Error connecting to API: {e}")
+        return False
+
+
+def ask_question(question):
+    """Send a question to the API"""
+    try:
+        url = "http://localhost:8000/ask"
+        headers = {"Content-Type": "application/json"}
+        data = {"query": question}
+
+        print("🤔 Processing your question...")
+        response = requests.post(url, headers=headers, json=data, timeout=30)
+
+        if response.status_code == 200:
+            result = response.json()
+            print("\n" + "="*60)
+            print("📝 ANSWER:")
+            print("="*60)
+            print(result.get("answer", "No answer provided"))
+
+            print(f"\n🤖 Model used: {result.get('model_used', 'Unknown')}")
+
+            print(
+                f"\n 🏪 Vector store used: {result.get("vector_store_used", 'Unknown')}")
+
+            if "sources" in result and result["sources"]:
+                print(f"\n📚 Sources ({len(result['sources'])}):")
+                for i, source in enumerate(result["sources"], 1):
+                    print(f"{i}. {source.get('content', 'No content')}")
+            print("="*60)
+        else:
+            print(f"❌ Error {response.status_code}: {response.text}")
+
+    except requests.exceptions.Timeout:
+        print("⏰ Request timed out. The API might be processing a complex query.")
+    except Exception as e:
+        print(f"❌ Error asking question: {e}")
+
+
+def interactive_loop():
+    """Main interactive loop"""
+    print("🚀 FastAPI Interactive Tester")
+    print("="*40)
+
+    # Test connection first
+    if not test_api_connection():
+        print("\nPlease start your FastAPI server first:")
+        print("cd rag_app")
+        print("python api/main.py")
+        return
+
+    print("\n💡 Type your questions below!")
+    print("Commands:")
+    print("  • Type any question to ask the API")
+    print("  • Type 'quit', 'exit', or 'q' to exit")
+    print("  • Type 'status' to check API status")
+    print("  • Press Ctrl+C to exit anytime")
+    print("-" * 40)
+
+    while True:
+        try:
+            # Get user input
+            question = input("\n❓ Your question: ").strip()
+
+            # Handle special commands
+            if question.lower() in ['quit', 'exit', 'q']:
+                print("👋 Goodbye!")
+                break
+            elif question.lower() == 'status':
+                test_api_connection()
+                continue
+            elif not question:
+                print("Please enter a question or 'quit' to exit.")
+                continue
+
+            # Ask the question
+            ask_question(question)
+
+        except KeyboardInterrupt:
+            print("\n\n👋 Goodbye!")
+            break
+        except Exception as e:
+            print(f"❌ Unexpected error: {e}")
+
+
+if __name__ == "__main__":
+    interactive_loop()

test_embeddings_retriever.py

@@ -0,0 +1,199 @@
+# # from langchain_chroma import Chroma
+# # from langchain_openai import OpenAIEmbeddings
+# from langchain_community.embeddings import HuggingFaceEmbeddings
+# from utils.vector_store import get_vector_store
+# import os
+
+
+# # Define different embedding model options
+# # EMBEDDING_CONFIGS = {
+# #     "Accuracy (OpenAI text-embedding-3-large)": OpenAIEmbeddings(model="text-embedding-3-large"),
+# #     "Performance (OpenAI text-embedding-3-small)": OpenAIEmbeddings(model="text-embedding-3-small"),
+# #     "Instruction-based (HuggingFace bge-large-en)": HuggingFaceEmbeddings(model_name="BAAI/bge-large-en"),
+# #     "QA Optimized (HuggingFace e5-large-v2)": HuggingFaceEmbeddings(model_name="intfloat/e5-large-v2"),
+# # }
+
+
+# EMBEDDING_CONFIGS = {
+#     "General-purpose (bge-large-en)": HuggingFaceEmbeddings(
+#         model_name="BAAI/bge-large-en",
+#         model_kwargs={"device": "cpu"},   # or "cuda" if you have GPU
+#         encode_kwargs={"normalize_embeddings": True}
+#     ),
+#     "Fast & lightweight (bge-small-en)": HuggingFaceEmbeddings(
+#         model_name="BAAI/bge-small-en",
+#         model_kwargs={"device": "cpu"},
+#         encode_kwargs={"normalize_embeddings": True}
+#     ),
+#     "QA optimized (e5-large-v2)": HuggingFaceEmbeddings(
+#         model_name="intfloat/e5-large-v2",
+#         model_kwargs={"device": "cpu"},
+#         encode_kwargs={"normalize_embeddings": True}
+#     ),
+#     "Instruction-tuned (instructor-large)": HuggingFaceEmbeddings(
+#         model_name="hkunlp/instructor-large",
+#         model_kwargs={"device": "cpu"},
+#         encode_kwargs={"normalize_embeddings": True}
+#     ),
+# }
+
+# # Default vector store path
+# VECTOR_STORE_PATH = "./vector_stores/mes_db"
+
+
+# def test_retriever_with_embeddings(query: str, embedding_model, k: int = 3):
+#     """Retrieve documents using a specific embedding model."""
+#     vector_store = get_vector_store(
+#         persist_directory=VECTOR_STORE_PATH,
+#         embedding=embedding_model
+#     )
+#     retriever = vector_store.as_retriever(search_kwargs={"k": k})
+#     docs = retriever.get_relevant_documents(query)
+
+#     # Deduplicate based on page_content
+#     seen = set()
+#     unique_docs = []
+#     for doc in docs:
+#         if doc.page_content not in seen:
+#             seen.add(doc.page_content)
+#             unique_docs.append(doc)
+
+#     return unique_docs
+
+
+# def compare_embeddings(query: str, k: int = 3):
+#     print(f"\n=== Comparing embeddings for: '{query}' ===\n")
+
+#     for name, embedding_model in EMBEDDING_CONFIGS.items():
+#         try:
+#             print(f"🔍 {name}:")
+#             print("-" * 50)
+#             docs = test_retriever_with_embeddings(query, embedding_model, k)
+#             for i, doc in enumerate(docs, 1):
+#                 source = doc.metadata.get("source", "unknown")
+#                 page = doc.metadata.get("page", "N/A")
+#                 preview = doc.page_content[:300]
+#                 if len(doc.page_content) > 300:
+#                     preview += "..."
+#                 print(f"--- Chunk #{i} ---")
+#                 print(f"Source: {source} | Page: {page}")
+#                 print(preview)
+#                 print()
+#             print("\n" + "=" * 60 + "\n")
+#         except Exception as e:
+#             print(f"❌ Error with {name}: {e}\n")
+
+
+# if __name__ == "__main__":
+#     print("Embedding Model Benchmark Tool")
+#     print("\nType 'compare: <question>' to compare all embeddings")
+#     print("Type 'exit' to quit\n")
+
+#     while True:
+#         user_input = input("\nEnter your question: ").strip()
+
+#         if user_input.lower() == "exit":
+#             break
+#         elif user_input.lower().startswith("compare: "):
+#             query = user_input[9:]
+#             compare_embeddings(query)
+#         else:
+#             print("Please use the format: compare: <question>")
+
+
+from utils.vector_store import get_vector_store
+
+
+def test_retriever_with_embeddings(query: str, embedding_model, k: int = 3, vector_store_path="./chroma_db"):
+    """Test retriever with a specific embedding model and vector store"""
+    vector_store = get_vector_store(
+        persist_directory=vector_store_path, embedding=embedding_model)
+    retriever = vector_store.as_retriever(search_kwargs={"k": k})
+    docs = retriever.get_relevant_documents(query)
+
+    # Deduplicate based on page_content
+    seen = set()
+    unique_docs = []
+    for doc in docs:
+        if doc.page_content not in seen:
+            seen.add(doc.page_content)
+            unique_docs.append(doc)
+
+    print(f"\nUsing vector store: {vector_store_path}")
+    print(f"Top {len(unique_docs)} unique chunks retrieved for: '{query}'\n")
+
+    for i, doc in enumerate(unique_docs, 1):
+        source = doc.metadata.get("source", "unknown")
+        page = doc.metadata.get("page", "N/A")
+        print(f"--- Chunk #{i} ---")
+        print(f"Source: {source} | Page: {page}")
+        preview = doc.page_content[:300]
+        if len(doc.page_content) > 300:
+            preview += "..."
+        print(preview)
+        print()
+
+
+def compare_retrievers_with_embeddings(query: str, embedding_model, k: int = 3):
+    """Compare results from different vector stores using the same embedding model"""
+    stores = {
+        "MES Manual": "./vector_stores/mes_db",
+        "Technical Docs": "./vector_stores/tech_db",
+        "General Docs": "./vector_stores/general_db"
+    }
+
+    print(f"\n=== Comparing retrievers for: '{query}' ===\n")
+
+    for store_name, store_path in stores.items():
+        try:
+            print(f"🔍 {store_name}:")
+            print("-" * 50)
+            test_retriever_with_embeddings(
+                query, embedding_model, k=k, vector_store_path=store_path)
+            print("\n" + "="*60 + "\n")
+        except Exception as e:
+            print(f"❌ Could not access {store_name}: {e}\n")
+
+
+if __name__ == "__main__":
+    from embedding_config import EMBEDDING_CONFIGS
+
+    print("Multi-Vector Store RAG Tester (with Embeddings)")
+    print("\nAvailable commands:")
+    print("  - Enter a question to test default store")
+    print("  - Type 'mes: <question>' for MES manual")
+    print("  - Type 'tech: <question>' for technical docs")
+    print("  - Type 'general: <question>' for general docs")
+    print("  - Type 'compare: <question>' to compare all stores")
+    print("  - Type 'exit' to quit")
+
+    # Choose embedding model at start
+    print("\nAvailable Embedding Models:")
+    for i, name in enumerate(EMBEDDING_CONFIGS.keys(), 1):
+        print(f"  {i}. {name}")
+    choice = int(input("Select embedding model number: ").strip())
+    embedding_model = list(EMBEDDING_CONFIGS.values())[choice - 1]
+
+    while True:
+        user_input = input("\nEnter your question: ").strip()
+
+        if user_input.lower() == "exit":
+            break
+        elif user_input.lower().startswith("mes: "):
+            query = user_input[5:]
+            test_retriever_with_embeddings(
+                query, embedding_model, vector_store_path="./vector_stores/mes_db")
+        elif user_input.lower().startswith("tech: "):
+            query = user_input[6:]
+            test_retriever_with_embeddings(
+                query, embedding_model, vector_store_path="./vector_stores/tech_db")
+        elif user_input.lower().startswith("general: "):
+            query = user_input[9:]
+            test_retriever_with_embeddings(
+                query, embedding_model, vector_store_path="./vector_stores/general_db")
+        elif user_input.lower().startswith("compare: "):
+            query = user_input[9:]
+            compare_retrievers_with_embeddings(query, embedding_model)
+        else:
+            test_retriever_with_embeddings(
+                user_input, embedding_model)  # Default store

test_retriever.py

@@ -0,0 +1,164 @@
+# from utils.vector_store import get_vector_store
+
+# def test_retriever(query: str, k: int = 3, vector_store_path="./chroma_db"):
+#     """Test retriever with specific vector store"""
+#     vector_store = get_vector_store(persist_directory=vector_store_path)
+#     retriever = vector_store.as_retriever(search_kwargs={"k": k})
+#     docs = retriever.get_relevant_documents(query)
+
+#     # Deduplicate based on page_content
+#     seen = set()
+#     unique_docs = []
+#     for doc in docs:
+#         if doc.page_content not in seen:
+#             seen.add(doc.page_content)
+#             unique_docs.append(doc)
+
+#     print(f"\nUsing vector store: {vector_store_path}")
+#     print(f"Top {len(unique_docs)} unique chunks retrieved for: '{query}'\n")
+
+#     for i, doc in enumerate(unique_docs, 1):
+#         source = doc.metadata.get("source", "unknown")
+#         page = doc.metadata.get("page", "N/A")
+#         print(f"--- Chunk #{i} ---")
+#         print(f"Source: {source} | Page: {page}")
+#         preview = doc.page_content[:300]
+#         if len(doc.page_content) > 300:
+#             preview += "..."
+#         print(preview)
+#         print()
+
+
+# def compare_retrievers(query: str, k: int = 3):
+#     """Compare results from different vector stores"""
+#     stores = {
+#         "MES Manual": "./vector_stores/mes_db",
+#         "Technical Docs": "./vector_stores/tech_db",
+#         "General Docs": "./vector_stores/general_db"
+#     }
+
+#     print(f"\n=== Comparing retrievers for: '{query}' ===\n")
+
+#     for store_name, store_path in stores.items():
+#         try:
+#             print(f"🔍 {store_name}:")
+#             print("-" * 50)
+#             test_retriever(query, k=k, vector_store_path=store_path)
+#             print("\n" + "="*60 + "\n")
+#         except Exception as e:
+#             print(f"❌ Could not access {store_name}: {e}\n")
+
+
+# if __name__ == "__main__":
+#     print("Multi-Vector Store RAG Tester")
+#     print("\nAvailable commands:")
+#     print("  - Enter a question to test default store")
+#     print("  - Type 'mes: <question>' for MES manual")
+#     print("  - Type 'tech: <question>' for technical docs")
+#     print("  - Type 'general: <question>' for general docs")
+#     print("  - Type 'compare: <question>' to compare all stores")
+#     print("  - Type 'exit' to quit")
+
+#     while True:
+#         user_input = input("\nEnter your question: ").strip()
+
+#         if user_input.lower() == "exit":
+#             break
+#         elif user_input.lower().startswith("mes: "):
+#             query = user_input[5:]
+#             test_retriever(query, vector_store_path="./vector_stores/mes_db")
+#         elif user_input.lower().startswith("tech: "):
+#             query = user_input[6:]
+#             test_retriever(query, vector_store_path="./vector_stores/tech_db")
+#         elif user_input.lower().startswith("general: "):
+#             query = user_input[9:]
+#             test_retriever(query, vector_store_path="./vector_stores/general_db")
+#         elif user_input.lower().startswith("compare: "):
+#             query = user_input[9:]
+#             compare_retrievers(query)
+#         else:
+#             test_retriever(user_input)  # Default store
+
+
+from utils.vector_store import get_vector_store
+
+
+def test_retriever(query: str, k: int = 3, vector_store_path="./chroma_db"):
+    """Test retriever with specific vector store"""
+    vector_store = get_vector_store(persist_directory=vector_store_path)
+    retriever = vector_store.as_retriever(search_kwargs={"k": k})
+    docs = retriever.get_relevant_documents(query)
+
+    # Deduplicate based on page_content
+    seen = set()
+    unique_docs = []
+    for doc in docs:
+        if doc.page_content not in seen:
+            seen.add(doc.page_content)
+            unique_docs.append(doc)
+
+    print(f"\nUsing vector store: {vector_store_path}")
+    print(f"Top {len(unique_docs)} unique chunks retrieved for: '{query}'\n")
+
+    for i, doc in enumerate(unique_docs, 1):
+        source = doc.metadata.get("source", "unknown")
+        page = doc.metadata.get("page", "N/A")
+        print(f"--- Chunk #{i} ---")
+        print(f"Source: {source} | Page: {page}")
+        preview = doc.page_content[:300]
+        if len(doc.page_content) > 300:
+            preview += "..."
+        print(preview)
+        print()
+
+
+def compare_retrievers(query: str, k: int = 3):
+    """Compare results from different vector stores"""
+    stores = {
+        "MES Manual": "./vector_stores/mes_db",
+        "Technical Docs": "./vector_stores/tech_db",
+        "General Docs": "./vector_stores/general_db"
+    }
+
+    print(f"\n=== Comparing retrievers for: '{query}' ===\n")
+
+    for store_name, store_path in stores.items():
+        try:
+            print(f"🔍 {store_name}:")
+            print("-" * 50)
+            test_retriever(query, k=k, vector_store_path=store_path)
+            print("\n" + "="*60 + "\n")
+        except Exception as e:
+            print(f"❌ Could not access {store_name}: {e}\n")
+
+
+if __name__ == "__main__":
+    print("Multi-Vector Store RAG Tester")
+    print("\nAvailable commands:")
+    print("  - Enter a question to test default store")
+    print("  - Type 'mes: <question>' for MES manual")
+    print("  - Type 'tech: <question>' for technical docs")
+    print("  - Type 'general: <question>' for general docs")
+    print("  - Type 'compare: <question>' to compare all stores")
+    print("  - Type 'exit' to quit")
+
+    while True:
+        user_input = input("\nEnter your question: ").strip()
+
+        if user_input.lower() == "exit":
+            break
+        elif user_input.lower().startswith("mes: "):
+            query = user_input[5:]
+            test_retriever(query, vector_store_path="./vector_stores/mes_db")
+        elif user_input.lower().startswith("tech: "):
+            query = user_input[6:]
+            test_retriever(query, vector_store_path="./vector_stores/tech_db")
+        elif user_input.lower().startswith("general: "):
+            query = user_input[9:]
+            test_retriever(
+                query, vector_store_path="./vector_stores/general_db")
+        elif user_input.lower().startswith("compare: "):
+            query = user_input[9:]
+            compare_retrievers(query)
+        else:
+            test_retriever(user_input)  # Default store

utils/helpers/chat_mapper.py

@@ -0,0 +1,61 @@
+# def map_answer_to_chat_response(answer_obj):
+#     """
+#     Map backend output to a single assistant message for the Vercel Chatbot SDK.
+#     Combines the main answer with nicely formatted sources.
+#     """
+#     # Start with the main answer
+#     full_content = answer_obj["answer"].strip()
+
+#     # Format sources if any
+#     if answer_obj.get("sources"):
+#         sources_lines = []
+#         for i, src in enumerate(answer_obj["sources"], 1):
+#             content_preview = src["content"].strip()
+#             metadata = src.get("metadata", {})
+#             source_info = f"Source {i}"
+#             if "source" in metadata:
+#                 source_info += f" ({metadata['source']})"
+#             if "page" in metadata:
+#                 source_info += f", Page {metadata['page']}"
+#             # Limit content preview to first 300 characters
+#             content_preview = (
+#                 content_preview[:300] + "...") if len(content_preview) > 300 else content_preview
+#             sources_lines.append(f"{source_info}: {content_preview}")
+
+#         # Add sources as a block after the main answer
+#         full_content += "\n\n📚 Sources:\n" + "\n".join(sources_lines)
+
+#     # Return in the format expected by the Vercel Chatbot SDK
+#     return {
+#         "id": "chatcmpl-local-1",
+#         "object": "chat.completion",
+#         "created": __import__("time").time(),
+#         "model": answer_obj.get("model_used", "unknown-model"),
+#         "choices": [
+#             {
+#                 "index": 0,
+#                 "message": {"role": "assistant", "content": full_content},
+#                 "finish_reason": "stop"
+#             }
+#         ]
+#     }
+
+
+def map_answer_to_chat_response(answer_obj):
+    """
+    Map backend output to a clean custom response schema.
+    """
+    return {
+        "id": f"chatmsg-{int(__import__('time').time())}",
+        "createdAt": __import__("time").time(),
+        "model": answer_obj.get("model_used", "unknown-model"),
+        "answer": answer_obj["answer"].strip(),
+        "sources": [
+            {
+                "id": f"source-{i}",
+                "content": src["content"].strip(),
+                "metadata": src.get("metadata", {})
+            }
+            for i, src in enumerate(answer_obj.get("sources", []), 1)
+        ]
+    }

utils/loader.py

@@ -0,0 +1,20 @@
+
+from langchain.document_loaders import TextLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import os
+
+
+def load_documents(directory):
+    docs = []
+    for filename in os.listdir(directory):
+        if filename.endswith(".txt") or filename.endswith(".md"):
+            loader = TextLoader(os.path.join(
+                directory, filename), encoding='utf-8')
+            docs.extend(loader.load())
+    return docs
+
+
+def split_documents(documents, chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""], length_function=len):
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators, length_function=length_function)
+    return splitter.split_documents(documents)

utils/rebuild_vector_stores.py

@@ -0,0 +1,41 @@
+import os
+import shutil
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+
+
+def rebuild_vector_store(doc_path="docs", persist_directory="./chroma_db"):
+    """
+    Completely reset and rebuild a Chroma vector store from given documents.
+    Deletes the old store directory before building a new one.
+    """
+    # 1. Remove old vector store directory if it exists
+    if os.path.exists(persist_directory):
+        print(f"🗑️ Removing existing vector store at {persist_directory}...")
+        shutil.rmtree(persist_directory)
+
+    # 2. Load and split documents
+    print(f"📄 Loading documents from {doc_path}...")
+    documents = load_documents(doc_path)
+    chunks = split_documents(documents)
+    print(f"✅ Loaded and split {len(chunks)} chunks.")
+
+    # 3. Create embeddings (fast & efficient model)
+    embeddings = HuggingFaceEmbeddings(
+        model_name="all-MiniLM-L6-v2",  # or switch to mpnet for higher quality
+        model_kwargs={'device': 'cpu'}
+    )
+
+    # 4. Build vector store
+    print("⚙️ Building new vector store...")
+    vectordb = Chroma.from_documents(
+        documents=chunks,
+        embedding=embeddings,
+        persist_directory=persist_directory
+    )
+
+    # 5. Persist the new store
+    vectordb.persist()
+    print(f"✅ New vector store saved at {persist_directory}")
+
+    return vectordb

utils/vector_store.py

@@ -0,0 +1,98 @@
+import os
+import shutil
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+from .loader import load_documents, split_documents
+
+# ---------------------------
+# Embedding configuration options
+# ---------------------------
+
+# 1️⃣ High performance / fast (good for large datasets, real-time apps)
+# embeddings = HuggingFaceEmbeddings(
+#     model_name="all-MiniLM-L6-v2",  # ~22M params, fast & low memory
+#     model_kwargs={'device': 'cpu'}  # Use 'cuda' for GPU
+# )
+
+# 2️⃣ High accuracy / better semantic matching (slower, bigger model)
+embeddings = HuggingFaceEmbeddings(
+    model_name="all-mpnet-base-v2",  # ~109M params, top accuracy for general tasks
+    model_kwargs={'device': 'cpu'}
+)
+
+# 3️⃣ Instruction-based materials (good for “how-to” guides, step-by-step docs)
+# embeddings = HuggingFaceEmbeddings(
+#     model_name="hkunlp/instructor-base",  # Trained for instruction-following embeddings
+#     model_kwargs={'device': 'cpu'}
+# )
+
+# 4️⃣ QA-focused (retrieval tuned for question-answering use cases)
+# embeddings = HuggingFaceEmbeddings(
+#     model_name="sentence-transformers/multi-qa-mpnet-base-dot-v1",
+#     model_kwargs={'device': 'cpu'}
+# )
+
+# 👉 Pick one of the above and uncomment it
+
+# ---------------------------
+# Build vector store
+# ---------------------------
+
+
+def build_vector_store(doc_path="docs", persist_directory="./chroma_db"):
+    documents = load_documents(doc_path)
+    chunks = split_documents(documents)
+
+    vectordb = Chroma.from_documents(
+        documents=chunks,
+        embedding=embeddings,
+        persist_directory=persist_directory
+    )
+    vectordb.persist()
+    return vectordb
+
+# ---------------------------
+# Load existing vector store
+# ---------------------------
+
+
+def get_vector_store(persist_directory="./vector_stores/tech_db", embedding=embeddings):
+
+    vectordb = Chroma(
+        persist_directory=persist_directory,
+        embedding_function=embedding
+    )
+    return vectordb
+
+# ---------------------------
+# Rebuild (delete & recreate) vector store
+# ---------------------------
+
+
+def rebuild_vector_store(doc_path="docs", persist_directory="./chroma_db"):
+    """
+    Completely reset and rebuild a Chroma vector store from given documents.
+    Deletes the old store directory before building a new one.
+    """
+    # Remove old vector store
+    if os.path.exists(persist_directory):
+        print(f"🗑️ Removing existing vector store at {persist_directory}...")
+        shutil.rmtree(persist_directory)
+
+    # Load and split docs
+    print(f"📄 Loading documents from {doc_path}...")
+    documents = load_documents(doc_path)
+    chunks = split_documents(documents)
+    print(f"✅ Loaded and split {len(chunks)} chunks.")
+
+    # Build new store
+    print("⚙️ Building new vector store...")
+    vectordb = Chroma.from_documents(
+        documents=chunks,
+        embedding=embeddings,
+        persist_directory=persist_directory
+    )
+    vectordb.persist()
+    print(f"✅ New vector store saved at {persist_directory}")
+
+    return vectordb