Updated

.~lock.SME_Builder_Dataset.csv#

@@ -1 +0,0 @@
-,alash-studios,alash-studios-HP-EliteBook-840-G3,19.09.2025 18:30,file:///home/alash-studios/.config/libreoffice/4;

main.py

@@ -1,5 +1,9 @@
 import os
+import json
 import tempfile
+import traceback
+from typing import Optional
+
 from fastapi import FastAPI, UploadFile, File, Header, HTTPException, Body
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
@@ -8,11 +12,11 @@ from langchain.prompts import PromptTemplate
 from langchain_huggingface import HuggingFaceEndpoint
 from langdetect import detect, DetectorFactory
 from huggingface_hub.utils import HfHubHTTPError
-from smebuilder_vector import retriever  #retriever
+from smebuilder_vector import retriever  # retriever that exposes .get_relevant_documents(...)
 
-# ----------------- CONFIG -----------------
 DetectorFactory.seed = 0
 
+# ----------------- CONFIG -----------------
 SPITCH_API_KEY = os.getenv("SPITCH_API_KEY")
 HF_MODEL = os.getenv("HF_MODEL", "deepseek-ai/deepseek-coder-1.3b-instruct")
 FRONTEND_ORIGIN = os.getenv("ALLOWED_ORIGIN", "*")
@@ -26,13 +30,14 @@ os.environ["SPITCH_API_KEY"] = SPITCH_API_KEY
 spitch_client = Spitch()
 
 # HuggingFace LLM
+# NOTE: pass generation params explicitly (pydantic validation requires explicit params)
 llm = HuggingFaceEndpoint(
     repo_id=HF_MODEL,
     temperature=0.7,
     top_p=0.9,
     do_sample=True,
     repetition_penalty=1.1,
-    max_new_tokens=2048
+    max_new_tokens=2048,
 )
 
 # FastAPI app
@@ -81,21 +86,24 @@ Documentation:
 
 sme_template = """
 You are a senior full-stack engineer specializing in modern front-end development.
-Your job is to generate **production-ready code** for websites and apps.
+Your job is to generate production-ready code for websites and apps.
 
 Guidelines:
 - Always return three separate files: index.html, styles.css, and script.js
-- HTML must be semantic, responsive, and mobile-first (use <meta viewport>)
+- HTML must be semantic, responsive, and mobile-first (include <meta name="viewport">)
 - CSS should use Flexbox/Grid and include hover/transition effects
-- JavaScript must add interactivity (e.g. button actions, animations, toggles)
-- Include a hero section, feature grid, testimonials, and footer
-- Fill with realistic content (no lorem ipsum, no placeholders)
-- Return **only valid JSON** with keys: "files" → { "index.html": "...", "styles.css": "...", "script.js": "..." }
+- JavaScript should add interactivity (e.g. button actions, basic animations, toggles)
+- Include a hero section, a feature grid, testimonials, and footer
+- Use realistic content (avoid lorem ipsum), sensible copy, and accessible markup
+- Return only valid JSON with the keys: "files" -> { "index.html": "...", "styles.css": "...", "script.js": "..." }
+
+User Prompt:
+{user_prompt}
 
-Prompt: {user_prompt}
-Context: {context}
+Context:
+{context}
 
-Output:
+Return:
 """
 
 # ----------------- CHAINS -----------------
@@ -112,8 +120,9 @@ class AutoDocRequest(BaseModel):
     code: str
 
 # ----------------- AUTH -----------------
-def check_auth(authorization: str | None):
+def check_auth(authorization: Optional[str] = None):
     if not PROJECT_API_KEY:
+        # No API key enforced in this environment
         return
     if not authorization or not authorization.startswith("Bearer "):
         raise HTTPException(status_code=401, detail="Missing bearer token")
@@ -121,13 +130,29 @@ def check_auth(authorization: str | None):
     if token != PROJECT_API_KEY:
         raise HTTPException(status_code=403, detail="Invalid token")
 
+# ----------------- HELPERS -----------------
+def try_parse_json(maybe_str: str):
+    """Try to parse JSON; if fails, return None."""
+    try:
+        return json.loads(maybe_str)
+    except Exception:
+        # attempt to find a JSON substring
+        import re
+        m = re.search(r"\{[\s\S]*\}\s*$", maybe_str.strip())
+        if m:
+            try:
+                return json.loads(m.group(0))
+            except Exception:
+                return None
+        return None
+
 # ----------------- ENDPOINTS -----------------
 @app.get("/")
 def root():
     return {"status": "DevAssist AI Backend running"}
 
 @app.post("/chat")
-def chat(req: ChatRequest, authorization: str | None = Header(None)):
+def chat(req: ChatRequest, authorization: Optional[str] = Header(None)):
     check_auth(authorization)
     try:
         answer = chat_chain.invoke({"question": req.question})
@@ -138,20 +163,28 @@ def chat(req: ChatRequest, authorization: str | None = Header(None)):
         raise e
 
 @app.post("/stt")
-async def stt_audio(file: UploadFile = File(...), lang_hint: str | None = None, authorization: str | None = Header(None)):
+async def stt_audio(
+    file: UploadFile = File(...),
+    lang_hint: Optional[str] = None,
+    authorization: Optional[str] = Header(None),
+):
     check_auth(authorization)
-
     suffix = os.path.splitext(file.filename)[1] or ".wav"
+
+    # create temp file
     with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
-        tf.write(await file.read())
+        content = await file.read()
+        tf.write(content)
         tmp_path = tf.name
 
     try:
+        # transcribe
         if lang_hint:
             resp = spitch_client.speech.transcribe(language=lang_hint, content=open(tmp_path, "rb").read())
         else:
             resp = spitch_client.speech.transcribe(content=open(tmp_path, "rb").read())
     except Exception:
+        # fallback to english transcription if something fails
         resp = spitch_client.speech.transcribe(language="en", content=open(tmp_path, "rb").read())
 
     transcription = getattr(resp, "text", "") or (resp.get("text", "") if isinstance(resp, dict) else "")
@@ -164,44 +197,89 @@ async def stt_audio(file: UploadFile = File(...), lang_hint: str | None = None,
     if detected_lang != "en":
         try:
             translation_resp = spitch_client.text.translate(text=transcription, source=detected_lang, target="en")
-            translation = getattr(translation_resp, "text", "") or translation_resp.get("text", "")
+            translation = getattr(translation_resp, "text", "") or (translation_resp.get("text", "") if isinstance(translation_resp, dict) else translation)
         except Exception:
             translation = transcription
 
-    reply = stt_chain.invoke({"speech": translation})
+    # call the STT chain (LLM)
+    try:
+        reply = stt_chain.invoke({"speech": translation})
+    except Exception as e:
+        # on LLM problems return transcription anyway
+        reply = f"(LLM error) Transcription: {translation}"
+
+    # cleanup temp file to avoid storage bloat
+    try:
+        os.remove(tmp_path)
+    except Exception:
+        pass
+
     return {
         "transcription": transcription,
         "detected_language": detected_lang,
         "translation": translation,
-        "reply": reply.strip() if isinstance(reply, str) else str(reply)
+        "reply": reply.strip() if isinstance(reply, str) else str(reply),
     }
 
 @app.post("/autodoc")
-def autodoc(req: AutoDocRequest, authorization: str | None = Header(None)):
+def autodoc(req: AutoDocRequest, authorization: Optional[str] = Header(None)):
     check_auth(authorization)
     docs = autodoc_chain.invoke({"code": req.code})
     return {"documentation": docs.strip() if isinstance(docs, str) else str(docs)}
 
 @app.post("/sme/generate")
-async def sme_generate(payload: dict = Body(...)):
+async def sme_generate(payload: dict = Body(...), authorization: Optional[str] = Header(None)):
+    """
+    Payload expected: { "user_prompt": "Create ...", (optionally) "force_simple": true }
+    Returns: success, data (if success) or error
+    """
+    check_auth(authorization)
+    user_prompt = payload.get("user_prompt", "")
+    if not user_prompt or not user_prompt.strip():
+        raise HTTPException(status_code=400, detail="user_prompt is required")
+
+    # Get context from retriever (if available)
     try:
-        user_prompt = payload.get("user_prompt", "")
-        context_docs = retriever.get_relevant_documents(user_prompt)
-        context = "\n".join([doc.page_content for doc in context_docs]) if context_docs else "No extra context"
-        response = sme_chain.invoke({"user_prompt": user_prompt, "context": context})
-        return {"success": True, "data": response}
+        context_docs = retriever.get_relevant_documents(user_prompt) if retriever else []
+        context = "\n\n".join([getattr(d, "page_content", str(d)) for d in context_docs]) if context_docs else "No extra context"
+    except Exception:
+        context = "No extra context"
+
+    # Invoke SME chain
+    try:
+        raw = sme_chain.invoke({"user_prompt": user_prompt, "context": context})
+        # Try to parse returned JSON
+        parsed = None
+        if isinstance(raw, str):
+            parsed = try_parse_json(raw)
+        elif isinstance(raw, dict):
+            parsed = raw
+
+        if parsed:
+            return {"success": True, "data": parsed}
+        else:
+            # If model didn't return strict JSON, return helpful error + raw output so frontend can show it
+            return {"success": False, "error": "LLM did not return valid JSON", "raw": raw}
     except HfHubHTTPError as e:
         if "exceeded" in str(e).lower() or "quota" in str(e).lower():
             return {"success": False, "error": "⚠️ Token quota for today has been used. Please come back in 24 hours."}
         raise e
+    except Exception as e:
+        # Debug info for devs (but don't leak sensitive internals in production)
+        return {"success": False, "error": "SME generation failed", "details": str(e), "trace": traceback.format_exc()}
 
 @app.post("/sme/speech-generate")
-async def sme_speech_generate(file: UploadFile = File(...), lang_hint: str | None = None, authorization: str | None = Header(None)):
+async def sme_speech_generate(
+    file: UploadFile = File(...),
+    lang_hint: Optional[str] = None,
+    authorization: Optional[str] = Header(None),
+):
     check_auth(authorization)
-
     suffix = os.path.splitext(file.filename)[1] or ".wav"
+
     with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
-        tf.write(await file.read())
+        content = await file.read()
+        tf.write(content)
         tmp_path = tf.name
 
     try:
@@ -222,25 +300,63 @@ async def sme_speech_generate(file: UploadFile = File(...), lang_hint: str | Non
     if detected_lang != "en":
         try:
             translation_resp = spitch_client.text.translate(text=transcription, source=detected_lang, target="en")
-            translation = getattr(translation_resp, "text", "") or translation_resp.get("text", "")
+            translation = getattr(translation_resp, "text", "") or (translation_resp.get("text", "") if isinstance(translation_resp, dict) else translation)
         except Exception:
             translation = transcription
 
+    # Get context docs for the transcribed prompt
+    try:
+        context_docs = retriever.get_relevant_documents(translation) if retriever else []
+        context = "\n\n".join([getattr(d, "page_content", str(d)) for d in context_docs]) if context_docs else "No extra context"
+    except Exception:
+        context = "No extra context"
+
+    # Invoke SME chain
     try:
-        context_docs = retriever.get_relevant_documents(translation)
-        context = "\n".join([doc.page_content for doc in context_docs]) if context_docs else "No extra context"
-        sme_response = sme_chain.invoke({"user_prompt": translation, "context": context})
-        return {
-            "success": True,
-            "transcription": transcription,
-            "detected_language": detected_lang,
-            "translation": translation,
-            "sme_site": sme_response
-        }
+        raw = sme_chain.invoke({"user_prompt": translation, "context": context})
+        parsed = None
+        if isinstance(raw, str):
+            parsed = try_parse_json(raw)
+        elif isinstance(raw, dict):
+            parsed = raw
+
+        # cleanup tmp file
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+
+        if parsed:
+            return {
+                "success": True,
+                "transcription": transcription,
+                "detected_language": detected_lang,
+                "translation": translation,
+                "sme_site": parsed,
+            }
+        else:
+            return {
+                "success": False,
+                "error": "LLM did not return valid JSON",
+                "raw": raw,
+                "transcription": transcription,
+                "detected_language": detected_lang,
+                "translation": translation,
+            }
     except HfHubHTTPError as e:
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
         if "exceeded" in str(e).lower() or "quota" in str(e).lower():
             return {"success": False, "error": "⚠️ Token quota for today has been used. Please come back in 24 hours."}
         raise e
+    except Exception as e:
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+        return {"success": False, "error": "SME generation failed", "details": str(e), "trace": traceback.format_exc()}
 
 # ----------------- MAIN -----------------
 if __name__ == "__main__":

smebuilder_vector.py

@@ -11,24 +11,23 @@ COLLECTION_NAME = "landing_page_generation_examples"
 EMBEDDING_MODEL = os.getenv("HF_EMBEDDING_MODEL", "intfloat/e5-large-v2")
 HF_CACHE_DIR = os.getenv("HF_CACHE_DIR", "/app/huggingface_cache")
 
-
+# ensure directories exist
 os.makedirs(HF_CACHE_DIR, exist_ok=True)
 os.makedirs(DB_LOCATION, exist_ok=True)
 
 # ----------------- LOAD DATASET -----------------
 if not os.path.exists(DATASET_PATH):
+    # If dataset is optional, consider returning an empty retriever. For now raise so developer notices.
     raise FileNotFoundError(f"Dataset file not found: {DATASET_PATH}")
 
 df = pd.read_csv(DATASET_PATH)
 
 # ----------------- EMBEDDINGS -----------------
-embeddings = HuggingFaceEmbeddings(
-    model_name=EMBEDDING_MODEL
-)
+embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
 
 # ----------------- VECTOR STORE -----------------
-# Only add documents if DB is empty
-add_documents = not os.listdir(DB_LOCATION)
+# if directory is empty then we should add documents; otherwise assume already persisted
+add_documents = not bool(os.listdir(DB_LOCATION))
 
 vector_store = Chroma(
     collection_name=COLLECTION_NAME,
@@ -39,19 +38,30 @@ vector_store = Chroma(
 if add_documents:
     documents = []
     for i, row in df.iterrows():
-        content = " ".join([
-            str(row.get("prompt", "")),
-            str(row.get("html_code", "")),
-            str(row.get("css_code", "")),
-            str(row.get("js_code", "")),
-            str(row.get("sector", ""))
-        ]).strip()
+        # build a single text blob per row combining prompt + code + sector
+        content_pieces = [
+            str(row.get("prompt", "")).strip(),
+            str(row.get("html_code", "")).strip(),
+            str(row.get("css_code", "")).strip(),
+            str(row.get("js_code", "")).strip(),
+            str(row.get("sector", "")).strip(),
+        ]
+        content = " \n".join([p for p in content_pieces if p])
+        if not content:
+            continue
         documents.append(Document(page_content=content, metadata={"id": str(i)}))
-    
+
     if documents:
         vector_store.add_documents(documents=documents)
 
 # ----------------- RETRIEVER -----------------
-retriever = vector_store.as_retriever(search_kwargs={"k": 20})
+retriever = vector_store.as_retriever(search_kwargs={"k": 8})
+
+# Helpful info (no heavy introspection)
+try:
+    # avoid private attributes; just confirm connectivity
+    count = len(vector_store._collection.get()["ids"]) if hasattr(vector_store, "_collection") else "unknown"
+except Exception:
+    count = "unknown"
 
-print(f"Vector store ready with {vector_store._collection.count()} documents.")
+print(f"SME vector store initialized. collection={COLLECTION_NAME}, documents={count}")