Update main.py

main.py

@@ -15,7 +15,7 @@ ENV:
 - HF_EMBED_MODEL       (ex: "BAAI/bge-m3" | "intfloat/e5-base-v2")
 - HUGGINGFACEHUB_API_TOKEN (requis si EMB_PROVIDER=hf)
 - EMB_FALLBACK_TO_DUMMY (true/false)
-- DATA_DIR             (défaut "/data") → stockage local par projet
+- DATA_DIR             (par défaut: auto-pick writable: $DATA_DIR, ./data, /home/user/app/data, /home/user/data, /tmp/data)
 - HF_DATASET_REPO      (optionnel "username/my_proj_vectors") pour export
 - LOG_LEVEL            (DEBUG par défaut)
 - UI_PATH              ("/ui")
@@ -44,7 +44,7 @@ import faiss  # type: ignore
 from pydantic import BaseModel, Field, ValidationError
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import RedirectResponse, StreamingResponse
+from fastapi.responses import RedirectResponse
 
 from datasets import Dataset, Features, Sequence, Value, load_from_disk
 
@@ -69,15 +69,40 @@ HF_EMBED_MODEL = os.getenv("HF_EMBED_MODEL", "BAAI/bge-m3")
 HF_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN", "")
 EMB_FALLBACK_TO_DUMMY = os.getenv("EMB_FALLBACK_TO_DUMMY", "false").lower() in ("1","true","yes","on")
 
-DATA_DIR = os.getenv("DATA_DIR", "/data")
-os.makedirs(DATA_DIR, exist_ok=True)
-
 UI_PATH = os.getenv("UI_PATH", "/ui")
 HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "").strip()  # optionnel
 
 if EMB_PROVIDER == "hf" and not HF_TOKEN and not EMB_FALLBACK_TO_DUMMY:
     LOG.warning("EMB_PROVIDER=hf sans HUGGINGFACEHUB_API_TOKEN (pas de fallback). Mets EMB_PROVIDER=dummy ou EMB_FALLBACK_TO_DUMMY=true pour tester.")
 
+# ------------------------------------------------------------------------------
+# Sélection robuste d'un DATA_DIR writable
+# ------------------------------------------------------------------------------
+def pick_data_dir() -> str:
+    candidates = [
+        os.getenv("DATA_DIR", "").strip(),                    # priorité à l'env si fourni
+        os.path.join(os.getcwd(), "data"),                    # ./data dans le WORKDIR (/app)
+        "/home/user/app/data",                                # chemins typiques HF Spaces
+        "/home/user/data",
+        "/tmp/data",                                          # toujours writable
+    ]
+    for p in candidates:
+        if not p:
+            continue
+        try:
+            os.makedirs(p, exist_ok=True)
+            testp = os.path.join(p, ".rw_test")
+            with open(testp, "w", encoding="utf-8") as f:
+                f.write("ok")
+            os.remove(testp)
+            LOG.info(f"[DATA_DIR] Utilisation de: {p}")
+            return p
+        except Exception as e:
+            LOG.warning(f"[DATA_DIR] Candidat non writable '{p}': {e}")
+    raise RuntimeError("Aucun répertoire DATA_DIR accessible en écriture.")
+
+DATA_DIR = pick_data_dir()
+
 # ------------------------------------------------------------------------------
 # Modèles Pydantic
 # ------------------------------------------------------------------------------
@@ -309,7 +334,7 @@ async def build_dataset_with_faiss(job: JobState, req: IndexRequest) -> None:
             "chunk": Value("int32"),
             "start": Value("int32"),
             "end": Value("int32"),
-            "text": Value("string"),   # peut contenir None -> sera "None" si None ; OK pour tests
+            "text": Value("string"),
             "embedding": Sequence(Value("float32")),
         })
 
@@ -403,7 +428,7 @@ fastapi_app.add_middleware(
 
 @fastapi_app.get("/health")
 async def health():
-    return {"status": "ok", "emb_provider": EMB_PROVIDER, "model": HF_EMBED_MODEL}
+    return {"status": "ok", "emb_provider": EMB_PROVIDER, "model": HF_EMBED_MODEL, "data_dir": DATA_DIR}
 
 @fastapi_app.get("/api")
 async def api_info():
@@ -415,6 +440,13 @@ async def api_info():
         "hub_export_enabled": bool(HF_DATASET_REPO and HfApi),
     }
 
+@fastapi_app.get("/debug/paths")
+async def debug_paths(project_id: Optional[str] = None):
+    res = {"DATA_DIR": DATA_DIR, "cwd": os.getcwd()}
+    if project_id:
+        res["project_paths"] = project_paths(project_id)
+    return res
+
 @fastapi_app.get("/")
 async def root_redirect():
     return RedirectResponse(url=UI_PATH, status_code=307)
@@ -454,7 +486,6 @@ async def query(req: QueryRequest):
     qvec = await embed_query(req.text)
     if len(qvec) != vec_dim:
         raise HTTPException(status_code=400, detail=f"Dim requête {len(qvec)} ≠ dim index {vec_dim}")
-    # get_nearest_examples renvoie (scores, examples)
     scores, ex = ds.get_nearest_examples("embedding", np.array(qvec, dtype=np.float32), k=req.top_k)
     results = []
     for s, path, chunk, text in zip(scores, ex["path"], ex["chunk"], ex["text"]):
@@ -464,10 +495,6 @@ async def query(req: QueryRequest):
 
 @fastapi_app.post("/export_hub")
 async def export_hub(project_id: str = Query(..., min_length=1), repo_id: Optional[str] = None):
-    """
-    Optionnel: push le dossier du projet (dataset + faiss + meta) dans un repo Dataset du Hub.
-    - HF_DATASET_REPO ou ?repo_id=... (ex: "chourmovs/deepweb_vectors")
-    """
     if not HfApi or not HF_TOKEN:
         raise HTTPException(status_code=400, detail="huggingface_hub non dispo ou HF token absent.")
     p = project_paths(project_id)
@@ -483,7 +510,7 @@ async def export_hub(project_id: str = Query(..., min_length=1), repo_id: Option
     except Exception:
         pass
 
-    # Zipper le dossier projet pour un upload rapide
+    # zip le dossier projet
     buf = io.BytesIO()
     base_dir = p["base"]
     zip_name = f"{project_id}_vectors.zip"
@@ -514,7 +541,7 @@ def _default_two_docs() -> List[Dict[str, str]]:
 
 async def ui_wipe(project: str):
     try:
-        resp = await wipe(project)  # appelle route interne
+        resp = await wipe(project)
         return f"✅ Wipe ok — projet {resp['project_id']} vidé."
     except Exception as e:
         LOG.exception("wipe UI error")