add namespace

Development/Debug/run-experiments-job-namespace-debug-2025-09-29.md

@@ -0,0 +1,54 @@
+# Debug Report: Job submission requires explicit Jobs token/namespace
+Date: 2025-09-29
+
+## Problem Statement
+Experiment submission returned a 403 Forbidden error: missing `job.write` permission when creating Jobs under namespace `tianhaowang`.
+
+## Symptoms
+- UI banner displayed the raw Hugging Face Jobs API error containing the request id and permission failure.
+- No compute job was created; submission stopped immediately after the API call.
+
+## Reproduction Steps
+1. Configure the Space with `SERVICE_HF_TOKEN` only (no `HF_JOBS_TOKEN` or namespace override).
+2. Upload a dataset and press "Run Experiments".
+3. Observe the 403 banner referencing missing `job.write` permissions for the default namespace.
+
+## Investigation Process
+### Hypotheses
+- The service token lacked the `jobs:write` scope even though it could push datasets.
+- The Jobs API defaulted to the user namespace (`whoami()['name']`) while the token carried permissions only for an organization namespace.
+
+### Tests Performed
+- Extended regression coverage (`tests/debug/test_run_experiments.py::test_submit_experiments_uses_configured_job_namespace`) to ensure the app forwards an explicit `job_namespace` when configured.
+- Re-ran the full debug suite (`pytest tests/debug`) after installing pytest; all 11 tests passed locally.
+
+### Findings
+- `submit_experiments` always created Jobs in the implicit namespace and reused the dataset service token. This broke deployments where the compute token differs from the dataset token or where Jobs permissions are tied to an organization.
+
+## Solution Approach
+### Proposed Fix
+- Enrich `SpaceConfig` with optional job-specific credentials (token from `HF_JOBS_TOKEN`, namespace from `HF_JOBS_NAMESPACE`, or inferred from `JOB_CODE_REPO` / org ownership).
+- Use `SpaceConfig.resolve_job_token` to pick the OAuth token, jobs token, or service token in that order (no new secrets required for the common case).
+- Deduce a namespace from the selected token (prefer organizations where the token has admin/write roles, optionally biasing toward the owner encoded in `JOB_CODE_REPO`) and pass it to `huggingface_hub.run_job` alongside the token so the API call runs under a namespace that actually grants `jobs:write`.
+
+### Sample Code
+```python
+job_token = CONFIG.resolve_job_token(user_token=user_token)
+job_namespace = CONFIG.resolve_job_namespace(job_token)
+job = run_job(
+    image="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel",
+    command=command,
+    flavor="a10g-small",
+    timeout=7200,
+    env=env,
+    token=job_token,
+    namespace=job_namespace,
+)
+```
+
+## Validation
+- `pytest tests/debug` (11 tests) — all green.
+
+## Prevention Recommendations
+- When issuing service tokens, include the Jobs scope or ensure the organization grants `jobs:write`; document the expected namespace (e.g., via `JOB_CODE_REPO`) in deployment runbooks.
+- Add CI coverage that mocks 403 responses and asserts the UI surfaces a descriptive remediation message.

app.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 import gradio as gr
 from huggingface_hub import inspect_job, run_job
+from huggingface_hub.utils import HfHubHTTPError
 
 from utils.config import ConfigError, load_space_config
 from utils.data import DatasetSpec, load_candidate_catalog
@@ -152,7 +153,7 @@ def submit_experiments(
         raise RuntimeError(f"Configuration error: {CONFIG_ERROR}")
     assert CONFIG is not None
     try:
-        service_token = CONFIG.require_service_token()
+        CONFIG.require_service_token()
     except ConfigError as exc:
         raise RuntimeError(
             "`SERVICE_HF_TOKEN` is required to submit experiments. Configure the secret "
@@ -211,6 +212,9 @@ def submit_experiments(
     jobs: List[Dict[str, Any]] = []
     env = CONFIG.job_env(user_token=user_token)
 
+    job_token = CONFIG.resolve_job_token(user_token=user_token)
+    job_namespace = CONFIG.resolve_job_namespace(job_token)
+
     for dk in dk_list:
         command = [
             "python",
@@ -234,14 +238,23 @@ def submit_experiments(
         ]
         if test_repo:
             command.extend(["--test_dataset", test_repo])
-        job = run_job(
-            image="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel",
-            command=command,
-            flavor="a10g-small",
-            timeout=7200,
-            env=env,
-            token=user_token or service_token,
-        )
+        try:
+            job = run_job(
+                image="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel",
+                command=command,
+                flavor="a10g-small",
+                timeout=7200,
+                env=env,
+                token=job_token,
+                namespace=job_namespace,
+            )
+        except HfHubHTTPError as exc:
+            if getattr(exc.response, "status_code", None) == 403:
+                raise RuntimeError(
+                    "`SERVICE_HF_TOKEN` lacks `jobs:write` or is not allowed to create Jobs in the selected "
+                    "namespace. Sign in with Hugging Face or configure a token with Jobs permissions."
+                ) from exc
+            raise
         jobs.append(
             {
                 "id": job.id,

tests/debug/test_run_experiments.py

@@ -1,9 +1,53 @@
 from types import SimpleNamespace
 
+import pytest
+import requests
+from huggingface_hub.utils import HfHubHTTPError
+
 import app
 from utils import hub
 
 
+class DummyConfig:
+    def __init__(
+        self,
+        service_token: str,
+        results_repo: str,
+        *,
+        job_token: str | None = None,
+        job_namespace: str | None = None,
+        preferred_job_namespace: str | None = None,
+    ) -> None:
+        self.service_token = service_token
+        self.results_repo = results_repo
+        self.job_token = job_token
+        self.job_namespace = job_namespace
+        self.preferred_job_namespace = preferred_job_namespace
+
+    def require_service_token(self) -> str:
+        return self.service_token
+
+    def job_env(self, user_token=None):  # noqa: ANN001 - mimic production signature
+        env = {
+            "SERVICE_HF_TOKEN": self.service_token,
+            "RESULTS_REPO": self.results_repo,
+        }
+        token = user_token or self.job_token or self.service_token
+        env["HF_TOKEN"] = token
+        env["HUGGINGFACEHUB_API_TOKEN"] = token
+        return env
+
+    def resolve_job_token(self, user_token=None):  # noqa: ANN001 - mimic production signature
+        if user_token:
+            return user_token
+        if self.job_token:
+            return self.job_token
+        return self.service_token
+
+    def resolve_job_namespace(self, token=None):  # noqa: ANN001 - mimic production signature
+        return self.job_namespace or self.preferred_job_namespace
+
+
 def test_environment_diagnostics_blocks_without_service_token(monkeypatch):
     monkeypatch.delenv("SERVICE_HF_TOKEN", raising=False)
     monkeypatch.delenv("OAUTH_CLIENT_ID", raising=False)
@@ -140,24 +184,6 @@ def test_ensure_results_repo_uses_resolved_owner(monkeypatch):
 
 
 def test_submit_experiments_uses_service_token_when_oauth_missing(monkeypatch):
-    class DummyConfig:
-        def __init__(self, service_token: str, results_repo: str):
-            self.service_token = service_token
-            self.results_repo = results_repo
-
-        def require_service_token(self) -> str:
-            return self.service_token
-
-        def job_env(self, user_token=None):  # noqa: ANN001 - mimic production signature
-            env = {
-                "SERVICE_HF_TOKEN": self.service_token,
-                "RESULTS_REPO": self.results_repo,
-            }
-            token = user_token or self.service_token
-            env["HF_TOKEN"] = token
-            env["HUGGINGFACEHUB_API_TOKEN"] = token
-            return env
-
     config = DummyConfig(service_token="service-token", results_repo="owner/results")
     monkeypatch.setattr(app, "CONFIG", config)
     monkeypatch.setattr(app, "CONFIG_ERROR", None)
@@ -168,6 +194,7 @@ def test_submit_experiments_uses_service_token_when_oauth_missing(monkeypatch):
 
     def fake_run_job(*, image, command, env=None, secrets=None, flavor=None, timeout=None, namespace=None, token=None):  # noqa: ANN001
         captured["token"] = token
+        captured["namespace"] = namespace
         return SimpleNamespace(id="job-1", status="queued", url="")
 
     monkeypatch.setattr(app, "run_job", fake_run_job)
@@ -187,27 +214,10 @@ def test_submit_experiments_uses_service_token_when_oauth_missing(monkeypatch):
 
     assert jobs[0]["id"] == "job-1"
     assert captured["token"] == "service-token"
+    assert captured["namespace"] is None
 
 
 def test_submit_experiments_prefers_oauth_token(monkeypatch):
-    class DummyConfig:
-        def __init__(self, service_token: str, results_repo: str):
-            self.service_token = service_token
-            self.results_repo = results_repo
-
-        def require_service_token(self) -> str:
-            return self.service_token
-
-        def job_env(self, user_token=None):  # noqa: ANN001 - mimic production signature
-            env = {
-                "SERVICE_HF_TOKEN": self.service_token,
-                "RESULTS_REPO": self.results_repo,
-            }
-            token = user_token or self.service_token
-            env["HF_TOKEN"] = token
-            env["HUGGINGFACEHUB_API_TOKEN"] = token
-            return env
-
     config = DummyConfig(service_token="service-token", results_repo="owner/results")
     monkeypatch.setattr(app, "CONFIG", config)
     monkeypatch.setattr(app, "CONFIG_ERROR", None)
@@ -218,6 +228,7 @@ def test_submit_experiments_prefers_oauth_token(monkeypatch):
 
     def fake_run_job(*, image, command, env=None, secrets=None, flavor=None, timeout=None, namespace=None, token=None):  # noqa: ANN001
         captured["token"] = token
+        captured["namespace"] = namespace
         return SimpleNamespace(id="job-1", status="queued", url="")
 
     monkeypatch.setattr(app, "run_job", fake_run_job)
@@ -239,3 +250,74 @@ def test_submit_experiments_prefers_oauth_token(monkeypatch):
     )
 
     assert captured["token"] == "user-token"
+    assert captured["namespace"] is None
+
+
+def test_submit_experiments_uses_configured_job_namespace(monkeypatch):
+    config = DummyConfig(
+        service_token="service-token",
+        results_repo="owner/results",
+        job_token="job-token",
+        job_namespace="curation-org",
+    )
+    monkeypatch.setattr(app, "CONFIG", config)
+    monkeypatch.setattr(app, "CONFIG_ERROR", None)
+    monkeypatch.setattr(app, "ensure_uploaded_dataset", lambda *_, **__: "user/dataset")
+    monkeypatch.setattr(app, "ensure_results_repo", lambda cfg: cfg.results_repo)
+
+    captured = {}
+
+    def fake_run_job(*, image, command, env=None, secrets=None, flavor=None, timeout=None, namespace=None, token=None):  # noqa: ANN001
+        captured["token"] = token
+        captured["namespace"] = namespace
+        return SimpleNamespace(id="job-1", status="queued", url="")
+
+    monkeypatch.setattr(app, "run_job", fake_run_job)
+
+    app.submit_experiments(
+        d0_files=[],
+        d0_id="user/dataset",
+        task="classification",
+        model="meta-llama/Llama-3.1-8B-Instruct",
+        metrics=["f1"],
+        dk_list=["candidate/dataset"],
+        sizes=[1000],
+        target_size=1000,
+        test_files=None,
+        test_id="",
+    )
+
+    assert captured["token"] == "job-token"
+    assert captured["namespace"] == "curation-org"
+
+
+def test_submit_experiments_handles_jobs_permission_error(monkeypatch):
+    config = DummyConfig(service_token="service-token", results_repo="owner/results")
+    monkeypatch.setattr(app, "CONFIG", config)
+    monkeypatch.setattr(app, "CONFIG_ERROR", None)
+    monkeypatch.setattr(app, "ensure_uploaded_dataset", lambda *_, **__: "user/dataset")
+    monkeypatch.setattr(app, "ensure_results_repo", lambda cfg: cfg.results_repo)
+
+    def fake_run_job(*args, **kwargs):  # noqa: ANN001 - signature compatibility
+        response = requests.Response()
+        response.status_code = 403
+        response.headers = {"x-request-id": "req"}
+        raise HfHubHTTPError("forbidden", response=response)
+
+    monkeypatch.setattr(app, "run_job", fake_run_job)
+
+    with pytest.raises(RuntimeError) as err:
+        app.submit_experiments(
+            d0_files=[],
+            d0_id="user/dataset",
+            task="classification",
+            model="meta-llama/Llama-3.1-8B-Instruct",
+            metrics=["f1"],
+            dk_list=["candidate/dataset"],
+            sizes=[1000],
+            target_size=1000,
+            test_files=None,
+            test_id="",
+        )
+
+    assert "jobs:write" in str(err.value)

utils/config.py

@@ -2,10 +2,12 @@
 from __future__ import annotations
 
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Dict, Optional
 
+from huggingface_hub import whoami
+
 
 class ConfigError(RuntimeError):
     """Raised when required configuration is missing."""
@@ -17,6 +19,11 @@ class SpaceConfig:
     results_repo: Optional[str]
     persist_dir: Optional[Path]
     hf_home: Optional[Path]
+    job_token: Optional[str]
+    job_namespace: Optional[str]
+    preferred_job_namespace: Optional[str]
+    _cached_job_namespace: Optional[str] = field(default=None, init=False, repr=False)
+    _cached_namespace_token: Optional[str] = field(default=None, init=False, repr=False)
 
     @property
     def persistent_storage_enabled(self) -> bool:
@@ -32,6 +39,7 @@ class SpaceConfig:
     def job_env(self, user_token: Optional[str] = None) -> Dict[str, str]:
         env: Dict[str, str] = {}
         service_token = self.service_token
+        job_token = self.job_token
 
         if service_token:
             env["SERVICE_HF_TOKEN"] = service_token
@@ -42,19 +50,59 @@ class SpaceConfig:
         if self.hf_home:
             env["HF_HOME"] = str(self.hf_home)
 
+        shared_token: Optional[str] = None
         if user_token:
-            env["HF_TOKEN"] = user_token
-            env["HUGGINGFACEHUB_API_TOKEN"] = user_token
+            shared_token = user_token
+        elif job_token:
+            shared_token = job_token
         elif service_token:
-            env["HF_TOKEN"] = service_token
-            env["HUGGINGFACEHUB_API_TOKEN"] = service_token
-        else:
+            shared_token = service_token
+        if not shared_token:
             raise ConfigError(
                 "SERVICE_HF_TOKEN is required when an OAuth token is not provided."
             )
+        env["HF_TOKEN"] = shared_token
+        env["HUGGINGFACEHUB_API_TOKEN"] = shared_token
 
         return env
 
+    def resolve_job_token(self, user_token: Optional[str] = None) -> str:
+        if user_token:
+            return user_token
+        if self.job_token:
+            return self.job_token
+        if self.service_token:
+            return self.service_token
+        raise ConfigError(
+            "A token with `jobs:write` permission is required. Set `HF_JOBS_TOKEN` or `SERVICE_HF_TOKEN`."
+        )
+
+    def resolve_job_namespace(self, token: Optional[str]) -> Optional[str]:
+        if self.job_namespace:
+            return self.job_namespace
+
+        if token and token not in {self.job_token, self.service_token}:
+            # OAuth token: let the API infer the namespace from the user identity.
+            return None
+
+        effective = token or self.job_token or self.service_token
+        if not effective:
+            return None
+
+        if (
+            self._cached_namespace_token == effective
+            and self._cached_job_namespace is not None
+        ):
+            return self._cached_job_namespace
+
+        namespace = _infer_namespace_from_token(
+            effective,
+            preferred=self.preferred_job_namespace,
+        )
+        self._cached_namespace_token = effective
+        self._cached_job_namespace = namespace
+        return namespace
+
 
 def load_space_config(strict: bool = False) -> SpaceConfig:
     service_token = os.getenv("SERVICE_HF_TOKEN")
@@ -69,6 +117,16 @@ def load_space_config(strict: bool = False) -> SpaceConfig:
     hf_home_env = _clean_env_value("HF_HOME")
     hf_home = Path(hf_home_env) if hf_home_env else None
 
+    job_token = _clean_env_value("HF_JOBS_TOKEN")
+
+    job_namespace = _clean_env_value("HF_JOBS_NAMESPACE")
+    if not job_namespace:
+        job_namespace = _clean_env_value("HF_ORG")
+    job_preferred_namespace: Optional[str] = None
+    job_code_repo = _clean_env_value("JOB_CODE_REPO")
+    if job_code_repo and "/" in job_code_repo:
+        job_preferred_namespace = job_code_repo.split("/", 1)[0].strip() or None
+
     if persist_dir and not persist_dir.exists():
         persist_dir.mkdir(parents=True, exist_ok=True)
 
@@ -77,9 +135,47 @@ def load_space_config(strict: bool = False) -> SpaceConfig:
         results_repo=results_repo,
         persist_dir=persist_dir,
         hf_home=hf_home,
+        job_token=job_token,
+        job_namespace=job_namespace,
+        preferred_job_namespace=job_preferred_namespace,
     )
 
 
 def _clean_env_value(key: str) -> Optional[str]:
     raw = os.getenv(key, "").strip()
     return raw or None
+
+
+def _infer_namespace_from_token(
+    token: str,
+    *,
+    preferred: Optional[str] = None,
+) -> Optional[str]:
+    try:
+        identity = whoami(token=token)
+    except Exception:
+        return None
+
+    orgs = identity.get("orgs") or []
+    identity_name = identity.get("name")
+    available_names = {
+        *(org.get("name") for org in orgs if org.get("name")),
+        identity_name,
+    }
+    available_names.discard(None)
+
+    if (
+        preferred
+        and preferred != identity_name
+        and preferred in available_names
+    ):
+        return preferred
+
+    for org in orgs:
+        role = (org.get("role") or "").lower()
+        if role in {"admin", "write"} or "job" in role:
+            name = org.get("name")
+            if name:
+                return name
+
+    return identity.get("name")