Repository: wandb/wandb

Issue:
[Q] Change staging dir
Currently when the run ends I am getting this error trace. I think it's because I do not have write access to where the staging directory is selected by default. How do I change that?

```python
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.
Thread SenderThread:
Traceback (most recent call last):
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/internal_util.py", line 49, in run
    self._run()
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/internal_util.py", line 100, in _run
    self._process(record)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/internal.py", line 329, in _process
    self._sm.send(record)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 343, in send
    send_handler(record)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 365, in send_request
    send_handler(record)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 615, in send_request_defer
    self._flush_job()
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 1567, in _flush_job
    artifact = self._job_builder.build()
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/internal/job_builder.py", line 214, in build
    f.write(json.dumps(source_info, indent=4))
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/contextlib.py", line 120, in __exit__
    next(self.gen)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/wandb_artifacts.py", line 412, in new_file
    self.add_file(path, name=name)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/wandb_artifacts.py", line 433, in add_file
    return self._add_local_file(name, local_path, digest=digest)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/wandb_artifacts.py", line 749, in _add_local_file
    with get_new_staging_file() as f:
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/interface/artifacts.py", line 999, in get_new_staging_file
    return tempfile.NamedTemporaryFile(dir=get_staging_dir(), delete=False)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/interface/artifacts.py", line 994, in get_staging_dir
    filesystem.mkdir_exists_ok(path)
  File "/arc/project/st-jiaruid-1/miniconda3/envs/mrl-2/lib/python3.8/site-packages/wandb/sdk/lib/filesystem.py", line 18, in mkdir_exists_ok
    raise PermissionError(f"{dir_name!s} is not writable")
PermissionError: /home/shenoy/.local/share/wandb/artifacts/staging is not writable
wandb: ERROR Internal wandb error: file data was not synced
```


Discussion:
I think I found the issue, I need to change the `env` variable `WANDB_DATA_DIR`.
Thank you @shenoynikhil for letting us know the solution to your issue. I will be closing it now. If you have any other issue in the future please feel free to contact us!

Resolving patch:
diff --git a/wandb/sdk/interface/artifacts.py b/wandb/sdk/interface/artifacts.py
--- a/wandb/sdk/interface/artifacts.py
+++ b/wandb/sdk/interface/artifacts.py
@@ -1013,7 +1013,14 @@ def get_artifacts_cache() -> ArtifactsCache:
 
 def get_staging_dir() -> FilePathStr:
     path = os.path.join(env.get_data_dir(), "artifacts", "staging")
-    mkdir_exists_ok(path)
+    try:
+        mkdir_exists_ok(path)
+    except OSError as e:
+        raise PermissionError(
+            f"Unable to write staging files to {path}. To fix this problem, please set "
+            f"{env.DATA_DIR} to a directory where you have the necessary write access."
+        ) from e
+
     return FilePathStr(os.path.abspath(os.path.expanduser(path)))
 
 


Test patch:
diff --git a/tests/pytest_tests/unit_tests/test_artifacts/test_storage.py b/tests/pytest_tests/unit_tests/test_artifacts/test_storage.py
--- a/tests/pytest_tests/unit_tests/test_artifacts/test_storage.py
+++ b/tests/pytest_tests/unit_tests/test_artifacts/test_storage.py
@@ -6,6 +6,7 @@
 import pytest
 import wandb
 from wandb.sdk import wandb_artifacts
+from wandb.sdk.interface import artifacts
 
 
 def test_opener_rejects_append_mode(cache):
@@ -345,3 +346,15 @@ class UnfinishedStorageHandler(wandb_artifacts.StorageHandler):
         ush.load_path(manifest_entry=None)
     with pytest.raises(NotImplementedError):
         ush.store_path(artifact=None, path="")
+
+
+def test_unwritable_staging_dir(monkeypatch):
+    # Use a non-writable directory as the staging directory.
+    # CI just doesn't care about permissions, so we're patching os.makedirs 🙃
+    def nope(*args, **kwargs):
+        raise OSError(13, "Permission denied")
+
+    monkeypatch.setattr(os, "makedirs", nope)
+
+    with pytest.raises(PermissionError, match="WANDB_DATA_DIR"):
+        _ = artifacts.get_new_staging_file()