Repository: wandb/wandb

Issue:
[Q] Setting WANDB_NOTEBOOK_NAME and storing Databricks notebooks in wandb
Hi, I'd like to save a Databricks notebook in `wandb` and display it in the Code menu of wandb's UI. I am trying the following:

```python
import wandb
import os

os.environ["WANDB_NOTEBOOK_NAME"] = "/Repos/my-repo/my-notebook"

api_key = dbutils.secrets.get("wandb", "my-api-key")
wandb.login(key=api_key)

wandb.init(project="my-project", save_code=True)

wandb.log({"my_var": 123})

wandb.finish()
```

It logs the run and its variables as expected, however, I receive the following warning when running `wandb.login(key=api_key)` and it doesn't create a ` _session_history.ipynb` object:

```
wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Repos/my-repo/my-notebook
```

Note: Regular jupyter notebooks work fine, I only have this issue with Databricks. Any ideas? 

Thanks a lot!


Discussion:
I'm assuming there's no file named `/Repos/my-repo/my-notebook` on that machine.  I haven't tried databricks notebooks for some time, but you should be setting `WANDB_NOTEBOOK_NAME` to the path for the `notebook.ipynb` file that's currently open.
Hi @vanpelt, thanks a lot for the reply! 

I was picking up the path with `dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()` which was removing `/Workspace/` from the full path. 

My initial error is fixed but I came across the following when running `wandb.init(project="my-project", save_code=True, dir="/tmp")`:

```
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
/databricks/python/lib/python3.8/site-packages/backcall/backcall.py in adapted(*args, **kwargs)
    102                 kwargs.pop(name)
    103 #            print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw)
--> 104             return callback(*args, **kwargs)
    105 
    106         return adapted

/databricks/python/lib/python3.8/site-packages/wandb/sdk/wandb_init.py in _pause_backend(self)
    284             logger.info("pausing backend")
    285             # Attempt to save the code on every execution
--> 286             if self.notebook.save_ipynb():
    287                 res = self.run.log_code(root=None)
    288                 logger.info("saved code: %s", res)

/databricks/python/lib/python3.8/site-packages/wandb/jupyter.py in save_ipynb(self)
    378         if relpath:
    379             if os.path.exists(relpath):
--> 380                 shutil.copy(
    381                     relpath,
    382                     os.path.join(

/usr/lib/python3.8/shutil.py in copy(src, dst, follow_symlinks)
    416     if os.path.isdir(dst):
    417         dst = os.path.join(dst, os.path.basename(src))
--> 418     copyfile(src, dst, follow_symlinks=follow_symlinks)
    419     copymode(src, dst, follow_symlinks=follow_symlinks)
    420     return dst

/usr/lib/python3.8/shutil.py in copyfile(src, dst, follow_symlinks)
    262         os.symlink(os.readlink(src), dst)
    263     else:
--> 264         with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
    265             # macOS
    266             if _HAS_FCOPYFILE:

OSError: [Errno 95] Operation not supported: '/Workspace/Repos/my-repo/my-notebook'
```

The error is caused when `save_code=True`. Initially, I thought it was due to this [Databricks design](https://kb.databricks.com/dbfs/errno95-operation-not-supported.html), however, the error persists with both local storage and a mounted S3 bucket. Any ideas? 
Thar error indicates we can't read that file from the filesystem.  Are we sure that's a file, or is it a directory.  Can you run `ls -al` in the `my-repo` and or `my-notebook` directories?
Hi @vanpelt, I ran `ls -al` in `my-repo`:

```
total 0
drwxrwxrwx 2 root root 0 Jan 10 15:40 .
drwxrwxrwx 2 root root 0 Jan 10 15:40 ..
-rwxrwxrwx 1 root root 0 Jan  1  1970 my-notebook
-rwxrwxrwx 1 root root 0 Jan  1  1970 my-notebook-2
```

In case it's helpful, I changed the `dir` to `wandb.init(project="my-project", save_code=True, dir="/Workspace/Repos/my-repo/")` and got a slightly extended error message:

```
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
/databricks/python/lib/python3.8/site-packages/wandb/sdk/wandb_init.py in init(job_type, dir, config, project, entity, reinit, tags, group, name, notes, magic, config_exclude_keys, config_include_keys, anonymous, mode, allow_val_change, resume, force, tensorboard, sync_tensorboard, monitor_gym, save_code, id, settings)
    927         wi = _WandbInit()
--> 928         wi.setup(kwargs)
    929         except_exit = wi.settings._except_exit

/databricks/python/lib/python3.8/site-packages/wandb/sdk/wandb_init.py in setup(self, kwargs)
    208         if not settings._noop:
--> 209             self._log_setup(settings)
    210 

/databricks/python/lib/python3.8/site-packages/wandb/sdk/wandb_init.py in _log_setup(self, settings)
    340         """Sets up logging from settings."""
--> 341         filesystem._safe_makedirs(os.path.dirname(settings.log_user))
    342         filesystem._safe_makedirs(os.path.dirname(settings.log_internal))

/databricks/python/lib/python3.8/site-packages/wandb/sdk/lib/filesystem.py in _safe_makedirs(dir_name)
     10     try:
---> 11         os.makedirs(dir_name)
     12     except OSError as e:

/usr/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    212         try:
--> 213             makedirs(head, exist_ok=exist_ok)
    214         except FileExistsError:

/usr/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    212         try:
--> 213             makedirs(head, exist_ok=exist_ok)
    214         except FileExistsError:

/usr/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    222     try:
--> 223         mkdir(name, mode)
    224     except OSError:

OSError: [Errno 38] Function not implemented: '/Workspace/Repos/my-repo/wandb'

The above exception was the direct cause of the following exception:

Exception                                 Traceback (most recent call last)
<command-3626329098573567> in <module>
----> 1 wandb.init(project="my-project", save_code=True, dir="/Workspace/Repos/my-repo/")

/databricks/python/lib/python3.8/site-packages/wandb/sdk/wandb_init.py in init(job_type, dir, config, project, entity, reinit, tags, group, name, notes, magic, config_exclude_keys, config_include_keys, anonymous, mode, allow_val_change, resume, force, tensorboard, sync_tensorboard, monitor_gym, save_code, id, settings)
    967             if except_exit:
    968                 os._exit(-1)
--> 969             six.raise_from(Exception("problem"), error_seen)
    970     return run

/databricks/python/lib/python3.8/site-packages/six.py in raise_from(value, from_value)

Exception: problem
```
@kstathou it looks like notebook code saving is unable to work in databricks notebooks because it's not providing a real filesystem at `/Workspace/Repos/...` we'll need to dig in and attempt to reproduce.  I wonder if `/tmp` is writable.  One thing to try would be setting `dir="/tmp"` and see if that still causes errors.
Hi @vanpelt, thanks again for the help! I tried `dir="/tmp"` but got this error: 
```
OSError: [Errno 95] Operation not supported: '/Workspace/Repos/my-repo/my-notebook
```

I confirmed we can write to `/tmp` but I think the problem is that Databricks doesn't allow copying from `/Workspace/Repos/...`. I tried to `%sh cp my-notebook /tmp/my-notebook` and received the following error: 

```
cp: cannot open 'my-notebook' for reading: Operation not supported
```


Thanks for the info. We're looking into this and will keep you updated here.
This issue is stale because it has been open 60 days with no activity.

Resolving patch:
diff --git a/wandb/jupyter.py b/wandb/jupyter.py
--- a/wandb/jupyter.py
+++ b/wandb/jupyter.py
@@ -5,11 +5,13 @@
 import shutil
 import sys
 from base64 import b64encode
+from typing import Dict
 
 import requests
 from requests.compat import urljoin
 
 import wandb
+import wandb.util
 from wandb.sdk.lib import filesystem
 
 try:
@@ -177,10 +179,10 @@ def notebook_metadata_from_jupyter_servers_and_kernel_id():
     return None
 
 
-def notebook_metadata(silent):
+def notebook_metadata(silent) -> Dict[str, str]:
     """Attempts to query jupyter for the path and name of the notebook file.
 
-    This can handle many different jupyter environments, specifically:
+    This can handle different jupyter environments, specifically:
 
     1. Colab
     2. Kaggle
@@ -193,30 +195,21 @@ def notebook_metadata(silent):
         "the WANDB_NOTEBOOK_NAME environment variable to enable code saving."
     )
     try:
-        # In colab we can request the most recent contents
+        jupyter_metadata = notebook_metadata_from_jupyter_servers_and_kernel_id()
+
+        # Colab:
+        # request the most recent contents
         ipynb = attempt_colab_load_ipynb()
-        if ipynb:
-            nb_name = ipynb["metadata"]["colab"]["name"]
-            if ".ipynb" not in nb_name:
-                nb_name += ".ipynb"
-            ret = {
+        if ipynb is not None and jupyter_metadata is not None:
+            return {
                 "root": "/content",
-                "path": nb_name,
-                "name": nb_name,
+                "path": jupyter_metadata["path"],
+                "name": jupyter_metadata["name"],
             }
 
-            try:
-                jupyter_metadata = (
-                    notebook_metadata_from_jupyter_servers_and_kernel_id()
-                )
-            except RuntimeError:
-                pass
-            else:
-                ret["path"] = jupyter_metadata["path"]
-            return ret
-
+        # Kaggle:
         if wandb.util._is_kaggle():
-            # In kaggle we can request the most recent contents
+            # request the most recent contents
             ipynb = attempt_kaggle_load_ipynb()
             if ipynb:
                 return {
@@ -225,7 +218,6 @@ def notebook_metadata(silent):
                     "name": ipynb["metadata"]["name"],
                 }
 
-        jupyter_metadata = notebook_metadata_from_jupyter_servers_and_kernel_id()
         if jupyter_metadata:
             return jupyter_metadata
         if not silent:
@@ -401,12 +393,14 @@ def _save_ipynb(self) -> bool:
         # TODO: likely only save if the code has changed
         colab_ipynb = attempt_colab_load_ipynb()
         if colab_ipynb:
-            nb_name = (
-                colab_ipynb.get("metadata", {})
-                .get("colab", {})
-                .get("name", "colab.ipynb")
-            )
-            if ".ipynb" not in nb_name:
+            try:
+                jupyter_metadata = (
+                    notebook_metadata_from_jupyter_servers_and_kernel_id()
+                )
+                nb_name = jupyter_metadata["name"]
+            except Exception:
+                nb_name = "colab.ipynb"
+            if not nb_name.endswith(".ipynb"):
                 nb_name += ".ipynb"
             with open(
                 os.path.join(


Test patch:
diff --git a/tests/pytest_tests/unit_tests_old/tests_s_nb/test_notebooks.py b/tests/pytest_tests/unit_tests_old/tests_s_nb/test_notebooks.py
--- a/tests/pytest_tests/unit_tests_old/tests_s_nb/test_notebooks.py
+++ b/tests/pytest_tests/unit_tests_old/tests_s_nb/test_notebooks.py
@@ -3,9 +3,12 @@
 import platform
 import subprocess
 import sys
+from unittest import mock
 
 import pytest
 import wandb
+import wandb.jupyter
+import wandb.util
 from wandb.errors import UsageError
 
 from tests.pytest_tests.unit_tests_old import utils
@@ -126,14 +129,23 @@ def test_notebook_metadata_no_servers(mocker, mocked_module):
 def test_notebook_metadata_colab(mocked_module):
     colab = mocked_module("google.colab")
     colab._message.blocking_request.return_value = {
-        "ipynb": {"metadata": {"colab": {"name": "colab.ipynb"}}}
-    }
-    meta = wandb.jupyter.notebook_metadata(False)
-    assert meta == {
-        "root": "/content",
-        "path": "colab.ipynb",
-        "name": "colab.ipynb",
+        "ipynb": {"metadata": {"colab": {"name": "koalab.ipynb"}}}
     }
+    with mock.patch.object(
+        wandb.jupyter,
+        "notebook_metadata_from_jupyter_servers_and_kernel_id",
+        lambda *args, **kwargs: {
+            "path": "colab.ipynb",
+            "root": "/consent",
+            "name": "colab.ipynb",
+        },
+    ):
+        meta = wandb.jupyter.notebook_metadata(False)
+        assert meta == {
+            "root": "/content",
+            "path": "colab.ipynb",
+            "name": "colab.ipynb",
+        }
 
 
 def test_notebook_metadata_kaggle(mocker, mocked_module):
@@ -144,12 +156,17 @@ def test_notebook_metadata_kaggle(mocker, mocked_module):
         "source": json.dumps({"metadata": {}, "cells": []})
     }
     kaggle.UserSessionClient.return_value = kaggle_client
-    meta = wandb.jupyter.notebook_metadata(False)
-    assert meta == {
-        "root": "/kaggle/working",
-        "path": "kaggle.ipynb",
-        "name": "kaggle.ipynb",
-    }
+    with mock.patch.object(
+        wandb.jupyter,
+        "notebook_metadata_from_jupyter_servers_and_kernel_id",
+        lambda *args, **kwargs: {},
+    ):
+        meta = wandb.jupyter.notebook_metadata(False)
+        assert meta == {
+            "root": "/kaggle/working",
+            "path": "kaggle.ipynb",
+            "name": "kaggle.ipynb",
+        }
 
 
 def test_databricks_notebook_doesnt_hang_on_wandb_login(mocked_module):