Repository: stanfordnlp/dspy

Issue:
[Bug] Disk caching is unconditionally enabled during initialization
### What happened?

See [here](https://github.com/stanfordnlp/dspy/blob/main/dspy/clients/__init__.py#L80-L86). In environments where disk access is forbidden (eg, AWS lambda), this causes the lambda invocation to fail. Note that this occurs prior to being able to set disk caching to false in application code, which I'm doing as follows:

```
        dspy.configure_cache(
            enable_disk_cache=False,
            enable_memory_cache=False,
        )
        lm = dspy.LM("bedrock/anthropic.claude-3-haiku-20240307-v1:0", cache=False)
```

Here's the stacktrace:

```
[ERROR] OSError: [Errno 30] Cache directory "/home/sbx_user1051/.dspy_cache/000" does not exist and could not be created
Traceback (most recent call last):
  File "/var/lang/lib/python3.12/importlib/__init__.py", line 90, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 999, in exec_module
  File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
  [redacted]
  File "/var/task/dspy/__init__.py", line 1, in <module>
    from dspy.predict import *
  File "/var/task/dspy/predict/__init__.py", line 1, in <module>
    from dspy.predict.aggregation import majority
  File "/var/task/dspy/predict/aggregation.py", line 1, in <module>
    from dspy.evaluate import normalize_text
  File "/var/task/dspy/evaluate/__init__.py", line 1, in <module>
    from dspy.evaluate.auto_evaluation import CompleteAndGrounded, SemanticF1
  File "/var/task/dspy/evaluate/auto_evaluation.py", line 1, in <module>
    from dspy.predict.chain_of_thought import ChainOfThought
  File "/var/task/dspy/predict/chain_of_thought.py", line 6, in <module>
    from dspy.primitives.module import Module
  File "/var/task/dspy/primitives/__init__.py", line 1, in <module>
    from dspy.primitives.base_module import BaseModule
  File "/var/task/dspy/primitives/base_module.py", line 10, in <module>
    from dspy.utils.saving import get_dependency_versions
  File "/var/task/dspy/utils/__init__.py", line 5, in <module>
    from dspy.streaming.messages import StatusMessage, StatusMessageProvider
  File "/var/task/dspy/streaming/__init__.py", line 2, in <module>
    from dspy.streaming.streamify import apply_sync_streaming, streamify, streaming_response
  File "/var/task/dspy/streaming/streamify.py", line 18, in <module>
    from dspy.streaming.streaming_listener import StreamListener, find_predictor_for_stream_listeners
  File "/var/task/dspy/streaming/streaming_listener.py", line 8, in <module>
    from dspy.adapters.chat_adapter import ChatAdapter
  File "/var/task/dspy/adapters/__init__.py", line 2, in <module>
    from dspy.adapters.chat_adapter import ChatAdapter
  File "/var/task/dspy/adapters/chat_adapter.py", line 16, in <module>
    from dspy.clients.lm import LM
  File "/var/task/dspy/clients/__init__.py", line 80, in <module>
    DSPY_CACHE = Cache(
  File "/var/task/dspy/clients/cache.py", line 50, in __init__
    self.disk_cache = FanoutCache(
  File "/var/task/diskcache/fanout.py", line 43, in __init__
    self._shards = tuple(
  File "/var/task/diskcache/fanout.py", line 44, in <genexpr>
    Cache(
  File "/var/task/diskcache/core.py", line 450, in __init__
``` 

### Steps to reproduce

Deploy a lambda handler to aws lambda that imports dspy.

### DSPy version

HEAD as of time of this writing (361473cb5fe29c91d0ee7b4fddc2967115f4f80f)


Discussion:
Thanks, will take a look



Patch:
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index 15917d34fa..a463249d55 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -63,9 +63,7 @@ def configure_cache(
     else:
         litellm.cache = None
 
-    import dspy
-
-    dspy.cache = Cache(
+    DSPY_CACHE = Cache(
         enable_disk_cache,
         enable_memory_cache,
         disk_cache_dir,
@@ -73,17 +71,39 @@ def configure_cache(
         memory_max_entries,
     )
 
+    import dspy
+    # Update the reference to point to the new cache
+    dspy.cache = DSPY_CACHE
+
 
 litellm.telemetry = False
 litellm.cache = None  # By default we disable litellm cache and use DSPy on-disk cache.
 
-DSPY_CACHE = Cache(
-    enable_disk_cache=True,
-    enable_memory_cache=True,
-    disk_cache_dir=DISK_CACHE_DIR,
-    disk_size_limit_bytes=DISK_CACHE_LIMIT,
-    memory_max_entries=1000000,
-)
+def _get_dspy_cache():
+    disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
+    disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))
+
+    try:
+        _dspy_cache = Cache(
+            enable_disk_cache=True,
+            enable_memory_cache=True,
+            disk_cache_dir=disk_cache_dir,
+            disk_size_limit_bytes=disk_cache_limit,
+            memory_max_entries=1000000,
+        )
+    except Exception as e:
+        # If cache creation fails (e.g., in AWS Lambda), create a memory-only cache
+        logger.warning("Failed to initialize disk cache, falling back to memory-only cache: %s", e)
+        _dspy_cache = Cache(
+            enable_disk_cache=False,
+            enable_memory_cache=True,
+            disk_cache_dir=disk_cache_dir,
+            disk_size_limit_bytes=disk_cache_limit,
+            memory_max_entries=1000000,
+        )
+    return _dspy_cache
+
+DSPY_CACHE = _get_dspy_cache()
 
 if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
     # Accessed at run time by litellm; i.e., fine to keep after import


Test patch:
diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.py
index 42a1213957..fb0ccf33e3 100644
--- a/tests/clients/test_cache.py
+++ b/tests/clients/test_cache.py
@@ -1,3 +1,4 @@
+import os
 from dataclasses import dataclass
 from unittest.mock import patch
 
@@ -317,3 +318,29 @@ def test_function(**kwargs):
             )
             is not None
         )
+
+
+def test_cache_fallback_on_restricted_environment():
+    """Test that DSPy gracefully falls back to memory-only cache when disk cache fails."""
+    old_env = os.environ.get("DSPY_CACHEDIR")
+    try:
+        # Set an invalid cache directory that can't be created
+        os.environ["DSPY_CACHEDIR"] = "/dev/null/invalid_path"
+
+        import dspy
+        from dspy.clients import _get_dspy_cache
+
+        dspy.cache = _get_dspy_cache()
+
+        # Cache should work with memory-only fallback despite invalid disk path
+        test_request = {"model": "test", "prompt": "hello"}
+        dspy.cache.put(test_request, "fallback_result")
+        result = dspy.cache.get(test_request)
+
+        assert result == "fallback_result", "Memory cache fallback should work"
+
+    finally:
+        if old_env is None:
+            os.environ.pop("DSPY_CACHEDIR", None)
+        else:
+            os.environ["DSPY_CACHEDIR"] = old_env