Repository: joblib/joblib
Pull request #1576 (2024-04-30 17:23:44)

FIX revert MemorizedFunc.call API change

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/joblib/joblib/pull/1576?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=joblib) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 95.21%. Comparing base [(`4db39eb`)](https://app.codecov.io/gh/joblib/joblib/commit/4db39ebf064ecacbf9a0d01627c80692fdf64ac5?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=joblib) to head [(`92147b4`)](https://app.codecov.io/gh/joblib/joblib/pull/1576?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=joblib).
> Report is 1 commits behind head on main.


<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #1576      +/-   ##
==========================================
+ Coverage   95.16%   95.21%   +0.04%     
==========================================
  Files          45       45              
  Lines        7674     7705      +31     
==========================================
+ Hits         7303     7336      +33     
+ Misses        371      369       -2     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/joblib/joblib/pull/1576?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=joblib).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=joblib).


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 08482b4..60affb8 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,6 +5,10 @@ Latest changes
 In development
 --------------
 
+- Fix a backward incompatible change in ``MemorizedFunc.call`` which needs to
+  return the metadata. Also make sure that ``NotMemorizedFunc.call`` return
+  an empty dict for metadata for consistency.
+  https://github.com/joblib/joblib/pull/1576
 
 Release 1.4.0 -- 2024/04/08
 ---------------------------
diff --git a/joblib/memory.py b/joblib/memory.py
index d7d5dbf..29e2649 100644
--- a/joblib/memory.py
+++ b/joblib/memory.py
@@ -322,7 +322,7 @@ class NotMemorizedFunc(object):
         pass
 
     def call(self, *args, **kwargs):
-        return self.func(*args, **kwargs)
+        return self.func(*args, **kwargs), {}
 
     def check_call_in_cache(self, *args, **kwargs):
         return False
@@ -476,8 +476,9 @@ class MemorizedFunc(Logger):
 
         Returns
         -------
-        Output of the wrapped function if shelving is false, or a
-        MemorizedResult reference to the value if shelving is true.
+        output: Output of the wrapped function if shelving is false, or a
+            MemorizedResult reference to the value if shelving is true.
+        metadata: dict containing the metadata associated with the call.
         """
         args_id = self._get_args_id(*args, **kwargs)
         call_id = (self.func_id, args_id)
@@ -506,7 +507,7 @@ class MemorizedFunc(Logger):
         # the cache.
         if self._is_in_cache_and_valid(call_id):
             if shelving:
-                return self._get_memorized_result(call_id)
+                return self._get_memorized_result(call_id), {}
 
             try:
                 start_time = time.time()
@@ -514,7 +515,7 @@ class MemorizedFunc(Logger):
                 if self._verbose > 4:
                     self._print_duration(time.time() - start_time,
                                          context='cache loaded ')
-                return output
+                return output, {}
             except Exception:
                 # XXX: Should use an exception logger
                 _, signature = format_signature(self.func, *args, **kwargs)
@@ -527,6 +528,7 @@ class MemorizedFunc(Logger):
                 f"in location {location}"
             )
 
+        # Returns the output but not the metadata
         return self._call(call_id, args, kwargs, shelving)
 
     @property
@@ -567,10 +569,12 @@ class MemorizedFunc(Logger):
             class "NotMemorizedResult" is used when there is no cache
             activated (e.g. location=None in Memory).
         """
-        return self._cached_call(args, kwargs, shelving=True)
+        # Return the wrapped output, without the metadata
+        return self._cached_call(args, kwargs, shelving=True)[0]
 
     def __call__(self, *args, **kwargs):
-        return self._cached_call(args, kwargs, shelving=False)
+        # Return the output, without the metadata
+        return self._cached_call(args, kwargs, shelving=False)[0]
 
     def __getstate__(self):
         # Make sure self.func's source is introspected prior to being pickled -
@@ -752,11 +756,16 @@ class MemorizedFunc(Logger):
         -------
         output : object
             The output of the function call.
+        metadata : dict
+            The metadata associated with the call.
         """
         call_id = (self.func_id, self._get_args_id(*args, **kwargs))
+
+        # Return the output and the metadata
         return self._call(call_id, args, kwargs)
 
     def _call(self, call_id, args, kwargs, shelving=False):
+        # Return the output and the metadata
         self._before_call(args, kwargs)
         start_time = time.time()
         output = self.func(*args, **kwargs)
@@ -774,13 +783,13 @@ class MemorizedFunc(Logger):
             self._print_duration(duration)
         metadata = self._persist_input(duration, call_id, args, kwargs)
         if shelving:
-            return self._get_memorized_result(call_id, metadata)
+            return self._get_memorized_result(call_id, metadata), metadata
 
         if self.mmap_mode is not None:
             # Memmap the output at the first call to be consistent with
             # later calls
             output = self._load_item(call_id, metadata)
-        return output
+        return output, metadata
 
     def _persist_input(self, duration, call_id, args, kwargs,
                        this_duration_limit=0.5):
@@ -861,12 +870,14 @@ class MemorizedFunc(Logger):
 ###############################################################################
 class AsyncMemorizedFunc(MemorizedFunc):
     async def __call__(self, *args, **kwargs):
-        out = super().__call__(*args, **kwargs)
-        return await out if asyncio.iscoroutine(out) else out
+        out = self._cached_call(args, kwargs, shelving=False)
+        out = await out if asyncio.iscoroutine(out) else out
+        return out[0]  # Don't return metadata
 
     async def call_and_shelve(self, *args, **kwargs):
-        out = super().call_and_shelve(*args, **kwargs)
-        return await out if asyncio.iscoroutine(out) else out
+        out = self._cached_call(args, kwargs, shelving=True)
+        out = await out if asyncio.iscoroutine(out) else out
+        return out[0]  # Don't return metadata
 
     async def call(self, *args, **kwargs):
         out = super().call(*args, **kwargs)
@@ -876,8 +887,9 @@ class AsyncMemorizedFunc(MemorizedFunc):
         self._before_call(args, kwargs)
         start_time = time.time()
         output = await self.func(*args, **kwargs)
-        return self._after_call(call_id, args, kwargs, shelving,
-                                output, start_time)
+        return self._after_call(
+            call_id, args, kwargs, shelving, output, start_time
+        )
 
 
 ###############################################################################


Test patch:
diff --git a/conftest.py b/conftest.py
index 53413de..ee0f66f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -8,6 +8,7 @@ from _pytest.doctest import DoctestItem
 
 from joblib.parallel import mp
 from joblib.backports import LooseVersion
+from joblib import Memory
 try:
     import lz4
 except ImportError:
@@ -84,3 +85,11 @@ def pytest_unconfigure(config):
     # Note that we also use a shorter timeout for the per-test callback
     # configured via the pytest-timeout extension.
     faulthandler.dump_traceback_later(60, exit=True)
+
+
+@pytest.fixture(scope='function')
+def memory(tmp_path):
+    "Fixture to get an independent and self-cleaning Memory"
+    mem = Memory(location=tmp_path, verbose=0)
+    yield mem
+    mem.clear()
diff --git a/joblib/test/test_memory.py b/joblib/test/test_memory.py
index 7093291..1a6c69c 100644
--- a/joblib/test/test_memory.py
+++ b/joblib/test/test_memory.py
@@ -1402,12 +1402,6 @@ def test_info_log(tmpdir, caplog):
 class TestCacheValidationCallback:
     "Tests on parameter `cache_validation_callback`"
 
-    @pytest.fixture()
-    def memory(self, tmp_path):
-        mem = Memory(location=tmp_path)
-        yield mem
-        mem.clear()
-
     def foo(self, x, d, delay=None):
         d["run"] = True
         if delay is not None:
@@ -1481,3 +1475,42 @@ class TestCacheValidationCallback:
         assert d1["run"]
         assert not d2["run"]
         assert d3["run"]
+
+
+class TestMemorizedFunc:
+    "Tests for the MemorizedFunc and NotMemorizedFunc classes"
+
+    @staticmethod
+    def f(x, counter):
+        counter[x] = counter.get(x, 0) + 1
+        return counter[x]
+
+    def test_call_method_memorized(self, memory):
+        "Test calling the function"
+
+        f = memory.cache(self.f, ignore=['counter'])
+
+        counter = {}
+        assert f(2, counter) == 1
+        assert f(2, counter) == 1
+
+        x, meta = f.call(2, counter)
+        assert x == 2, "f has not been called properly"
+        assert isinstance(meta, dict), (
+            "Metadata are not returned by MemorizedFunc.call."
+        )
+
+    def test_call_method_not_memorized(self, memory):
+        "Test calling the function"
+
+        f = NotMemorizedFunc(self.f)
+
+        counter = {}
+        assert f(2, counter) == 1
+        assert f(2, counter) == 2
+
+        x, meta = f.call(2, counter)
+        assert x == 3, "f has not been called properly"
+        assert isinstance(meta, dict), (
+            "Metadata are not returned by MemorizedFunc.call."
+        )
diff --git a/joblib/test/test_memory_async.py b/joblib/test/test_memory_async.py
index ecad0c9..53ddeff 100644
--- a/joblib/test/test_memory_async.py
+++ b/joblib/test/test_memory_async.py
@@ -147,3 +147,24 @@ async def test_call_and_shelve_async(tmpdir):
         with raises(KeyError):
             result.get()
         result.clear()  # Do nothing if there is no cache.
+
+
+@pytest.mark.asyncio
+async def test_memorized_func_call_async(memory):
+
+    async def ff(x, counter):
+        await asyncio.sleep(0.1)
+        counter[x] = counter.get(x, 0) + 1
+        return counter[x]
+
+    gg = memory.cache(ff, ignore=['counter'])
+
+    counter = {}
+    assert await gg(2, counter) == 1
+    assert await gg(2, counter) == 1
+
+    x, meta = await gg.call(2, counter)
+    assert x == 2, "f has not been called properly"
+    assert isinstance(meta, dict), (
+        "Metadata are not returned by MemorizedFunc.call."
+    )