Repository: dask/zict

Issue:
Memory flare on Func.update() with File backend
Consider:
```python
d = Func(pickle.dumps, pickle.loads, File(somedir))
d.update(mydata)
```
### Current behaviour
1. call ``pickle.dumps`` on every element of mydata and store all output in memory
2. call ``File.__setitem__`` on each pickled element
3. descope the pickled data all at once

### Expected behaviour
File does not have an optimized update method, unlike LMDB or Sieve, as it does not benefit from updating everything in a single call.
Therefore, it should be possible to create a pickle buffer for every value, write it to disk, and then release it straight away.

The issue is not in Func, but actually in File.update and to be precise in ZictBase.update, which converts iterables into lists.

### Mitigation
- Instead of ``pickle.dumps``, write a function that returns a tuple of (pickle5 output, *buffers). 
- If you wrap Func in a zict.Buffer, Func.update will never be called.

dask.distributed does both of the above.


Comments:

Patch:
diff --git a/doc/source/changelog.rst b/doc/source/changelog.rst
index 0375754..e493d3f 100644
--- a/doc/source/changelog.rst
+++ b/doc/source/changelog.rst
@@ -4,6 +4,8 @@ Changelog
 2.2.0 - Unreleased
 ------------------
 - Added type annotations (:pr:`62`) `Guido Imperiale`_
+- If you call Func.update() and Func wraps around File, do not store all dump outputs in
+  memory (:pr:`64`) `Guido Imperiale`_
 
 
 2.1.0 - 2022-02-25
diff --git a/zict/common.py b/zict/common.py
index 6ed9e94..3d94a2e 100644
--- a/zict/common.py
+++ b/zict/common.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Iterable, Mapping
+from itertools import chain
 from typing import MutableMapping  # TODO move to collections.abc (needs Python >=3.9)
 from typing import Any, TypeVar, overload
 
@@ -39,12 +40,12 @@ class ZictBase(MutableMapping[KT, VT]):
         if args:
             other = args[0]
             if isinstance(other, Mapping) or hasattr(other, "items"):
-                items += other.items()
+                items = other.items()
             else:
                 # Assuming (key, value) pairs
-                items += other
+                items = other
         if kwds:
-            items += kwds.items()
+            items = chain(items, kwds.items())
         self._do_update(items)
 
     def _do_update(self, items: Iterable[tuple[KT, VT]]) -> None:


Test patch:
diff --git a/zict/tests/test_func.py b/zict/tests/test_func.py
index d2fbd34..5345045 100644
--- a/zict/tests/test_func.py
+++ b/zict/tests/test_func.py
@@ -1,4 +1,10 @@
+import gc
+from collections.abc import MutableMapping
+
+import pytest
+
 from zict import Func
+from zict.common import ZictBase
 
 from . import utils_test
 
@@ -46,3 +52,40 @@ def test_mapping():
     z = Func(rotl, rotr, d)
     utils_test.check_mapping(z)
     utils_test.check_closing(z)
+
+
+@pytest.mark.parametrize("wrapped_cls", [MutableMapping, ZictBase])
+def test_update_descopes_early(wrapped_cls):
+    """Test that Func.update() descopes the output of self.dump as soon as it can, if
+    the wrapped mapping allows, and doesn't store everything into a list.
+    """
+
+    class Dumped:
+        n = 0
+
+        def __init__(self):
+            gc.collect()  # Only necessary on pypy
+            Dumped.n += 1
+            assert Dumped.n < 3
+
+        def __del__(self):
+            Dumped.n -= 1
+
+    class Dummy(wrapped_cls):
+        def __setitem__(self, key, value):
+            pass
+
+        def __getitem__(self, key, value):
+            raise KeyError(key)
+
+        def __delitem__(self, key):
+            raise KeyError(key)
+
+        def __iter__(self):
+            return iter(())
+
+        def __len__(self):
+            return 0
+
+    d = Func(lambda v: Dumped(), lambda w: None, Dummy())
+    d.update(dict.fromkeys(range(10)))