Repository: huggingface/accelerate

Issue:
Feature request: Add support for NamedTuples in dataloaders
In order to produce self-documenting code, our team has the habit of using NamedTuples instead of plain tuples as the return type of our datasets.

Since they are sub-classes of tuples, they are handled by every PyTorch mechanism we've encountered as if they are tuple, so it all works smoothly.

When using Accelerate, I get an error in `send_to_device` at line 114. `type(tensor)(send_to_device(t, device) for t in tensor)` raises a type error because `type(tensor)` returns my NamedTuple, which cannot use the generator `(send_to_device(t, device) for t in tensor)` as argument. We would need some way to transform the generator into a type that is accepted by the NamedTuple.


Discussion:

Resolving patch:
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -17,6 +17,7 @@
 from packaging import version
 
 from .state import AcceleratorState, DistributedType, is_tpu_available
+from .utils import honor_type
 
 
 if is_tpu_available():
@@ -25,7 +26,7 @@
 
 def move_to_device(state, device):
     if isinstance(state, (list, tuple)):
-        return type(state)(move_to_device(t, device) for t in state)
+        return honor_type(state, (move_to_device(t, device) for t in state))
     elif isinstance(state, dict):
         return type(state)({k: move_to_device(v, device) for k, v in state.items()})
     elif isinstance(state, torch.Tensor):
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -99,6 +99,17 @@ def synchronize_rng_states(rng_types: List[Union[str, RNGType]], generator: Opti
         synchronize_rng_state(RNGType(rng_type), generator=generator)
 
 
+def honor_type(obj, generator):
+    """
+    Cast a generator to the same type as obj (list, tuple or namedtuple)
+    """
+    # There is no direct check whether an object if of type namedtuple sadly, this is a workaround.
+    if isinstance(obj, tuple) and hasattr(obj, "_fields"):
+        # Can instantiate a namedtuple from a generator directly, contrary to a tuple/list.
+        return type(obj)(*list(generator))
+    return type(obj)(generator)
+
+
 def send_to_device(tensor, device):
     """
     Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device.
@@ -113,7 +124,7 @@ def send_to_device(tensor, device):
         The same data structure as :obj:`tensor` with all tensors sent to the proper device.
     """
     if isinstance(tensor, (list, tuple)):
-        return type(tensor)(send_to_device(t, device) for t in tensor)
+        return honor_type(tensor, (send_to_device(t, device) for t in tensor))
     elif isinstance(tensor, dict):
         return type(tensor)({k: send_to_device(v, device) for k, v in tensor.items()})
     elif not hasattr(tensor, "to"):
@@ -138,7 +149,7 @@ def extract_model_from_parallel(model):
 
 def _tpu_gather(tensor, name="tensor"):
     if isinstance(tensor, (list, tuple)):
-        return type(tensor)(_tpu_gather(t, name=f"{name}_{i}") for i, t in enumerate(tensor))
+        return honor_type(tensor, (_tpu_gather(t, name=f"{name}_{i}") for i, t in enumerate(tensor)))
     elif isinstance(tensor, dict):
         return type(tensor)({k: _tpu_gather(v, name=f"{name}_{k}") for k, v in tensor.items()})
     elif not isinstance(tensor, torch.Tensor):
@@ -148,7 +159,7 @@ def _tpu_gather(tensor, name="tensor"):
 
 def _gpu_gather(tensor):
     if isinstance(tensor, (list, tuple)):
-        return type(tensor)(_gpu_gather(t) for t in tensor)
+        return honor_type(tensor, (_gpu_gather(t) for t in tensor))
     elif isinstance(tensor, dict):
         return type(tensor)({k: _gpu_gather(v) for k, v in tensor.items()})
     elif not isinstance(tensor, torch.Tensor):
@@ -198,7 +209,7 @@ def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
             Whether to pad at the beginning or the end.
     """
     if isinstance(tensor, (list, tuple)):
-        return type(tensor)(pad_across_processes(t, dim=dim, pad_index=pad_index) for t in tensor)
+        return honor_type(tensor, (pad_across_processes(t, dim=dim, pad_index=pad_index) for t in tensor))
     elif isinstance(tensor, dict):
         return type(tensor)({k: pad_across_processes(v, dim=dim, pad_index=pad_index) for k, v in tensor.items()})
     elif not isinstance(tensor, torch.Tensor):


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,53 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from collections import namedtuple
+
+import torch
+
+from accelerate.utils import send_to_device
+
+
+TestNamedTuple = namedtuple("TestNamedTuple", "a b")
+
+
+class UtilsTester(unittest.TestCase):
+    def test_send_to_device(self):
+        tensor = torch.randn(5, 2)
+        device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+        result1 = send_to_device(tensor, device)
+        self.assertTrue(torch.equal(result1.cpu(), tensor))
+
+        result2 = send_to_device((tensor, [tensor, tensor]), device)
+        self.assertIsInstance(result2, tuple)
+        self.assertTrue(torch.equal(result2[0].cpu(), tensor))
+        self.assertIsInstance(result2[1], list)
+        self.assertTrue(torch.equal(result2[1][0].cpu(), tensor))
+        self.assertTrue(torch.equal(result2[1][1].cpu(), tensor))
+
+        result2 = send_to_device({"a": tensor, "b": [tensor, tensor]}, device)
+        self.assertIsInstance(result2, dict)
+        self.assertTrue(torch.equal(result2["a"].cpu(), tensor))
+        self.assertIsInstance(result2["b"], list)
+        self.assertTrue(torch.equal(result2["b"][0].cpu(), tensor))
+        self.assertTrue(torch.equal(result2["b"][1].cpu(), tensor))
+
+        result3 = send_to_device(TestNamedTuple(a=tensor, b=[tensor, tensor]), device)
+        self.assertIsInstance(result3, TestNamedTuple)
+        self.assertTrue(torch.equal(result3.a.cpu(), tensor))
+        self.assertIsInstance(result3.b, list)
+        self.assertTrue(torch.equal(result3.b[0].cpu(), tensor))
+        self.assertTrue(torch.equal(result3.b[1].cpu(), tensor))