Repository: huggingface/transformers
Pull request: Improve BatchFeature (.to() works on lists/nested lists of tensors, auto-skip non array conversion)

# What does this PR do?

Fixes https://github.com/huggingface/transformers/issues/42205

Also fixes processors handling videos crashing when return_metadata=True and return_tensors="pt" .

Cc @zucchini-nlp , for now I have hardcoded all the skip conversions. I think it's safer to do this for now instead of silently not converting any inputs to BatchFeature for which tensor conversion fails. The main issue is with arrays, where non ragged arrays would be converted to tensors, and ragged arrays silently wouldn't, which could cause some unexpected behavior.
Open to alternative solutions so happy to hear your thoughts!

Edit: Added auto-skip, except for ragged arrays

Changed files:
filename: src/transformers/feature_extraction_utils.py
patch: @@ -30,6 +30,7 @@
     PROCESSOR_NAME,
     PushToHubMixin,
     TensorType,
+    _is_tensor_or_array_like,
     copy_func,
     is_numpy_array,
     is_torch_available,
@@ -167,6 +168,11 @@ def convert_to_tensors(
                 `None`, no modification is done.
             skip_tensor_conversion (`list[str]` or `set[str]`, *optional*):
                 List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified.
+
+        Note:
+            Values that don't have an array-like structure (e.g., strings, dicts, lists of strings) are
+            automatically skipped and won't be converted to tensors. Ragged arrays (lists of arrays with
+            different lengths) are still attempted, though they may raise errors during conversion.
         """
         if tensor_type is None:
             return self
@@ -179,6 +185,10 @@ def convert_to_tensors(
             if skip_tensor_conversion and key in skip_tensor_conversion:
                 continue
 
+            # Skip values that are not array-like
+            if not _is_tensor_or_array_like(value):
+                continue
+
             try:
                 if not is_tensor(value):
                     tensor = as_tensor(value)
@@ -233,12 +243,15 @@ def to(self, *args, **kwargs) -> "BatchFeature":
 
         # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
         def maybe_to(v):
-            # check if v is a floating point
+            # check if v is a floating point tensor
             if isinstance(v, torch.Tensor) and torch.is_floating_point(v):
                 # cast and send to device
                 return v.to(*args, **kwargs)
             elif isinstance(v, torch.Tensor) and device is not None:
                 return v.to(device=device, non_blocking=non_blocking)
+            # recursively handle lists and tuples
+            elif isinstance(v, (list, tuple)):
+                return type(v)(maybe_to(item) for item in v)
             else:
                 return v
 

filename: src/transformers/utils/__init__.py
patch: @@ -49,6 +49,7 @@
     PaddingStrategy,
     TensorType,
     TransformersKwargs,
+    _is_tensor_or_array_like,
     can_return_loss,
     can_return_tuple,
     expand_dims,

filename: src/transformers/utils/generic.py
patch: @@ -155,6 +155,26 @@ def is_torch_dtype(x):
     return isinstance(x, torch.dtype)
 
 
+def _is_tensor_or_array_like(value):
+    """
+    Check if a value is array-like (includes ragged arrays)
+    """
+    if is_numpy_array(value):
+        return True
+    if is_torch_tensor(value):
+        return True
+    if isinstance(value, (int, float, bool, np.number)):
+        return True
+
+    if isinstance(value, (list, tuple)):
+        if len(value) == 0:
+            # consider empty list or nested list as array-like
+            return True
+        return _is_tensor_or_array_like(value[0])
+
+    return False
+
+
 def maybe_autocast(
     device_type: str,
     dtype: Optional["_dtype"] = None,

filename: tests/utils/test_feature_extraction_utils.py
patch: @@ -135,11 +135,28 @@ def test_batch_feature_error_handling(self):
         self.assertIn("inhomogeneous", error_msg.lower())
         self.assertIn("return_tensors=None", error_msg)
 
-        # Unconvertible type (dict)
-        data_dict = {"values": [[1, 2]], "metadata": {"key": "val"}}
-        with self.assertRaises(ValueError) as context:
-            BatchFeature(data_dict, tensor_type="pt")
-        self.assertIn("metadata", str(context.exception))
+    @require_torch
+    def test_batch_feature_auto_skip_non_array_like(self):
+        """Test that non-array-like values are automatically skipped during tensor conversion."""
+        data = {
+            "values": [[1, 2]],
+            "metadata": {"key": "val"},
+            "image_path": "/path/to/image.jpg",
+            "tags": ["tag1", "tag2"],
+            "extra": None,
+        }
+        batch = BatchFeature(data, tensor_type="pt")
+
+        # values should be converted
+        self.assertIsInstance(batch["values"], torch.Tensor)
+
+        # Non-array-like values should remain unchanged
+        self.assertIsInstance(batch["metadata"], dict)
+        self.assertEqual(batch["metadata"], {"key": "val"})
+        self.assertIsInstance(batch["image_path"], str)
+        self.assertIsInstance(batch["tags"], list)
+        self.assertEqual(batch["tags"], ["tag1", "tag2"])
+        self.assertIsNone(batch["extra"])
 
     @require_torch
     def test_batch_feature_skip_tensor_conversion(self):
@@ -169,6 +186,32 @@ def test_batch_feature_convert_to_tensors_method(self):
         self.assertIsInstance(batch["input_values"], torch.Tensor)
         self.assertIsInstance(batch["metadata"], list)
 
+    @require_torch
+    def test_batch_feature_to_with_nested_tensors(self):
+        """Test .to() method works recursively with nested lists and tuples of tensors."""
+        batch = BatchFeature(
+            {
+                "list_tensors": [torch.tensor([1.0, 2.0]), torch.tensor([3.0, 4.0])],
+                "nested_list": [[torch.tensor([1.0]), torch.tensor([2.0])]],
+                "tuple_tensors": (torch.tensor([5.0]), torch.tensor([6.0])),
+            }
+        )
+
+        batch_fp16 = batch.to(torch.float16)
+
+        # Check lists of tensors are converted
+        self.assertIsInstance(batch_fp16["list_tensors"], list)
+        self.assertEqual(batch_fp16["list_tensors"][0].dtype, torch.float16)
+        self.assertEqual(batch_fp16["list_tensors"][1].dtype, torch.float16)
+
+        # Check nested lists are converted
+        self.assertIsInstance(batch_fp16["nested_list"][0], list)
+        self.assertEqual(batch_fp16["nested_list"][0][0].dtype, torch.float16)
+
+        # Check tuples are preserved and converted
+        self.assertIsInstance(batch_fp16["tuple_tensors"], tuple)
+        self.assertEqual(batch_fp16["tuple_tensors"][0].dtype, torch.float16)
+
 
 class FeatureExtractorUtilTester(unittest.TestCase):
     def test_cached_files_are_used_when_internet_is_down(self):