Repository: huggingface/accelerate
Pull request: Introduce listify, fix tensorboard silently failing
Solves https://github.com/huggingface/accelerate/issues/1546 by introducing a utility which will recursively convert all tensors in a dictionary/list of dictionaries/etc/etc into pure lists of digits. This is needed for tensorboard to log, as it also can't take in numpy arrays either (tried this first, to just bring in `numpify` from `transformers`). Should be a useful util in the future however, which is why I decided to bring it out fully rather than just have it specific to tensorboard

Problem statement:
accelerator.log(...) silently does nothing when logging a torch.Tensor to TensorBoard
### System Info

```Shell
accelerate==0.16.0
```


### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] One of the scripts in the examples/ folder of Accelerate or an officially supported `no_trainer` script in the `examples` folder of the `transformers` repo (such as `run_no_trainer_glue.py`)
- [X] My own task or dataset (give details below)

### Reproduction

When passing a `torch.Tensor` as a value in the dictionary passed to `accelerator.log`, the tensor is ignored and its value is not included in the log file:

```python
import torch
from accelerate import Accelerator

accelerator = Accelerator(log_with="tensorboard", logging_dir="some_path")
accelerator.init_trackers("train")
accelerator.log({"loss": torch.tensor(1.0)}, step=1) # This line does nothing, but throws nothing!
accelerator.end_training()

list(SummaryIterator("some_path/train/events.out.tfevents.1686081050.18920.0"))
# => Returns a single item:
#  [wall_time: 1686081050.30094
#   file_version: "brain.Event:2"
#   source_metadata {
#     writer: "tensorboard.summary.writer.event_file_writer"
#   }]
```

Passing a single float value instead of a scalar tensor does successfully cause the value to be saved.

### Expected behavior

Accelerate should pass the provided tensor's scalar value through to the TensorBoard log file, _or_ should throw an exception to indicate that the provided data type is invalid:

```python
import torch
from accelerate import Accelerator

accelerator = Accelerator(log_with="tensorboard", logging_dir="some_path")
accelerator.init_trackers("train")
accelerator.log({"loss": torch.tensor(1.0)}, step=1)
accelerator.end_training()

list(SummaryIterator("some_path/train/events.out.tfevents.1686081050.18920.0"))
# => Returns a single item:
# [wall_time: 1686081291.2548301
#  file_version: "brain.Event:2"
#  source_metadata {
#    writer: "tensorboard.summary.writer.event_file_writer"
#  },
#  wall_time: 1686081291.255583
#  step: 1
#  summary {
#    value {
#      tag: "loss"
#      simple_value: 1.0
#    }
#  }]
```

Hints from issue discussion:

Commit message: Clean imports and make note

Patch to review:
diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index d1a502ceb0a..553938eb2ff 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -32,6 +32,7 @@
     is_mlflow_available,
     is_tensorboard_available,
     is_wandb_available,
+    untensorify,
 )
 
 
@@ -236,6 +237,7 @@ def log(self, values: dict, step: Optional[int] = None, **kwargs):
                 Additional key word arguments passed along to either `SummaryWriter.add_scaler`,
                 `SummaryWriter.add_text`, or `SummaryWriter.add_scalers` method based on the contents of `values`.
         """
+        values = untensorify(values)
         for k, v in values.items():
             if isinstance(v, (int, float)):
                 self.writer.add_scalar(k, v, global_step=step, **kwargs)
diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index 067ac795a0c..9f562e86492 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -95,6 +95,7 @@
     reduce,
     send_to_device,
     slice_tensors,
+    untensorify,
 )
 from .versions import compare_versions, is_torch_version
 
diff --git a/src/accelerate/utils/operations.py b/src/accelerate/utils/operations.py
index 4dc3e5d8e39..448d081bf48 100644
--- a/src/accelerate/utils/operations.py
+++ b/src/accelerate/utils/operations.py
@@ -224,6 +224,29 @@ def find_batch_size(data):
     return data.shape[0]
 
 
+def untensorify(data):
+    """
+    Recursively finds tensors in a nested list/tuple/dictionary and converts them to a list of numbers.
+
+    Args:
+        data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to convert to regular numbers.
+
+    Returns:
+        The same data structure as `data` with lists of numbers instead of `torch.Tensor`.
+    """
+
+    def _convert_to_list(tensor):
+        tensor = tensor.detach().cpu()
+        if tensor.dtype == torch.bfloat16:
+            # As of Numpy 1.21.4, NumPy does not support bfloat16 (see
+            # https://github.com/numpy/numpy/blob/a47ecdea856986cd60eabbd53265c2ca5916ad5d/doc/source/user/basics.types.rst ).
+            # Until Numpy adds bfloat16, we must convert float32.
+            tensor = tensor.to(torch.float32)
+        return tensor.tolist()
+
+    return recursively_apply(_convert_to_list, data)
+
+
 def _tpu_gather(tensor):
     def _tpu_gather_one(tensor):
         if tensor.ndim == 0:
diff --git a/tests/test_tracking.py b/tests/test_tracking.py
index c0d729b99ad..98d65886c3e 100644
--- a/tests/test_tracking.py
+++ b/tests/test_tracking.py
@@ -25,6 +25,8 @@
 from typing import Optional
 from unittest import mock
 
+import torch
+
 # We use TF to parse the logs
 from accelerate import Accelerator
 from accelerate.test_utils.testing import (
@@ -36,12 +38,17 @@
     skip,
 )
 from accelerate.tracking import CometMLTracker, GeneralTracker
-from accelerate.utils import is_comet_ml_available
+from accelerate.utils import is_comet_ml_available, is_tensorboard_available
 
 
 if is_comet_ml_available():
     from comet_ml import OfflineExperiment
 
+if is_tensorboard_available():
+    import struct
+
+    import tensorboard.compat.proto.event_pb2 as event_pb2
+
 logger = logging.getLogger(__name__)
 
 
@@ -71,6 +78,35 @@ def test_log(self):
             log = list(filter(lambda x: x.is_file(), Path(f"{dirpath}/{project_name}").iterdir()))[0]
             self.assertNotEqual(str(log), "")
 
+    def test_log_with_tensor(self):
+        project_name = "test_project_with_log"
+        with tempfile.TemporaryDirectory() as dirpath:
+            accelerator = Accelerator(log_with="tensorboard", project_dir=dirpath)
+            accelerator.init_trackers(project_name)
+            values = {"tensor": torch.tensor(1)}
+            accelerator.log(values, step=0)
+            accelerator.end_training()
+            # Logged values are stored in the outermost-tfevents file and can be read in as a TFRecord
+            # Names are randomly generated each time
+            log = list(filter(lambda x: x.is_file(), Path(f"{dirpath}/{project_name}").iterdir()))[0]
+            # Reading implementation based on https://github.com/pytorch/pytorch/issues/45327#issuecomment-703757685
+            with open(log, "rb") as f:
+                data = f.read()
+            found_tensor = False
+            while data:
+                header = struct.unpack("Q", data[:8])
+
+                event_str = data[12 : 12 + int(header[0])]  # 8+4
+                data = data[12 + int(header[0]) + 4 :]
+                event = event_pb2.Event()
+
+                event.ParseFromString(event_str)
+                if event.HasField("summary"):
+                    for value in event.summary.value:
+                        if value.simple_value == 1.0 and value.tag == "tensor":
+                            found_tensor = True
+            self.assertTrue(found_tensor, "Converted tensor was not found in the log file!")
+
     def test_project_dir(self):
         with self.assertRaisesRegex(ValueError, "Logging with `tensorboard` requires a `logging_dir`"):
             _ = Accelerator(log_with="tensorboard")
diff --git a/tests/test_utils.py b/tests/test_utils.py
index cd7e1cb3a3f..4db611008e2 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -28,6 +28,7 @@
     patch_environment,
     recursively_apply,
     send_to_device,
+    untensorify,
 )
 
 
@@ -82,6 +83,18 @@ def test_honor_type(self):
             "Unsupported types (<class 'int'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.",
         )
 
+    def test_untensorify(self):
+        tensor = torch.tensor([1, 2, 3, 4, 5])
+        self.assertEqual(untensorify(tensor), [1, 2, 3, 4, 5])
+
+        tensor = torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
+        self.assertEqual(untensorify(tensor), [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
+
+        tensor = torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]])
+        self.assertEqual(
+            untensorify(tensor), [[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]]
+        )
+
     def test_patch_environment(self):
         with patch_environment(aa=1, BB=2):
             self.assertEqual(os.environ.get("AA"), "1")


Code review comments:
diff_hunk: @@ -224,6 +224,29 @@ def find_batch_size(data):
     return data.shape[0]
 
 
+def untensorify(data):
path: src/accelerate/utils/operations.py
text: @user1:
Should be `listify` since it converts to a list (or tries to)