muellerzr/github-pr-history huggingface dataset:
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index fadbe6211..58db9b24e 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -139,6 +139,13 @@ class ClusterConfig(BaseConfig):
     # args for fsdp
     fsdp_config: dict = None
 
+    def __post_init__(self):
+        if self.deepspeed_config is None:
+            self.deepspeed_config = {}
+        if self.fsdp_config is None:
+            self.fsdp_config = {}
+        return super().__post_init__()
+
 
 @dataclass
 class SageMakerConfig(BaseConfig):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/tracking.mdx b/docs/source/tracking.mdx
index 959317887..6d4ca437c 100644
--- a/docs/source/tracking.mdx
+++ b/docs/source/tracking.mdx
@@ -28,12 +28,12 @@ Currently `Accelerate` supports three trackers out-of-the-box:
 
 To use any of them, pass in the selected type(s) to the `log_with` parameter in [`Accelerate`]:
 ```python
-from accelerate import Accelerate
+from accelerate import Accelerator
 from accelerate.utils import LoggerType
 
-accelerator = Accelerate(log_with="all")  # For all available trackers in the environment
-accelerator = Accelerate(log_with="wandb")
-accelerator = Accelerate(log_with=["wandb", LoggerType.TENSORBOARD])
+accelerator = Accelerator(log_with="all")  # For all available trackers in the environment
+accelerator = Accelerator(log_with="wandb")
+accelerator = Accelerator(log_with=["wandb", LoggerType.TENSORBOARD])
 ```
 
 At the start of your experiment [`~Accelerator.init_trackers`] should be used to setup your project, and potentially add any experiment hyperparameters to be logged:
@@ -123,14 +123,14 @@ be used with the API:
 
 ```python
 tracker = MyCustomTracker("some_run_name")
-accelerator = Accelerate(log_with=tracker)
+accelerator = Accelerator(log_with=tracker)
 ```
 
 These also can be mixed with existing trackers, including with `"all"`:
 
 ```python
 tracker = MyCustomTracker("some_run_name")
-accelerator = Accelerate(log_with=[tracker, "all"])
+accelerator = Accelerator(log_with=[tracker, "all"])
 ```
 
 ## When a wrapper cannot work



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index a79012d5c..eb3b24ce9 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -725,6 +725,13 @@ def free_memory(self):
         gc.collect()
         torch.cuda.empty_cache()
 
+    def clear(self):
+        """
+        Alias for [`Accelerate.free_memory`], releases all references to the internal objects stored and call the
+        garbage collector. You should call this method between two trainings with different models/optimizers.
+        """
+        self.free_memory()
+
     def _get_named_parameters(self, *args):
         named_parameters = {}
         for obj in args:


Review comments: body: ```suggestion
        Alias for [`Accelerate.free_memory`], releases all references to the internal objects stored and call the garbage
```
diff_hunk: @@ -725,6 +725,13 @@ def free_memory(self):
         gc.collect()
         torch.cuda.empty_cache()
 
+    def clear(self):
+        """
+        Alias for `Accelerate.free_memory`, releases all references to the internal objects stored and call the garbage

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index e18686e12..514b3a4f4 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -440,7 +440,10 @@ def _prepare_deepspeed(self, *args):
             batch_size_per_device * deepspeed_plugin.gradient_accumulation_steps * self.num_processes
         )
 
-        result = [self._prepare_one(obj) if isinstance(obj, torch.utils.data.DataLoader) else obj for obj in args]
+        result = [
+            self._prepare_one(obj, first_pass=True) if isinstance(obj, torch.utils.data.DataLoader) else obj
+            for obj in args
+        ]
 
         model = None
         optimizer = None



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 15eaaecb6..d72680bea 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -54,9 +54,10 @@ def get_cluster_input():
 
     if distributed_type == DistributedType.NO:
         use_cpu = _ask_field(
-            "Do you want to run your training on CPU only (even if a GPU is available)? [no]:",
-            lambda x: bool(x),
+            "Do you want to run your training on CPU only (even if a GPU is available)? [yes/NO]:",
+            _convert_yes_no_to_bool,
             default=False,
+            error_message="Please enter yes or no.",
         )
     elif distributed_type == DistributedType.MULTI_CPU:
         use_cpu = True
diff --git a/tests/test_tracking.py b/tests/test_tracking.py
index b958dbbfc..b3974afe2 100644
--- a/tests/test_tracking.py
+++ b/tests/test_tracking.py
@@ -159,10 +159,10 @@ def test_log(self):
                 break
         # Check HPS through careful parsing and cleaning
         cleaned_log = re.sub(r"[\x00-\x1f]+", " ", content.decode("utf8", "ignore"))
-        self.assertEqual(self.get_value_from_log("total_loss", cleaned_log), "0.1")
-        self.assertEqual(self.get_value_from_log("iteration", cleaned_log), "1")
-        self.assertEqual(self.get_value_from_log("my_text", cleaned_log), "some_value")
-        self.assertEqual(self.get_value_from_log("_step", cleaned_log), "0")
+        self.assertTrue("0.1" in self.get_value_from_log("total_loss", cleaned_log))
+        self.assertTrue("1" in self.get_value_from_log("iteration", cleaned_log))
+        self.assertTrue("some_value" in self.get_value_from_log("my_text", cleaned_log))
+        self.assertTrue("0" in self.get_value_from_log("_step", cleaned_log))
 
 
 # Comet has a special `OfflineExperiment` we need to use for testing



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/README.md b/README.md
index 3e21a3ae1..a3fc74cef 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ mpirun -np 2 python examples/nlp_example.py
 
 ## Launching training using DeepSpeed
 
-🤗 Accelerate supports training on single/multiple GPUs using DeepSpeed. to use it, you don't need to change anything in your training code; you can set everything using just `accelerate config`. However, if you desire to tweak your DeepSpeed related args from your python script, we provide you the `DeepSpeedPlugin`.
+🤗 Accelerate supports training on single/multiple GPUs using DeepSpeed. To use it, you don't need to change anything in your training code; you can set everything using just `accelerate config`. However, if you desire to tweak your DeepSpeed related args from your python script, we provide you the `DeepSpeedPlugin`.
 
 ```python
 from accelerator import Accelerator, DeepSpeedPlugin
diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx
index 2b405e8ec..1c62f05f5 100644
--- a/docs/source/quicktour.mdx
+++ b/docs/source/quicktour.mdx
@@ -398,10 +398,6 @@ DeepSpeed support is experimental, so the underlying API will evolve in the near
 breaking changes. In particular, 🤗 Accelerate does not support DeepSpeed config you have written yourself yet, this
 will be added in a next version.
 
-One main caveat for the DeepSpeed integration is that the DeepSpeed launcher always passes a `local_rank` variable to
-the training script, so your training script should accept it (whether you launch training with the DeepSpeed launcher
-or `accelerate launch`).
-
 <Tip warning={true}>
 
 The [`notebook_launcher`] does not support the DeepSpeed integration yet.
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 0cbd66884..f2cab1ccb 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -233,7 +233,7 @@ def multi_gpu_launcher(args):
 
 
 def deepspeed_launcher(args):
-    cmd = ["deepspeed"]
+    cmd = ["deepspeed", "--no_local_rank"]
     if args.num_machines > 1:
         cmd.extend(
             [



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 6e0742e02..e18686e12 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -284,9 +284,10 @@ def use_fp16(self):
     @property
     def mixed_precision(self):
         if self.distributed_type == DistributedType.DEEPSPEED:
-            if self.state.deepspeed_plugin.deepspeed_config["fp16"]["enabled"]:
+            config = self.state.deepspeed_plugin.deepspeed_config
+            if config.get("fp16", {}).get("enabled", False):
                 mixed_precision = "fp16"
-            elif self.state.deepspeed_plugin.deepspeed_config["bf16"]["enabled"]:
+            elif config.get("bf16", {}).get("enabled", False):
                 mixed_precision = "bf16"
             else:
                 mixed_precision = "no"


Review comments: body: ```suggestion
            if config.get("fp16", {}).get("enabled", False):
```
diff_hunk: @@ -284,9 +284,10 @@ def use_fp16(self):
     @property
     def mixed_precision(self):
         if self.distributed_type == DistributedType.DEEPSPEED:
-            if self.state.deepspeed_plugin.deepspeed_config["fp16"]["enabled"]:
+            config = self.state.deepspeed_plugin.deepspeed_config
+            if "fp16" in config and config["fp16"].get("enabled", False):
body: ```suggestion
            elif config.get("bf16", {}).get("enabled", False):
```
diff_hunk: @@ -284,9 +284,10 @@ def use_fp16(self):
     @property
     def mixed_precision(self):
         if self.distributed_type == DistributedType.DEEPSPEED:
-            if self.state.deepspeed_plugin.deepspeed_config["fp16"]["enabled"]:
+            config = self.state.deepspeed_plugin.deepspeed_config
+            if "fp16" in config and config["fp16"].get("enabled", False):
                 mixed_precision = "fp16"
-            elif self.state.deepspeed_plugin.deepspeed_config["bf16"]["enabled"]:
+            elif "bf16" in config and config["bf16"].get("enabled", False):

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index dd946c85d..0cbd66884 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -56,7 +56,6 @@ def launch_command_parser(subparsers=None):
     )
     parser.add_argument(
         "--mixed_precision",
-        default="no",
         type=str,
         choices=["no", "fp16", "bf16"],
         help="Whether or not to use mixed precision training. "



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/checkpointing.py b/src/accelerate/checkpointing.py
index 0636dffaa..99f18be43 100644
--- a/src/accelerate/checkpointing.py
+++ b/src/accelerate/checkpointing.py
@@ -107,14 +107,14 @@ def load_accelerator_state(input_dir, models, optimizers, process_index, scaler=
     for i, model in enumerate(models):
         weights_name = f"{MODEL_NAME}.bin" if i == 0 else f"{MODEL_NAME}_{i}.bin"
         input_model_file = os.path.join(input_dir, weights_name)
-        models[i].load_state_dict(torch.load(input_model_file))
+        models[i].load_state_dict(torch.load(input_model_file, map_location="cpu"))
     logger.info("All model weights loaded successfully")
 
     # Optimizer states
     for i, opt in enumerate(optimizers):
         optimizer_name = f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin"
         input_optimizer_file = os.path.join(input_dir, optimizer_name)
-        optimizers[i].load_state_dict(torch.load(input_optimizer_file))
+        optimizers[i].load_state_dict(torch.load(input_optimizer_file, map_location="cpu"))
     logger.info("All optimizer states loaded successfully")
 
     # GradScaler state



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: LG2M! Nice catch! Looks like we might be able to link https://github.com/huggingface/accelerate/issues/296?
body: That's no linked per se since in this issue, the user was using `optimizer.load_state_dict` and not `accelerator.load_state`.
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 2f869bd0f..432d78ae8 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -78,7 +78,7 @@ def tokenize_function(examples):
 
     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
-    tokenized_datasets.rename_column_("label", "labels")
+    tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index fabd3e9fc..f01d06b98 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -14,6 +14,7 @@
 
 import gc
 import os
+import sys
 import warnings
 from contextlib import contextmanager
 from typing import List, Optional, Union
@@ -705,7 +706,7 @@ def autocast(self):
 
             autocast_context.__enter__()
             yield
-            autocast_context.__exit__()
+            autocast_context.__exit__(*sys.exc_info())
         else:
             yield
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 1254c19e9..f59d3e8bd 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -166,6 +166,10 @@ def __init__(
         self.device_placement = device_placement
         self.split_batches = split_batches
         self.dispatch_batches = dispatch_batches
+        if dispatch_batches is True and version.parse(torch.__version__) < version.parse("1.8.0"):
+            raise ImportError(
+                "Using `DataLoaderDispatcher` requires PyTorch 1.8.0 minimum. You have {torch.__version__}."
+            )
 
         # Mixed precision attributes
         self.scaler = None
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index b28f24fc6..cc46e5577 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -328,6 +328,10 @@ class DataLoaderDispatcher(DataLoader):
     def __init__(self, dataset, split_batches: bool = False, **kwargs):
         super().__init__(dataset, **kwargs)
         self.split_batches = split_batches
+        if version.parse(torch.__version__) < version.parse("1.8.0"):
+            raise ImportError(
+                "Using `DataLoaderDispatcher` requires PyTorch 1.8.0 minimum. You have {torch.__version__}."
+            )
 
     def __iter__(self):
         state = AcceleratorState()
@@ -474,7 +478,10 @@ def prepare_data_loader(
         This does not support :obj:`BatchSampler` with varying batch size yet.
     """
     if dispatch_batches is None:
-        dispatch_batches = False if not put_on_device else isinstance(dataloader.dataset, IterableDataset)
+        if version.parse(torch.__version__) < version.parse("1.8.0") or not put_on_device:
+            dispatch_batches = False
+        else:
+            dispatch_batches = isinstance(dataloader.dataset, IterableDataset)
 
     if dispatch_batches and not put_on_device:
         raise ValueError("Using `dispatch_batches=True` requires `put_on_device=True`.")
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 2d9937642..62b1291ee 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -63,13 +63,16 @@ class TensorInformation:
     dtype: torch.dtype
 
 
-def set_seed(seed: int):
+def set_seed(seed: int, device_specific: bool = False):
     """
     Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch``.
 
     Args:
         seed (:obj:`int`): The seed to set.
+        device_specific (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether to differ the seed on each device slightly with `self.process_index`
     """
+    if device_specific:
+        seed += AcceleratorState().process_index
     random.seed(seed)
     np.random.seed(seed)
     torch.manual_seed(seed)


Review comments: body: ```suggestion
        device_specific (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether to differ the seed on each device slightly with `self.process_index`
```
diff_hunk: @@ -63,13 +63,16 @@ class TensorInformation:
     dtype: torch.dtype
 
 
-def set_seed(seed: int):
+def set_seed(seed: int, device_specific: bool = False):
     """
     Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch``.
 
     Args:
         seed (:obj:`int`): The seed to set.
+        device_specific (:obj:`bool`): Whether to differ the seed on each device slightly with `self.process_index`
diff --git a/setup.py b/setup.py
index f06ab3c99..bec3b9805 100644
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,7 @@
 from setuptools import find_packages
 
 extras = {}
-extras["quality"] = ["black == 21.4b0", "isort >= 5.5.4", "flake8 >= 3.8.3"]
+extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3"]
 extras["docs"] = [
     "docutils==0.16.0",
     "recommonmark",
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 76c3067a3..d3e4fa603 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -302,10 +302,7 @@ def prepare_model(self, model):
         if self.distributed_type == DistributedType.MULTI_GPU:
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(
-                model,
-                device_ids=[self.local_process_index],
-                output_device=self.local_process_index,
-                **kwargs,
+                model, device_ids=[self.local_process_index], output_device=self.local_process_index, **kwargs
             )
         elif self.distributed_type == DistributedType.MULTI_CPU:
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
@@ -352,8 +349,12 @@ def _prepare_deepspeed(self, *args):
             is_adamw = isinstance(optimizer, torch.optim.AdamW)
             if (is_adam or is_adamw) and deepspeed_plugin.offload_optimizer_device == "cpu":
                 defaults = optimizer.defaults
+                params = []
+                for group in optimizer.param_groups:
+                    params.extend(group["params"])
+
                 optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam(
-                    model.parameters(),
+                    params,
                     lr=defaults["lr"],
                     bias_correction=True,
                     betas=defaults["betas"],



Discussion: body: Could you just run `make style` on your branch to fix the quality issue? Thanks!
body: is this okay? it looks like there are a few other lines in there that also got style fixed
body: Looks good, thanks again!
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 443c55622..6eba4b295 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -118,11 +118,11 @@ def honor_type(obj, generator):
     """
     Cast a generator to the same type as obj (list, tuple or namedtuple)
     """
-    # There is no direct check whether an object if of type namedtuple sadly, this is a workaround.
-    if isinstance(obj, tuple) and hasattr(obj, "_fields"):
-        # Can instantiate a namedtuple from a generator directly, contrary to a tuple/list.
+    try:
+        return type(obj)(generator)
+    except TypeError:
+        # Some objects may not be able to instantiate from a generator directly
         return type(obj)(*list(generator))
-    return type(obj)(generator)
 
 
 def is_torch_tensor(tensor):



Discussion: body: Sorry I was on my Chinese new year leave and just get back.
Yeah I do agree this should raise exact error with enough information for debugging.
body: Could you just run `male style` on your branch so we can merge this?
body: hi, I just noticed the existence of this pr......
I've fixed the linting issue related to this pr, but there seems to be more on other code
body: Seems like the checks are all passing now, thanks again!
diff --git a/docs/source/installation.md b/docs/source/installation.md
index 6dac579e9..ca2f88286 100644
--- a/docs/source/installation.md
+++ b/docs/source/installation.md
@@ -73,7 +73,7 @@ If you want to constantly use the bleeding edge `master` version of the source c
 
 ``` bash
 git clone https://github.com/huggingface/accelerate.git
-cd transformers
+cd accelerate
 pip install -e .
 ```
 



Discussion: body: Sorry I forgot to merge for some reason...
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index aad2b8b65..a1f310271 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -125,7 +125,7 @@ def collate_fn(examples):
     lr_scheduler = get_linear_schedule_with_warmup(
         optimizer=optimizer,
         num_warmup_steps=100,
-        num_training_steps=len(train_dataloader) * num_epochs,
+        num_training_steps=(len(train_dataloader) * num_epochs) // gradient_accumulation_steps,
     )
 
     # Now we train the model



Discussion: body: :+1:
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 4269242b4..52217ba6a 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -53,19 +53,27 @@ class AcceleratedOptimizer(torch.optim.Optimizer):
     def __init__(self, optimizer, device_placement=True, scaler=None):
         self.optimizer = optimizer
         self.scaler = scaler
-        self.state = AcceleratorState()
+        self.accelerator_state = AcceleratorState()
         self.device_placement = device_placement
         self._is_overflow = False
 
         # Handle device placement
         if device_placement:
             state_dict = self.optimizer.state_dict()
-            if self.state.distributed_type == DistributedType.TPU:
-                xm.send_cpu_data_to_device(state_dict, self.state.device)
+            if self.accelerator_state.distributed_type == DistributedType.TPU:
+                xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device)
             else:
-                state_dict = move_to_device(state_dict, self.state.device)
+                state_dict = move_to_device(state_dict, self.accelerator_state.device)
             self.optimizer.load_state_dict(state_dict)
 
+    @property
+    def state(self):
+        return self.optimizer.state
+
+    @state.setter
+    def state(self, state):
+        self.optimizer.state = state
+
     @property
     def param_groups(self):
         return self.optimizer.param_groups
@@ -86,8 +94,8 @@ def add_param_group(self, param_group):
         self.optimizer.add_param_group(param_group)
 
     def load_state_dict(self, state_dict):
-        if self.state.distributed_type == DistributedType.TPU and self.device_placement:
-            xm.send_cpu_data_to_device(state_dict, self.state.device)
+        if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
+            xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device)
         self.optimizer.load_state_dict(state_dict)
 
     def state_dict(self):
@@ -113,7 +121,7 @@ def zero_grad(self, set_to_none=None):
                 self.optimizer.zero_grad()
 
     def step(self, closure=None):
-        if self.state.distributed_type == DistributedType.TPU:
+        if self.accelerator_state.distributed_type == DistributedType.TPU:
             optimizer_args = {"closure": closure} if closure is not None else {}
             xm.optimizer_step(self.optimizer, optimizer_args=optimizer_args)
         elif self.scaler is not None:
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index ac79d3e0a..f94b2cd99 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -150,7 +150,8 @@ def __init__(
         self._models = []
 
         # RNG Types
-        if rng_types is None:
+        self.rng_types = rng_types
+        if self.rng_types is None:
             self.rng_types = ["torch"] if version.parse(torch.__version__) <= version.parse("1.5.1") else ["generator"]
 
     @property
diff --git a/README.md b/README.md
index f944eea2d..cbc23ed10 100644
--- a/README.md
+++ b/README.md
@@ -206,6 +206,13 @@ You should use 🤗 Accelerate when you want to easily run your training scripts
 
 You shouldn't use 🤗 Accelerate if you don't want to write a training loop yourself. There are plenty of high-level libraries above PyTorch that will offer you that, 🤗 Accelerate is not one of them.
 
+## Frameworks using 🤗 Accelerate
+
+If you like the simplicity of 🤗 Accelerate but would prefer a higher-level abstraction around your training loop, some frameworks that are built on top of 🤗 Accelerate are listed below:
+
+* [pytorch-accelerated](https://github.com/Chris-hughes10/pytorch-accelerated) is a lightweight training library, with a streamlined feature set centred around a general-purpose [Trainer](https://pytorch-accelerated.readthedocs.io/en/latest/trainer.html), that places a huge emphasis on simplicity and transparency; enabling users to understand exactly what is going on under the hood, but without having to write and maintain the boilerplate themselves!
+* [Kornia](https://kornia.readthedocs.io/en/latest/get-started/introduction.html) is a differentiable library that allows classical computer vision to be integrated into deep learning models. Kornia provides a [Trainer](https://kornia.readthedocs.io/en/latest/x.html#kornia.x.Trainer) with the specific purpose to train and fine-tune the supported deep learning algorithms within the library.
+
 ## Installation
 
 This repository is tested on Python 3.6+ and PyTorch 1.4.0+
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index fef2f0c08..b848e13d4 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -114,7 +114,7 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
 
             launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
             try:
-                print(f"Launching a training on {num_processes} GPUs.")
+                print(f"Launching training on {num_processes} GPUs.")
                 start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
             finally:
                 # Clean up the environment variables set.


Review comments: body: No need for those two new lines here :-)
diff_hunk: @@ -220,6 +227,8 @@ Then, you will need to install PyTorch: refer to the [official installation page
 pip install accelerate
 ```
 
+
+
body: Removed.
diff_hunk: @@ -220,6 +227,8 @@ Then, you will need to install PyTorch: refer to the [official installation page
 pip install accelerate
 ```
 
+
+

Discussion: body: > Thanks for your PR! I was more thinking of having a separate section named "Frameworks using Accelerate" where you could put the same content. There is also kornia using Accelerate internally in their trainer I would add there :-)

Sure thing, I can create an additional section, migrate the content, and will also look into and add Kornia. 
body: @sgugger I have updated the README, hopefully that is closer to what you had in mind. Would you also like to me add a corresponding section to the docs?
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 3b1281c84..384db658a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -69,7 +69,7 @@ Changing it to work with accelerate is really easy and only adds a few lines of
           loss = my_loss_function(outputs, targets)
           # Just a small change for the backward instruction
     -     loss.backward()
-    +     accelerate.backward(loss)
+    +     accelerator.backward(loss)
           my_optimizer.step()
 
 and with this, your script can now run in a distributed environment (multi-GPU, TPU).
@@ -97,7 +97,7 @@ especially for TPU training):
           loss = my_loss_function(outputs, targets)
           # Just a small change for the backward instruction
     -     loss.backward()
-    +     accelerate.backward(loss)
+    +     accelerator.backward(loss)
           my_optimizer.step()
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 37ca78833..4269242b4 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import inspect
+
 import torch
 
 from packaging import version
@@ -100,9 +102,15 @@ def zero_grad(self, set_to_none=None):
                 )
             self.optimizer.zero_grad()
         else:
-            if set_to_none is None:
-                set_to_none = False
-            self.optimizer.zero_grad(set_to_none=set_to_none)
+            accept_arg = "set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters
+            if accept_arg:
+                if set_to_none is None:
+                    set_to_none = False
+                self.optimizer.zero_grad(set_to_none=set_to_none)
+            else:
+                if set_to_none is not None:
+                    raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.")
+                self.optimizer.zero_grad()
 
     def step(self, closure=None):
         if self.state.distributed_type == DistributedType.TPU:
diff --git a/README.md b/README.md
index 281325bfe..f944eea2d 100644
--- a/README.md
+++ b/README.md
@@ -155,6 +155,8 @@ For instance, here is how you would run the GLUE example on the MRPC task (from
 accelerate launch examples/nlp_example.py
 ```
 
+This CLI tool is **optional**, and you can still use `python my_script.py` or `python -m torch.distributed.launch my_script.py` at your convenance.
+
 ## Launching multi-CPU run using MPI
 
 🤗 Here is another way to launch multi-CPU run using MPI. You can learn how to install Open MPI on [this page](https://www.open-mpi.org/faq/?category=building#easy-build). You can use Intel MPI or MVAPICH as well.
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 220c6ddcc..aad2b8b65 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -162,8 +162,8 @@ def collate_fn(examples):
 
 def main():
     parser = argparse.ArgumentParser(description="Simple example of training script.")
-    parser.add_argument("--fp16", type=bool, default=False, help="If passed, will use FP16 training.")
-    parser.add_argument("--cpu", type=bool, default=False, help="If passed, will train on the CPU.")
+    parser.add_argument("--fp16", action="store_true", help="If passed, will use FP16 training.")
+    parser.add_argument("--cpu", action="store_true", help="If passed, will train on the CPU.")
     args = parser.parse_args()
     config = {"lr": 2e-5, "num_epochs": 3, "correct_bias": True, "seed": 42, "batch_size": 16}
     training_function(config, args)
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index e25b80c49..28a38539a 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -201,7 +201,7 @@ def _send_to_device(t, device):
     def _has_to_method(t):
         return hasattr(t, "to")
 
-    return recursively_apply(_send_to_device, tensor, device, test_type=_has_to_method, error_on_other_type=True)
+    return recursively_apply(_send_to_device, tensor, device, test_type=_has_to_method)
 
 
 def get_data_structure(data):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index ca617634d..9b16aba7b 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -20,7 +20,7 @@
 from accelerate.utils import send_to_device
 
 
-TestNamedTuple = namedtuple("TestNamedTuple", "a b")
+TestNamedTuple = namedtuple("TestNamedTuple", "a b c")
 
 
 class UtilsTester(unittest.TestCase):
@@ -31,23 +31,26 @@ def test_send_to_device(self):
         result1 = send_to_device(tensor, device)
         self.assertTrue(torch.equal(result1.cpu(), tensor))
 
-        result2 = send_to_device((tensor, [tensor, tensor]), device)
+        result2 = send_to_device((tensor, [tensor, tensor], 1), device)
         self.assertIsInstance(result2, tuple)
         self.assertTrue(torch.equal(result2[0].cpu(), tensor))
         self.assertIsInstance(result2[1], list)
         self.assertTrue(torch.equal(result2[1][0].cpu(), tensor))
         self.assertTrue(torch.equal(result2[1][1].cpu(), tensor))
+        self.assertEqual(result2[2], 1)
 
-        result2 = send_to_device({"a": tensor, "b": [tensor, tensor]}, device)
+        result2 = send_to_device({"a": tensor, "b": [tensor, tensor], "c": 1}, device)
         self.assertIsInstance(result2, dict)
         self.assertTrue(torch.equal(result2["a"].cpu(), tensor))
         self.assertIsInstance(result2["b"], list)
         self.assertTrue(torch.equal(result2["b"][0].cpu(), tensor))
         self.assertTrue(torch.equal(result2["b"][1].cpu(), tensor))
+        self.assertEqual(result2["c"], 1)
 
-        result3 = send_to_device(TestNamedTuple(a=tensor, b=[tensor, tensor]), device)
+        result3 = send_to_device(TestNamedTuple(a=tensor, b=[tensor, tensor], c=1), device)
         self.assertIsInstance(result3, TestNamedTuple)
         self.assertTrue(torch.equal(result3.a.cpu(), tensor))
         self.assertIsInstance(result3.b, list)
         self.assertTrue(torch.equal(result3.b[0].cpu(), tensor))
         self.assertTrue(torch.equal(result3.b[1].cpu(), tensor))
+        self.assertEqual(result3.c, 1)
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 582066414..73f8ecc0b 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import math
 from typing import List, Optional, Union
 
 import torch
@@ -388,6 +389,14 @@ def __iter__(self):
                 xm.mark_step()
             yield slice_tensors(batch, data_slice)
 
+    def __len__(self):
+        state = AcceleratorState()
+        whole_length = super().__len__()
+        if self.drop_last:
+            return whole_length // state.num_processes
+        else:
+            return math.ceil(whole_length / state.num_processes)
+
 
 def prepare_data_loader(
     dataloader: DataLoader,
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 8df05578a..e25b80c49 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -255,7 +255,7 @@ def _convert_to_fp32(tensor):
     def _is_fp16_tensor(tensor):
         return hasattr(tensor, "dtype") and tensor.dtype == torch.float16
 
-    return recursively_apply(_is_fp16_tensor, tensor, test_type=_is_fp16_tensor)
+    return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_tensor)
 
 
 def convert_outputs_to_fp32(model_forward):



Discussion: body: Thanks for fixing this!
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 03e38baf8..582066414 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import warnings
 from typing import List, Optional, Union
 
 import torch
@@ -228,8 +227,8 @@ def __init__(
         process_index: int = 0,
         split_batches: bool = False,
     ):
-        if split_batches and batch_size % num_processes != 0:
-            warnings.warn(
+        if split_batches and batch_size > 1 and batch_size % num_processes != 0:
+            raise ValueError(
                 f"To use `IterableDatasetShard` in `split_batches` mode, the batch size ({batch_size}) "
                 f"needs to be a round multiple of the number of processes ({num_processes})."
             )
@@ -466,9 +465,9 @@ def prepare_data_loader(
         process_index = state.process_index
 
     # Sanity check
-    if split_batches and dataloader.batch_size % num_processes != 0:
-        warnings.warn(
-            f"To use `IterableDatasetShard` in `split_batches` mode, the batch size ({dataloader.batch_size}) "
+    if split_batches and dataloader.batch_size > 1 and dataloader.batch_size % num_processes != 0:
+        raise ValueError(
+            f"To use a `DataLoader` in `split_batches` mode, the batch size ({dataloader.batch_size}) "
             f"needs to be a round multiple of the number of processes ({num_processes})."
         )
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index d81e4ab61..ca49adf45 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -203,6 +203,8 @@ def _tpu_gather(tensor, name="tensor"):
         return type(tensor)({k: _tpu_gather(v, name=f"{name}_{k}") for k, v in tensor.items()})
     elif not isinstance(tensor, torch.Tensor):
         raise TypeError(f"Can't gather the values of type {type(tensor)}, only of nested list/tuple/dicts of tensors.")
+    if tensor.ndim == 0:
+        tensor = tensor.clone()[None]
     return xm.mesh_reduce(name, tensor, torch.cat)
 
 
@@ -213,6 +215,8 @@ def _gpu_gather(tensor):
         return type(tensor)({k: _gpu_gather(v) for k, v in tensor.items()})
     elif not isinstance(tensor, torch.Tensor):
         raise TypeError(f"Can't gather the values of type {type(tensor)}, only of nested list/tuple/dicts of tensors.")
+    if tensor.ndim == 0:
+        tensor = tensor.clone()[None]
     output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())]
     torch.distributed.all_gather(output_tensors, tensor)
     return torch.cat(output_tensors, dim=0)
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index a6b7a7d74..c5238369c 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -388,4 +388,5 @@ def __post_init__(self):
                 },
             },
             "steps_per_print": float("inf"),  # this will stop deepspeed from logging @ stdout
+            "zero_allow_untested_optimizer": True,
         }



Discussion: body: FYI, I'm not involved with accelerate's deepspeed integration. I suggested that it will be done in same way as transformers when it was first contributed, but it was still committed in a different way. Thus now the user has to use a completely different setup when switching from transformers and developers have to maintain something totally different. So I'm not participating in this particular endeavour.
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index a6b7a7d74..d81e4ab61 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -151,7 +151,7 @@ def convert_to_fp32(tensor):
     if isinstance(tensor, (list, tuple)):
         return honor_type(tensor, (convert_to_fp32(t) for t in tensor))
     elif isinstance(tensor, dict):
-        return type(tensor)({k: convert_to_fp32(v) for k, v in tensor.items()})
+        return type(tensor)(**{k: convert_to_fp32(v) for k, v in tensor.items()})
     elif not hasattr(tensor, "dtype") or tensor.dtype != torch.float16:
         return tensor
     return tensor.float()
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 8707ed373..3b1281c84 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -136,7 +136,7 @@ Supported integrations
 - multi-GPU on several nodes (machines)
 - TPU
 - FP16 with native AMP (apex on the roadmap)
-- DeepSpeed (experimental)
+- DeepSpeed (experimental support)
 
 .. toctree::
     :maxdepth: 2
diff --git a/docs/source/quicktour.rst b/docs/source/quicktour.rst
index 68a23fff5..3e9f5f470 100644
--- a/docs/source/quicktour.rst
+++ b/docs/source/quicktour.rst
@@ -381,6 +381,22 @@ this:
         lr_scheduler.step()
 
 
+DeepSpeed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DeepSpeed support is experimental, so the underlying API will evolve in the near future and may have some slight
+breaking changes. In particular, 🤗 Accelerate does not support DeepSpeed config you have written yourself yet, this
+will be added in a next version.
+
+One main caveat for the DeepSpeed integration is that the DeepSpeed launcher always passes a ``local_rank`` variable to
+the training script, so your training script should accept it (whether you launch training with the DeepSpeed launcher
+or ``accelerate launch``).
+
+.. Warning::
+
+    The :func:`~accelerate.notebook_launcher` does not support the DeepSpeed integration yet.
+
+
 Internal mechanism
 -----------------------------------------------------------------------------------------------------------------------
 


Review comments: body: ```suggestion
or ``accelerate launch``).
```
diff_hunk: @@ -381,6 +381,22 @@ this:
         lr_scheduler.step()
 
 
+DeepSpeed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DeepSpeed support is experimental, so the underlying API will evolve in the near future and may have some slight
+breaking changes. In particular, 🤗 Accelerate does not support DeepSpeed config you have written yourself yet, this
+will be added in a next version.
+
+One main caveat for the DeepSpeed integration is that the DeepSpeed launcher always passes a ``local_rank`` variable to
+the training script, so your training script should accept it (whether you launch training with the DeepSpeed launcher
+or ``accelerate launch`` Accelerate.
diff --git a/docs/source/quicktour.rst b/docs/source/quicktour.rst
index 7ccdf5daf..2efdf6a82 100644
--- a/docs/source/quicktour.rst
+++ b/docs/source/quicktour.rst
@@ -251,6 +251,13 @@ To introduce special behavior in your script for TPUs you can check the :obj:`di
 The `NLP example <https://github.com/huggingface/accelerate/blob/main/examples/nlp_example.py>`__ shows an example in
 situation with dynamic padding.
 
+One last thing to pay close attnetion to: if your model has tied weights (such as language models which tie the weights
+of the embedding matrix with the weights of the decoder), moving this model to the TPU (either yourself or after you
+passed your model to :meth:`~accelerate.Accelerator.prepare`) will break the tying. You will need to retie the weights
+after. You can find an example of this in the `run_clm_no_trainer
+<https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_clm.py>`__ script in
+the Transformers repository.
+
 
 Other caveats
 -----------------------------------------------------------------------------------------------------------------------
@@ -345,6 +352,7 @@ If you are using gradient clipping in your script, you should replace the calls
 :obj:`torch.nn.utils.clip_grad_norm_` or :obj:`torch.nn.utils.clip_grad_value_` with :obj:`accelerator.clip_grad_norm_`
 and :obj:`accelerator.clip_grad_value_` respectively.
 
+
 Mixed Precision training
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 30fd8314e..57ea686f9 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -52,6 +52,7 @@ def __init__(self, optimizer, device_placement=True, scaler=None):
         self.optimizer = optimizer
         self.scaler = scaler
         self.state = AcceleratorState()
+        self.device_placement = device_placement
 
         # Handle device placement
         if device_placement:
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 44909ab3c..96648ca65 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -104,6 +104,8 @@ def __init__(
         self.drop_last = batch_sampler.drop_last
 
     def __len__(self):
+        if self.split_batches:
+            return len(self.batch_sampler)
         if len(self.batch_sampler) % self.num_processes == 0:
             return len(self.batch_sampler) // self.num_processes
         length = len(self.batch_sampler) // self.num_processes
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 20274b671..281f4e9a7 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -413,8 +413,10 @@ def unscale_gradients(self, optimizer=None):
                 optimizer = self._optimizers
             elif not isinstance(optimizer, (tuple, list)):
                 optimizer = [optimizer]
-            for optimizer in optimizer:
-                self.scaler.unscale_(optimizer)
+            for opt in optimizer:
+                while isinstance(opt, AcceleratedOptimizer):
+                    opt = opt.optimizer
+                self.scaler.unscale_(opt)
 
     def clip_grad_norm_(self, parameters, max_norm, norm_type=2):
         """



Discussion: body: The tests are not properly setup :see_no_evil:
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index 4c785b12f..fef2f0c08 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -16,7 +16,8 @@
 import sys
 
 import torch
-from torch.multiprocessing import start_processes
+
+from packaging import version
 
 from .state import AcceleratorState
 from .utils import PrepareForLaunch
@@ -82,6 +83,14 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
 
         if num_processes > 1:
             # Multi-GPU launch
+            if version.parse(torch.__version__) < version.parse("1.5.0"):
+                raise ImportError(
+                    "Using `notebook_launcher` for distributed training on GPUs require torch >= 1.5.0, got "
+                    f"{torch.__version__}."
+                )
+
+            from torch.multiprocessing import start_processes
+
             if len(AcceleratorState._shared_state) > 0:
                 raise ValueError(
                     "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
diff --git a/README.md b/README.md
index ffbf36bbd..78a2ceb2a 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,10 @@ limitations under the License.
 <p>Run your *raw* PyTorch training script on any kind of device
 </h3>
 
+<h3 align="center">
+    <a href="https://hf.co/course"><img src="https://raw.githubusercontent.com/huggingface/transformers/master/docs/source/imgs/course_banner.png"></a>
+</h3>
+
 ## Easy to integrate
 
 🤗 Accelerate was created for PyTorch users who like to write the training loop of PyTorch models but are reluctant to write and maintain the boilerplate code needed to use multi-GPUs/TPU/fp16.
diff --git a/docs/source/imgs/course_banner.png b/docs/source/imgs/course_banner.png
new file mode 100644
index 000000000..45773d164
Binary files /dev/null and b/docs/source/imgs/course_banner.png differ
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 2bc1459fd..a700cad62 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -387,16 +387,16 @@ def prepare_data_loader(self, data_loader):
     def prepare_optimizer(self, optimizer):
         return AcceleratedOptimizer(optimizer, device_placement=self.device_placement, scaler=self.scaler)
 
-    def backward(self, loss):
+    def backward(self, loss, **kwargs):
         """
         Use :obj:`accelerator.backward(loss)` in lieu of :obj:`loss.backward()`.
         """
         if self.distributed_type == DistributedType.DEEPSPEED:
-            self.deepspeed_engine.backward(loss)
+            self.deepspeed_engine.backward(loss, **kwargs)
         elif self.scaler is not None:
-            self.scaler.scale(loss).backward()
+            self.scaler.scale(loss).backward(**kwargs)
         else:
-            loss.backward()
+            loss.backward(**kwargs)
 
     def unscale_gradients(self, optimizer=None):
         """