muellerzr/github-pr-history huggingface dataset:
diff --git a/examples/by_feature/deepspeed_with_config_support.py b/examples/by_feature/deepspeed_with_config_support.py
index 36ace84b9..0fdff58ae 100755
--- a/examples/by_feature/deepspeed_with_config_support.py
+++ b/examples/by_feature/deepspeed_with_config_support.py
@@ -285,10 +285,9 @@ def evaluate(args, model, eval_dataloader, accelerator, eval_dataset):
             outputs = model(**batch)
 
         loss = outputs.loss
-        losses.append(accelerator.gather(loss.repeat(args.per_device_eval_batch_size)))
+        losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))
 
     losses = torch.cat(losses)
-    losses = losses[: len(eval_dataset)]
     try:
         eval_loss = torch.mean(losses)
         perplexity = math.exp(eval_loss)


Review comments: body: This line (291) becomes redundant when using `gather_for_metrics`
diff_hunk: @@ -285,7 +285,7 @@ def evaluate(args, model, eval_dataloader, accelerator, eval_dataset):
             outputs = model(**batch)
 
         loss = outputs.loss
-        losses.append(accelerator.gather(loss.repeat(args.per_device_eval_batch_size)))
+        losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))
 
     losses = torch.cat(losses)
     losses = losses[: len(eval_dataset)]
body: Ah yeah, makes sense thank you!
diff_hunk: @@ -285,7 +285,7 @@ def evaluate(args, model, eval_dataloader, accelerator, eval_dataset):
             outputs = model(**batch)
 
         loss = outputs.loss
-        losses.append(accelerator.gather(loss.repeat(args.per_device_eval_batch_size)))
+        losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))
 
     losses = torch.cat(losses)
     losses = losses[: len(eval_dataset)]

Discussion: body: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/accelerate/pr_821). All of your documentation changes will be reflected on that endpoint.
diff --git a/src/accelerate/hooks.py b/src/accelerate/hooks.py
index 120bb7f42..ef429a369 100644
--- a/src/accelerate/hooks.py
+++ b/src/accelerate/hooks.py
@@ -160,17 +160,19 @@ def new_forward(*args, **kwargs):
     return module
 
 
-def remove_hook_from_module(module: nn.Module):
+def remove_hook_from_module(module: nn.Module, recurse=False):
     """
     Removes any hook attached to a module via `add_hook_to_module`.
 
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
+        recurse (`bool`, **optional**): Whether to remove the hooks recursively
 
     Returns:
         `torch.nn.Module`: The same module, with the hook detached (the module is modified in place, so the result can
         be discarded).
     """
+
     if hasattr(module, "_hf_hook"):
         module._hf_hook.detach_hook(module)
         delattr(module, "_hf_hook")
@@ -179,6 +181,10 @@ def remove_hook_from_module(module: nn.Module):
         module.forward = module._old_forward
         delattr(module, "_old_forward")
 
+    if recurse:
+        for child in module.children():
+            remove_hook_from_module(child, recurse)
+
     return module
 
 


Review comments: body: Not sure the check is useful (should be >0 but the loop will just be empty)
```suggestion
    if recurse:
```
diff_hunk: @@ -179,6 +181,10 @@ def remove_hook_from_module(module: nn.Module):
         module.forward = module._old_forward
         delattr(module, "_old_forward")
 
+    if recurse and len(list(module.children())) > 1:
body: ```suggestion
        for child in module.children():
```
No need to ask for the names if you don't want them ;-)
diff_hunk: @@ -179,6 +181,10 @@ def remove_hook_from_module(module: nn.Module):
         module.forward = module._old_forward
         delattr(module, "_old_forward")
 
+    if recurse and len(list(module.children())) > 1:
+        for _, child in module.named_children():
body: Nice catch! Yes indeed ;) 
diff_hunk: @@ -179,6 +181,10 @@ def remove_hook_from_module(module: nn.Module):
         module.forward = module._old_forward
         delattr(module, "_old_forward")
 
+    if recurse and len(list(module.children())) > 1:

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thanks for the suggestion! Can confirm the `BetterTransformers` tests pass on my side ;)
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 4fad12c72..d5eeef99a 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -157,3 +157,9 @@ def is_overflow(self):
     def step_was_skipped(self):
         """Whether or not the optimizer step was skipped."""
         return self._is_overflow
+
+    def __getstate__(self):
+        return self.__dict__.copy()
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
diff --git a/tests/test_optimizer.py b/tests/test_optimizer.py
new file mode 100644
index 000000000..15a095bf7
--- /dev/null
+++ b/tests/test_optimizer.py
@@ -0,0 +1,36 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+import unittest
+
+import torch
+
+from accelerate import Accelerator
+from accelerate.state import AcceleratorState
+from accelerate.test_utils import require_cpu
+
+
+@require_cpu
+class OptimizerTester(unittest.TestCase):
+    def test_accelerated_optimizer_pickling(self):
+        model = torch.nn.Linear(10, 10)
+        optimizer = torch.optim.SGD(model.parameters(), 0.1)
+        accelerator = Accelerator()
+        optimizer = accelerator.prepare(optimizer)
+        try:
+            pickle.loads(pickle.dumps(optimizer))
+        except Exception as e:
+            self.fail(f"Accelerated optimizer pickling failed with {e}")
+        AcceleratorState._reset_state()



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Perfect, thanks!
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index b9f6c1089..7c8534e93 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -592,6 +592,12 @@ def deepspeed_launcher(args):
     num_machines = getattr(args, "num_machines")
     main_process_ip = getattr(args, "main_process_ip")
     main_process_port = getattr(args, "main_process_port")
+
+    # make sure launcher is not None
+    if args.deepspeed_multinode_launcher is None:
+        # set to default pdsh
+        setattr(args, "deepspeed_multinode_launcher", DEEPSPEED_MULTINODE_LAUNCHERS[0])
+
     if num_machines > 1 and args.deepspeed_multinode_launcher != DEEPSPEED_MULTINODE_LAUNCHERS[1]:
         cmd = ["deepspeed", "--no_local_rank"]
         cmd.extend(["--hostfile", str(args.deepspeed_hostfile), "--launcher", str(args.deepspeed_multinode_launcher)])



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/offload.py b/src/accelerate/utils/offload.py
index 750ff9d57..bc0fcdef1 100644
--- a/src/accelerate/utils/offload.py
+++ b/src/accelerate/utils/offload.py
@@ -178,5 +178,13 @@ def extract_submodules_state_dict(state_dict: Dict[str, torch.Tensor], submodule
     """
     result = {}
     for module_name in submodule_names:
-        result.update({key: param for key, param in state_dict.items() if key.startswith(module_name)})
+        # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
+        # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
+        result.update(
+            {
+                key: param
+                for key, param in state_dict.items()
+                if key == module_name or key.startswith(module_name + ".")
+            }
+        )
     return result
diff --git a/tests/test_offload.py b/tests/test_offload.py
index 765a9685d..a9ebd36ee 100644
--- a/tests/test_offload.py
+++ b/tests/test_offload.py
@@ -21,6 +21,7 @@
 
 from accelerate.utils import (
     OffloadedWeightsLoader,
+    extract_submodules_state_dict,
     is_torch_version,
     load_offloaded_weight,
     offload_state_dict,
@@ -105,3 +106,12 @@ def test_offload_weights_loader(self):
             self.assertEqual(sorted(weight_map), sorted(state_dict.keys()))
             for key, param in state_dict.items():
                 self.assertTrue(torch.allclose(param, weight_map[key]))
+
+    def test_extract_submodules_state_dict(self):
+        state_dict = {"a.1": 0, "a.10": 1, "a.2": 2}
+        extracted = extract_submodules_state_dict(state_dict, ["a.1", "a.2"])
+        self.assertDictEqual(extracted, {"a.1": 0, "a.2": 2})
+
+        state_dict = {"a.1.a": 0, "a.10.a": 1, "a.2.a": 2}
+        extracted = extract_submodules_state_dict(state_dict, ["a.1", "a.2"])
+        self.assertDictEqual(extracted, {"a.1.a": 0, "a.2.a": 2})



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index fa7588b0a..071c86a1d 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -470,7 +470,7 @@ def multi_gpu_launcher(args):
         setattr(args, "nproc_per_node", str(num_processes // num_machines))
         setattr(args, "nnodes", str(num_machines))
         setattr(args, "node_rank", int(args.machine_rank))
-        if getattr(args, "same_network"):
+        if getattr(args, "same_network", False):
             setattr(args, "master_addr", str(main_process_ip))
             setattr(args, "master_port", str(main_process_port))
         else:
@@ -488,7 +488,7 @@ def multi_gpu_launcher(args):
         setattr(args, "no_python", True)
 
     current_env = os.environ.copy()
-    gpu_ids = getattr(args, "gpu_ids")
+    gpu_ids = getattr(args, "gpu_ids", "all")
     if gpu_ids != "all" and args.gpu_ids is not None:
         current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     mixed_precision = args.mixed_precision.lower()
@@ -618,7 +618,7 @@ def deepspeed_launcher(args):
         setattr(args, "nproc_per_node", str(num_processes // num_machines))
         setattr(args, "nnodes", str(num_machines))
         setattr(args, "node_rank", int(args.machine_rank))
-        if getattr(args, "same_network"):
+        if getattr(args, "same_network", False):
             setattr(args, "master_addr", str(main_process_ip))
             setattr(args, "master_port", str(main_process_port))
         else:
@@ -636,7 +636,7 @@ def deepspeed_launcher(args):
         setattr(args, "no_python", True)
 
     current_env = os.environ.copy()
-    gpu_ids = getattr(args, "gpu_ids")
+    gpu_ids = getattr(args, "gpu_ids", "all")
     if gpu_ids != "all" and args.gpu_ids is not None:
         current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     try:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index 08ee3a65a..1760e72bc 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -365,7 +365,7 @@ def get_balanced_memory(
     if not torch.cuda.is_available():
         return max_memory
 
-    num_devices = len([d for d in max_memory if torch.device(d).type == "cuda"])
+    num_devices = len([d for d in max_memory if torch.device(d).type == "cuda" and max_memory[d] > 0])
     module_sizes = compute_module_sizes(model, dtype=dtype)
     per_gpu = module_sizes[""] // (num_devices - 1 if low_zero else num_devices)
 
@@ -409,7 +409,7 @@ def get_balanced_memory(
 
     max_memory = get_max_memory(max_memory)
     # The last device is left with max_memory just in case the buffer is not enough.
-    for i in range(num_devices - 1):
+    for i in range(len(max_memory) - 1):
         max_memory[i] = min(0 if low_zero and i == 0 else per_gpu, max_memory[i])
 
     if low_zero:
diff --git a/tests/test_modeling_utils.py b/tests/test_modeling_utils.py
index 1c6f6088d..02ab867d7 100644
--- a/tests/test_modeling_utils.py
+++ b/tests/test_modeling_utils.py
@@ -375,3 +375,7 @@ def test_get_balanced_memory(self):
         # Last device always get max memory to give more buffer and avoid accidental CPU offload
         max_memory = get_balanced_memory(model, max_memory={0: 300, 1: 500})
         self.assertDictEqual({0: 215, 1: 500}, max_memory)
+
+        # If we set a device to 0, it's not counted.
+        max_memory = get_balanced_memory(model, max_memory={0: 0, 1: 300, 2: 300})
+        self.assertDictEqual({0: 0, 1: 215, 2: 300}, max_memory)



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index e1282c0bb..bf17f57ce 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -777,10 +777,11 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
         if device_placement:
             model = model.to(self.device)
         if self.distributed_type == DistributedType.MULTI_GPU:
-            kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
-            model = torch.nn.parallel.DistributedDataParallel(
-                model, device_ids=[self.local_process_index], output_device=self.local_process_index, **kwargs
-            )
+            if any(p.requires_grad for p in model.parameters()):
+                kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
+                model = torch.nn.parallel.DistributedDataParallel(
+                    model, device_ids=[self.local_process_index], output_device=self.local_process_index, **kwargs
+                )
         elif self.distributed_type == DistributedType.FSDP:
             from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index e40797b39..e1282c0bb 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -1194,6 +1194,9 @@ def clip_grad_norm_(self, parameters, max_norm, norm_type=2):
         """
         Should be used in place of `torch.nn.utils.clip_grad_norm_`.
 
+        Returns:
+            `torch.Tensor`: Total norm of the parameter gradients (viewed as a single vector).
+
         Example:
 
         ```python
@@ -1217,13 +1220,13 @@ def clip_grad_norm_(self, parameters, max_norm, norm_type=2):
             parameters = [p for p in parameters]
             for model in self._models:
                 if parameters == [p for p in model.parameters()]:
-                    model.clip_grad_norm_(max_norm, norm_type)
-                    return
+                    return model.clip_grad_norm_(max_norm, norm_type)
         elif self.distributed_type == DistributedType.DEEPSPEED:
             # `accelerator.backward(loss)` is doing that automatically. Therefore, it's implementation is not needed
-            return
+            # We cannot return the gradient norm because DeepSpeed does it.
+            return None
         self.unscale_gradients()
-        torch.nn.utils.clip_grad_norm_(parameters, max_norm, norm_type=norm_type)
+        return torch.nn.utils.clip_grad_norm_(parameters, max_norm, norm_type=norm_type)
 
     def clip_grad_value_(self, parameters, clip_value):
         """



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/imports.py b/src/accelerate/utils/imports.py
index 58009d03b..bdb53988a 100644
--- a/src/accelerate/utils/imports.py
+++ b/src/accelerate/utils/imports.py
@@ -18,7 +18,9 @@
 
 import torch
 
-from .versions import is_torch_version
+from packaging.version import parse
+
+from .versions import compare_versions, is_torch_version
 
 
 # The package importlib_metadata is in a different place, depending on the Python version.
@@ -88,7 +90,11 @@ def is_bf16_available(ignore_tpu=False):
 
 
 def is_megatron_lm_available():
-    return importlib.util.find_spec("megatron") is not None
+    package_exists = importlib.util.find_spec("megatron") is not None
+    if package_exists:
+        megatron_version = parse(importlib_metadata.version("megatron-lm"))
+        return compare_versions(megatron_version, ">=", "2.2.0")
+    return False
 
 
 def is_transformers_available():


Review comments: body: Redundant `else` :)
diff_hunk: @@ -88,7 +90,12 @@ def is_bf16_available(ignore_tpu=False):
 
 
 def is_megatron_lm_available():
-    return importlib.util.find_spec("megatron") is not None
+    package_exists = importlib.util.find_spec("megatron") is not None
+    if package_exists:
+        megatron_version = parse(importlib_metadata.version("megatron-lm"))
+        return compare_versions(megatron_version, ">=", "2.2.0")
+    else:
body: Good point, thanks @tjruwase :)
diff_hunk: @@ -88,7 +90,12 @@ def is_bf16_available(ignore_tpu=False):
 
 
 def is_megatron_lm_available():
-    return importlib.util.find_spec("megatron") is not None
+    package_exists = importlib.util.find_spec("megatron") is not None
+    if package_exists:
+        megatron_version = parse(importlib_metadata.version("megatron-lm"))
+        return compare_versions(megatron_version, ">=", "2.2.0")
+    else:

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thank you for the fix ✨. LGTM!
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 43faf45ca..716d5f602 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -165,6 +165,7 @@ class SageMakerConfig(BaseConfig):
     profile: Optional[str] = None
     region: str = "us-east-1"
     num_machines: int = 1
+    gpu_ids: str = "all"
     base_job_name: str = f"accelerate-sagemaker-{num_machines}"
     pytorch_version: str = SAGEMAKER_PYTORCH_VERSION
     transformers_version: str = SAGEMAKER_TRANSFORMERS_VERSION



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/index.mdx b/docs/source/index.mdx
index 1664fd7a2..a87bd6e32 100644
--- a/docs/source/index.mdx
+++ b/docs/source/index.mdx
@@ -51,19 +51,19 @@ accelerate launch {my_script.py}
 
 <div class="mt-10">
   <div class="w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-2 md:gap-y-4 md:gap-x-5">
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/basic_tutorials/overview"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./basic_tutorials/overview"
       ><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Tutorials</div>
       <p class="text-gray-700">Learn the basics and become familiar with using 🤗 Accelerate. Start here if you are using 🤗 Accelerate for the first time!</p>
     </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/usage_guides/gradient_accumulation"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./usage_guides/gradient_accumulation"
       ><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">How-to guides</div>
       <p class="text-gray-700">Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 Accelerate to solve real-world problems.</p>
     </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/concept_guides/gradient_synchronization"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./concept_guides/gradient_synchronization"
       ><div class="w-full text-center bg-gradient-to-br from-pink-400 to-pink-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Conceptual guides</div>
       <p class="text-gray-700">High-level explanations for building a better understanding of important topics such as avoiding subtle nuances and pitfalls in distributed training and DeepSpeed.</p>
    </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/package_reference/accelerator"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./package_reference/accelerator"
       ><div class="w-full text-center bg-gradient-to-br from-purple-400 to-purple-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Reference</div>
       <p class="text-gray-700">Technical descriptions of how 🤗 Accelerate classes and methods work.</p>
     </a>



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index 0b57ae6d8..08ee3a65a 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -258,7 +258,7 @@ def get_max_layer_size(
     modules_to_treat = modules.copy()
     while len(modules_to_treat) > 0:
         module_name, module = modules_to_treat.pop(0)
-        modules_children = list(module.named_children())
+        modules_children = list(module.named_children()) if isinstance(module, torch.nn.Module) else []
         if len(modules_children) == 0 or module.__class__.__name__ in no_split_module_classes:
             # No splitting this one so we compare to the max_size
             size = module_sizes[module_name]


Review comments: body: ```suggestion
        modules_children = list(module.named_children()) if isinstance(module, torch.nn.Module) else []
```
diff_hunk: @@ -258,7 +258,7 @@ def get_max_layer_size(
     modules_to_treat = modules.copy()
     while len(modules_to_treat) > 0:
         module_name, module = modules_to_treat.pop(0)
-        modules_children = list(module.named_children())
+        modules_children = list(module.named_children()) if not isinstance(module, torch.nn.Parameter) else []

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: > What's a reproducer? I'm not sure how you get to a parameter here since we only apply the `children` method.

Haha that was fast. Here a repro:

```py
from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker

StableDiffusionSafetyChecker._no_split_modules = ["CLIPEncoderLayer"]
pipe = StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker', device_map='auto')
```

(sorry, it's 1GB of download)
body: Which branch should I check? On main I just get an error telling me device_map auto is not supported yet
body: Ah sorry just updated it above, can you try:

```python
from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker

StableDiffusionSafetyChecker._no_split_modules = ["CLIPEncoderLayer"]
pipe = StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker', device_map='auto')
```
body: Awesome! Can I merge after one :heavy_check_mark: here or wait for another reviewer? 
body: Nope, you can go ahead and merge!
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 01c419f4f..e86a0abff 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -35,6 +35,7 @@ def get_cluster_input():
 
     machine_rank = 0
     num_machines = 1
+    num_processes = 1
     gpu_ids = None
     main_process_ip = None
     main_process_port = None
@@ -294,12 +295,6 @@ def get_cluster_input():
             default=1,
             error_message="Please enter an integer.",
         )
-
-    if distributed_type in [DistributedType.MULTI_GPU, DistributedType.NO] and not use_cpu:
-        gpu_ids = _ask_field(
-            "What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:",
-            default="all",
-        )
     elif distributed_type in [DistributedType.FSDP, DistributedType.DEEPSPEED]:
         num_processes = _ask_field(
             "How many GPU(s) should be used for distributed training? [1]:",
@@ -310,6 +305,12 @@ def get_cluster_input():
     else:
         num_processes = 1
 
+    if distributed_type in [DistributedType.MULTI_GPU, DistributedType.NO] and not use_cpu:
+        gpu_ids = _ask_field(
+            "What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:",
+            default="all",
+        )
+
     if distributed_type != DistributedType.TPU:
         if distributed_type == DistributedType.DEEPSPEED and use_deepspeed_config:
             mixed_precision = "no"



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5d8ff0c1b..593eec741 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,6 +6,7 @@ on:
       - "src/**"
       - "tests/**"
       - ".github/**"
+      - "examples/**"
       - "setup.py"
     types: [opened, synchronize, reopened]
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/training_zoo.mdx b/docs/source/usage_guides/training_zoo.mdx
index d0510bf93..b3fff25e3 100644
--- a/docs/source/usage_guides/training_zoo.mdx
+++ b/docs/source/usage_guides/training_zoo.mdx
@@ -100,3 +100,7 @@ These are tutorials from libraries that integrate with 🤗 Accelerate:
 ### Tez 
 
 - [Leaf disease detection with Tez and Accelerate](https://www.kaggle.com/code/abhishek/tez-faster-and-easier-training-for-leaf-detection/notebook)
+
+### trlx 
+
+- [How to implement a sentiment learning task with trlx](https://github.com/CarperAI/trlx#example-how-to-add-a-task)
\ No newline at end of file



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 69a101307..3ed39ac4b 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -1,6 +1,5 @@
 name: "\U0001F41B Bug Report"
 description: Submit a bug report to help us improve Accelerate
-labels: [ "bug" ]
 body:
   - type: textarea
     id: system-info



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/workflows/build_and_run_tests.yml b/.github/workflows/build_and_run_tests.yml
index a5378353d..abf9f3883 100644
--- a/.github/workflows/build_and_run_tests.yml
+++ b/.github/workflows/build_and_run_tests.yml
@@ -34,8 +34,8 @@ jobs:
           done
           
   build-docker-containers:
-    needs: check-for-setup
-    if: (github.event_name == 'push') && (needs.check-for-setup.outputs.changed == '1')
+    needs: check-for-source
+    if: (github.event_name == 'push') && (needs.check-for-source.outputs.changed == '1')
     uses: ./.github/workflows/build_docker_images.yml
     secrets: inherit
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index bf57d11a4..d1905d1c0 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -844,8 +844,8 @@ def launch_command(args):
                 args.mixed_precision = defaults.mixed_precision
     else:
         if args.num_processes is None:
-            warned.append("\t`--num_processes` was set to a value of `1`")
-            args.num_processes = 1
+            args.num_processes = torch.cuda.device_count() if args.multi_gpu else 1
+            warned.append("\t`--num_processes` was set to a value of `{args.num_processes}`")
         if args.num_machines is None:
             warned.append("\t`--num_machines` was set to a value of `1`")
             args.num_machines = 1
@@ -854,14 +854,6 @@ def launch_command(args):
             args.mixed_precision = "no"
         if not hasattr(args, "use_cpu"):
             args.use_cpu = args.cpu
-    if args.multi_gpu and args.num_processes == 1:
-        args.num_processes = torch.cuda.device_count()
-        if not any("--num_processes" in warn for warn in warned):
-            warned.append(f"\t`--num_processes` was set to `{args.num_processes}`")
-        else:
-            for i, warn in enumerate(warned):
-                if "--num_processes" in warn:
-                    warned[i] = warn.replace("`1`", f"`{args.num_processes}`")
 
     if args.num_cpu_threads_per_process is None:
         local_size = get_int_from_env(


Review comments: body: ```suggestion
            args.num_processes = torch.cuda.device_count() if args.multi_gpu else 1
            warned.append("\t`--num_processes` was set to a value of `{args.num_processes}`")
```
diff_hunk: @@ -845,7 +845,7 @@ def launch_command(args):
     else:
         if args.num_processes is None:
             warned.append("\t`--num_processes` was set to a value of `1`")
body: ```suggestion
```
diff_hunk: @@ -845,7 +845,7 @@ def launch_command(args):
     else:
         if args.num_processes is None:
             warned.append("\t`--num_processes` was set to a value of `1`")
-            args.num_processes = 1
+            args.num_processes = torch.cuda.device_count() if args.multi_gpu else 1

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/workflows/build-docker-images-release.yml b/.github/workflows/build-docker-images-release.yml
new file mode 100644
index 000000000..654259f87
--- /dev/null
+++ b/.github/workflows/build-docker-images-release.yml
@@ -0,0 +1,64 @@
+name: Build Docker images (releases)
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+concurrency:
+  group: docker-image-builds
+  cancel-in-progress: false
+
+jobs:
+  get-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.step1.outputs.version }}
+    steps:
+      - uses: actions/checkout@v3
+      - id: step1
+        run: echo "::set-output name=version::$(python setup.py --version)"
+
+  version-cpu:
+    name: "Latest Accelerate CPU [version]"
+    runs-on: ubuntu-latest
+    needs: get-version
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and Push CPU
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/accelerate-cpu
+          push: true
+          tags: huggingface/accelerate-cpu:${{needs.get-version.outputs.version}}
+
+  version-cuda:
+    name: "Latest Accelerate GPU [version]"
+    runs-on: ubuntu-latest
+    needs: get-version
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and Push GPU
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/accelerate-gpu
+          push: true
+          tags: huggingface/accelerate-gpu:${{needs.get-version.outputs.version}}
\ No newline at end of file



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: > FFT, should we just merge these two release workflows together and check if 'dev' is in the version to automatically tag the `latest` tag? Or maybe just publish the prerelease tags? WDYT?

I guess you are talking about the existing `build_docker_images.yml` here. I don't have particular opinion about if merging these 2 workflow files or not.
body: @ydshieh by other folks, not for testing. We've had people use the latest images already, so made sense to also do releases 

Okay, will think on this and do a follow up most likely with the merge
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 1eb8ff79d..fac1b50e2 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -347,7 +347,6 @@ def __iter__(self):
         try:
             current_batch = next(dataloader_iter)
         except StopIteration:
-            self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
             yield
         while True:
             try:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index ab4b48660..2b6558944 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -619,6 +619,12 @@ def prepare(self, *args, device_placement=None):
             device_placement (`List[bool]`, *optional*):
                 Used to customize whether automatic device placement should be performed for each object passed. Needs
                 to be a list of the same length as `args`.
+
+        <Tip>
+
+          You don't need to prepare a model if you only use it for inference without any kind of mixed precision
+
+        </Tip>
         """
         if device_placement is None:
             device_placement = [None for _ in args]
@@ -699,7 +705,8 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
 
         Args:
             model (`torch.nn.Module`):
-                A PyTorch model to prepare
+                A PyTorch model to prepare. You don't need to prepare a model if it is used only for inference without
+                any kind of mixed precision
             device_placement (`bool`, *optional*):
                 Whether or not to place the model on the proper device. Will default to `self.device_placement`.
         """



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/hooks.py b/src/accelerate/hooks.py
index 5c04b8621..493444daa 100644
--- a/src/accelerate/hooks.py
+++ b/src/accelerate/hooks.py
@@ -71,7 +71,7 @@ def post_forward(self, module, output):
 
     def detach_hook(self, module):
         """
-        To be executed when the hook is deached from a module.
+        To be executed when the hook is detached from a module.
 
         Args:
             module (`torch.nn.Module`): The module detached from this hook.
@@ -182,7 +182,7 @@ class AlignDevicesHook(ModelHook):
     Args:
         execution_device (`torch.device`, *optional*):
             The device on which inputs and model weights should be placed before the forward pass.
-        offload (`bool`, *optional*, defauts to `False`):
+        offload (`bool`, *optional*, defaults to `False`):
             Whether or not the weights should be offloaded after the forward pass.
         io_same_device (`bool`, *optional*, defaults to `False`):
             Whether or not the output should be placed on the same device as the input was.
@@ -319,7 +319,7 @@ def attach_align_device_hook(
             The module where we want to attach the hooks.
         execution_device (`torch.device`, *optional*):
             The device on which inputs and model weights should be placed before the forward pass.
-        offload (`bool`, *optional*, defauts to `False`):
+        offload (`bool`, *optional*, defaults to `False`):
             Whether or not the weights should be offloaded after the forward pass.
         weights_map (`Mapping[str, torch.Tensor]`, *optional*):
             When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
@@ -402,7 +402,7 @@ def attach_align_device_hook_on_blocks(
         execution_device (`torch.device` or `Dict[str, torch.device]`, *optional*):
             The device on which inputs and model weights should be placed before the forward pass. It can be one device
             for the whole module, or a dictionary mapping module name to device.
-        offload (`bool`, *optional*, defauts to `False`):
+        offload (`bool`, *optional*, defaults to `False`):
             Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole
             module, or a dictionary mapping module name to boolean.
         weights_map (`Mapping[str, torch.Tensor]`, *optional*):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/utils/style_doc.py b/utils/style_doc.py
index 2a325805f..0422ebeb4 100644
--- a/utils/style_doc.py
+++ b/utils/style_doc.py
@@ -28,7 +28,7 @@
 # Regexes
 # Re pattern that catches list introduction (with potential indent)
 _re_list = re.compile(r"^(\s*-\s+|\s*\*\s+|\s*\d+\.\s+)")
-# Re pattern that catches code block introduction (with potentinal indent)
+# Re pattern that catches code block introduction (with potential indent)
 _re_code = re.compile(r"^(\s*)```(.*)$")
 # Re pattern that catches rst args blocks of the form `Parameters:`.
 _re_args = re.compile("^\s*(Args?|Arguments?|Params?|Parameters?):\s*$")
@@ -62,7 +62,7 @@ def parse_code_example(code_lines):
 
     Args:
         code_lines (`List[str]`): The code lines to parse.
-        max_len (`int`): The maximum lengh per line.
+        max_len (`int`): The maximum length per line.
 
     Returns:
         (List[`str`], List[`str`]): The list of code samples and the list of outputs.
@@ -109,7 +109,7 @@ def format_code_example(code: str, max_len: int, in_docstring: bool = False):
 
     Args:
         code (`str`): The code example to format.
-        max_len (`int`): The maximum lengh per line.
+        max_len (`int`): The maximum length per line.
         in_docstring (`bool`, *optional*, defaults to `False`): Whether or not the code example is inside a docstring.
 
     Returns:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/tests/test_tracking.py b/tests/test_tracking.py
index 917ee329d..5e26eb9b5 100644
--- a/tests/test_tracking.py
+++ b/tests/test_tracking.py
@@ -85,12 +85,12 @@ def setUp(self):
         self.add_mocks(mock.patch.dict(os.environ, {"WANDB_DIR": self.tmpdir}))
 
     @staticmethod
-    def get_value_from_log(key: str, log: str, key_occurance: int = 0):
+    def get_value_from_log(key: str, log: str, key_occurrence: int = 0):
         """
         Parses wandb log for `key` and returns the value.
-        If parsing through multiple calls to .log, pass in a `key_occurance`
+        If parsing through multiple calls to .log, pass in a `key_occurrence`
         """
-        res = re.findall(rf"(?<={key} )[^\s]+", log)[key_occurance]
+        res = re.findall(rf"(?<={key} )[^\s]+", log)[key_occurrence]
         if '"' in res:
             return re.findall(r'"([^"]*)"', res)[0]
         else:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index 707c93c4c..87f804ce6 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -58,7 +58,7 @@
 EVAL_BATCH_SIZE = 32
 
 # New Code #
-# We need a different `get_dataloaders` function that will build dataloaders by indexs
+# We need a different `get_dataloaders` function that will build dataloaders by index
 
 
 def get_fold_dataloaders(
@@ -71,9 +71,9 @@ def get_fold_dataloaders(
         accelerator (`Accelerator`):
             The main `Accelerator` object
         train_idxs (list of `int`):
-            The split indicies for the training dataset
+            The split indices for the training dataset
         valid_idxs (list of `int`):
-            The split indicies for the validation dataset
+            The split indices for the validation dataset
         batch_size (`int`):
             The size of the minibatch. Default is 16
     """
diff --git a/examples/by_feature/deepspeed_with_config_support.py b/examples/by_feature/deepspeed_with_config_support.py
index 89234a911..36ace84b9 100755
--- a/examples/by_feature/deepspeed_with_config_support.py
+++ b/examples/by_feature/deepspeed_with_config_support.py
@@ -525,7 +525,7 @@ def group_texts(examples):
         },
     ]
     # New Code #
-    # Creates Dummy Optimizer if `optimizer` was spcified in the config file else creates Adam Optimizer
+    # Creates Dummy Optimizer if `optimizer` was specified in the config file else creates Adam Optimizer
     optimizer_cls = (
         torch.optim.AdamW
         if accelerator.state.deepspeed_plugin is None
@@ -554,7 +554,7 @@ def group_texts(examples):
         args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
 
     # New Code #
-    # Creates Dummy Scheduler if `scheduler` was spcified in the config file else creates `args.lr_scheduler_type` Scheduler
+    # Creates Dummy Scheduler if `scheduler` was specified in the config file else creates `args.lr_scheduler_type` Scheduler
     if (
         accelerator.state.deepspeed_plugin is None
         or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
diff --git a/examples/by_feature/memory.py b/examples/by_feature/memory.py
index 9f0d55c69..82dc34b8a 100644
--- a/examples/by_feature/memory.py
+++ b/examples/by_feature/memory.py
@@ -29,7 +29,7 @@
 ########################################################################
 # This is a fully working simple example to use Accelerate,
 # specifically showcasing how to ensure out-of-memory errors never
-# iterrupt training, and builds off the `nlp_example.py` script.
+# interrupt training, and builds off the `nlp_example.py` script.
 #
 # This example trains a Bert base model on GLUE MRPC
 # in any of the following settings (with the same script):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/sagemaker.mdx b/docs/source/usage_guides/sagemaker.mdx
index 6195783e9..0afe52e29 100644
--- a/docs/source/usage_guides/sagemaker.mdx
+++ b/docs/source/usage_guides/sagemaker.mdx
@@ -129,7 +129,26 @@ You can find your model data at: s3://your-bucket/accelerate-sagemaker-1-2021-04
 
 ### Distributed Training: Data Parallelism
 
-*currently in development, will be supported soon.*
+Set up the accelerate config by running `accelerate config` and answer the SageMaker questions and set it up.
+To use SageMaker DDP, select it when asked 
+`What is the distributed mode? ([0] No distributed training, [1] data parallelism):`.
+Example config below:
+```yaml
+base_job_name: accelerate-sagemaker-1
+compute_environment: AMAZON_SAGEMAKER
+distributed_type: DATA_PARALLEL
+ec2_instance_type: ml.p3.16xlarge
+iam_role_name: xxxxx
+image_uri: null
+mixed_precision: fp16
+num_machines: 1
+profile: xxxxx
+py_version: py38
+pytorch_version: 1.10.2
+region: us-east-1
+transformers_version: 4.17.0
+use_cpu: false
+```
 
 ### Distributed Training: Model Parallelism
 
diff --git a/examples/README.md b/examples/README.md
index c4f050b3f..6a3c0a11a 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -187,6 +187,9 @@ To run it in each of these various modes, use the following commands:
 ### Simple vision example (GANs)
 
 - [huggan project](https://github.com/huggingface/community-events/tree/main/huggan)
+
+### Using AWS SageMaker integration
+- [Examples showcasing AWS SageMaker integration of 🤗 Accelerate.](https://github.com/pacman100/accelerate-aws-sagemaker)
     
 ## Finer Examples
 
diff --git a/src/accelerate/commands/config/sagemaker.py b/src/accelerate/commands/config/sagemaker.py
index 786881edf..b3a45c9e4 100644
--- a/src/accelerate/commands/config/sagemaker.py
+++ b/src/accelerate/commands/config/sagemaker.py
@@ -157,7 +157,7 @@ def get_sagemaker_input():
         )
 
     distributed_type = _ask_field(
-        "Which type of machine are you using? ([0] No distributed training, [1] data parallelism): ",
+        "What is the distributed mode? ([0] No distributed training, [1] data parallelism): ",
         _convert_sagemaker_distributed_mode,
         error_message="Please enter 0 or 1",
     )



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/memory.mdx b/docs/source/usage_guides/memory.mdx
index 9c5674977..8841cd584 100644
--- a/docs/source/usage_guides/memory.mdx
+++ b/docs/source/usage_guides/memory.mdx
@@ -48,4 +48,4 @@ def training_function(args):
 +   inner_training_loop()
 ```
 
-To find out more, check the documentation [here](package_reference/utilities#accelerate.find_executable_batch_size)
\ No newline at end of file
+To find out more, check the documentation [here](../package_reference/utilities#accelerate.find_executable_batch_size).



Discussion: body: Hello @tomaarsen, looks good, thanks for your contribution 😄
body: _The documentation is not available anymore as the PR was closed or merged._
body: Thanks a lot!