muellerzr/github-pr-history huggingface dataset:
diff --git a/src/accelerate/big_modeling.py b/src/accelerate/big_modeling.py
index 311111ad4..0750248de 100644
--- a/src/accelerate/big_modeling.py
+++ b/src/accelerate/big_modeling.py
@@ -75,15 +75,35 @@ def register_empty_buffer(module, name, buffer):
         if buffer is not None:
             module._buffers[name] = module._buffers[name].to(torch.device("meta"))
 
+    # Patch tensor creation
+    if include_buffers:
+        tensor_constructors_to_patch = {
+            torch_function_name: getattr(torch, torch_function_name)
+            for torch_function_name in ["empty", "zeros", "ones", "full"]
+        }
+    else:
+        tensor_constructors_to_patch = {}
+
+    def patch_tensor_constructor(fn):
+        def wrapper(*args, **kwargs):
+            kwargs["device"] = torch.device("meta")
+            return fn(*args, **kwargs)
+
+        return wrapper
+
     try:
         nn.Module.register_parameter = register_empty_parameter
         if include_buffers:
             nn.Module.register_buffer = register_empty_buffer
+        for torch_function_name in tensor_constructors_to_patch.keys():
+            setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name)))
         yield
     finally:
         nn.Module.register_parameter = old_register_parameter
         if include_buffers:
             nn.Module.register_buffer = old_register_buffer
+        for torch_function_name, old_torch_function in tensor_constructors_to_patch.items():
+            setattr(torch, torch_function_name, old_torch_function)
 
 
 def cpu_offload(



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: I don't really understand why this is needed: while load a pretrained model inside the context manager and complain it takes time?
```py
from accelerate import init_empty_weights
from transformers import AutoConfig, AutoModel

config = AutoConfig.from_pretrained("gpt2")

with init_empty_weights():
    model = AutoModel.from_config(config)
```
is way faster than 6s
body: Hum it's doing it 10 times, so 0.6 sec per load. Benchmarking your solution displays the same order of magnitude: `14.567796499999996` (I think the difference is that the use of config doesn't require to read from disk the checkpoint anymore)

Though your workaround removed the need to override `torch.load` and especially the hack that I introduced.
```
Using config + this PR: 4.1660165419999995
```
body: Also I'm not sure why this wasn't detected, but the test `tests/test_big_modeling.py::BigModelingTester::test_init_empty_weights` pass on my MAC contrary to CI ... Moving back to draft as I need to figure this one out.
body: Actually if we activate this feature only when `include_buffers=True` (I'm guessing the assumption is that all pytorch tensors are expected to be `meta`) then that should be fine.
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index b17ea387d..552c8392c 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -747,6 +747,12 @@ def _prepare_deepspeed(self, *args):
             batch_size_per_device = deepspeed_plugin.deepspeed_config["train_micro_batch_size_per_gpu"]
             result = [obj for obj in args]
 
+        if self.gradient_accumulation_steps != deepspeed_plugin.deepspeed_config["gradient_accumulation_steps"]:
+            logger.info(
+                f"Updating DeepSpeed's gradient accumulation steps to {self.gradient_accumulation_steps} from "
+                f"{deepspeed_plugin.deepspeed_config['gradient_accumulation_steps']}."
+            )
+            deepspeed_plugin.deepspeed_config["gradient_accumulation_steps"] = self.gradient_accumulation_steps
         config_kwargs = {
             "train_micro_batch_size_per_gpu": batch_size_per_device,
             "train_batch_size": batch_size_per_device


Review comments: body: No need for a warning here, I think a logging info is sufficient (the fact that args to the `Accelerator` supercede the config is a common behavior, you can do it to force CPU or a given mixed precision already).
diff_hunk: @@ -747,6 +747,12 @@ def _prepare_deepspeed(self, *args):
             batch_size_per_device = deepspeed_plugin.deepspeed_config["train_micro_batch_size_per_gpu"]
             result = [obj for obj in args]
 
+        if self.gradient_accumulation_steps != deepspeed_plugin.deepspeed_config["gradient_accumulation_steps"]:
+            warnings.warn(
body: Done.
diff_hunk: @@ -747,6 +747,12 @@ def _prepare_deepspeed(self, *args):
             batch_size_per_device = deepspeed_plugin.deepspeed_config["train_micro_batch_size_per_gpu"]
             result = [obj for obj in args]
 
+        if self.gradient_accumulation_steps != deepspeed_plugin.deepspeed_config["gradient_accumulation_steps"]:
+            warnings.warn(

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/mps.mdx b/docs/source/usage_guides/mps.mdx
index 3648bdce8..7a7f8a628 100644
--- a/docs/source/usage_guides/mps.mdx
+++ b/docs/source/usage_guides/mps.mdx
@@ -19,7 +19,7 @@ This will map computational graphs and primitives on the MPS Graph framework and
 For more information please refer official documents [Introducing Accelerated PyTorch Training on Mac](https://pytorch.org/blog/introducing-accelerated-pytorch-training-on-mac/)
 and [MPS BACKEND](https://pytorch.org/docs/stable/notes/mps.html).
 
-### Benefits of Training and Inference using Apple M1 Chips
+### Benefits of Training and Inference using Apple Silicon Chips
 
 1. Enables users to train larger networks or batch sizes locally
 2. Reduces data retrieval latency and provides the GPU with direct access to the full memory store due to unified memory architecture. 
diff --git a/src/accelerate/launchers.py b/src/accelerate/launchers.py
index dcdead956..0387109b6 100644
--- a/src/accelerate/launchers.py
+++ b/src/accelerate/launchers.py
@@ -121,12 +121,17 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mix
                 start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
 
         else:
-            # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            if torch.cuda.is_available():
+            # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
+            use_mps_device = "false"
+            if torch.backends.mps.is_available():
+                print("Launching training on MPS.")
+                use_mps_device = "true"
+            elif torch.cuda.is_available():
                 print("Launching training on one GPU.")
             else:
                 print("Launching training on CPU.")
-            function(*args)
+            with patch_environment(use_mps_device=use_mps_device):
+                function(*args)
 
 
 def debug_launcher(function, args=(), num_processes=2):



Discussion: body: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/accelerate/pr_695). All of your documentation changes will be reflected on that endpoint.
diff --git a/src/accelerate/utils/random.py b/src/accelerate/utils/random.py
index 51b7c3719..e95ed03bf 100644
--- a/src/accelerate/utils/random.py
+++ b/src/accelerate/utils/random.py
@@ -64,7 +64,7 @@ def synchronize_rng_state(rng_type: Optional[RNGType] = None, generator: Optiona
     state = AcceleratorState()
     if state.distributed_type == DistributedType.TPU:
         rng_state = xm.mesh_reduce("random_seed", rng_state, lambda x: x[0])
-    elif state.distributed_type in [DistributedType.DEEPSPEED, DistributedType.MULTI_GPU]:
+    elif state.distributed_type in [DistributedType.DEEPSPEED, DistributedType.MULTI_GPU, DistributedType.FSDP]:
         rng_state = rng_state.to(state.device)
         torch.distributed.broadcast(rng_state, 0)
         rng_state = rng_state.cpu()



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index fb5393d34..84c2ebd67 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -622,6 +622,7 @@ def prepare_data_loader(
     new_dataset = dataloader.dataset
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
+    sampler_is_batch_sampler = False
     generator = getattr(dataloader, "generator", None)
     # No change if no multiprocess
     if num_processes != 1 and not dispatch_batches:
@@ -645,8 +646,10 @@ def prepare_data_loader(
                     generator.manual_seed(int(torch.empty((), dtype=torch.int64).random_().item()))
             elif getattr(dataloader.batch_sampler, "generator", None) is not None:
                 generator = dataloader.batch_sampler.generator
+            sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler)
+            batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler
             new_batch_sampler = BatchSamplerShard(
-                dataloader.batch_sampler,
+                batch_sampler,
                 num_processes=num_processes,
                 process_index=process_index,
                 split_batches=split_batches,
@@ -684,6 +687,15 @@ def prepare_data_loader(
             _drop_last=dataloader.drop_last,
             **kwargs,
         )
+    elif sampler_is_batch_sampler:
+        dataloader = DataLoaderShard(
+            new_dataset,
+            device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
+            sampler=new_batch_sampler,
+            rng_types=rng_types,
+            generator=generator,
+            **kwargs,
+        )
     else:
         dataloader = DataLoaderShard(
             new_dataset,



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index cbe39efa3..fb5393d34 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -462,10 +462,6 @@ def _fetch_batches(self, iterator):
                 else:
                     batch_info = [None, True]
                 broadcast_object_list(batch_info)
-                if batch_info[1]:
-                    return batch, batch_info
-            else:
-                return batch, batch_info
         return batch, batch_info
 
     def __iter__(self):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 989774148..b17ea387d 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -745,6 +745,7 @@ def _prepare_deepspeed(self, *args):
                 )
         else:
             batch_size_per_device = deepspeed_plugin.deepspeed_config["train_micro_batch_size_per_gpu"]
+            result = [obj for obj in args]
 
         config_kwargs = {
             "train_micro_batch_size_per_gpu": batch_size_per_device,



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index c7f235843..fad42792d 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -270,6 +270,25 @@ def launch_command_parser(subparsers=None):
         default=None,
         help="The port to use to communicate with the machine of rank 0.",
     )
+    # Rendezvous related arguments
+    parser.add_argument(
+        "--rdzv_conf",
+        type=str,
+        default="",
+        help="Additional rendezvous configuration (<key1>=<value1>,<key2>=<value2>,...).",
+    )
+    parser.add_argument(
+        "--max_restarts",
+        type=int,
+        default=0,
+        help="Maximum number of worker group restarts before failing.",
+    )
+    parser.add_argument(
+        "--monitor_interval",
+        type=float,
+        default=5,
+        help="Interval, in seconds, to monitor the state of workers.",
+    )
     parser.add_argument(
         "--main_training_function",
         type=str,



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: > It looks like those new fields are not used anywhere afterwards?

Hello, it is being used via below lines:

https://github.com/huggingface/accelerate/blob/4400eb90b29638deec117f95fecdcb79bd7684cb/src/accelerate/commands/launch.py#L465

https://github.com/huggingface/accelerate/blob/4400eb90b29638deec117f95fecdcb79bd7684cb/src/accelerate/utils/launch.py#L51
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 9b14aeaed..a174115b1 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -59,7 +59,7 @@ def get_cluster_input():
                 lambda x: int(x),
             )
             same_network = _ask_field(
-                "Are all the machines on the same network? [YES/no]: ",
+                "Are all the machines on the same local network? Answer `no` if nodes are on the cloud and/or on different network hosts [YES/no]: ",
                 _convert_yes_no_to_bool,
                 default=True,
                 error_message="Please enter yes or no.",


Review comments: body: ```suggestion
                "Are all the machines on the same local network? Answer `no` if nodes are on the cloud and/or on the different network hosts [YES/no]: ",
```

diff_hunk: @@ -59,7 +59,7 @@ def get_cluster_input():
                 lambda x: int(x),
             )
             same_network = _ask_field(
-                "Are all the machines on the same network? [YES/no]: ",
+                "Are all the machines on the same local network? [YES/no]: ",

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/imports.py b/src/accelerate/utils/imports.py
index 074d02e4a..3efb61034 100644
--- a/src/accelerate/utils/imports.py
+++ b/src/accelerate/utils/imports.py
@@ -14,6 +14,7 @@
 
 import importlib
 import sys
+from functools import lru_cache
 
 import torch
 
@@ -50,6 +51,7 @@ def is_apex_available():
     return importlib.util.find_spec("apex") is not None
 
 
+@lru_cache()
 def is_tpu_available(check_device=True):
     "Checks if `torch_xla` is installed and potentially if a TPU is in the environment"
     if _tpu_available and check_device:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/README.md b/README.md
index f15f442a7..deddf2986 100644
--- a/README.md
+++ b/README.md
@@ -243,3 +243,16 @@ pip install accelerate
 - FP16 with native AMP (apex on the roadmap)
 - DeepSpeed support (Experimental)
 - PyTorch Fully Sharded Data Parallel (FSDP) support (Experimental)
+
+## Citing 🤗 Accelerate
+
+If you use 🤗 Accelerate in your publication, please cite it by using the following BibTeX entry.
+
+```bibtex
+@Misc{accelerate,
+  title =        {Accelerate: Training and inference at scale made simple, efficient and adaptable.},
+  author =       {Sylvain Gugger, Lysandre Debut, Thomas Wolf, Philipp Schmid, Zachary Mueller, Sourab Mangrulkar},
+  howpublished = {\url{https://github.com/huggingface/accelerate}},
+  year =         {2022}
+}
+```



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/concept_guides/gradient_synchronization.mdx b/docs/source/concept_guides/gradient_synchronization.mdx
index 02b5adf0d..ea4de3d72 100644
--- a/docs/source/concept_guides/gradient_synchronization.mdx
+++ b/docs/source/concept_guides/gradient_synchronization.mdx
@@ -114,4 +114,6 @@ for batch in dataloader:
         outputs = model(inputs)
         loss = loss_function(outputs, targets)
         accelerator.backward(loss)
-```
\ No newline at end of file
+```
+
+As a result, you should either use *`accelerator.accumulate` or `accelerator.no_sync`* when it comes to API choice. 
\ No newline at end of file
diff --git a/docs/source/usage_guides/gradient_accumulation.mdx b/docs/source/usage_guides/gradient_accumulation.mdx
index bd06efed9..798e15847 100644
--- a/docs/source/usage_guides/gradient_accumulation.mdx
+++ b/docs/source/usage_guides/gradient_accumulation.mdx
@@ -126,3 +126,5 @@ for batch in training_dataloader:
         scheduler.step()
         optimizer.zero_grad()
 ```
+
+To learn more about what magic this wraps around, read the [Gradient Synchronization concept guide](/concept_guides/gradient_synchronization)
\ No newline at end of file


Review comments: body: ```suggestion
As a result, you should either use *`accelerator.accumulate` or `accelerator.no_sync`* when it comes to API choice. 
```
diff_hunk: @@ -114,4 +114,6 @@ for batch in dataloader:
         outputs = model(inputs)
         loss = loss_function(outputs, targets)
         accelerator.backward(loss)
-```
\ No newline at end of file
+```
+
+As a result, you should either use *one or the other* when it comes to API choice. 

Discussion: body: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/accelerate/pr_656). All of your documentation changes will be reflected on that endpoint.
diff --git a/README.md b/README.md
index c67d5dd50..f15f442a7 100644
--- a/README.md
+++ b/README.md
@@ -196,7 +196,7 @@ from accelerate import notebook_launcher
 notebook_launcher(training_function)
 ```
 
-An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb). [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb)
+An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb). [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb)
 
 ## Why should I use 🤗 Accelerate?
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 8261524fd..4991ddbe4 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -1032,6 +1032,7 @@ def wait_for_everyone(self):
         """
         wait_for_everyone()
 
+    @on_main_process
     def init_trackers(self, project_name: str, config: Optional[dict] = None, init_kwargs: Optional[dict] = {}):
         """
         Initializes a run for all trackers stored in `self.log_with`, potentially with starting configurations



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @Gladiator07 could you adjust the following code in the examples as well? Since this removes the need for the check:

https://github.com/huggingface/accelerate/blob/main/examples/by_feature/tracking.py#L168-L172
body: @muellerzr adjust as in? Should I remove the `accelerator.is_main_process` check?
body: @Gladiator07 correct. We're putting this in place so that is no longer needed, so the examples should show this :)
body: Cool, doing this in a separate PR.
diff --git a/src/accelerate/utils/dataclasses.py b/src/accelerate/utils/dataclasses.py
index cbbaeae16..5f425f200 100644
--- a/src/accelerate/utils/dataclasses.py
+++ b/src/accelerate/utils/dataclasses.py
@@ -60,6 +60,8 @@ class DistributedDataParallelKwargs(KwargsHandler):
 
     `gradient_as_bucket_view` is only available in PyTorch 1.7.0 and later versions.
 
+    `static_graph` is only available in PyTorch 1.11.0 and later versions.
+
     </Tip>"""
 
     dim: int = 0
@@ -68,6 +70,7 @@ class DistributedDataParallelKwargs(KwargsHandler):
     find_unused_parameters: bool = False
     check_reduction: bool = False
     gradient_as_bucket_view: bool = False
+    static_graph: bool = False
 
 
 @dataclass



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thanks! Could we add a warning to this please similar to the other one that exists for these kwargs in the docstring mentioning its only available on pytorch>=1.11?
body: Sure, done
body: @rom1504 looks like the last thing we need is to run `make style; make quality` :) 
body: @muellerzr fixed
body: Great! Thanks!!
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 11aadc8e2..fcc80bb7f 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -26,7 +26,6 @@
 from typing import Dict, List
 
 import torch
-import torch.distributed.run as distrib_run
 
 import psutil
 from accelerate.commands.config import default_config_file, load_config_from_file
@@ -50,6 +49,10 @@
 from rich.logging import RichHandler
 
 
+if is_torch_version(">=", "1.9.0"):
+    import torch.distributed.run as distrib_run
+
+
 FORMAT = "%(message)s"
 logging.basicConfig(format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thanks for fixing!
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index 99d109bd1..1b86e66ee 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -105,8 +105,6 @@ def training_function(config, args):
     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking and accelerator.is_main_process:
         run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
         accelerator.init_trackers(run, config)
 
     # Grab all the image filenames
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index 094f6cfb5..93e6764f6 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -77,8 +77,6 @@ def training_function(config, args):
     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking and accelerator.is_main_process:
         run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
         accelerator.init_trackers(run, config)
 
     tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 4e0900eb4..051cf8bc6 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -251,7 +251,7 @@ def __repr__(self):
         if self.distributed_type == DistributedType.DEEPSPEED:
             repr += f"ds_config: {self.deepspeed_plugin.deepspeed_config}\n"
         else:
-            f"Mixed precision type: {mixed_precision}\n"
+            repr += f"Mixed precision type: {mixed_precision}\n"
         return repr
 
     # For backward compatibility



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/mps.mdx b/docs/source/usage_guides/mps.mdx
index 43a00f104..3648bdce8 100644
--- a/docs/source/usage_guides/mps.mdx
+++ b/docs/source/usage_guides/mps.mdx
@@ -72,8 +72,9 @@ accelerate launch /examples/cv_example.py --data_dir images
 
 ## A few caveats to be aware of
 
-1. For `nlp_example.py` the metrics are too bad when compared to CPU-only training. 
-This means certain operations in BERT model are going wrong using mps device and this needs to be fixed by PyTorch.
+1. We strongly recommend to install PyTorch >= 1.13 (nightly version at the time of writing) on your MacOS machine. 
+It has major fixes related to model correctness and performance improvements for transformer based models.
+Please refer to https://github.com/pytorch/pytorch/issues/82707 for more details.
 2. Distributed setups `gloo` and `nccl` are not working with `mps` device. 
 This means that currently only single GPU of `mps` device type can be used.
 



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 7cb4529d9..541bda48e 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -367,7 +367,7 @@ def multi_gpu_launcher(args):
     if num_machines > 1:
         setattr(args, "nproc_per_node", str(num_processes // num_machines))
         setattr(args, "nnodes", str(num_machines))
-        setattr(args, "machine_rank", str(args.machine_rank))
+        setattr(args, "node_rank", str(args.machine_rank))
         setattr(args, "master_addr", str(args.main_process_ip))
         setattr(args, "master_port", str(args.main_process_port))
     else:
@@ -441,16 +441,16 @@ def multi_gpu_launcher(args):
             current_env["FSDP_STATE_DICT_TYPE"] = str(args.fsdp_state_dict_type)
     current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
     if is_torch_version(">=", "1.9.0"):
-        distrib_args = _filter_args(args)
+        args = _filter_args(args)
         with patch_environment(**current_env):
-            distrib_run.run(distrib_args)
+            distrib_run.run(args)
     else:
         # We still have to use subprocess, the user won't get a clean traceback as a result
         cmd = get_launch_prefix()
         for k, v in vars(args).items():
             if k in TORCH_LAUNCH_PARAMS and v:
                 param = [f"--{k}"]
-                if not v:
+                if type(v) != bool:
                     param.append(v)
                 cmd.extend(param)
         cmd.append(args.training_script)
diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py
index c6605ee26..1a452ca63 100644
--- a/src/accelerate/utils/launch.py
+++ b/src/accelerate/utils/launch.py
@@ -46,16 +46,12 @@ def _filter_args(args):
     Filters out all `accelerate` specific args
     """
     distrib_args = distrib_run.get_args_parser()
-    known_args, _ = distrib_args.parse_known_args()
-    for arg in list(vars(args).keys()):
-        if arg not in vars(known_args).keys():
-            delattr(args, arg)
-    distrib_args = distrib_run.parse_args(vars(args))
+    new_args, _ = distrib_args.parse_known_args()
+
     for key, value in vars(args).items():
-        setattr(distrib_args, key, value)
-    if is_torch_version("<", "1.9.0"):
-        setattr(distrib_args, "use_env", True)
-    return distrib_args
+        if key in vars(new_args).keys():
+            setattr(new_args, key, value)
+    return new_args
 
 
 class PrepareForLaunch:



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 7cb4529d9..117932f2b 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -334,6 +334,8 @@ def simple_launcher(args):
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu or args.use_cpu)
     current_env["USE_MPS_DEVICE"] = str(args.use_mps_device)
+    if args.use_mps_device:
+        current_env["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
     if args.num_machines > 1:
         current_env["MASTER_ADDR"] = args.main_process_ip
         current_env["MASTER_PORT"] = str(args.main_process_port)
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index bd339048f..4e0900eb4 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import warnings
 from distutils.util import strtobool
 
 import torch
@@ -221,6 +222,14 @@ def __init__(
                                 "and/or you do not have an MPS-enabled device on this machine."
                             )
                     else:
+                        from .utils import is_torch_version
+
+                        if not is_torch_version(">", "1.12.0"):
+                            warnings.warn(
+                                "We strongly recommend to install PyTorch >= 1.13 (nightly version at the time of writing) on your MacOS machine. "
+                                "It has major fixes related to model correctness and performance improvements for transformer based models. "
+                                "Please refer to https://github.com/pytorch/pytorch/issues/82707 for more details."
+                            )
                         self.device = torch.device("mps")
                 elif cpu or not torch.cuda.is_available():
                     self.device = torch.device("cpu")


Review comments: body: That warning may not age well once PyTorch 1.13 is released. Maybe:
```suggestion
                                "We strongly recommend to install PyTorch >= 1.13 (nightly version at the time of writing) on your MacOS machine. "
```
diff_hunk: @@ -221,6 +222,14 @@ def __init__(
                                 "and/or you do not have an MPS-enabled device on this machine."
                             )
                     else:
+                        from .utils import is_torch_version
+
+                        if not is_torch_version(">", "1.12.0"):
+                            warnings.warn(
+                                "We strongly recommend to install the latest PyTorch nightly version (1.13.0.dev...) on your MacOS machine. "

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/examples/README.md b/examples/README.md
index 150afb75c..c4f050b3f 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -136,7 +136,7 @@ To run it in each of these various modes, use the following commands:
         ```
 - single GPU:
     ```bash
-    python ./nlp_example.py  # from a server with a GPU
+    python ./cv_example.py  # from a server with a GPU
     ```
 - with fp16 (mixed-precision)
     * from any server by passing `fp16=True` to the `Accelerator`.



Discussion: body: Thanks for the fix!
body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/docs/source/usage_guides/gradient_accumulation.mdx b/docs/source/usage_guides/gradient_accumulation.mdx
index f9f5dc62a..bd06efed9 100644
--- a/docs/source/usage_guides/gradient_accumulation.mdx
+++ b/docs/source/usage_guides/gradient_accumulation.mdx
@@ -31,7 +31,6 @@ model.to(device)
 gradient_accumulation_steps = 2
 
 for index, batch in enumerate(training_dataloader):
-    optimizer.zero_grad()
     inputs, targets = batch
     inputs = inputs.to(device)
     targets = targets.to(device)
@@ -42,6 +41,7 @@ for index, batch in enumerate(training_dataloader):
     if (index + 1) % gradient_accumulation_steps == 0:
         optimizer.step()
         scheduler.step()
+        optimizer.zero_grad()
 ```
 
 ## Converting it to 🤗 Accelerate
@@ -57,7 +57,6 @@ First the code shown earlier will be converted to utilize 🤗 Accelerate withou
 + )
 
   for index, batch in enumerate(training_dataloader):
-      optimizer.zero_grad()
       inputs, targets = batch
 -     inputs = inputs.to(device)
 -     targets = targets.to(device)
@@ -68,6 +67,7 @@ First the code shown earlier will be converted to utilize 🤗 Accelerate withou
       if (index+1) % gradient_accumulation_steps == 0:
           optimizer.step()
           scheduler.step()
+          optimizer.zero_grad()
 ```
 
 <Tip warning={true}>
@@ -94,7 +94,6 @@ You just wrap it around the entire training part of our code:
 - for index, batch in enumerate(training_dataloader):
 + for batch in training_dataloader:
 +     with accelerator.accumulate(model):
-          optimizer.zero_grad()
           inputs, targets = batch
           outputs = model(inputs)
 ```
@@ -107,6 +106,7 @@ You can remove all the special checks for the step number and the loss adjustmen
 - if (index+1) % gradient_accumulation_steps == 0:
   optimizer.step()
   scheduler.step()
+  optimizer.zero_grad()
 ```
 
 As you can see the [`Accelerator`] is able to keep track of the batch number you are on and it will automatically know whether to step through the prepared optimizer and how to adjust the loss. 
@@ -118,11 +118,11 @@ Below is the finished implementation for performing gradient accumulation with 
 ```python
 for batch in training_dataloader:
     with accelerator.accumulate(model):
-        optimizer.zero_grad()
         inputs, targets = batch
         outputs = model(inputs)
         loss = loss_function(outputs, targets)
         accelerator.backward(loss)
         optimizer.step()
         scheduler.step()
+        optimizer.zero_grad()
 ```



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/examples/README.md b/examples/README.md
index 271ffd3c5..150afb75c 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -184,6 +184,10 @@ To run it in each of these various modes, use the following commands:
     * In PyTorch:
         Add an `xmp.spawn` line in your script as you usually do.
 
+### Simple vision example (GANs)
+
+- [huggan project](https://github.com/huggingface/community-events/tree/main/huggan)
+    
 ## Finer Examples
 
 While the first two scripts are extremely barebones when it comes to what you can do with accelerate, more advanced features are documented in two other locations.


Review comments: body: ```suggestion
- [huggan project](https://github.com/huggingface/community-events/tree/main/huggan)
```
diff_hunk: @@ -184,6 +184,10 @@ To run it in each of these various modes, use the following commands:
     * In PyTorch:
         Add an `xmp.spawn` line in your script as you usually do.
 
+### Simple vision example (GANs)
+
+- Please see: https://github.com/huggingface/community-events/tree/main/huggan

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index c0c9c62f2..cb0a7ee42 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -57,10 +57,16 @@ jobs:
           git fetch && git checkout ${{ github.sha }}
           pip install -e . --no-deps
 
-      - name: Run test on GPUs
+      - name: Run core and big modeling tests on GPUs
         run: |
           source activate accelerate
-          make test
+          make test_big_modeling
+          make test_core
+
+      - name: Run Integration tests on GPUs
+        run: |
+          source activate accelerate
+          make test_integrations
 
       - name: Run examples on GPUs
         run: |
diff --git a/Makefile b/Makefile
index 100a1484a..c1c79d6e8 100644
--- a/Makefile
+++ b/Makefile
@@ -44,6 +44,8 @@ test_examples:
 	python -m pytest -s -v ./tests/test_examples.py
 
 # Broken down example tests for the CI runners
+test_integrations:
+	python -m pytest -s -v ./tests/deepspeed ./tests/fsdp
 test_example_differences:
 	python -m pytest -s -v ./tests/test_examples.py::ExampleDifferenceTests
 
diff --git a/tests/deepspeed/test_deepspeed.py b/tests/deepspeed/test_deepspeed.py
index 30eb2a4d8..1be794421 100644
--- a/tests/deepspeed/test_deepspeed.py
+++ b/tests/deepspeed/test_deepspeed.py
@@ -35,7 +35,6 @@
     require_cuda,
     require_deepspeed,
     require_multi_gpu,
-    skip,
     slow,
 )
 from accelerate.test_utils.training import RegressionDataset
@@ -697,7 +696,6 @@ def test_performance(self):
             with patch_environment(omp_num_threads=1):
                 execute_subprocess_async(cmd_stage, env=os.environ.copy())
 
-    @skip
     def test_checkpointing(self):
         self.test_file_path = os.path.join(self.test_scripts_folder, "test_checkpointing.py")
         cmd = [



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thank you 😄. This is neat and finally all the tests would run as expected 🤗.
diff --git a/docs/source/index.mdx b/docs/source/index.mdx
index 3f0f58802..1664fd7a2 100644
--- a/docs/source/index.mdx
+++ b/docs/source/index.mdx
@@ -55,7 +55,7 @@ accelerate launch {my_script.py}
       ><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Tutorials</div>
       <p class="text-gray-700">Learn the basics and become familiar with using 🤗 Accelerate. Start here if you are using 🤗 Accelerate for the first time!</p>
     </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/utility_guides/gradient_accumulation"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="/docs/accelerate/usage_guides/gradient_accumulation"
       ><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">How-to guides</div>
       <p class="text-gray-700">Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 Accelerate to solve real-world problems.</p>
     </a>
@@ -68,4 +68,4 @@ accelerate launch {my_script.py}
       <p class="text-gray-700">Technical descriptions of how 🤗 Accelerate classes and methods work.</p>
     </a>
   </div>
-</div>
\ No newline at end of file
+</div>



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index 188800527..467a757b4 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -677,7 +677,7 @@ def load_checkpoint_in_model(
         else:
             for param_name, param in checkpoint.items():
                 module_name = param_name
-                if dtype is not None:
+                if dtype is not None and not str(param.dtype).startswith(("torch.uint", "torch.int", "torch.bool")):
                     param = param.to(dtype)
                 while len(module_name) > 0 and module_name not in device_map:
                     module_name = ".".join(module_name.split(".")[:-1])



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index 673975fc7..839a7a2a9 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -209,7 +209,7 @@ def __init__(self, run_name: str, **kwargs):
 
     @property
     def tracker(self):
-        return self.run.run
+        return self.run
 
     def store_init_configuration(self, values: dict):
         """



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Thanks!
diff --git a/tests/deepspeed/test_deepspeed.py b/tests/deepspeed/test_deepspeed.py
index a8c1aedac..30eb2a4d8 100644
--- a/tests/deepspeed/test_deepspeed.py
+++ b/tests/deepspeed/test_deepspeed.py
@@ -35,6 +35,7 @@
     require_cuda,
     require_deepspeed,
     require_multi_gpu,
+    skip,
     slow,
 )
 from accelerate.test_utils.training import RegressionDataset
@@ -642,7 +643,9 @@ def setUp(self):
             "deepspeed_stage_1_fp16": 1600,
             "deepspeed_stage_2_fp16": 2500,
             "deepspeed_stage_3_zero_init_fp16": 2800,
-            "deepspeed_stage_3_cpu_offload_fp16": 1900,
+            # Disabling below test as it overwhelms the RAM memory usage
+            # on CI self-hosted runner leading to tests getting killed.
+            # "deepspeed_stage_3_cpu_offload_fp16": 1900,
         }
         self.n_train = 160
         self.n_val = 160
@@ -694,6 +697,7 @@ def test_performance(self):
             with patch_environment(omp_num_threads=1):
                 execute_subprocess_async(cmd_stage, env=os.environ.copy())
 
+    @skip
     def test_checkpointing(self):
         self.test_file_path = os.path.join(self.test_scripts_folder, "test_checkpointing.py")
         cmd = [
diff --git a/tests/fsdp/test_fsdp.py b/tests/fsdp/test_fsdp.py
index 628e377c3..249d2b692 100644
--- a/tests/fsdp/test_fsdp.py
+++ b/tests/fsdp/test_fsdp.py
@@ -191,7 +191,9 @@ def setUp(self):
             "multi_gpu_fp16": 3200,
             "fsdp_shard_grad_op_transformer_based_wrap_fp16": 2000,
             "fsdp_full_shard_transformer_based_wrap_fp16": 1900,
-            "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
+            # Disabling below test as it overwhelms the RAM memory usage
+            # on CI self-hosted runner leading to tests getting killed.
+            # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
         }
         self.n_train = 160
         self.n_val = 160



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 599e710f0..3930745dc 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -339,17 +339,6 @@ def simple_launcher(args):
         mixed_precision = "fp16"
 
     current_env["MIXED_PRECISION"] = str(mixed_precision)
-    if args.num_cpu_threads_per_process is None:
-        local_size = get_int_from_env(
-            ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
-        )
-        args.num_cpu_threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
-        if args.num_cpu_threads_per_process == 0:
-            args.num_cpu_threads_per_process = 1
-        logger.info(
-            f"num_cpu_threads_per_process unset, we set it at {args.num_cpu_threads_per_process} to improve oob performance."
-        )
-
     current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
 
     process = subprocess.Popen(cmd, env=current_env)
@@ -447,9 +436,6 @@ def multi_gpu_launcher(args):
             current_env["FSDP_BACKWARD_PREFETCH"] = str(args.fsdp_backward_prefetch_policy)
         if args.fsdp_state_dict_type is not None:
             current_env["FSDP_STATE_DICT_TYPE"] = str(args.fsdp_state_dict_type)
-    if args.num_cpu_threads_per_process is None:
-        args.num_cpu_threads_per_process = 1
-        logger.info(f"num_cpu_threads_per_process unset, we set it at {args.num_cpu_threads_per_process}.")
     current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
     process = subprocess.Popen(cmd, env=current_env)
     process.wait()
@@ -803,6 +789,17 @@ def launch_command(args):
                 if "--num_processes" in warn:
                     warned[i] = warn.replace("`1`", f"`{args.num_processes}`")
 
+    if args.num_cpu_threads_per_process is None:
+        local_size = get_int_from_env(
+            ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
+        )
+        args.num_cpu_threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
+        if args.num_cpu_threads_per_process == 0:
+            args.num_cpu_threads_per_process = 1
+        warned.append(
+            f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance"
+        )
+
     if any(warned):
         message = "The following values were not passed to `accelerate launch` and had defaults used instead:\n"
         message += "\n".join(warned)



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/setup.py b/setup.py
index a59d1ecaf..ec015f276 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
 extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3", "hf-doc-builder >= 0.3.0"]
 extras["docs"] = []
 extras["test_prod"] = ["pytest", "pytest-xdist", "pytest-subtests", "parameterized"]
-extras["test_dev"] = ["datasets", "evaluate", "transformers", "scipy", "sklearn", "deepspeed", "tqdm"]
+extras["test_dev"] = ["datasets", "evaluate", "transformers", "scipy", "sklearn", "deepspeed<0.7.0", "tqdm"]
 extras["testing"] = extras["test_prod"] + extras["test_dev"]
 
 extras["test_trackers"] = ["wandb", "comet-ml", "tensorboard"]



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index c2fd566b7..093d4c00f 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -412,7 +412,7 @@ def wrapper(self, *args, **kwargs):
 
     def on_local_process(local_process_idx):
         """
-        Run func on certain local process only
+        A decorator that will run the decorated function on a given local process index only.
         """
 
         def decorator(func):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0278218b..806f0094d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,6 +1,6 @@
 name: Run Tests
 
-# on: [pull_request]
+on: [pull_request]
 
 env:
   HF_HOME: ~/hf_cache



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index 7100d2c88..188800527 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -405,17 +405,12 @@ def get_balanced_memory(
     leaves = [n for n in module_sizes if len([p for p in module_sizes if p.startswith(n) and len(p) > len(n)]) == 0]
     mean_leaves = int(sum([module_sizes[n] for n in leaves]) / len(leaves))
     buffer = int(1.25 * max(buffer, mean_leaves))
-    if low_zero:
-        per_gpu += buffer
-        gpu_zero = 0
-    else:
-        gpu_zero = per_gpu
-        per_gpu += buffer
+    per_gpu += buffer
 
     max_memory = get_max_memory(max_memory)
-    for i in range(num_devices):
-        # We still leave slightly more space on GPU 0 and only apply the buffer on the other devices.
-        max_memory[i] = min(gpu_zero if i == 0 else per_gpu, max_memory[i])
+    # The last device is left with max_memory just in case the buffer is not enough.
+    for i in range(num_devices - 1):
+        max_memory[i] = min(0 if low_zero and i == 0 else per_gpu, max_memory[i])
 
     if low_zero:
         min_zero = max(0, module_sizes[""] - sum([max_memory[i] for i in range(1, num_devices)]))
diff --git a/tests/test_modeling_utils.py b/tests/test_modeling_utils.py
index 799170257..f5c36ee3d 100644
--- a/tests/test_modeling_utils.py
+++ b/tests/test_modeling_utils.py
@@ -365,7 +365,11 @@ def test_get_balanced_memory(self):
         model = ModelForTest()
         # model has size 236: linear1 64, batchnorm 72, linear2 100
         max_memory = get_balanced_memory(model, max_memory={0: 200, 1: 200})
-        self.assertDictEqual({0: 118, 1: 200}, max_memory)
+        self.assertDictEqual({0: 200, 1: 200}, max_memory)
 
         max_memory = get_balanced_memory(model, max_memory={0: 300, 1: 300})
-        self.assertDictEqual({0: 118, 1: 215}, max_memory)
+        self.assertDictEqual({0: 215, 1: 300}, max_memory)
+
+        # Last device always get max memory to give more buffer and avoid accidental CPU offload
+        max_memory = get_balanced_memory(model, max_memory={0: 300, 1: 500})
+        self.assertDictEqual({0: 215, 1: 500}, max_memory)



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._