muellerzr/github-pr-history huggingface dataset:
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index d119013d2..2bc1459fd 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -14,6 +14,7 @@
 
 import gc
 import os
+from contextlib import contextmanager
 from typing import List, Optional, Union
 
 import torch
@@ -183,6 +184,33 @@ def use_fp16(self):
             use_fp16 = self.state.use_fp16
         return use_fp16
 
+    @contextmanager
+    def local_main_process_first(self):
+        """
+        Lets the local main process go inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_local_main_process)
+
+    @contextmanager
+    def main_process_first(self):
+        """
+        Lets the main process go first inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_main_process)
+
+    def _goes_first(self, is_main):
+        if not is_main:
+            self.wait_for_everyone()
+
+        yield
+
+        if is_main:
+            self.wait_for_everyone()
+
     def print(self, *args, **kwargs):
         """
         Use in replacement of :obj:`print()` to only print once per server.


Review comments: body: Small nit: can we use `is_main` for consistency here (as well as at line 211 below)?
diff_hunk: @@ -183,6 +184,33 @@ def use_fp16(self):
             use_fp16 = self.state.use_fp16
         return use_fp16
 
+    @contextmanager
+    def local_main_process_first(self):
+        """
+        Lets the local main process go inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_local_main_process)
+
+    @contextmanager
+    def main_process_first(self):
+        """
+        Lets the main process go first inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_main_process)
+
+    def _goes_first(self, is_master):
+        if not is_master:
body: Yes sure!
diff_hunk: @@ -183,6 +184,33 @@ def use_fp16(self):
             use_fp16 = self.state.use_fp16
         return use_fp16
 
+    @contextmanager
+    def local_main_process_first(self):
+        """
+        Lets the local main process go inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_local_main_process)
+
+    @contextmanager
+    def main_process_first(self):
+        """
+        Lets the main process go first inside a with block.
+
+        The other processes will enter the with block after the main process exits.
+        """
+        yield from self._goes_first(self.is_main_process)
+
+    def _goes_first(self, is_master):
+        if not is_master:

Discussion: body: Addressed the request.
body: Thanks!
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index c7706d390..7c12b18e6 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import gc
 from typing import List, Optional, Union
 
 import torch
@@ -338,6 +339,15 @@ def save(self, obj, f):
         """
         save(obj, f)
 
+    def free_memory(self):
+        """
+        Will release all references to the internal objects stored and call the garbage collector. You should call this
+        method between two trainings with different models/optimizers.
+        """
+        self._optimizers = []
+        gc.collect()
+        torch.cuda.empty_cache()
+
     def _get_named_parameters(self, *args):
         named_parameters = {}
         for obj in args:
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index c7706d390..5a4bbe217 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -253,24 +253,36 @@ def backward(self, loss):
         else:
             loss.backward()
 
-    def clip_grad_norm_(self, parameters, max_norm, norm_type=2):
+    def unscale_gradients(self, optimizer=None):
         """
-        Should be used in place of :func:`torch.nn.utils.clip_grad_norm_`.
+        Unscale the gradients in mixed precision training with AMP. This is a noop in all other settings.
+
+        Args:
+            optimizer (:obj:`torch.optim.Optimizer` or :obj:`List[torch.optim.Optimizer]`, `optional`):
+                The optimizer(s) for which to unscale gradients. If not set, will unscale gradients on all optimizers
+                that were passed to :meth:`~accelerate.Accelerator.prepare`.
         """
-        # TODO: this unscales all optimizers where we should only unscale the one where parameters are.
         if self.state.use_fp16 and self.native_amp:
-            for optimizer in self._optimizers:
+            if optimizer is None:
+                # TODO: this unscales all optimizers where we should only unscale the one where parameters are.
+                optimizer = self._optimizers
+            elif not isinstance(optimizer, (tuple, list)):
+                optimizer = [optimizer]
+            for optimizer in optimizer:
                 self.scaler.unscale_(optimizer)
+
+    def clip_grad_norm_(self, parameters, max_norm, norm_type=2):
+        """
+        Should be used in place of :func:`torch.nn.utils.clip_grad_norm_`.
+        """
+        self.unscale_gradients()
         torch.nn.utils.clip_grad_norm_(parameters, max_norm, norm_type=norm_type)
 
     def clip_grad_value_(self, parameters, clip_value):
         """
         Should be used in place of :func:`torch.nn.utils.clip_grad_value_`.
         """
-        # TODO: this unscales all optimizers where we should only unscale the one where parameters are.
-        if self.state.use_fp16 and self.native_amp:
-            for optimizer in self._optimizers:
-                self.scaler.unscale_(optimizer)
+        self.unscale_gradients()
         torch.nn.utils.clip_grad_value_(parameters, clip_value)
 
     def gather(self, tensor):
diff --git a/examples/README.md b/examples/README.md
index 4d941b5d3..19995fc0a 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -23,7 +23,7 @@ The [nlp_example.py](./nlp_example.py) script is a simple example to train a Ber
 Prior to running it you should install 🤗 Dataset and 🤗 Transformers:
 
 ```bash
-pip install datasets, transformers
+pip install datasets transformers
 ```
 
 The same script can be run in any of the following configurations:
diff --git a/README.md b/README.md
index 9c044ce9e..4f6cf5f40 100644
--- a/README.md
+++ b/README.md
@@ -63,12 +63,12 @@ Here is an example:
 + device = accelerator.device
 
   model = torch.nn.Transformer().to(device)
-  optim = torch.optim.Adam(model.parameters())
+  optimizer = torch.optim.Adam(model.parameters())
 
   dataset = load_dataset('my_dataset')
   data = torch.utils.data.DataLoader(dataset, shuffle=True)
 
-+ model, optim, data = accelerator.prepare(model, optim, data)
++ model, optimizer, data = accelerator.prepare(model, optimizer, data)
 
   model.train()
   for epoch in range(10):
@@ -104,12 +104,12 @@ In particular, the same code can then be run without modification on your local
 
 - model = torch.nn.Transformer().to(device)
 + model = torch.nn.Transformer()
-  optim = torch.optim.Adam(model.parameters())
+  optimizer = torch.optim.Adam(model.parameters())
 
   dataset = load_dataset('my_dataset')
   data = torch.utils.data.DataLoader(dataset, shuffle=True)
 
-+ model, optim, data = accelerator.prepare(model, optim, data)
++ model, optimizer, data = accelerator.prepare(model, optimizer, data)
 
   model.train()
   for epoch in range(10):
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 0e115bdd4..8cc993b59 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -39,7 +39,8 @@
 
 
 def load_config_from_file(config_file):
-    config_file = config_file if config_file is not None else default_config_file
+    config_file_exists = config_file is not None and os.path.isfile(config_file)
+    config_file = config_file if config_file_exists else default_config_file
     with open(config_file, "r", encoding="utf-8") as f:
         if config_file.endswith(".json"):
             if (
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index 3a9b03e28..4c785b12f 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -106,7 +106,7 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
             try:
                 print(f"Launching a training on {num_processes} GPUs.")
-                start_processes(launcher, nprocs=num_processes, start_method="fork")
+                start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
             finally:
                 # Clean up the environment variables set.
                 del os.environ["WORLD_SIZE"]
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index 57f253267..3a9b03e28 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -42,10 +42,10 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             The port to use to communicate between processes when launching a multi-GPU training.
     """
     # Are we in a google colab or a Kaggle Kernel?
-    if "IPython" in sys.modules:
-        in_colab_or_kaggle = "google.colab" in str(sys.modules["IPython"].get_ipython())
-    elif any(key.startswith("KAGGLE") for key in os.environ.keys()):
+    if any(key.startswith("KAGGLE") for key in os.environ.keys()):
         in_colab_or_kaggle = True
+    elif "IPython" in sys.modules:
+        in_colab_or_kaggle = "google.colab" in str(sys.modules["IPython"].get_ipython())
     else:
         in_colab_or_kaggle = False
diff --git a/examples/README.md b/examples/README.md
index 434a230ea..4d941b5d3 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -110,7 +110,7 @@ The same script can be run in any of the following configurations:
 Prior to running it you should install timm and torchvision:
 
 ```bash
-pip install timm, torchvision
+pip install timm torchvision
 ```
 
 and you should download the data with the following commands:
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 7a8422a05..44909ab3c 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -364,7 +364,8 @@ def prepare_data_loader(
         )
 
     new_dataset = dataloader.dataset
-    new_batch_sampler = dataloader.batch_sampler
+    # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
+    new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     generator = getattr(dataloader, "generator", None)
     # No change if no multiprocess
     if num_processes != 1:
@@ -413,6 +414,11 @@ def prepare_data_loader(
         for k in _PYTORCH_DATALOADER_KWARGS
         if k not in ignore_kwargs
     }
+
+    # Need to provide batch_size as batch_sampler is None for Iterable dataset
+    if new_batch_sampler is None:
+        kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
+
     return DataLoaderShard(
         new_dataset,
         device=device if put_on_device else None,


Review comments: body: ```suggestion
    # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
    new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
```
Let's group the two lines together.
diff_hunk: @@ -365,6 +365,9 @@ def prepare_data_loader(
 
     new_dataset = dataloader.dataset
     new_batch_sampler = dataloader.batch_sampler
+    if isinstance(new_dataset, IterableDataset):
+        # Iterable dataset doesn't like batch_sampler, data_loader creates a default one for it
+        new_batch_sampler = None
body: ```suggestion
    # Need to provide batch_size as batch_sampler is None for Iterable dataset
    if new_batch_sampler is None:
        kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
```
diff_hunk: @@ -413,6 +416,10 @@ def prepare_data_loader(
         for k in _PYTORCH_DATALOADER_KWARGS
         if k not in ignore_kwargs
     }
+
+    # Need to provide batch_size as batch_sampler is None for Iterable dataset
+    if new_batch_sampler is None: kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index f1242d3fd..44cff4d55 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -137,6 +137,7 @@ def multi_gpu_launcher(args):
         )
     else:
         cmd.extend(["--nproc_per_node", str(args.num_processes)])
+        cmd.extend(["--master_port", str(args.main_process_port)])
     cmd.append(args.training_script)
     cmd.extend(args.training_script_args)
diff --git a/src/accelerate/commands/test.py b/src/accelerate/commands/test.py
index 1940c5039..ec6d2524b 100644
--- a/src/accelerate/commands/test.py
+++ b/src/accelerate/commands/test.py
@@ -46,7 +46,7 @@ def test_command(args):
     script_name = os.path.sep.join(__file__.split(os.path.sep)[:-2] + ["test_utils", "test_script.py"])
 
     test_args = f"""
-        {script_name} --config_file={args.config_file}
+        --config_file={args.config_file} {script_name}
     """.split()
     cmd = ["accelerate-launch"] + test_args
     result = execute_subprocess_async(cmd, env=os.environ.copy())
diff --git a/README.md b/README.md
index 25bc92c13..280473ee5 100644
--- a/README.md
+++ b/README.md
@@ -151,6 +151,20 @@ For instance, here is how you would run the GLUE example on the MRPC task (from
 accelerate launch examples/nlp_example.py
 ```
 
+## Launching your training from a notebook
+
+🤗 Accelerate also provides a `notebook_launcher` function you can use in a notebook to launch a distributed training. This is especially useful for Colab or Kaggle notebooks with a TPU backend. Just define your training loop in a `training_function` then in your last cell, add:
+
+```python
+from accelerate import notebook_launcher
+
+notebook_launcher(training_function)
+```
+
+An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb). [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb)
+
+Note that this launcher does not work in Jupyter Notebook on a machine with multiple GPUs (yet). This feature will be released in a later version.
+
 ## Why should I use 🤗 Accelerate?
 
 You should use 🤗 Accelerate when you want to easily run your training scripts in a distributed environment without having to renounce full control over your training loop. This is not a high-level framework above PyTorch, just a thin wrapper so you don't have to learn a new library, In fact the whole API of 🤗 Accelerate is in one class, the `Accelerator` object.
diff --git a/docs/source/launcher.rst b/docs/source/launcher.rst
index 1b0520d1b..da28ad920 100644
--- a/docs/source/launcher.rst
+++ b/docs/source/launcher.rst
@@ -19,6 +19,9 @@ Colab](https://colab.research.google.com/) and [Kaggle kernels](https://www.kagg
 but support for training on several GPUs (if the machine on which you are running your notebook has them) is planned
 for a future release.
 
+An example can be found in `this notebook
+<https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb>`__.
+
 .. warning::
 
     If you are training on Colab or a Kaggle kernel with TPUs, your :obj:`Accelerator` object should only be defined
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 56ba51f64..c8bf1bcd7 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -46,7 +46,7 @@ def get_cluster_input():
             main_process_ip = _ask_field(
                 "What is the IP address of the machine that will host the main process? ",
             )
-            main_process_ip = _ask_field(
+            main_process_port = _ask_field(
                 "What is the port you will use to communicate with the main process? ",
                 lambda x: int(x),
             )
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 55d00ab5d..63b53eccb 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -14,6 +14,8 @@
 
 import torch
 
+from packaging import version
+
 from .state import AcceleratorState, DistributedType, is_tpu_available
 
 
@@ -86,8 +88,18 @@ def load_state_dict(self, state_dict):
     def state_dict(self):
         return self.optimizer.state_dict()
 
-    def zero_grad(self):
-        self.optimizer.zero_grad()
+    def zero_grad(self, set_to_none=None):
+        if version.parse(torch.__version__) < version.parse("1.7.0"):
+            if set_to_none is not None:
+                raise ValueError(
+                    "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
+                    f"earlier versions (found version {torch.__version__})."
+                )
+            self.optimizer.zero_grad()
+        else:
+            if set_to_none is not None:
+                set_to_none = False
+            self.optimizer.zero_grad(set_to_none=set_to_none)
 
     def step(self):
         if self.state.distributed_type == DistributedType.TPU:


Review comments: body: if set_to_none is None:
diff_hunk: @@ -86,8 +88,18 @@ def load_state_dict(self, state_dict):
     def state_dict(self):
         return self.optimizer.state_dict()
 
-    def zero_grad(self):
-        self.optimizer.zero_grad()
+    def zero_grad(self, set_to_none=None):
+        if version.parse(torch.__version__) < version.parse("1.7.0"):
+            if set_to_none is not None:
+                raise ValueError(
+                    "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
+                    f"earlier versions (found version {torch.__version__})."
+                )
+            self.optimizer.zero_grad()
+        else:
+            if set_to_none is not None:
body: Good catch!
diff_hunk: @@ -86,8 +88,18 @@ def load_state_dict(self, state_dict):
     def state_dict(self):
         return self.optimizer.state_dict()
 
-    def zero_grad(self):
-        self.optimizer.zero_grad()
+    def zero_grad(self, set_to_none=None):
+        if version.parse(torch.__version__) < version.parse("1.7.0"):
+            if set_to_none is not None:
+                raise ValueError(
+                    "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
+                    f"earlier versions (found version {torch.__version__})."
+                )
+            self.optimizer.zero_grad()
+        else:
+            if set_to_none is not None:
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 382636253..903f2b0e0 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -66,10 +66,10 @@ def launch_command_parser(subparsers=None):
         "--num_processes", type=int, default=None, help="The total number of processes to be launched in parallel."
     )
     parser.add_argument(
-        "--num_machines", type=int, default=1, help="The total number of machines used in this training."
+        "--num_machines", type=int, default=None, help="The total number of machines used in this training."
     )
     parser.add_argument(
-        "--machine_rank", type=int, default=0, help="The rank of the machine on which this script is launched."
+        "--machine_rank", type=int, default=None, help="The rank of the machine on which this script is launched."
     )
     parser.add_argument("--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0.")
     parser.add_argument(
@@ -298,6 +298,16 @@ def launch_command(args):
         if not args.multi_gpu and not args.tpu:
             args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU
             args.tpu = defaults.distributed_type == DistributedType.TPU
+        if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
+            # Update args with the defaults
+            for name, attr in defaults.__dict__.items():
+                # Those args are handled separately
+                if (
+                    name not in ["compute_environment", "fp16", "distributed_type"]
+                    and getattr(args, name, None) is None
+                ):
+                    setattr(args, name, attr)
+
         if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
             args.num_processes = defaults.num_processes
         if not args.fp16:
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index a9e412f7d..56ba51f64 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -58,22 +58,22 @@ def get_cluster_input():
     else:
         main_training_function = "main"
 
-        num_processes = _ask_field(
-            "How many processes in total will you use? [1]: ",
-            lambda x: int(x),
-            default=1,
-            error_message="Please enter an integer.",
-        )
+    num_processes = _ask_field(
+        "How many processes in total will you use? [1]: ",
+        lambda x: int(x),
+        default=1,
+        error_message="Please enter an integer.",
+    )
 
-        if distributed_type != DistributedType.TPU:
-            fp16 = _ask_field(
-                "Do you wish to use FP16 (mixed precision)? [yes/NO]: ",
-                _convert_yes_no_to_bool,
-                default=False,
-                error_message="Please enter yes or no.",
-            )
-        else:
-            fp16 = False
+    if distributed_type != DistributedType.TPU:
+        fp16 = _ask_field(
+            "Do you wish to use FP16 (mixed precision)? [yes/NO]: ",
+            _convert_yes_no_to_bool,
+            default=False,
+            error_message="Please enter yes or no.",
+        )
+    else:
+        fp16 = False
 
     return ClusterConfig(
         compute_environment=ComputeEnvironment.LOCAL_MACHINE,
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index a05debfda..0e115bdd4 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -44,7 +44,7 @@ def load_config_from_file(config_file):
         if config_file.endswith(".json"):
             if (
                 json.load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE)
-                is ComputeEnvironment.LOCAL_MACHINE
+                == ComputeEnvironment.LOCAL_MACHINE
             ):
                 config_class = ClusterConfig
             else:
@@ -53,7 +53,7 @@ def load_config_from_file(config_file):
         else:
             if (
                 yaml.safe_load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE)
-                is ComputeEnvironment.LOCAL_MACHINE
+                == ComputeEnvironment.LOCAL_MACHINE
             ):
                 config_class = ClusterConfig
             else:
diff --git a/examples/README.md b/examples/README.md
index be636e63c..434a230ea 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -28,7 +28,7 @@ pip install datasets, transformers
 
 The same script can be run in any of the following configurations:
 - single CPU or single GPU
-- multi GPUS (using PyTorch distributed mode)
+- multi GPUs (using PyTorch distributed mode)
 - (multi) TPUs
 - fp16 (mixed-precision) or fp32 (normal precision)
 
@@ -57,8 +57,8 @@ To run it in each of these various modes, use the following commands:
         ```
     * from any server with Accelerate launcher
         ```bash
-        accelerate launch --fb16 ./nlp_example.py
-- multi GPUS (using PyTorch distributed mode)
+        accelerate launch --fp16 ./nlp_example.py
+- multi GPUs (using PyTorch distributed mode)
     * With Accelerate config and launcher
         ```bash
         accelerate config  # This will create a config file on your server
@@ -103,7 +103,7 @@ The [cv_example.py](./cv_example.py) script is a simple example to fine-tune a R
 
 The same script can be run in any of the following configurations:
 - single CPU or single GPU
-- multi GPUS (using PyTorch distributed mode)
+- multi GPUs (using PyTorch distributed mode)
 - (multi) TPUs
 - fp16 (mixed-precision) or fp32 (normal precision)
 
@@ -145,8 +145,8 @@ To run it in each of these various modes, use the following commands:
         ```
     * from any server with Accelerate launcher
         ```bash
-        accelerate launch --fb16 ./cv_example.py --data_dir path_to_data
-- multi GPUS (using PyTorch distributed mode)
+        accelerate launch --fp16 ./cv_example.py --data_dir path_to_data
+- multi GPUs (using PyTorch distributed mode)
     * With Accelerate config and launcher
         ```bash
         accelerate config  # This will create a config file on your server
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 5b0dd00ba..fe2cb122a 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -42,13 +42,13 @@ def load_config_from_file(config_file):
     config_file = config_file if config_file is not None else default_config_file
     with open(config_file, "r", encoding="utf-8") as f:
         if config_file.endswith(".json"):
-            if json.load(f)["compute_environment"] == ComputeEnvironment.LOCAL_MACHINE:
+            if json.load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE):
                 config_class = ClusterConfig
             else:
                 config_class = SageMakerConfig
             return config_class.from_json_file(json_file=config_file)
         else:
-            if yaml.safe_load(f)["compute_environment"] == ComputeEnvironment.LOCAL_MACHINE:
+            if yaml.safe_load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE):
                 config_class = ClusterConfig
             else:
                 config_class = SageMakerConfig
@@ -73,7 +73,10 @@ def to_dict(self):
     def from_json_file(cls, json_file=None):
         json_file = default_json_config_file if json_file is None else json_file
         with open(json_file, "r", encoding="utf-8") as f:
-            return cls(**json.load(f))
+            config_dict = json.load(f)
+        if "compute_environment" not in config_dict:
+            config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        return cls(**config_dict)
 
     def to_json_file(self, json_file):
         with open(json_file, "w", encoding="utf-8") as f:
@@ -84,7 +87,10 @@ def to_json_file(self, json_file):
     def from_yaml_file(cls, yaml_file=None):
         yaml_file = default_yaml_config_file if yaml_file is None else yaml_file
         with open(yaml_file, "r", encoding="utf-8") as f:
-            return cls(**yaml.safe_load(f))
+            config_dict = yaml.safe_load(f)
+        if "compute_environment" not in config_dict:
+            config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        return cls(**config_dict)
 
     def to_yaml_file(self, yaml_file):
         with open(yaml_file, "w", encoding="utf-8") as f:



Discussion: body: No problem @philschmid, I should have caught it during the review :-)
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index e67a4cee1..a8c11b887 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -198,10 +198,10 @@ def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
     new_tensor = tensor.new_zeros(tuple(new_size)) + pad_index
     if pad_first:
         indices = tuple(
-            slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(size))
+            slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(new_size))
         )
     else:
-        indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(size)))
+        indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(new_size)))
     new_tensor[indices] = tensor
     return new_tensor
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 54ef3c12f..61a39fc95 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -105,10 +105,7 @@ def send_to_device(tensor, device):
     elif isinstance(tensor, dict):
         return type(tensor)({k: send_to_device(v, device) for k, v in tensor.items()})
     elif not hasattr(tensor, "to"):
-        raise TypeError(
-            f"Can't send the values of type {type(tensor)} to device {device}, only of nested list/tuple/dicts "
-            "of tensors or objects having a `to` method."
-        )
+        return tensor
     return tensor.to(device)
diff --git a/README.md b/README.md
index 0b28b6a50..219b93248 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ for epoch in range(10):
 
 + from accelerate import Accelerator
 + accelerator = Accelerator()
-+ device = accelerator.device
+-
 
 + model = torch.nn.Transformer()
   optim = torch.optim.Adam(
diff --git a/README.md b/README.md
index 794176758..728976692 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ for epoch in range(10):
               output, targets
           )
 
-+         accelerate.backward(loss)
++         accelerator.backward(loss)
 
           optimizer.step()
 ```
@@ -243,7 +243,7 @@ for epoch in range(10):
               output, targets
           )
 
-+         accelerate.backward(loss)
++         accelerator.backward(loss)
 
           optimizer.step()
 ```



Discussion: body: Thanks a lot!
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index e20426857..859309a7a 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -49,6 +49,9 @@
 
 
 def training_function(config, args):
+    # Initialize accelerator
+    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu)
+
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     lr = config["lr"]
     num_epochs = int(config["num_epochs"])
@@ -97,14 +100,13 @@ def collate_fn(examples):
     )
 
     set_seed(seed)
-    # Initialize accelerator
-    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu)
 
     # Instantiate the model (we build the model here so that the seed also control new weights initialization)
     model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", return_dict=True)
-    # We could avoid this line since we set the accelerator with `device_placement=True`.
-    # If setting devices manually, this line absolutely needs to be before the optimizer creation otherwise training
-    # will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
+
+    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
+    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
+    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
     model = model.to(accelerator.device)
 
     # Instantiate optimizer



Discussion: body: Ah thanks a lot for fixing!
diff --git a/README.md b/README.md
index 3776c7cc2..a9d6c5704 100644
--- a/README.md
+++ b/README.md
@@ -270,7 +270,7 @@ accelerate launch my_script.py --args_to_my_script
 For instance, here is how you would run the GLUE example on the MRPC task (from the root of the repo):
 
 ```bash
-accelerate launch examples/glue_example.py --task_name mrpc --model_name_or_path bert-base-cased
+accelerate launch examples/nlp_example.py
 ```
 
 ## Why should I use 🤗 Accelerate?


Review comments: body: ```suggestion
accelerate launch examples/nlp_example.py
```
This one does not take any arg however.
diff_hunk: @@ -270,7 +270,7 @@ accelerate launch my_script.py --args_to_my_script
 For instance, here is how you would run the GLUE example on the MRPC task (from the root of the repo):
 
 ```bash
-accelerate launch examples/glue_example.py --task_name mrpc --model_name_or_path bert-base-cased
+accelerate launch examples/nlp_example.py --task_name mrpc --model_name_or_path bert-base-cased

Discussion: body: Thanks a lot!
diff --git a/README.md b/README.md
index 3776c7cc2..3fa28ba40 100644
--- a/README.md
+++ b/README.md
@@ -254,7 +254,7 @@ for epoch in range(10):
 
 ## Launching script
 
-🤗 Accelerate also provides a CLI tool that allows you to quickly configure and test your training environment then launch the scripts. No need to remember how to use `torch.distributed.launch` or to write a specific launcher for TPU training!
+🤗 Accelerate also provides an optional CLI tool that allows you to quickly configure and test your training environment then launch the scripts. No need to remember how to use `torch.distributed.launch` or to write a specific launcher for TPU training!
 On your machine(s) just run:
 
 ```bash
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 32ff600a9..e20426857 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -39,7 +39,7 @@
 #
 # To run it in each of these various modes, follow the instructions
 # in the readme for examples:
-# https://github.com/huggingface/accelerate/examples
+# https://github.com/huggingface/accelerate/tree/main/examples
 #
 ########################################################################
 



Discussion: body: Thanks!