diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index bf17f57ce..81759962c 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -153,6 +153,10 @@ class Accelerator:
             If set to `True`, the dataloader prepared by the Accelerator is only iterated through on the main process
             and then the batches are split and broadcast to each process. Will default to `True` for `DataLoader` whose
             underlying dataset is an `IterableDataset`, `False` otherwise.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            If set to `True`, in cases where the total batch size across all processes does not exactly divide the
+            dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
+            all workers.
         step_scheduler_with_optimizer (`bool`, *optional`, defaults to `True`):
             Set `True` if the learning rate scheduler is stepped at the same time as the optimizer, `False` if only
             done under certain circumstances (at the end of each epoch, for instance).
@@ -191,6 +195,7 @@ def __init__(
         log_with: Optional[List[Union[str, LoggerType, GeneralTracker]]] = None,
         logging_dir: Optional[Union[str, os.PathLike]] = None,
         dispatch_batches: Optional[bool] = None,
+        even_batches: bool = True,
         step_scheduler_with_optimizer: bool = True,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
@@ -305,6 +310,7 @@ def __init__(
             raise ImportError(
                 "Using `DataLoaderDispatcher` requires PyTorch 1.8.0 minimum. You have {torch.__version__}."
             )
+        self.even_batches = even_batches
         self.step_scheduler_with_optimizer = step_scheduler_with_optimizer
 
         # Mixed precision attributes
@@ -1109,6 +1115,7 @@ def prepare_data_loader(self, data_loader: torch.utils.data.DataLoader, device_p
             put_on_device=device_placement,
             rng_types=self.rng_types.copy(),
             dispatch_batches=self.dispatch_batches,
+            even_batches=self.even_batches,
         )
 
     def prepare_optimizer(self, optimizer: torch.optim.Optimizer, device_placement=None):
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index cdae54af9..39f843430 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -118,7 +118,8 @@ class BatchSamplerShard(BatchSampler):
 
     <Tip warning={true}>
 
-    This does not support `BatchSampler` with varying batch size yet.
+    `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches`
+    equal to `False`
 
     </Tip>"""
 
@@ -570,6 +571,7 @@ def prepare_data_loader(
     put_on_device: bool = False,
     rng_types: Optional[List[Union[str, RNGType]]] = None,
     dispatch_batches: Optional[bool] = None,
+    even_batches: bool = True,
 ) -> DataLoader:
     """
     Wraps a PyTorch `DataLoader` to generate batches for one of the processes only.
@@ -615,15 +617,21 @@ def prepare_data_loader(
             If set to `True`, the datalaoder prepared is only iterated through on the main process and then the batches
             are split and broadcast to each process. Will default to `True` when the underlying dataset is an
             `IterableDataset`, `False` otherwise.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            If set to `True`, in cases where the total batch size across all processes does not exactly divide the
+            dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
+            all workers.
 
     Returns:
         `torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
 
     <Tip warning={true}>
 
-    This does not support `BatchSampler` with varying batch size yet.
+    `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches`
+    equal to `False`
 
-    </Tip>"""
+    </Tip>
+    """
     if dispatch_batches is None:
         if is_torch_version("<", "1.8.0") or not put_on_device:
             dispatch_batches = False
@@ -683,6 +691,7 @@ def prepare_data_loader(
                 num_processes=num_processes,
                 process_index=process_index,
                 split_batches=split_batches,
+                even_batches=even_batches,
             )
 
     # We ignore all of those since they are all dealt with by our new_batch_sampler
diff --git a/src/accelerate/test_utils/scripts/test_distributed_data_loop.py b/src/accelerate/test_utils/scripts/test_distributed_data_loop.py
new file mode 100644
index 000000000..eaf7c6a34
--- /dev/null
+++ b/src/accelerate/test_utils/scripts/test_distributed_data_loop.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import torch
+from torch.utils.data import DataLoader, TensorDataset
+
+from accelerate.accelerator import Accelerator
+
+
+def create_accelerator(even_batches=True):
+    accelerator = Accelerator(even_batches=even_batches)
+    assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
+    return accelerator
+
+
+def create_dataloader(accelerator: Accelerator, dataset_size: int, batch_size: int):
+    """
+    Create a simple DataLoader to use during the test cases
+    """
+    dataset = TensorDataset(torch.as_tensor(range(dataset_size)))
+
+    dl = DataLoader(dataset, batch_size=batch_size)
+    dl = accelerator.prepare(dl)
+
+    return dl
+
+
+def verify_dataloader_batch_sizes(
+    accelerator: Accelerator,
+    dataset_size: int,
+    batch_size: int,
+    process_0_expected_batch_sizes: List[int],
+    process_1_expected_batch_sizes: List[int],
+):
+    """
+    A helper function for verifying the batch sizes coming from a prepared dataloader in each process
+    """
+    dl = create_dataloader(accelerator=accelerator, dataset_size=dataset_size, batch_size=batch_size)
+
+    batch_sizes = [len(batch[0]) for batch in dl]
+
+    if accelerator.process_index == 0:
+        assert batch_sizes == process_0_expected_batch_sizes
+    elif accelerator.process_index == 1:
+        assert batch_sizes == process_1_expected_batch_sizes
+
+
+def test_default_ensures_even_batch_sizes():
+
+    accelerator = create_accelerator()
+
+    # without padding, we would expect a different number of batches
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=3,
+        batch_size=1,
+        process_0_expected_batch_sizes=[1, 1],
+        process_1_expected_batch_sizes=[1, 1],
+    )
+
+    # without padding, we would expect the same number of batches, but different sizes
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=7,
+        batch_size=2,
+        process_0_expected_batch_sizes=[2, 2],
+        process_1_expected_batch_sizes=[2, 2],
+    )
+
+
+def test_can_disable_even_batches():
+    accelerator = create_accelerator(even_batches=False)
+
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=3,
+        batch_size=1,
+        process_0_expected_batch_sizes=[1, 1],
+        process_1_expected_batch_sizes=[1],
+    )
+
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=7,
+        batch_size=2,
+        process_0_expected_batch_sizes=[2, 2],
+        process_1_expected_batch_sizes=[2, 1],
+    )
+
+
+if __name__ == "__main__":
+    accelerator = create_accelerator()
+
+    accelerator.print("Test that even_batches variable ensures uniform batches across processes")
+    test_default_ensures_even_batch_sizes()
+
+    accelerator.print("Run tests with even_batches disabled")
+    test_can_disable_even_batches()
diff --git a/tests/test_multigpu.py b/tests/test_multigpu.py
index 2c0403ed5..54a5bca75 100644
--- a/tests/test_multigpu.py
+++ b/tests/test_multigpu.py
@@ -28,6 +28,9 @@ class MultiGPUTester(unittest.TestCase):
     def setUp(self):
         mod_file = inspect.getfile(accelerate.test_utils)
         self.test_file_path = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "test_script.py"])
+        self.data_loop_file_path = os.path.sep.join(
+            mod_file.split(os.path.sep)[:-1] + ["scripts", "test_distributed_data_loop.py"]
+        )
 
     @require_multi_gpu
     def test_multi_gpu(self):
@@ -42,6 +45,17 @@ def test_pad_across_processes(self):
         with patch_environment(omp_num_threads=1):
             execute_subprocess_async(cmd, env=os.environ.copy())
 
+    @require_multi_gpu
+    def test_distributed_data_loop(self):
+        """
+        This TestCase checks the behaviour that occurs during distributed training or evaluation,
+        when the batch size does not evenly divide the dataset size.
+        """
+        print(f"Found {torch.cuda.device_count()} devices, using 2 devices only")
+        cmd = get_launch_prefix() + [f"--nproc_per_node={torch.cuda.device_count()}", self.data_loop_file_path]
+        with patch_environment(omp_num_threads=1, cuda_visible_devices="0,1"):
+            execute_subprocess_async(cmd, env=os.environ.copy())
+
 
 if __name__ == "__main__":
     accelerator = Accelerator()


Review comments: body: ```suggestion
        even_batches (`bool`, *optional*, defaults to `True`):
```
diff_hunk: @@ -153,6 +153,10 @@ class Accelerator:
             If set to `True`, the dataloader prepared by the Accelerator is only iterated through on the main process
             and then the batches are split and broadcast to each process. Will default to `True` for `DataLoader` whose
             underlying dataset is an `IterableDataset`, `False` otherwise.
+        even_batches ('bool', defaults to `True`):
body: ```suggestion
        even_batches (`bool`, *optional*, defaults to `True`):
```
diff_hunk: @@ -615,15 +616,15 @@ def prepare_data_loader(
             If set to `True`, the datalaoder prepared is only iterated through on the main process and then the batches
             are split and broadcast to each process. Will default to `True` when the underlying dataset is an
             `IterableDataset`, `False` otherwise.
+        even_batches ('bool', defaults to `True`):
body: You should leave the warning but rephrase to state a `BatchSampler` with varying batch size will require `even_batches=False`.
diff_hunk: @@ -615,15 +616,15 @@ def prepare_data_loader(
             If set to `True`, the datalaoder prepared is only iterated through on the main process and then the batches
             are split and broadcast to each process. Will default to `True` when the underlying dataset is an
             `IterableDataset`, `False` otherwise.
+        even_batches ('bool', defaults to `True`):
+            If set to `True`, in cases where the total batch size across all processes does not exactly divide the
+            dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
+            all workers.
 
     Returns:
         `torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
 
-    <Tip warning={true}>
-
-    This does not support `BatchSampler` with varying batch size yet.
body: Sure, I can update that
diff_hunk: @@ -615,15 +616,15 @@ def prepare_data_loader(
             If set to `True`, the datalaoder prepared is only iterated through on the main process and then the batches
             are split and broadcast to each process. Will default to `True` when the underlying dataset is an
             `IterableDataset`, `False` otherwise.
+        even_batches ('bool', defaults to `True`):
+            If set to `True`, in cases where the total batch size across all processes does not exactly divide the
+            dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
+            all workers.
 
     Returns:
         `torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
 
-    <Tip warning={true}>
-
-    This does not support `BatchSampler` with varying batch size yet.
body: You're currently not testing anything at all, it's never being ran. Please see here to learn how: https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/scripts/test_sync.py#L229-L274

I also highly recommend using print statements so we can see just what is being tested at a given moment, and we know if they've run 😄 
diff_hunk: @@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import torch
+from torch.utils.data import DataLoader, TensorDataset
+
+from accelerate.accelerator import Accelerator
+
+
+def create_accelerator(even_batches=True):
+    accelerator = Accelerator(even_batches=even_batches)
+    assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
+    return accelerator
+
+
+def create_dataloader(accelerator: Accelerator, dataset_size: int, batch_size: int):
+    """
+    Create a simple DataLoader to use during the test cases
+    """
+    dataset = TensorDataset(torch.as_tensor(range(dataset_size)))
+
+    dl = DataLoader(dataset, batch_size=batch_size)
+    dl = accelerator.prepare(dl)
+
+    return dl
+
+
+def verify_dataloader_batch_sizes(
+    accelerator: Accelerator,
+    dataset_size: int,
+    batch_size: int,
+    process_0_expected_batch_sizes: List[int],
+    process_1_expected_batch_sizes: List[int],
+):
+    """
+    A helper function for verifying the batch sizes coming from a prepared dataloader in each process
+    """
+    dl = create_dataloader(accelerator=accelerator, dataset_size=dataset_size, batch_size=batch_size)
+
+    batch_sizes = [len(batch[0]) for batch in dl]
+
+    if accelerator.process_index == 0:
+        assert batch_sizes == process_0_expected_batch_sizes
+    elif accelerator.process_index == 1:
+        assert batch_sizes == process_1_expected_batch_sizes
+
+
+def test_default_ensures_even_batch_sizes():
+
+    accelerator = create_accelerator()
+
+    # without padding, we would expect a different number of batches
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=3,
+        batch_size=1,
+        process_0_expected_batch_sizes=[1, 1],
+        process_1_expected_batch_sizes=[1, 1],
+    )
+
+    # without padding, we would expect the same number of batches, but different sizes
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=7,
+        batch_size=2,
+        process_0_expected_batch_sizes=[2, 2],
+        process_1_expected_batch_sizes=[2, 2],
+    )
+
+
+def test_can_disable_even_batches():
body: Gah! I think I misinterpreted what you meant when you said PyTest format. Looking at the other example, it looks like it still needs a main method to drive running the tests; just what I removed a few commits back :satisfied:! I'll reinsert the main method and some print statements.

That said, when I run it with PyTest locally, using `CUDA_VISIBLE_DEVICES="0,1" pytest -sv tests/test_multigpu.py` it does seem to pick them up.
diff_hunk: @@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import torch
+from torch.utils.data import DataLoader, TensorDataset
+
+from accelerate.accelerator import Accelerator
+
+
+def create_accelerator(even_batches=True):
+    accelerator = Accelerator(even_batches=even_batches)
+    assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
+    return accelerator
+
+
+def create_dataloader(accelerator: Accelerator, dataset_size: int, batch_size: int):
+    """
+    Create a simple DataLoader to use during the test cases
+    """
+    dataset = TensorDataset(torch.as_tensor(range(dataset_size)))
+
+    dl = DataLoader(dataset, batch_size=batch_size)
+    dl = accelerator.prepare(dl)
+
+    return dl
+
+
+def verify_dataloader_batch_sizes(
+    accelerator: Accelerator,
+    dataset_size: int,
+    batch_size: int,
+    process_0_expected_batch_sizes: List[int],
+    process_1_expected_batch_sizes: List[int],
+):
+    """
+    A helper function for verifying the batch sizes coming from a prepared dataloader in each process
+    """
+    dl = create_dataloader(accelerator=accelerator, dataset_size=dataset_size, batch_size=batch_size)
+
+    batch_sizes = [len(batch[0]) for batch in dl]
+
+    if accelerator.process_index == 0:
+        assert batch_sizes == process_0_expected_batch_sizes
+    elif accelerator.process_index == 1:
+        assert batch_sizes == process_1_expected_batch_sizes
+
+
+def test_default_ensures_even_batch_sizes():
+
+    accelerator = create_accelerator()
+
+    # without padding, we would expect a different number of batches
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=3,
+        batch_size=1,
+        process_0_expected_batch_sizes=[1, 1],
+        process_1_expected_batch_sizes=[1, 1],
+    )
+
+    # without padding, we would expect the same number of batches, but different sizes
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=7,
+        batch_size=2,
+        process_0_expected_batch_sizes=[2, 2],
+        process_1_expected_batch_sizes=[2, 2],
+    )
+
+
+def test_can_disable_even_batches():
body: I've just pushed some changes that hopefully resolve this. Just FYI, it is getting pretty late here in the UK, and I am heading off on vacation tomorrow until the 31st. If there are other changes required, and you would like this merged before I'm back, please feel free to update the branch with any further changes!
diff_hunk: @@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import torch
+from torch.utils.data import DataLoader, TensorDataset
+
+from accelerate.accelerator import Accelerator
+
+
+def create_accelerator(even_batches=True):
+    accelerator = Accelerator(even_batches=even_batches)
+    assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
+    return accelerator
+
+
+def create_dataloader(accelerator: Accelerator, dataset_size: int, batch_size: int):
+    """
+    Create a simple DataLoader to use during the test cases
+    """
+    dataset = TensorDataset(torch.as_tensor(range(dataset_size)))
+
+    dl = DataLoader(dataset, batch_size=batch_size)
+    dl = accelerator.prepare(dl)
+
+    return dl
+
+
+def verify_dataloader_batch_sizes(
+    accelerator: Accelerator,
+    dataset_size: int,
+    batch_size: int,
+    process_0_expected_batch_sizes: List[int],
+    process_1_expected_batch_sizes: List[int],
+):
+    """
+    A helper function for verifying the batch sizes coming from a prepared dataloader in each process
+    """
+    dl = create_dataloader(accelerator=accelerator, dataset_size=dataset_size, batch_size=batch_size)
+
+    batch_sizes = [len(batch[0]) for batch in dl]
+
+    if accelerator.process_index == 0:
+        assert batch_sizes == process_0_expected_batch_sizes
+    elif accelerator.process_index == 1:
+        assert batch_sizes == process_1_expected_batch_sizes
+
+
+def test_default_ensures_even_batch_sizes():
+
+    accelerator = create_accelerator()
+
+    # without padding, we would expect a different number of batches
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=3,
+        batch_size=1,
+        process_0_expected_batch_sizes=[1, 1],
+        process_1_expected_batch_sizes=[1, 1],
+    )
+
+    # without padding, we would expect the same number of batches, but different sizes
+    verify_dataloader_batch_sizes(
+        accelerator,
+        dataset_size=7,
+        batch_size=2,
+        process_0_expected_batch_sizes=[2, 2],
+        process_1_expected_batch_sizes=[2, 2],
+    )
+
+
+def test_can_disable_even_batches():

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Hi, I have made a start on the work discussed in https://github.com/huggingface/accelerate/issues/684, and added an argument to the accelerator. 

I am slightly unsure about your testing strategy though, and which tests you require for this; I couldn't seem to find any integration tests for accelerator arguments. So far, I have tried to follow similar ideas to those seen in the end-to-end multi-GPU tests. 

I am slightly confused by exactly how the Multi-GPU tests are triggered though; whilst I see that there is a decorator to skip certain tests when multiple GPUs are not available, when I try to run this locally (on a machine with 2 GPUs), those tests only seem to execute on a single process.

Any guidance here would be appreciated! cc: @muellerzr @sgugger 
body: Hey @Chris-hughes10! Thanks!

You're absolutely right, I should probably write this all up in a contribution doc and will add it to my list of tasks!

Accelerates tests are setup in a few ways, but the key idea to remember is:

If it's something to be tested across multiple devices/workers, such as CPU, TPU, and mutli-GPU (such as your tests you want to add), these all get thrown into a script located in `accelerate/test_utils/scripts`

For example, the major integration script that checks the dataloaders operate properly is located at `accelerate/test_utils/scripts/test_script.py` (I wonder if we should section this out a bit more @sgugger since we now have a whole folder dedicated to it and we can clean it up some),  which you can find here: https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/scripts/test_script.py

These tests are setup like how a `pytest` test would be, where you have `def some_test` and `assert` statements to check if things are working properly. 

From here your script should be added as a test case to `tests/test.py`, `tests/test_cpu.py` (if applicable for multi-cpu),  `tests/test_multigpu.py`, and `tests/test_tpu.py`.

It has to be done this way because each platform has different launching configurations :) 

Finally, when running pytest you need to set the cuda visible devices for it to be able to pick up on the multiple GPUs, such as:

```bash
CUDA_VISIBLE_DEVICES="0,1" pytest -sv tests/test_multigpu.py
```
Or:
```bash
CUDA_VISIBLE_DEVICES="" pytest -sv tests/test_multigpu.py
```
(the latter will have all their tests skipped). 

All skipping decorator conditionals can be found here: https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/testing.py

Let me know if there are any other questions you have on this @Chris-hughes10, afterwards I can throw this all up into some documentation :) 
body: Hey @muellerzr, Thanks for the info!

So, it looks like I was on the right track regarding the structure, but I was running with unittests and not Pytest. I have refactored it to pytest, and it seems to work alright using `CUDA_VISIBLE_DEVICES="0,1" pytest -sv tests/test_multigpu.py, so it is probably ready for an initial review. 

I noticed that the main integration script seems to be doing a lot, so I created a new script to be super clear on what it is testing. The tests for the context manager and joining can probably live here too when implemented.

As an aside, how do you guys set up your development environment? I have set up a dev container, using the accelerate-gpu image, if it is useful, I could push that in another PR.
body: @Chris-hughes10 I use conda for mine, and the CI's all run based off the docker images as there's hosted computes we use
body: > @Chris-hughes10 I use conda for mine, and the CI's all run based off the conda images as there's hosted computes we use

Ok, if you are using conda envs, I think that this could be useful. I'll create a small PR and you guys can see what you think.
body: Hey @muellerzr, are there any further changes required on this?
body: @Chris-hughes10 close, your tests still weren't passing, and this is because `nproc_per_node` needed to be specified in the test function. (Discussing with Sylvain if this should be *everywhere* or not) but basically without that `Accelerator.num_processes` was only ever equal to one. 

Went ahead and pushed the fix :)
body: Thanks for all your work on this!