diff --git a/docs/source/index.rst b/docs/source/index.rst
index 065329a1d..57933888d 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -145,4 +145,5 @@ Supported integrations
     :caption: API reference
 
     accelerator
+    kwargs
     internal
diff --git a/docs/source/kwargs.rst b/docs/source/kwargs.rst
new file mode 100644
index 000000000..61f42cd11
--- /dev/null
+++ b/docs/source/kwargs.rst
@@ -0,0 +1,30 @@
+.. 
+    Copyright 2021 The HuggingFace Team. All rights reserved.
+
+    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+    the License. You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations under the License.
+
+
+Kwargs Handlers
+=======================================================================================================================
+
+The following objects can be passed to the main :class:`~accelerate.Accelerator` to customize how some PyTorch objects
+related to distributed training or mixed precision are created.
+
+
+DistributedDataParallelKwargs
+-----------------------------------------------------------------------------------------------------------------------
+
+.. autoclass:: accelerate.DistributedDataParallelKwargs
+
+
+GradScalerKwargs
+-----------------------------------------------------------------------------------------------------------------------
+
+.. autoclass:: accelerate.GradScalerKwargs
diff --git a/examples/cv_example.py b/examples/cv_example.py
index 862fbce15..f4164613f 100644
--- a/examples/cv_example.py
+++ b/examples/cv_example.py
@@ -22,7 +22,6 @@
 from torch.utils.data import DataLoader, Dataset
 
 import PIL
-import torchvision
 from accelerate import Accelerator
 from timm import create_model
 from torchvision.transforms import Compose, RandomResizedCrop, Resize, ToTensor
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 13333c4a4..aad2b8b65 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 
+import torch
 from torch.utils.data import DataLoader
 
 from accelerate import Accelerator, DistributedType
diff --git a/src/accelerate/__init__.py b/src/accelerate/__init__.py
index 1cc471903..da4f7402e 100644
--- a/src/accelerate/__init__.py
+++ b/src/accelerate/__init__.py
@@ -5,5 +5,6 @@
 __version__ = "0.1.0"
 
 from .accelerator import Accelerator
+from .kwargs_handlers import DistributedDataParallelKwargs, GradScalerKwargs
 from .state import DistributedType
 from .utils import synchronize_rng_states
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 4707e0afe..d9f7cf1bc 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -19,6 +19,7 @@
 from packaging import version
 
 from .data_loader import prepare_data_loader
+from .kwargs_handlers import DistributedDataParallelKwargs, GradScalerKwargs, KwargsHandler
 from .optimizer import AcceleratedOptimizer
 from .state import AcceleratorState, DistributedType
 from .utils import RNGType, extract_model_from_parallel, gather, save, wait_for_everyone
@@ -56,6 +57,9 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
+        kwargs_handlers (list of kwargs handlers, `optional`)
+            A list of :obj:`KwargHandler` to customize how the objects related to distributed training or mixed
+            precision are created. See :doc:`kwargs` for more information.
 
     Attributes
 
@@ -70,18 +74,37 @@ def __init__(
         fp16: bool = None,
         cpu: bool = False,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
+        kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
         self.state = AcceleratorState(fp16=fp16, cpu=cpu, _from_accelerator=True)
 
         self.device_placement = device_placement
         self.split_batches = split_batches
 
+        # Kwargs handlers
+        self.ddp_handler = None
+        self.scaler_handler = None
+        if kwargs_handlers is not None:
+            for handler in kwargs_handlers:
+                assert isinstance(handler, KwargsHandler), f"Unsupported kwargs handler passed: {handler}."
+                if isinstance(handler, DistributedDataParallelKwargs):
+                    if self.ddp_handler is not None:
+                        raise ValueError("You can only pass one `DistributedDataParallelKwargs` in `kwargs_handler`.")
+                    else:
+                        self.ddp_handler = handler
+                elif isinstance(handler, GradScalerKwargs):
+                    if self.scaler_handler is not None:
+                        raise ValueError("You can only pass one `GradScalerKwargs` in `kwargs_handler`.")
+                    else:
+                        self.scaler_handler = handler
+
         # Mixed precision attributes
         self.scaler = None
         self.native_amp = False
         if self.state.use_fp16:
             self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
-            self.scaler = torch.cuda.amp.GradScaler()
+            kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
+            self.scaler = torch.cuda.amp.GradScaler(**kwargs)
 
         # Internal references to the training objects
         self._optimizers = []
@@ -187,16 +210,18 @@ def prepare(self, *args):
                 if isinstance(obj, torch.optim.Optimizer):
                     obj._switch_parameters(mapping)
 
-        return result
+        return result if len(result) > 1 else result[0]
 
     def prepare_model(self, model):
         if self.device_placement:
             model = model.to(self.device)
         if self.distributed_type == DistributedType.MULTI_GPU:
+            kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(
                 model,
                 device_ids=[self.local_process_index],
                 output_device=self.local_process_index,
+                **kwargs,
             )
         if self.native_amp:
             model.forward = torch.cuda.amp.autocast()(model.forward)
diff --git a/src/accelerate/kwargs_handlers.py b/src/accelerate/kwargs_handlers.py
new file mode 100644
index 000000000..7f52d6639
--- /dev/null
+++ b/src/accelerate/kwargs_handlers.py
@@ -0,0 +1,73 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+from dataclasses import dataclass
+
+
+class KwargsHandler:
+    """
+    Internal mixin that implements a :obj:`to_kwargs()` method for a dataclass.
+    """
+
+    def to_dict(self):
+        return copy.deepcopy(self.__dict__)
+
+    def to_kwargs(self):
+        """
+        Returns a dictionary containing the attributes with values different from the default of this class.
+        """
+        default_dict = self.__class__().to_dict()
+        this_dict = self.to_dict()
+        return {k: v for k, v in this_dict.items() if default_dict[k] != v}
+
+
+@dataclass
+class DistributedDataParallelKwargs(KwargsHandler):
+    """
+    Use this object in your :class:`~accelerate.Accelerator` to customize how your model is wrapped in a
+    :obj:`torch.nn.parallel.DistributedDataParallel`. Please refer to the documentation of this `wrapper
+    <https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html>`__ for more information
+    on each argument.
+
+    .. warning::
+
+        :obj:`gradient_as_bucket_view` is only available in PyTorch 1.7.0 and later versions.
+    """
+
+    dim: int = 0
+    broadcast_buffers: bool = True
+    bucket_cap_mb: int = 25
+    find_unused_parameters: bool = False
+    check_reduction: bool = False
+    gradient_as_bucket_view: bool = False
+
+
+@dataclass
+class GradScalerKwargs(KwargsHandler):
+    """
+    Use this object in your :class:`~accelerate.Accelerator` to customize the behavior of mixed precision, specifically
+    how the :obj:`torch.cuda.amp.GradScaler` used is created. Please refer to the documentation of this `scaler
+    <https://pytorch.org/docs/stable/amp.html?highlight=gradscaler>`__ for more information on each argument.
+
+    .. warning::
+
+        :obj:`GradScaler` is only available in PyTorch 1.5.0 and later versions.
+    """
+
+    init_scale: float = 65536.0
+    growth_factor: float = 2.0
+    backoff_factor: float = 0.5
+    growth_interval: int = 2000
+    enabled: bool = True
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index a8b95d2d2..55d00ab5d 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -64,7 +64,7 @@ def param_groups(self):
         return self.optimizer.param_groups
 
     @param_groups.setter
-    def param_group(self, param_groups):
+    def param_groups(self, param_groups):
         self.optimizer.param_groups = param_groups
 
     @property
@@ -72,7 +72,7 @@ def defaults(self):
         return self.optimizer.defaults
 
     @defaults.setter
-    def param_group(self, defaults):
+    def defaults(self, defaults):
         self.optimizer.defaults = defaults
 
     def add_param_group(self, param_group):
diff --git a/src/accelerate/test_utils/__init__.py b/src/accelerate/test_utils/__init__.py
index bb9f0f66e..03aef45b3 100644
--- a/src/accelerate/test_utils/__init__.py
+++ b/src/accelerate/test_utils/__init__.py
@@ -2,5 +2,5 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-from .testing import are_the_same_tensors, execute_subprocess_async, require_multi_gpu, require_tpu
+from .testing import are_the_same_tensors, execute_subprocess_async, require_cuda, require_multi_gpu, require_tpu
 from .training import RegressionDataset, RegressionModel
diff --git a/src/accelerate/test_utils/testing.py b/src/accelerate/test_utils/testing.py
index d01b5276e..507f16eb8 100644
--- a/src/accelerate/test_utils/testing.py
+++ b/src/accelerate/test_utils/testing.py
@@ -33,10 +33,19 @@ def are_the_same_tensors(tensor):
     return True
 
 
+def require_cuda(test_case):
+    """
+    Decorator marking a test that requires CUDA. These tests are skipped when there are no GPU available.
+    """
+    if not torch.cuda.is_available():
+        return unittest.skip("test requires a GPU")(test_case)
+    else:
+        return test_case
+
+
 def require_tpu(test_case):
     """
     Decorator marking a test that requires TPUs. These tests are skipped when there are no TPUs available.
-
     """
     if not is_tpu_available():
         return unittest.skip("test requires TPU")(test_case)
diff --git a/tests/test_kwargs_handlers.py b/tests/test_kwargs_handlers.py
new file mode 100644
index 000000000..aaf15de0c
--- /dev/null
+++ b/tests/test_kwargs_handlers.py
@@ -0,0 +1,96 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+import sys
+import unittest
+from dataclasses import dataclass
+
+import torch
+
+from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
+from accelerate.kwargs_handlers import KwargsHandler
+from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
+
+
+@dataclass
+class MockClass(KwargsHandler):
+    a: int = 0
+    b: bool = False
+    c: float = 3.0
+
+
+class DataLoaderTester(unittest.TestCase):
+    def test_kwargs_handler(self):
+        # If no defaults are changed, `to_kwargs` returns an empty dict.
+        self.assertDictEqual(MockClass().to_kwargs(), {})
+        self.assertDictEqual(MockClass(a=2).to_kwargs(), {"a": 2})
+        self.assertDictEqual(MockClass(a=2, b=True).to_kwargs(), {"a": 2, "b": True})
+        self.assertDictEqual(MockClass(a=2, c=2.25).to_kwargs(), {"a": 2, "c": 2.25})
+
+    @require_cuda
+    def test_grad_scaler_kwargs(self):
+        # If no defaults are changed, `to_kwargs` returns an empty dict.
+        scaler_handler = GradScalerKwargs(init_scale=1024, growth_factor=2)
+        accelerator = Accelerator(fp16=True, kwargs_handlers=[scaler_handler])
+        print(accelerator.use_fp16)
+        scaler = accelerator.scaler
+
+        # Check the kwargs have been applied
+        self.assertEqual(scaler._init_scale, 1024.0)
+        self.assertEqual(scaler._growth_factor, 2.0)
+
+        # Check the other values are at the default
+        self.assertEqual(scaler._backoff_factor, 0.5)
+        self.assertEqual(scaler._growth_interval, 2000)
+        self.assertEqual(scaler._enabled, True)
+
+    @require_multi_gpu
+    def test_ddp_kwargs(self):
+        distributed_args = f"""
+            -m torch.distributed.launch
+            --nproc_per_node={torch.cuda.device_count()}
+            --use_env
+            {inspect.getfile(self.__class__)}
+        """.split()
+        cmd = [sys.executable] + distributed_args
+        execute_subprocess_async(cmd, env=os.environ.copy())
+
+
+if __name__ == "__main__":
+    ddp_scaler = DistributedDataParallelKwargs(bucket_cap_mb=15, find_unused_parameters=True)
+    accelerator = Accelerator(kwargs_handlers=[ddp_scaler])
+    model = torch.nn.Linear(100, 200)
+    model = accelerator.prepare(model)
+
+    # Check the values changed in kwargs
+    error_msg = ""
+    observed_bucket_cap_map = model.bucket_bytes_cap // (1024 * 1024)
+    if observed_bucket_cap_map != 15:
+        error_msg += f"Kwargs badly passed, should have `15` but found {observed_bucket_cap_map}.\n"
+    if model.find_unused_parameters is not True:
+        error_msg += f"Kwargs badly passed, should have `True` but found {model.find_unused_parameters}.\n"
+
+    # Check the values of the defaults
+    if model.dim != 0:
+        error_msg += f"Default value not respected, should have `0` but found {model.dim}.\n"
+    if model.broadcast_buffers is not True:
+        error_msg += f"Default value not respected, should have `True` but found {model.broadcast_buffers}.\n"
+    if model.gradient_as_bucket_view is not False:
+        error_msg += f"Default value not respected, should have `False` but found {model.gradient_as_bucket_view}.\n"
+
+    # Raise error at the end to make sure we don't stop at the first failure.
+    if len(error_msg) > 0:
+        raise ValueError(error_msg)


Review comments: body: Unrelated to this PR, this is to make quality pass.
diff_hunk: @@ -22,7 +22,6 @@
 from torch.utils.data import DataLoader, Dataset
 
 import PIL
-import torchvision
body: Unrelated to this PR, this is to make quality pass.
diff_hunk: @@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 
+import torch
body: Unrelated to this PR, this is to make quality pass.
diff_hunk: @@ -64,15 +64,15 @@ def param_groups(self):
         return self.optimizer.param_groups
 
     @param_groups.setter
-    def param_group(self, param_groups):
+    def param_groups(self, param_groups):
body: Unrelated to this PR, this is to make quality pass.
diff_hunk: @@ -64,15 +64,15 @@ def param_groups(self):
         return self.optimizer.param_groups
 
     @param_groups.setter
-    def param_group(self, param_groups):
+    def param_groups(self, param_groups):
         self.optimizer.param_groups = param_groups
 
     @property
     def defaults(self):
         return self.optimizer.defaults
 
     @defaults.setter
-    def param_group(self, defaults):
+    def defaults(self, defaults):