diff --git a/README.md b/README.md
index 280473ee5..6e2211143 100644
--- a/README.md
+++ b/README.md
@@ -163,8 +163,6 @@ notebook_launcher(training_function)
 
 An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb). [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb)
 
-Note that this launcher does not work in Jupyter Notebook on a machine with multiple GPUs (yet). This feature will be released in a later version.
-
 ## Why should I use 🤗 Accelerate?
 
 You should use 🤗 Accelerate when you want to easily run your training scripts in a distributed environment without having to renounce full control over your training loop. This is not a high-level framework above PyTorch, just a thin wrapper so you don't have to learn a new library, In fact the whole API of 🤗 Accelerate is in one class, the `Accelerator` object.
diff --git a/docs/source/launcher.rst b/docs/source/launcher.rst
index da28ad920..1c2279946 100644
--- a/docs/source/launcher.rst
+++ b/docs/source/launcher.rst
@@ -15,16 +15,15 @@ Notebook Launcher
 =======================================================================================================================
 
 Launch your training function inside a notebook. Currently supports launching a training with TPUs on [Google
-Colab](https://colab.research.google.com/) and [Kaggle kernels](https://www.kaggle.com/code), or training on one GPU,
-but support for training on several GPUs (if the machine on which you are running your notebook has them) is planned
-for a future release.
+Colab](https://colab.research.google.com/) and [Kaggle kernels](https://www.kaggle.com/code), as well as training on
+several GPUs (if the machine on which you are running your notebook has them).
 
 An example can be found in `this notebook
 <https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb>`__.
 
 .. warning::
 
-    If you are training on Colab or a Kaggle kernel with TPUs, your :obj:`Accelerator` object should only be defined
-    inside the training function. This is because the initialization should be done inside the launcher only.
+    Your :obj:`Accelerator` object should only be defined inside the training function. This is because the
+    initialization should be done inside the launcher only.
 
 .. autofunction:: accelerate.notebook_launcher
diff --git a/docs/source/quicktour.rst b/docs/source/quicktour.rst
index d9846b777..1e28fed85 100644
--- a/docs/source/quicktour.rst
+++ b/docs/source/quicktour.rst
@@ -199,9 +199,8 @@ Launching training from a notebook
 -----------------------------------------------------------------------------------------------------------------------
 
 In Accelerate 0.3.0, a new :class:`~accelerate.notebook_launcher` has been introduced to help you launch your training
-function from a notebook. Currently supports launching a training with TPUs on Colab, or training on one GPU, but
-support for training on several GPUs (if the machine on which you are running your notebook has them) is planned for a
-future release.
+function from a notebook. This launcher supports launching a training with TPUs on Colab or Kaggle, as well as training
+on several GPUs (if the machine on which you are running your notebook has them).
 
 Just define a function responsible for your whole training and/or evaluation in a cell of the notebook, then execute a
 cell with the following code:
@@ -214,8 +213,8 @@ cell with the following code:
 
 .. warning::
 
-    If you are training on Colab with TPUs, your :obj:`Accelerator` object should only be defined inside the training
-    function. This is because the initialization should be done inside the launcher only.
+    Your :obj:`Accelerator` object should only be defined inside the training function. This is because the
+    initialization should be done inside the launcher only.
 
 
 Training on TPU
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index e9dce7677..57f253267 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -14,15 +14,15 @@
 
 import os
 import sys
-import warnings
 
 import torch
+from torch.multiprocessing import start_processes
 
 from .state import AcceleratorState
 from .utils import PrepareForLaunch
 
 
-def notebook_launcher(function, args=(), num_processes=None, start_method="fork", **kwargs):
+def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use_port="29500"):
     """
     Launches a training function, using several processes if it's possible in the current environment (TPU with
     multiple cores for instance).
@@ -32,17 +32,15 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             The training function to execute. If it accepts arguments, the first argument should be the index of the
             process run.
         args (:obj:`Tuple`):
-            Tuple of arguments to pass to the function (it will receive :obj:`(index, *args)`).
+            Tuple of arguments to pass to the function (it will receive :obj:`*args`).
         num_processes (:obj:`int`, `optional`):
             The number of processes to use for training. Will default to 8 in Colab/Kaggle if a TPU is available, to
             the number of GPUs available otherwise.
-
-    .. warning::
-
-        Multiple GPUs is not yet supported.
+        use_fp16 (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If :obj:`True`, will use mixed precision training on multi-GPU.
+        use_port (:obj:`str`, `optional`, defaults to :obj:`"29500"`):
+            The port to use to communicate between processes when launching a multi-GPU training.
     """
-    launcher = PrepareForLaunch(function)
-
     # Are we in a google colab or a Kaggle Kernel?
     if "IPython" in sys.modules:
         in_colab_or_kaggle = "google.colab" in str(sys.modules["IPython"].get_ipython())
@@ -65,15 +63,60 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             if num_processes is None:
                 num_processes = 8
 
+            launcher = PrepareForLaunch(function, distributed_type="TPU")
+            print(f"Launching a training on {num_processes} TPU cores.")
             xmp.spawn(launcher, args=args, nprocs=num_processes, start_method="fork")
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            launcher(0, *args)
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
 
     else:
         if num_processes is None:
-            num_processes = torch.cuda.device_count() if torch.cuda.is_available() else 1
+            raise ValueError(
+                "You have to specify the number of GPUs you would like to use, add `num_process=...` to your call."
+            )
 
         if num_processes > 1:
-            warnings.warn("`notebook_launcher` does not support multiple GPUs yet, launching the training on one GPU.")
-        launcher(0, *args)
+            # Multi-GPU launch
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+
+            if torch.cuda.is_initialized():
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
+                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
+                    "function."
+                )
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            os.environ["WORLD_SIZE"] = str(num_processes)
+            os.environ["MASTER_ADDR"] = "127.0.0.1"
+            os.environ["MASTER_PORT"] = str(use_port)
+            os.environ["USE_FP16"] = str(use_fp16)
+
+            launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
+            try:
+                print(f"Launching a training on {num_processes} GPUs.")
+                start_processes(launcher, nprocs=num_processes, start_method="fork")
+            finally:
+                # Clean up the environment variables set.
+                del os.environ["WORLD_SIZE"]
+                del os.environ["MASTER_ADDR"]
+                del os.environ["MASTER_PORT"]
+
+        else:
+            # No need for a distributed launch otherwise as it's either CPU or one GPU.
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 0c2282b2c..5dfe85e54 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import importlib
-import inspect
+import os
 import random
 from enum import Enum
 from typing import List, Optional, Union
@@ -252,15 +252,23 @@ def save(obj, f):
 
 class PrepareForLaunch:
     """
-    Prepare a function that launches a script.
+    Prepare a function that will launched in a distributed setup.
+
+    Args:
+        launcher (:obj:`Callable`):
+            The function to launch.
+        distributed_type (:class:`~accelerate.state.DistributedType`):
+            The distributed type to prepare for.
     """
 
-    def __init__(self, launcher):
+    def __init__(self, launcher, distributed_type="NO"):
         self.launcher = launcher
+        self.distributed_type = DistributedType(distributed_type)
+
+    def __call__(self, index, *args):
+        if self.distributed_type == DistributedType.MULTI_GPU:
+            # Prepare the environment for torch.distributed
+            os.environ["LOCAL_RANK"] = str(index)
+            os.environ["RANK"] = str(index)
 
-    def __call__(self, index):
-        launcher_sig = inspect.signature(self.launcher)
-        if len(launcher_sig.parameters) == 0:
-            self.launcher()
-        else:
-            self.launcher(index)
+        self.launcher(*args)


Review comments: body: I am not that experienced with `torch.distributed`, but is it good to have them hardcoded and not overrideable? We could add parameters to adjust them. 

diff_hunk: @@ -65,15 +61,60 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             if num_processes is None:
                 num_processes = 8
 
+            launcher = PrepareForLaunch(function, distributed_type="TPU")
+            print(f"Launching a training on {num_processes} TPU cores.")
             xmp.spawn(launcher, args=args, nprocs=num_processes, start_method="fork")
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            launcher(0, *args)
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
 
     else:
         if num_processes is None:
-            num_processes = torch.cuda.device_count() if torch.cuda.is_available() else 1
+            raise ValueError(
+                "You have to specify the number of GPUs you would like to use, add `num_process=...` to your call."
+            )
 
         if num_processes > 1:
-            warnings.warn("`notebook_launcher` does not support multiple GPUs yet, launching the training on one GPU.")
-        launcher(0, *args)
+            # Multi-GPU launch
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+
+            if torch.cuda.is_initialized():
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
+                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
+                    "function."
+                )
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            os.environ["WORLD_SIZE"] = str(num_processes)
+            os.environ["MASTER_ADDR"] = "127.0.0.1"
+            os.environ["MASTER_PORT"] = "29500"
body: Those are the default values in `torch.distributed.launch`. The master address should not change since this does not support multi-node training. I can add a kwarg for the master port.
diff_hunk: @@ -65,15 +61,60 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             if num_processes is None:
                 num_processes = 8
 
+            launcher = PrepareForLaunch(function, distributed_type="TPU")
+            print(f"Launching a training on {num_processes} TPU cores.")
             xmp.spawn(launcher, args=args, nprocs=num_processes, start_method="fork")
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            launcher(0, *args)
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
 
     else:
         if num_processes is None:
-            num_processes = torch.cuda.device_count() if torch.cuda.is_available() else 1
+            raise ValueError(
+                "You have to specify the number of GPUs you would like to use, add `num_process=...` to your call."
+            )
 
         if num_processes > 1:
-            warnings.warn("`notebook_launcher` does not support multiple GPUs yet, launching the training on one GPU.")
-        launcher(0, *args)
+            # Multi-GPU launch
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+
+            if torch.cuda.is_initialized():
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
+                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
+                    "function."
+                )
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            os.environ["WORLD_SIZE"] = str(num_processes)
+            os.environ["MASTER_ADDR"] = "127.0.0.1"
+            os.environ["MASTER_PORT"] = "29500"
body: Yeah saw the default [here](https://github.com/pytorch/pytorch/blob/master/torch/distributed/elastic_launch.py#L394). But who knows what setup customers are using, e.g. IPV6, custom cnames etc. 
diff_hunk: @@ -65,15 +61,60 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             if num_processes is None:
                 num_processes = 8
 
+            launcher = PrepareForLaunch(function, distributed_type="TPU")
+            print(f"Launching a training on {num_processes} TPU cores.")
             xmp.spawn(launcher, args=args, nprocs=num_processes, start_method="fork")
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            launcher(0, *args)
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
 
     else:
         if num_processes is None:
-            num_processes = torch.cuda.device_count() if torch.cuda.is_available() else 1
+            raise ValueError(
+                "You have to specify the number of GPUs you would like to use, add `num_process=...` to your call."
+            )
 
         if num_processes > 1:
-            warnings.warn("`notebook_launcher` does not support multiple GPUs yet, launching the training on one GPU.")
-        launcher(0, *args)
+            # Multi-GPU launch
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+
+            if torch.cuda.is_initialized():
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
+                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
+                    "function."
+                )
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            os.environ["WORLD_SIZE"] = str(num_processes)
+            os.environ["MASTER_ADDR"] = "127.0.0.1"
+            os.environ["MASTER_PORT"] = "29500"
body: ```suggestion
            os.environ["MASTER_PORT"] = str(use_port)
```
Would add the extra conversion, might be else error prone
diff_hunk: @@ -65,15 +63,60 @@ def notebook_launcher(function, args=(), num_processes=None, start_method="fork"
             if num_processes is None:
                 num_processes = 8
 
+            launcher = PrepareForLaunch(function, distributed_type="TPU")
+            print(f"Launching a training on {num_processes} TPU cores.")
             xmp.spawn(launcher, args=args, nprocs=num_processes, start_method="fork")
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
-            launcher(0, *args)
+            if torch.cuda.is_available():
+                print("Launching training on one GPU.")
+            else:
+                print("Launching training on CPU.")
+            function(*args)
 
     else:
         if num_processes is None:
-            num_processes = torch.cuda.device_count() if torch.cuda.is_available() else 1
+            raise ValueError(
+                "You have to specify the number of GPUs you would like to use, add `num_process=...` to your call."
+            )
 
         if num_processes > 1:
-            warnings.warn("`notebook_launcher` does not support multiple GPUs yet, launching the training on one GPU.")
-        launcher(0, *args)
+            # Multi-GPU launch
+            if len(AcceleratorState._shared_state) > 0:
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
+                    "inside your training function. Restart your notebook and make sure no cells initializes an "
+                    "`Accelerator`."
+                )
+
+            if torch.cuda.is_initialized():
+                raise ValueError(
+                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
+                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
+                    "function."
+                )
+
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            os.environ["WORLD_SIZE"] = str(num_processes)
+            os.environ["MASTER_ADDR"] = "127.0.0.1"
+            os.environ["MASTER_PORT"] = use_port