diff --git a/.gitignore b/.gitignore
index 3b89c957d..f41377320 100644
--- a/.gitignore
+++ b/.gitignore
@@ -130,3 +130,6 @@ dmypy.json
 
 # VSCode
 .vscode
+
+# IntelliJ
+.idea
\ No newline at end of file
diff --git a/examples/cv_example.py b/examples/cv_example.py
index f4164613f..bb7c7f5b1 100644
--- a/examples/cv_example.py
+++ b/examples/cv_example.py
@@ -73,7 +73,7 @@ def __getitem__(self, idx):
 
 def training_function(config, args):
     # Initialize accelerator
-    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu)
+    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu, mixed_precision=args.mix_precision)
 
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     lr = config["lr"]
@@ -187,6 +187,15 @@ def main():
     parser = argparse.ArgumentParser(description="Simple example of training script.")
     parser.add_argument("--data_dir", required=True, help="The data folder on disk.")
     parser.add_argument("--fp16", action="store_true", help="If passed, will use FP16 training.")
+    parser.add_argument(
+        "--mixed_precision",
+        type=str,
+        default="no",
+        choices=["no", "fp16", "bf16"],
+        help="Whether to use mixed precision. Choose"
+        "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
+        "and an Nvidia Ampere GPU.",
+    )
     parser.add_argument("--cpu", action="store_true", help="If passed, will train on the CPU.")
     args = parser.parse_args()
     config = {"lr": 3e-2, "num_epochs": 3, "seed": 42, "batch_size": 64, "image_size": 224}
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index a1f310271..2f869bd0f 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -51,7 +51,7 @@
 
 def training_function(config, args):
     # Initialize accelerator
-    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu)
+    accelerator = Accelerator(fp16=args.fp16, cpu=args.cpu, mixed_precision=args.mixed_precision)
 
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     lr = config["lr"]
@@ -163,6 +163,15 @@ def collate_fn(examples):
 def main():
     parser = argparse.ArgumentParser(description="Simple example of training script.")
     parser.add_argument("--fp16", action="store_true", help="If passed, will use FP16 training.")
+    parser.add_argument(
+        "--mixed_precision",
+        type=str,
+        default="no",
+        choices=["no", "fp16", "bf16"],
+        help="Whether to use mixed precision. Choose"
+        "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
+        "and an Nvidia Ampere GPU.",
+    )
     parser.add_argument("--cpu", action="store_true", help="If passed, will train on the CPU.")
     args = parser.parse_args()
     config = {"lr": 2e-5, "num_epochs": 3, "correct_bias": True, "seed": 42, "batch_size": 16}
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 76c3067a3..ed3b2acb7 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -14,6 +14,7 @@
 
 import gc
 import os
+import warnings
 from contextlib import contextmanager
 from typing import List, Optional, Union
 
@@ -60,10 +61,11 @@ class Accelerator:
             :obj:`True` the actual batch size used will be the same on any kind of distributed processes, but it must
             be a round multiple of the :obj:`num_processes` you are using. If :obj:`False`, actual batch size used will
             be the one set in your script multiplied by the number of processes.
-        fp16 (:obj:`bool`, `optional`):
-            Whether or not to use mixed precision training. Will default to the value in the environment variable
-            :obj:`USE_FP16`, which will use the default value in the accelerate config of the current system or the
-            flag passed with the :obj:`accelerate.launch` command.
+        mixed_precision (:obj:`str`, `optional`):
+            Whether or not to use mixed precision training (fp16 or bfloat16). Choose from 'no','fp16','bf16'. Will
+            default to the value in the environment variable :obj:`MIXED_PRECISION`, which will use the default value
+            in the accelerate config of the current system or the flag passed with the :obj:`accelerate.launch`
+            command. 'fp16' requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
         cpu (:obj:`bool`, `optional`):
             Whether or not to force the script to execute on CPU. Will ignore GPU available if set to :obj:`True` and
             force the execution on one process only.
@@ -101,12 +103,25 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = None,
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
+
+        if mixed_precision is not None:
+            mixed_precision = mixed_precision.lower()
+            if mixed_precision not in ["no", "fp16", "bf16"]:
+                raise ValueError(
+                    f"Unknown mixed_precision mode: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
+                )
+
+        if fp16:
+            warnings.warn('fp16=True is deprecated. Use mixed_precision="fp16" instead.', DeprecationWarning)
+            mixed_precision = "fp16"
+
         if deepspeed_plugin is None:  # init from env variables
             deepspeed_plugin = DeepSpeedPlugin() if os.environ.get("USE_DEEPSPEED", "false") == "true" else None
         else:
@@ -139,7 +154,11 @@ def __init__(
 
         kwargs = self.init_handler.to_kwargs() if self.init_handler is not None else {}
         self.state = AcceleratorState(
-            fp16=fp16, cpu=cpu, deepspeed_plugin=deepspeed_plugin, _from_accelerator=True, **kwargs
+            mixed_precision=mixed_precision,
+            cpu=cpu,
+            deepspeed_plugin=deepspeed_plugin,
+            _from_accelerator=True,
+            **kwargs,
         )
 
         self.device_placement = device_placement
@@ -149,8 +168,18 @@ def __init__(
         # Mixed precision attributes
         self.scaler = None
         self.native_amp = False
-        if self.state.use_fp16:
+        if self.state.mixed_precision == "fp16":
             self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
+            if version.parse(torch.__version__) < version.parse("1.6"):
+                raise ValueError("fp16 mixed precision requires PyTorch >= 1.6")
+
+            kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
+            self.scaler = torch.cuda.amp.GradScaler(**kwargs)
+        elif self.state.mixed_precision == "bf16":
+            self.native_amp = version.parse(torch.__version__) >= version.parse("1.10")
+            if mixed_precision == "bf16" and version.parse(torch.__version__) < version.parse("1.10"):
+                raise ValueError("bf16 mixed precision requires PyTorch >= 1.10")
+
             kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
             self.scaler = torch.cuda.amp.GradScaler(**kwargs)
 
@@ -195,11 +224,20 @@ def is_local_main_process(self):
 
     @property
     def use_fp16(self):
+        return self.mixed_precision != "no"
+
+    @property
+    def mixed_precision(self):
         if self.distributed_type == DistributedType.DEEPSPEED:
-            use_fp16 = self.state.deepspeed_plugin.deepspeed_config["fp16"]["enabled"]
+            if self.state.deepspeed_plugin.deepspeed_config["fp16"]["enabled"]:
+                mixed_precision = "fp16"
+            elif self.state.deepspeed_plugin.deepspeed_config["bf16"]["enabled"]:
+                mixed_precision = "bf16"
+            else:
+                mixed_precision = "no"
         else:
-            use_fp16 = self.state.use_fp16
-        return use_fp16
+            mixed_precision = self.state.mixed_precision
+        return mixed_precision
 
     @contextmanager
     def local_main_process_first(self):
@@ -311,7 +349,12 @@ def prepare_model(self, model):
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(model, **kwargs)
         if self.native_amp:
-            model.forward = torch.cuda.amp.autocast()(model.forward)
+            if self.mixed_precision == "fp16" and version.parse(torch.__version__) >= version.parse("1.10"):
+                model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
+            elif self.mixed_precision == "bf16":
+                model.forward = torch.cuda.amp.autocast(dtype=torch.bfloat16)(model.forward)
+            else:
+                model.forward = torch.cuda.amp.autocast()(model.forward)
             model.forward = convert_outputs_to_fp32(model.forward)
         return model
 
@@ -577,7 +620,13 @@ def autocast(self):
         different will happen otherwise.
         """
         if self.native_amp:
-            autocast_context = torch.cuda.amp.autocast()
+            if self.mixed_precision == "fp16" and version.parse(torch.__version__) >= version.parse("1.10"):
+                autocast_context = torch.cuda.amp.autocast(dtype=torch.float16)
+            elif self.mixed_precision == "bf16":
+                autocast_context = torch.cuda.amp.autocast(dtype=torch.bfloat16)
+            else:
+                autocast_context = torch.cuda.amp.autocast()
+
             autocast_context.__enter__()
             yield
             autocast_context.__exit__()
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index eaaad20f2..534ea1526 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -103,20 +103,19 @@ def get_cluster_input():
     )
 
     if distributed_type != DistributedType.TPU:
-        fp16 = _ask_field(
-            "Do you wish to use FP16 (mixed precision)? [yes/NO]: ",
-            _convert_yes_no_to_bool,
-            default=False,
-            error_message="Please enter yes or no.",
+        mixed_precision = _ask_field(
+            "Do you wish to use FP16 or BF16 (mixed precision)? [NO/fp16/bf16]: ",
+            lambda x: str(x).lower(),
+            default="no",
         )
     else:
-        fp16 = False
+        mixed_precision = "no"
 
     return ClusterConfig(
         compute_environment=ComputeEnvironment.LOCAL_MACHINE,
         distributed_type=distributed_type,
         num_processes=num_processes,
-        fp16=fp16,
+        mixed_precision=mixed_precision,
         machine_rank=machine_rank,
         num_machines=num_machines,
         main_process_ip=main_process_ip,
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 39c00c280..891580ab5 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -66,7 +66,7 @@ def load_config_from_file(config_file):
 class BaseConfig:
     compute_environment: ComputeEnvironment
     distributed_type: Union[DistributedType, SageMakerDistributedType]
-    fp16: bool
+    mixed_precision: str
 
     def to_dict(self):
         result = self.__dict__
@@ -83,6 +83,10 @@ def from_json_file(cls, json_file=None):
             config_dict = json.load(f)
         if "compute_environment" not in config_dict:
             config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        if "mixed_precision" not in config_dict:
+            config_dict["mixed_precision"] = "fp16" if ("fp16" in config_dict and config_dict["fp16"]) else "no"
+        if "fp16" in config_dict:  # Convert the config to the new format.
+            del config_dict["fp16"]
         return cls(**config_dict)
 
     def to_json_file(self, json_file):
@@ -97,6 +101,12 @@ def from_yaml_file(cls, yaml_file=None):
             config_dict = yaml.safe_load(f)
         if "compute_environment" not in config_dict:
             config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+
+        if "mixed_precision" not in config_dict:
+            config_dict["mixed_precision"] = "fp16" if ("fp16" in config_dict and config_dict["fp16"]) else "no"
+        if "fp16" in config_dict:  # Convert the config to the new format.
+            del config_dict["fp16"]
+
         return cls(**config_dict)
 
     def to_yaml_file(self, yaml_file):
diff --git a/src/accelerate/commands/config/sagemaker.py b/src/accelerate/commands/config/sagemaker.py
index feff0aecf..3b0fc78dd 100644
--- a/src/accelerate/commands/config/sagemaker.py
+++ b/src/accelerate/commands/config/sagemaker.py
@@ -20,7 +20,7 @@
 from accelerate.utils import is_boto3_available
 
 from .config_args import SageMakerConfig
-from .config_utils import _ask_field, _convert_sagemaker_distributed_mode, _convert_yes_no_to_bool
+from .config_utils import _ask_field, _convert_sagemaker_distributed_mode
 
 
 if is_boto3_available():
@@ -139,11 +139,11 @@ def get_sagemaker_input():
             lambda x: int(x),
             default=2,
         )
-    fp16 = _ask_field(
-        "Do you wish to use FP16 (mixed precision)? [yes/NO]: ",
-        _convert_yes_no_to_bool,
-        default=False,
-        error_message="Please enter yes or no.",
+
+    mixed_precision = _ask_field(
+        "Do you wish to use FP16 or BF16 (mixed precision)? [No/FP16/BF16]: ",
+        lambda x: str(x),
+        default="No",
     )
 
     return SageMakerConfig(
@@ -153,6 +153,6 @@ def get_sagemaker_input():
         profile=aws_profile,
         region=aws_region,
         iam_role_name=iam_role_name,
-        fp16=fp16,
+        mixed_precision=mixed_precision,
         num_machines=num_machines,
     )
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 4f392d6fa..77db4546b 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -19,6 +19,7 @@
 import os
 import subprocess
 import sys
+import warnings
 from ast import literal_eval
 from pathlib import Path
 from typing import Dict, List
@@ -53,6 +54,16 @@ def launch_command_parser(subparsers=None):
     parser.add_argument(
         "--tpu", default=False, action="store_true", help="Whether or not this should launch a TPU training."
     )
+    parser.add_argument(
+        "--mixed_precision",
+        default="no",
+        type=str,
+        choices=["no", "fp16", "bf16"],
+        help="Whether or not to use mixed precision training. "
+        "Choose between FP16 and BF16 (bfloat16) training. "
+        "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.",
+    )
+
     parser.add_argument(
         "--fp16", default=False, action="store_true", help="Whether or not to use mixed precision training."
     )
@@ -134,7 +145,16 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
-    current_env["USE_FP16"] = str(args.fp16)
+
+    mixed_precision = args.mixed_precision.lower()
+    if mixed_precision not in ["no", "fp16", "bf16"]:
+        raise ValueError(f"Unknown mixed_precision mode: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'.")
+
+    if args.fp16:
+        warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
+        mixed_precision = "fp16"
+
+    current_env["MIXED_PRECISION"] = str(mixed_precision)
 
     process = subprocess.Popen(cmd, env=current_env)
     process.wait()
@@ -168,7 +188,16 @@ def multi_gpu_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in ["no", "fp16", "bf16"]:
+        raise ValueError(f"Unknown mixed_precision mode: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'.")
+
+    if args.fp16:
+        warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
+        mixed_precision = "fp16"
+
+    current_env["MIXED_PRECISION"] = str(mixed_precision)
 
     process = subprocess.Popen(cmd, env=current_env)
     process.wait()
@@ -201,7 +230,16 @@ def deepspeed_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in ["no", "fp16", "bf16"]:
+        raise ValueError(f"Unknown mixed_precision mode: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'.")
+
+    if args.fp16:
+        warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
+        mixed_precision = "fp16"
+
+    current_env["MIXED_PRECISION"] = str(mixed_precision)
     current_env["USE_DEEPSPEED"] = "true"
     current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage)
     current_env["GRADIENT_ACCUMULATION_STEPS"] = str(args.gradient_accumulation_steps)
@@ -309,8 +347,17 @@ def sagemaker_launcher(sagemaker_config: SageMakerConfig, args):
     print("Converting Arguments to Hyperparameters")
     hyperparameters = _convert_nargs_to_dict(args.training_script_args)
 
-    environment = {"USE_FP16": args.fp16}  # Environment variables to be set for use during training job
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in ["no", "fp16", "bf16"]:
+        raise ValueError(f"Unknown mixed_precision mode: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'.")
 
+    if args.fp16:
+        warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
+        mixed_precision = "fp16"
+
+    # Environment variables to be set for use during training job
+    environment = {"MIXED_PRECISION": str(mixed_precision)}
     # configure distribution set up
     distribution = None  # TODO: not yet implemented
 
@@ -360,13 +407,16 @@ def launch_command(args):
 
                 # Those args are handled separately
                 if (
-                    name not in ["compute_environment", "fp16", "distributed_type"]
+                    name not in ["compute_environment", "fp16", "mixed_precision", "distributed_type"]
                     and getattr(args, name, None) is None
                 ):
                     setattr(args, name, attr)
 
-        if not args.fp16:
-            args.fp16 = defaults.fp16
+        if not args.mixed_precision:
+            if args.fp16:
+                args.mixed_precision = "fp16"
+            else:
+                args.mixed_precision = defaults.mixed_precision
     else:
         if args.num_processes is None:
             args.num_processes = 1
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/notebook_launcher.py
index b848e13d4..90b4f826a 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/notebook_launcher.py
@@ -14,6 +14,7 @@
 
 import os
 import sys
+import warnings
 
 import torch
 
@@ -23,7 +24,7 @@
 from .utils import PrepareForLaunch
 
 
-def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use_port="29500"):
+def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mixed_precision="no", use_port="29500"):
     """
     Launches a training function, using several processes if it's possible in the current environment (TPU with
     multiple cores for instance).
@@ -37,8 +38,8 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
         num_processes (:obj:`int`, `optional`):
             The number of processes to use for training. Will default to 8 in Colab/Kaggle if a TPU is available, to
             the number of GPUs available otherwise.
-        use_fp16 (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            If :obj:`True`, will use mixed precision training on multi-GPU.
+        mixed_precision (:obj:`str`, `optional`, defaults to :obj:`no`):
+            If :obj:`fp16` or :obj:`bf16`, will use mixed precision training on multi-GPU.
         use_port (:obj:`str`, `optional`, defaults to :obj:`"29500"`):
             The port to use to communicate between processes when launching a multi-GPU training.
     """
@@ -110,7 +111,18 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             os.environ["WORLD_SIZE"] = str(num_processes)
             os.environ["MASTER_ADDR"] = "127.0.0.1"
             os.environ["MASTER_PORT"] = str(use_port)
-            os.environ["USE_FP16"] = str(use_fp16)
+
+            mixed_precision = mixed_precision.lower()
+            if mixed_precision not in ["no", "fp16", "bf16"]:
+                raise ValueError(
+                    f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
+                )
+
+            if use_fp16:
+                warnings.warn('use_fp16=True is deprecated. Use mixed_precision="fp16" instead.', DeprecationWarning)
+                mixed_precision = "fp16"
+
+            os.environ["MIXED_PRECISION"] = str(mixed_precision)
 
             launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
             try:
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index e4d536e14..6693a34ca 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -66,6 +66,11 @@ def parse_flag_from_env(key, default=False):
     return strtobool(value) == 1  # As its name indicates `strtobool` actually returns an int...
 
 
+def parse_choice_from_env(key, default="no"):
+    value = os.environ.get(key, str(default))
+    return value
+
+
 class DistributedType(str, Enum):
     """
     Represents a type of distributed environment.
@@ -133,18 +138,25 @@ class AcceleratorState:
         - **num_processes** (:obj:`int`) -- The number of processes currently launched in parallel.
         - **process_index** (:obj:`int`) -- The index of the current process.
         - **local_process_index** (:obj:`int`) -- The index of the current process on the current server.
-        - **use_fp16** (:obj:`bool`) -- Whether or not the current script will use mixed precision.
+        - **mixed_precision** (:obj:`str`) -- Whether or not the current script will use mixed precision. If you are
+          using mixed precision, define if you want to use FP16 or BF16 (bfloat16) as the floating point.
     """
 
     _shared_state = {}
 
     def __init__(
-        self, fp16: bool = None, cpu: bool = False, deepspeed_plugin=None, _from_accelerator: bool = False, **kwargs
+        self,
+        mixed_precision: str = None,
+        cpu: bool = False,
+        deepspeed_plugin=None,
+        _from_accelerator: bool = False,
+        **kwargs,
     ):
         self.__dict__ = self._shared_state
         if not getattr(self, "initialized", False):
             self.backend = None
             self.deepspeed_plugin = None
+            mixed_precision = mixed_precision.lower() if mixed_precision else None
             if not _from_accelerator:
                 raise ValueError(
                     "Please make sure to properly initialize your accelerator via `accelerator = Accelerator()` "
@@ -156,7 +168,7 @@ def __init__(
                 self.process_index = xm.get_ordinal()
                 self.local_process_index = xm.get_local_ordinal()
                 self.device = xm.xla_device()
-                self.use_fp16 = False
+                self.mixed_precision = "no"
             elif os.environ.get("USE_DEEPSPEED", "false") == "true" and not cpu:
                 assert (
                     is_deepspeed_available()
@@ -170,9 +182,14 @@ def __init__(
                 self.local_process_index = int(os.environ.get("LOCAL_RANK", -1))
                 self.device = torch.device("cuda", self.local_process_index)
                 torch.cuda.set_device(self.device)
-                self.use_fp16 = False  # deepspeed handles fp16 using deepspeed_config
-                fp16 = parse_flag_from_env("USE_FP16", False) if fp16 is None else fp16
-                deepspeed_plugin.deepspeed_config.update({"fp16": {"enabled": fp16}})
+                self.mixed_precision = "no"  # deepspeed handles mixed_precision using deepspeed_config
+                mixed_precision = (
+                    parse_choice_from_env("MIXED_PRECISION", "no") if mixed_precision is None else mixed_precision
+                )
+                if mixed_precision == "fp16":
+                    deepspeed_plugin.deepspeed_config.update({"fp16": {"enabled": True}})
+                elif mixed_precision == "bf16":
+                    deepspeed_plugin.deepspeed_config.update({"bfloat16": {"enabled": True}})
                 self.deepspeed_plugin = deepspeed_plugin
             elif int(os.environ.get("LOCAL_RANK", -1)) != -1 and not cpu:
                 self.distributed_type = DistributedType.MULTI_GPU
@@ -184,7 +201,10 @@ def __init__(
                 self.local_process_index = int(os.environ.get("LOCAL_RANK", -1))
                 self.device = torch.device("cuda", self.local_process_index)
                 torch.cuda.set_device(self.device)
-                self.use_fp16 = parse_flag_from_env("USE_FP16", False) if fp16 is None else fp16
+                self.mixed_precision = (
+                    parse_choice_from_env("MIXED_PRECISION", "no") if mixed_precision is None else mixed_precision
+                )
+
             elif get_int_from_env(["PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE", "WORLD_SIZE"], 1) > 1:
                 self.distributed_type = DistributedType.MULTI_CPU
                 if is_ccl_available() and get_int_from_env(["CCL_WORKER_COUNT"], 0) > 0:
@@ -221,24 +241,32 @@ def __init__(
                 self.process_index = torch.distributed.get_rank()
                 self.local_process_index = local_rank
                 self.device = torch.device("cpu")
-                self.use_fp16 = False
+                self.mixed_precision = "no"
             else:
                 self.distributed_type = DistributedType.NO
                 self.num_processes = 1
                 self.process_index = self.local_process_index = 0
                 self.device = torch.device("cuda" if torch.cuda.is_available() and not cpu else "cpu")
-                self.use_fp16 = parse_flag_from_env("USE_FP16", False) if fp16 is None else fp16
+                self.mixed_precision = (
+                    parse_choice_from_env("MIXED_PRECISION", "no") if mixed_precision is None else mixed_precision
+                )
             self.initialized = True
 
     def __repr__(self):
-        use_fp16 = self.deepspeed_plugin.fp16 if self.distributed_type == DistributedType.DEEPSPEED else self.use_fp16
+        mixed_precision = self.mixed_precision
+        if self.distributed_type == DistributedType.DEEPSPEED:
+            if self.deepspeed_plugin.fp16:
+                mixed_precision = "fp16"
+            if self.deepspeed_plugin.bflaot16:
+                mixed_precision = "bf16"
+
         repr = (
             f"Distributed environment: {self.distributed_type}{('  Backend: ' + self.backend) if self.backend else ''}\n"
             f"Num processes: {self.num_processes}\n"
             f"Process index: {self.process_index}\n"
             f"Local process index: {self.local_process_index}\n"
             f"Device: {self.device}\n"
-            f"Use FP16 precision: {use_fp16}\n"
+            f"Mixed precision type: {mixed_precision}\n"
         )
         if self.distributed_type == DistributedType.DEEPSPEED:
             repr += f"ds_config: {self.deepspeed_plugin.ds_config}\n"
diff --git a/src/accelerate/test_utils/test_script.py b/src/accelerate/test_utils/test_script.py
index 41efcf61c..ecd8e4f02 100644
--- a/src/accelerate/test_utils/test_script.py
+++ b/src/accelerate/test_utils/test_script.py
@@ -245,6 +245,29 @@ def training_check():
     accelerator.print("Training yielded the same results on one CPU or distributes setup with batch split.")
 
     # Mostly a test that FP16 doesn't crash as the operation inside the model is not converted to FP16
+    print("FP16 training check.")
+    accelerator = Accelerator(mixed_precision="fp16")
+    train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
+    model = RegressionModel()
+    optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
+
+    train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer)
+    set_seed(42)
+    generator.manual_seed(42)
+    for _ in range(3):
+        for batch in train_dl:
+            model.zero_grad()
+            output = model(batch["x"])
+            loss = torch.nn.functional.mse_loss(output, batch["y"])
+            accelerator.backward(loss)
+            optimizer.step()
+
+    model = accelerator.unwrap_model(model).cpu()
+    assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
+    assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
+
+    # TEST that previous fp16 flag still works
+    print("Legacy FP16 training check.")
     accelerator = Accelerator(fp16=True)
     train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
     model = RegressionModel()
@@ -265,6 +288,28 @@ def training_check():
     assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
     assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
 
+    # Mostly a test that BF16 doesn't crash as the operation inside the model is not converted to BF16
+    print("BF16 training check.")
+    accelerator = Accelerator(mixed_precision="bf16")
+    train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
+    model = RegressionModel()
+    optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
+
+    train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer)
+    set_seed(42)
+    generator.manual_seed(42)
+    for _ in range(3):
+        for batch in train_dl:
+            model.zero_grad()
+            output = model(batch["x"])
+            loss = torch.nn.functional.mse_loss(output, batch["y"])
+            accelerator.backward(loss)
+            optimizer.step()
+
+    model = accelerator.unwrap_model(model).cpu()
+    assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
+    assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
+
 
 def main():
     accelerator = Accelerator()
diff --git a/src/accelerate/test_utils/training.py b/src/accelerate/test_utils/training.py
index 3cdea4517..2ea86fbb1 100644
--- a/src/accelerate/test_utils/training.py
+++ b/src/accelerate/test_utils/training.py
@@ -36,6 +36,10 @@ def __init__(self, a=0, b=0, double_output=False):
         super().__init__()
         self.a = torch.nn.Parameter(torch.tensor(a).float())
         self.b = torch.nn.Parameter(torch.tensor(b).float())
+        self.first_batch = True
 
     def forward(self, x=None):
+        if self.first_batch:
+            print(f"Model dtype: {self.a.dtype}, {self.b.dtype}. Input dtype: {x.dtype}")
+            self.first_batch = False
         return x * self.a + self.b
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 991997282..3af792f1f 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -23,6 +23,8 @@
 import numpy as np
 import torch
 
+from packaging import version
+
 from .state import AcceleratorState, DistributedType, is_deepspeed_available, is_tpu_available
 
 
@@ -240,23 +242,26 @@ def _initialize_tensor(tensor_info):
 
 def convert_to_fp32(tensor):
     """
-    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16 precision to FP32.
+    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16/BF16 precision to FP32.
 
     Args:
         tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
-            The data to convert from FP16 to FP32.
+            The data to convert from FP16/BF16 to FP32.
 
     Returns:
-        The same data structure as :obj:`tensor` with all tensors that were in FP16 precision converted to FP32.
+        The same data structure as :obj:`tensor` with all tensors that were in FP16/BF16 precision converted to FP32.
     """
 
     def _convert_to_fp32(tensor):
         return tensor.float()
 
-    def _is_fp16_tensor(tensor):
-        return hasattr(tensor, "dtype") and tensor.dtype == torch.float16
+    def _is_fp16_bf16_tensor(tensor):
+        return hasattr(tensor, "dtype") and (
+            tensor.dtype == torch.float16
+            or (version.parse(torch.__version__) >= version.parse("1.10") and tensor.dtype == torch.bfloat16)
+        )
 
-    return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_tensor)
+    return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_bf16_tensor)
 
 
 def convert_outputs_to_fp32(model_forward):


Review comments: body: Default here should be `None`, and then get the value of `MIXED_PRECISION` (as is done for `fp16`).
diff_hunk: @@ -101,12 +106,24 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = "no",
body: ```suggestion
            warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
```
We're not inside a script, so I would adapt to use arguments: `fp16=True`is deprecated, use `mixed_precision="fp16"` instead.
diff_hunk: @@ -101,12 +106,24 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = "no",
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
+        mixed_precision = mixed_precision.lower()
+        assert mixed_precision in [
+            "no",
+            "fp16",
+            "bf16",
+        ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
+
+        if fp16:
+            warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
body: Rephrase as a test and raise a ValueError:
```
if mixed_precision not in [...]:
    raise ValueError(...)
```
diff_hunk: @@ -101,12 +106,24 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = "no",
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
+        mixed_precision = mixed_precision.lower()
+        assert mixed_precision in [
+            "no",
+            "fp16",
+            "bf16",
+        ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: ```suggestion
        return self.mixed_precision != "no":
```
Seems easier this way.
diff_hunk: @@ -195,11 +220,23 @@ def is_local_main_process(self):
 
     @property
     def use_fp16(self):
+        if self.mixed_precision != "no":
+            return True
+        else:
+            return False
body: We have to be a bit more careful here and add a version test, as the argument `dtype` was only introduce in PyTorch 1.10.  So this first test should include
```
and version.parse(torch.__version__) >= version.parse("1.10")
```
otherwise we will fall in the else branch at the end.
diff_hunk: @@ -311,7 +348,12 @@ def prepare_model(self, model):
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(model, **kwargs)
         if self.native_amp:
-            model.forward = torch.cuda.amp.autocast()(model.forward)
+            if self.mixed_precision == "fp16":
+                model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
body: Same here.
diff_hunk: @@ -577,7 +619,13 @@ def autocast(self):
         different will happen otherwise.
         """
         if self.native_amp:
-            autocast_context = torch.cuda.amp.autocast()
+            if self.mixed_precision == "fp16":
body: It should also be added in the `from_json_file` method above for backward compatibility, and I think we should remove the fp16 key as well, to fully convert the config to the new format.
diff_hunk: @@ -97,6 +98,12 @@ def from_yaml_file(cls, yaml_file=None):
             config_dict = yaml.safe_load(f)
         if "compute_environment" not in config_dict:
             config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        if "mixed_precision" not in config_dict:
body: As before, use a test and raise a ValueError here.
diff_hunk: @@ -134,7 +146,19 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
-    current_env["USE_FP16"] = str(args.fp16)
+
+    mixed_precision = args.mixed_precision.lower()
+    assert mixed_precision in [
+        "no",
+        "fp16",
+        "bf16",
+    ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: Same here.
diff_hunk: @@ -168,7 +192,18 @@ def multi_gpu_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+    assert mixed_precision in [
+        "no",
+        "fp16",
+        "bf16",
+    ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: Same here.
diff_hunk: @@ -201,7 +236,18 @@ def deepspeed_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+    assert mixed_precision in [
+        "no",
+        "fp16",
+        "bf16",
+    ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: And same here.
diff_hunk: @@ -309,8 +355,20 @@ def sagemaker_launcher(sagemaker_config: SageMakerConfig, args):
     print("Converting Arguments to Hyperparameters")
     hyperparameters = _convert_nargs_to_dict(args.training_script_args)
 
-    environment = {"USE_FP16": args.fp16}  # Environment variables to be set for use during training job
+    mixed_precision = args.mixed_precision.lower()
+    assert mixed_precision in [
+        "no",
+        "fp16",
+        "bf16",
+    ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: Nit, put the comment before and have the dict on one line:
```suggestion
    # Environment variables to be set for use during training job
    environment = {"MIXED_PRECISION": str(mixed_precision)}
```
diff_hunk: @@ -309,8 +355,20 @@ def sagemaker_launcher(sagemaker_config: SageMakerConfig, args):
     print("Converting Arguments to Hyperparameters")
     hyperparameters = _convert_nargs_to_dict(args.training_script_args)
 
-    environment = {"USE_FP16": args.fp16}  # Environment variables to be set for use during training job
+    mixed_precision = args.mixed_precision.lower()
+    assert mixed_precision in [
+        "no",
+        "fp16",
+        "bf16",
+    ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
+
+    if args.fp16:
+        warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
+        mixed_precision = "fp16"
 
+    environment = {
+        "MIXED_PRECISION": str(mixed_precision)
+    }  # Environment variables to be set for use during training job
body: As before, raise a value error here.
diff_hunk: @@ -110,7 +113,19 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             os.environ["WORLD_SIZE"] = str(num_processes)
             os.environ["MASTER_ADDR"] = "127.0.0.1"
             os.environ["MASTER_PORT"] = str(use_port)
-            os.environ["USE_FP16"] = str(use_fp16)
+
+            mixed_precision = mixed_precision.lower()
+            assert mixed_precision in [
+                "no",
+                "fp16",
+                "bf16",
+            ], f"Unknown mixed_precision: {mixed_precision}. Choose between 'no', 'fp16' and 'bf16'."
body: Should be `None` here
diff_hunk: @@ -133,18 +138,25 @@ class AcceleratorState:
         - **num_processes** (:obj:`int`) -- The number of processes currently launched in parallel.
         - **process_index** (:obj:`int`) -- The index of the current process.
         - **local_process_index** (:obj:`int`) -- The index of the current process on the current server.
-        - **use_fp16** (:obj:`bool`) -- Whether or not the current script will use mixed precision.
+        - **mixed_precision** (:obj:`str`) -- Whether or not the current script will use mixed precision. If
+          you are using mixed precision, define if you want to use FP16 or BF16 (bfloat16) as the floating point.
     """
 
     _shared_state = {}
 
     def __init__(
-        self, fp16: bool = None, cpu: bool = False, deepspeed_plugin=None, _from_accelerator: bool = False, **kwargs
+        self,
+        mixed_precision: str = "no",
body: Test for None before lowering there.
diff_hunk: @@ -133,18 +138,25 @@ class AcceleratorState:
         - **num_processes** (:obj:`int`) -- The number of processes currently launched in parallel.
         - **process_index** (:obj:`int`) -- The index of the current process.
         - **local_process_index** (:obj:`int`) -- The index of the current process on the current server.
-        - **use_fp16** (:obj:`bool`) -- Whether or not the current script will use mixed precision.
+        - **mixed_precision** (:obj:`str`) -- Whether or not the current script will use mixed precision. If
+          you are using mixed precision, define if you want to use FP16 or BF16 (bfloat16) as the floating point.
     """
 
     _shared_state = {}
 
     def __init__(
-        self, fp16: bool = None, cpu: bool = False, deepspeed_plugin=None, _from_accelerator: bool = False, **kwargs
+        self,
+        mixed_precision: str = "no",
+        cpu: bool = False,
+        deepspeed_plugin=None,
+        _from_accelerator: bool = False,
+        **kwargs,
     ):
         self.__dict__ = self._shared_state
         if not getattr(self, "initialized", False):
             self.backend = None
             self.deepspeed_plugin = None
+            mixed_precision = mixed_precision.lower()
body: Get the env variable when `mixed_precision` is None
diff_hunk: @@ -170,9 +182,14 @@ def __init__(
                 self.local_process_index = int(os.environ.get("LOCAL_RANK", -1))
                 self.device = torch.device("cuda", self.local_process_index)
                 torch.cuda.set_device(self.device)
-                self.use_fp16 = False  # deepspeed handles fp16 using deepspeed_config
-                fp16 = parse_flag_from_env("USE_FP16", False) if fp16 is None else fp16
-                deepspeed_plugin.deepspeed_config.update({"fp16": {"enabled": fp16}})
+                self.mixed_precision = "no"  # deepspeed handles mixed_precision using deepspeed_config
+                mixed_precision = (
+                    parse_choice_from_env("MIXED_PRECISION", "no") if mixed_precision == "no" else mixed_precision
body: This would also need a version check, as I'm not sure the `torch.bfloat16` was there in PyTorch 1.6.
diff_hunk: @@ -240,23 +240,23 @@ def _initialize_tensor(tensor_info):
 
 def convert_to_fp32(tensor):
     """
-    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16 precision to FP32.
+    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16/BF16 precision to FP32.
 
     Args:
         tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
-            The data to convert from FP16 to FP32.
+            The data to convert from FP16/BF16 to FP32.
 
     Returns:
-        The same data structure as :obj:`tensor` with all tensors that were in FP16 precision converted to FP32.
+        The same data structure as :obj:`tensor` with all tensors that were in FP16/BF16 precision converted to FP32.
     """
 
     def _convert_to_fp32(tensor):
         return tensor.float()
 
-    def _is_fp16_tensor(tensor):
-        return hasattr(tensor, "dtype") and tensor.dtype == torch.float16
+    def _is_fp16_bf16_tensor(tensor):
+        return hasattr(tensor, "dtype") and (tensor.dtype == torch.float16 or tensor.dtype == torch.bfloat16)
body: Version test added 
diff_hunk: @@ -311,7 +348,12 @@ def prepare_model(self, model):
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(model, **kwargs)
         if self.native_amp:
-            model.forward = torch.cuda.amp.autocast()(model.forward)
+            if self.mixed_precision == "fp16":
+                model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
body: I forgot about the "from_json_file" method, i have updated it too and I added a line to delete the fp16 key from old config files
diff_hunk: @@ -97,6 +98,12 @@ def from_yaml_file(cls, yaml_file=None):
             config_dict = yaml.safe_load(f)
         if "compute_environment" not in config_dict:
             config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        if "mixed_precision" not in config_dict:
body: ```suggestion
        if mixed_precision is not None:
```
We usually prefer explicit tests instead of relying on Python's bool conversion.
diff_hunk: @@ -101,12 +106,29 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = None,
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
+
+        if mixed_precision:
body: ```suggestion
            if mixed_precision not in ["no", "fp16", "bf16"]:
```
Should fit in one line now.
diff_hunk: @@ -101,12 +106,29 @@ def __init__(
         device_placement: bool = True,
         split_batches: bool = False,
         fp16: bool = None,
+        mixed_precision: str = None,
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
+
+        if mixed_precision:
+            mixed_precision = mixed_precision.lower()
+            if mixed_precision not in [
+                "no",
+                "fp16",
+                "bf16",
+            ]:
body: I realize we are silently not training in mixed precision when:
- `mixed_precision="fp16"` and torch version is < 1.6
- `mixed_precision="bf16"` and torch version is < 1.10

I think we should raise an error message in both cases, or at least a warning.
diff_hunk: @@ -149,10 +175,14 @@ def __init__(
         # Mixed precision attributes
         self.scaler = None
         self.native_amp = False
-        if self.state.use_fp16:
+        if self.state.mixed_precision == "fp16":
             self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
             kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
             self.scaler = torch.cuda.amp.GradScaler(**kwargs)
+        elif self.state.mixed_precision == "bf16":
+            self.native_amp = version.parse(torch.__version__) >= version.parse("1.10")
+            kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
+            self.scaler = torch.cuda.amp.GradScaler(**kwargs)
body: Nice!
diff_hunk: @@ -97,6 +98,12 @@ def from_yaml_file(cls, yaml_file=None):
             config_dict = yaml.safe_load(f)
         if "compute_environment" not in config_dict:
             config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE
+        if "mixed_precision" not in config_dict:
body: No need for this new line I think.
```suggestion
```
diff_hunk: @@ -53,9 +54,20 @@ def launch_command_parser(subparsers=None):
     parser.add_argument(
         "--tpu", default=False, action="store_true", help="Whether or not this should launch a TPU training."
     )
+    parser.add_argument(
+        "--mixed_precision",
+        default="no",
+        type=str,
+        choices=["no", "fp16", "bf16"],
+        help="Whether or not to use mixed precision training. "
+        "Choose between FP16 and BF16 (bfloat16) training. "
+        "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.",
+    )
+
     parser.add_argument(
         "--fp16", default=False, action="store_true", help="Whether or not to use mixed precision training."
     )
+
body: ```suggestion
    if mixed_precision not in ["no", "fp16", "bf16"]:
```
diff_hunk: @@ -134,7 +146,20 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
-    current_env["USE_FP16"] = str(args.fp16)
+
+    mixed_precision = args.mixed_precision.lower()
+    if mixed_precision not in [
+        "no",
+        "fp16",
+        "bf16",
+    ]:
body: ```suggestion
    if mixed_precision not in ["no", "fp16", "bf16"]:
```
diff_hunk: @@ -168,7 +193,20 @@ def multi_gpu_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in [
+        "no",
+        "fp16",
+        "bf16",
+    ]:
body: ```suggestion
    if mixed_precision not in ["no", "fp16", "bf16"]:
```
diff_hunk: @@ -201,7 +239,20 @@ def deepspeed_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_FP16"] = str(args.fp16)
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in [
+        "no",
+        "fp16",
+        "bf16",
+    ]:
body: ```suggestion
    if mixed_precision not in ["no", "fp16", "bf16"]:
```
diff_hunk: @@ -309,8 +360,21 @@ def sagemaker_launcher(sagemaker_config: SageMakerConfig, args):
     print("Converting Arguments to Hyperparameters")
     hyperparameters = _convert_nargs_to_dict(args.training_script_args)
 
-    environment = {"USE_FP16": args.fp16}  # Environment variables to be set for use during training job
+    mixed_precision = args.mixed_precision.lower()
+
+    if mixed_precision not in [
+        "no",
+        "fp16",
+        "bf16",
+    ]:
body: You can remove this from the documentation (we try to avoid documenting deprecated args so users don't start using them).
diff_hunk: @@ -39,6 +40,8 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             the number of GPUs available otherwise.
         use_fp16 (:obj:`bool`, `optional`, defaults to :obj:`False`):
             If :obj:`True`, will use mixed precision training on multi-GPU.
body: ```suggestion
            if mixed_precision not in ["no", "fp16", "bf16"]:
```
diff_hunk: @@ -110,7 +113,22 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, use
             os.environ["WORLD_SIZE"] = str(num_processes)
             os.environ["MASTER_ADDR"] = "127.0.0.1"
             os.environ["MASTER_PORT"] = str(use_port)
-            os.environ["USE_FP16"] = str(use_fp16)
+
+            mixed_precision = mixed_precision.lower()
+            if mixed_precision not in [
+                "no",
+                "fp16",
+                "bf16",
+            ]:

Discussion: body: I have updated the PR with the fixes you mention 
body: Let me know if you need any help with the last standing comments. When you're finished, make sure to run `make style` on your branch so that the quality check passes :-)
body: I think they are solved now. Maybe we should add a warning in 'cluster.py' to warn if the user chooses the "fp16" or "bf16" option but doesn't have the correct version of Pytorch installed?
body: This [comment](https://github.com/huggingface/accelerate/pull/247/files#r800669214) and this [one](https://github.com/huggingface/accelerate/pull/247/files#r800671521) don't seem to have been addressed. Could you take care of them?
Quality check is also still failing.

We can also add warnings when the options are chosen, yes!
body: Sorry the previous commit to solve the problem of silently not training in mixed precision was incorrect. I have a question about implementation. If  the version is correct we set self.native_amp to True. 
```Python
self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
```
If the version is incorrect we disable mixed precision by setting that variable to False. Is it used for anything else? If we are going to raise an error, should we remove it? 
Should we raise a error always if the pytorch version is not correct? I'm not sure I understand this part of the code 100% and I don't know if I've implemented right the errors if the pytorch version is not compatible with the chosen mixed precision. 


body: We should at least raise a warning, to tell the user that we are not delivering what they asked for.
body: What I mean is that what I have done is adding the following code (same for bf16 but it checks pytorch 1.10).  This raises an error if you try to run "fp16" with Pytorch < 1.6. However, the previous line sets self.native_amp=True with the oposite condition. 
We will never run the training if native_amp = False because we raise an error. Is this the correct behavior? If it is, should we keep the native_amp variable?

```Python
if self.state.mixed_precision == "fp16":
    self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
    if version.parse(torch.__version__) < version.parse("1.6"):
        raise ValueError("fp16 mixed precision requires PyTorch >= 1.6")
```
body: The way you did it is great! Thanks again for your awesome contribution :-)