diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index cb904a9b5..6dc1f847c 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -21,8 +21,6 @@
 
 import torch
 
-from packaging import version
-
 from .checkpointing import load_accelerator_state, load_custom_state, save_accelerator_state, save_custom_state
 from .data_loader import prepare_data_loader
 from .logging import get_logger
@@ -46,6 +44,7 @@
     gather,
     get_pretty_name,
     is_deepspeed_available,
+    is_torch_version,
     pad_across_processes,
     reduce,
     save,
@@ -173,7 +172,7 @@ def __init__(
             os.environ["USE_FSDP"] = "true"  # use FSDP if plugin is provided
 
         if os.environ.get("USE_FSDP", "false") == "true":
-            if version.parse(torch.__version__) < version.parse("1.12.0.dev20220418+cu113"):
+            if is_torch_version("<", "1.12.0.dev20220418+cu113"):
                 raise ValueError("FSDP requires PyTorch >= 1.12.0.dev20220418+cu113")
 
         # Kwargs handlers
@@ -212,7 +211,7 @@ def __init__(
         self.device_placement = device_placement
         self.split_batches = split_batches
         self.dispatch_batches = dispatch_batches
-        if dispatch_batches is True and version.parse(torch.__version__) < version.parse("1.8.0"):
+        if dispatch_batches is True and is_torch_version("<", "1.8.0"):
             raise ImportError(
                 "Using `DataLoaderDispatcher` requires PyTorch 1.8.0 minimum. You have {torch.__version__}."
             )
@@ -222,15 +221,15 @@ def __init__(
         self.scaler = None
         self.native_amp = False
         if self.state.mixed_precision == "fp16":
-            self.native_amp = version.parse(torch.__version__) >= version.parse("1.6")
-            if version.parse(torch.__version__) < version.parse("1.6"):
+            self.native_amp = is_torch_version(">=", "1.6")
+            if not self.native_amp:
                 raise ValueError("fp16 mixed precision requires PyTorch >= 1.6")
 
             kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
             self.scaler = torch.cuda.amp.GradScaler(**kwargs)
         elif self.state.mixed_precision == "bf16":
-            self.native_amp = version.parse(torch.__version__) >= version.parse("1.10")
-            if mixed_precision == "bf16" and version.parse(torch.__version__) < version.parse("1.10"):
+            self.native_amp = is_torch_version(">=", "1.10")
+            if mixed_precision == "bf16" and not self.native_amp:
                 raise ValueError("bf16 mixed precision requires PyTorch >= 1.10")
 
             kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
@@ -245,7 +244,7 @@ def __init__(
         # RNG Types
         self.rng_types = rng_types
         if self.rng_types is None:
-            self.rng_types = ["torch"] if version.parse(torch.__version__) <= version.parse("1.5.1") else ["generator"]
+            self.rng_types = ["torch"] if is_torch_version("<=", "1.5.1") else ["generator"]
 
     @property
     def distributed_type(self):
@@ -486,7 +485,7 @@ def prepare_model(self, model):
             kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {}
             model = torch.nn.parallel.DistributedDataParallel(model, **kwargs)
         if self.native_amp:
-            if self.mixed_precision == "fp16" and version.parse(torch.__version__) >= version.parse("1.10"):
+            if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
             elif self.mixed_precision == "bf16":
                 model.forward = torch.cuda.amp.autocast(dtype=torch.bfloat16)(model.forward)
@@ -924,7 +923,7 @@ def autocast(self):
         different will happen otherwise.
         """
         if self.native_amp:
-            if self.mixed_precision == "fp16" and version.parse(torch.__version__) >= version.parse("1.10"):
+            if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 autocast_context = torch.cuda.amp.autocast(dtype=torch.float16)
             elif self.mixed_precision == "bf16":
                 autocast_context = torch.cuda.amp.autocast(dtype=torch.bfloat16)
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index ff4522162..22c8e621c 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -33,8 +33,7 @@
     PrepareForLaunch,
     is_sagemaker_available,
 )
-from accelerate.utils.versions import torch_version
-from packaging import version
+from accelerate.utils.versions import is_torch_version
 
 
 def launch_command_parser(subparsers=None):
@@ -219,9 +218,9 @@ def simple_launcher(args):
 
 
 def multi_gpu_launcher(args):
-    if torch_version >= version.parse("1.10.0"):
+    if is_torch_version(">=", "1.10.0"):
         cmd = ["torchrun"]
-    elif torch_version >= version.parse("1.9.0"):
+    elif is_torch_version(">=", "1.9.0"):
         cmd = [sys.executable, "-m", "torch.distributed.run"]
     else:
         cmd = [sys.executable, "-m", "torch.distributed.launch", "--use_env"]
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 50aeb57b6..79ba6b3f3 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -18,8 +18,6 @@
 import torch
 from torch.utils.data import BatchSampler, DataLoader, IterableDataset
 
-from packaging import version
-
 from .state import AcceleratorState, DistributedType, is_tpu_available
 from .utils import (
     RNGType,
@@ -29,6 +27,7 @@
     find_batch_size,
     get_data_structure,
     initialize_tensors,
+    is_torch_version,
     send_to_device,
     slice_tensors,
     synchronize_rng_states,
@@ -61,7 +60,7 @@
 }
 
 for v, additional_kwargs in _PYTORCH_DATALOADER_ADDITIONAL_KWARGS.items():
-    if version.parse(torch.__version__) >= version.parse(v):
+    if is_torch_version(">=", v):
         _PYTORCH_DATALOADER_KWARGS.update(additional_kwargs)
 
 
@@ -327,7 +326,7 @@ class DataLoaderDispatcher(DataLoader):
 
     def __init__(self, dataset, split_batches: bool = False, **kwargs):
         shuffle = False
-        if version.parse(torch.__version__) >= version.parse("1.11.0"):
+        if is_torch_version(">=", "1.11.0"):
             from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe
 
             # We need to save the shuffling state of the DataPipe
@@ -335,7 +334,7 @@ def __init__(self, dataset, split_batches: bool = False, **kwargs):
                 shuffle = dataset._shuffle_enabled
         super().__init__(dataset, **kwargs)
         self.split_batches = split_batches
-        if version.parse(torch.__version__) < version.parse("1.8.0"):
+        if is_torch_version("<", "1.8.0"):
             raise ImportError(
                 "Using `DataLoaderDispatcher` requires PyTorch 1.8.0 minimum. You have {torch.__version__}."
             )
@@ -487,7 +486,7 @@ def prepare_data_loader(
 
     </Tip>"""
     if dispatch_batches is None:
-        if version.parse(torch.__version__) < version.parse("1.8.0") or not put_on_device:
+        if is_torch_version("<", "1.8.0") or not put_on_device:
             dispatch_batches = False
         else:
             dispatch_batches = isinstance(dataloader.dataset, IterableDataset)
diff --git a/src/accelerate/launchers.py b/src/accelerate/launchers.py
index d3cd2e1a2..298db439b 100644
--- a/src/accelerate/launchers.py
+++ b/src/accelerate/launchers.py
@@ -19,10 +19,8 @@
 
 import torch
 
-from packaging import version
-
 from .state import AcceleratorState
-from .utils import PrecisionType, PrepareForLaunch, patch_environment
+from .utils import PrecisionType, PrepareForLaunch, is_torch_version, patch_environment
 
 
 def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mixed_precision="no", use_port="29500"):
@@ -85,7 +83,7 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mix
 
         if num_processes > 1:
             # Multi-GPU launch
-            if version.parse(torch.__version__) < version.parse("1.5.0"):
+            if is_torch_version("<", "1.5.0"):
                 raise ImportError(
                     "Using `notebook_launcher` for distributed training on GPUs require torch >= 1.5.0, got "
                     f"{torch.__version__}."
@@ -156,7 +154,7 @@ def debug_launcher(function, args=(), num_processes=2):
         num_processes (`int`, *optional*, defaults to 2):
             The number of processes to use for training.
     """
-    if version.parse(torch.__version__) < version.parse("1.5.0"):
+    if is_torch_version("<", "1.5.0"):
         raise ImportError(
             "Using `debug_launcher` for distributed training on GPUs require torch >= 1.5.0, got "
             f"{torch.__version__}."
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index dcf792592..3d5977968 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -17,10 +17,8 @@
 
 import torch
 
-from packaging import version
-
 from .state import AcceleratorState
-from .utils import DistributedType, honor_type, is_tpu_available
+from .utils import DistributedType, honor_type, is_torch_version, is_tpu_available
 
 
 if is_tpu_available():
@@ -103,7 +101,7 @@ def state_dict(self):
         return self.optimizer.state_dict()
 
     def zero_grad(self, set_to_none=None):
-        if version.parse(torch.__version__) < version.parse("1.7.0"):
+        if is_torch_version("<", "1.7.0"):
             if set_to_none is not None:
                 raise ValueError(
                     "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
diff --git a/src/accelerate/test_utils/test_script.py b/src/accelerate/test_utils/test_script.py
index 48e7bcc49..a23179bd7 100644
--- a/src/accelerate/test_utils/test_script.py
+++ b/src/accelerate/test_utils/test_script.py
@@ -21,8 +21,7 @@
 from accelerate.data_loader import prepare_data_loader
 from accelerate.state import AcceleratorState
 from accelerate.test_utils import RegressionDataset, RegressionModel, are_the_same_tensors
-from accelerate.utils import DistributedType, gather, set_seed, synchronize_rng_states
-from packaging import version
+from accelerate.utils import DistributedType, gather, is_torch_version, set_seed, synchronize_rng_states
 
 
 def init_state_check():
@@ -40,7 +39,7 @@ def rng_sync_check():
     if state.distributed_type == DistributedType.MULTI_GPU:
         synchronize_rng_states(["cuda"])
         assert are_the_same_tensors(torch.cuda.get_rng_state()), "RNG states improperly synchronized on GPU."
-    if version.parse(torch.__version__) >= version.parse("1.6.0"):
+    if is_torch_version(">=", "1.6.0"):
         generator = torch.Generator()
         synchronize_rng_states(["generator"], generator=generator)
         assert are_the_same_tensors(generator.get_state()), "RNG states improperly synchronized in generator."
diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index 19d8b81d4..186f27bb7 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -73,6 +73,7 @@
     send_to_device,
     slice_tensors,
 )
+from .versions import compare_versions, is_torch_version
 
 
 if is_deepspeed_available():
diff --git a/src/accelerate/utils/constants.py b/src/accelerate/utils/constants.py
index c2c36a486..af8a6bf35 100644
--- a/src/accelerate/utils/constants.py
+++ b/src/accelerate/utils/constants.py
@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import operator as op
+
+
 SCALER_NAME = "scaler.pt"
 MODEL_NAME = "pytorch_model"
 RNG_STATE_NAME = "random_states"
 OPTIMIZER_NAME = "optimizer"
 SCHEDULER_NAME = "scheduler"
+
+STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt}
diff --git a/src/accelerate/utils/operations.py b/src/accelerate/utils/operations.py
index 3fc9d969c..954bb5dfd 100644
--- a/src/accelerate/utils/operations.py
+++ b/src/accelerate/utils/operations.py
@@ -23,11 +23,10 @@
 import torch
 from torch.distributed import ReduceOp
 
-from packaging import version
-
 from ..state import AcceleratorState
 from .dataclasses import DistributedType, TensorInformation
 from .imports import is_tpu_available
+from .versions import is_torch_version
 
 
 if is_tpu_available():
@@ -459,8 +458,7 @@ def _convert_to_fp32(tensor):
 
     def _is_fp16_bf16_tensor(tensor):
         return hasattr(tensor, "dtype") and (
-            tensor.dtype == torch.float16
-            or (version.parse(torch.__version__) >= version.parse("1.10") and tensor.dtype == torch.bfloat16)
+            tensor.dtype == torch.float16 or (is_torch_version(">=", "1.10") and tensor.dtype == torch.bfloat16)
         )
 
     return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_bf16_tensor)
diff --git a/src/accelerate/utils/versions.py b/src/accelerate/utils/versions.py
index 36dd7ac48..38674d4b3 100644
--- a/src/accelerate/utils/versions.py
+++ b/src/accelerate/utils/versions.py
@@ -13,8 +13,11 @@
 # limitations under the License.
 
 import sys
+from typing import Union
 
-from packaging.version import parse
+from packaging.version import Version, parse
+
+from .constants import STR_OPERATION_TO_FUNC
 
 
 if sys.version_info < (3, 8):
@@ -23,3 +26,36 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """
+    Compares a library version to some requirement using a given operation.
+
+    Args:
+        library_or_version (`str` or `packaging.version.Version`):
+            A library name or a version to check.
+        operation (`str`):
+            A string representation of an operator, such as `">"` or `"<="`.
+        requirement_version (`str`):
+            The version to compare the library version against
+    """
+    if operation not in STR_OPERATION_TO_FUNC.keys():
+        raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}")
+    operation = STR_OPERATION_TO_FUNC[operation]
+    if isinstance(library_or_version, str):
+        library_or_version = parse(importlib_metadata.version(library_or_version))
+    return operation(library_or_version, parse(requirement_version))
+
+
+def is_torch_version(operation: str, version: str):
+    """
+    Compares the current PyTorch version to a given reference with an operation.
+
+    Args:
+        operation (`str`):
+            A string representation of an operator, such as `">"` or `"<="`
+        version (`str`):
+            A string version of PyTorch
+    """
+    return compare_versions(torch_version, operation, version)


Review comments: body: Not super fan of the name, wdyt of `torch_version_is("<", "1.12.0.dev20220418+cu113")` or `is_torch_version("<", "1.12.0.dev20220418+cu113")`?
diff_hunk: @@ -173,7 +172,7 @@ def __init__(
             os.environ["USE_FSDP"] = "true"  # use FSDP if plugin is provided
 
         if os.environ.get("USE_FSDP", "false") == "true":
-            if version.parse(torch.__version__) < version.parse("1.12.0.dev20220418+cu113"):
+            if check_torch_version("<", "1.12.0.dev20220418+cu113"):
body: Meant to be `compare_version` here?
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        requirement_version (`str`):
+            The version to compare the library version against
+    """
+    if operation not in STR_OPERATION_TO_FUNC.keys():
+        raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}")
+    operation = STR_OPERATION_TO_FUNC[operation]
+    if isinstance(library_or_version, str):
+        library_or_version = parse(importlib_metadata.version(library_or_version))
+    return operation(library_or_version, parse(requirement_version))
+
+
+def check_torch_version(operation: str, version: str):
+    """Compares the current PyTorch version to `version` with an `operation`
+
+    Args:
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        version (`str`):
+            A string version of PyTorch
+    """
+    return check_torch_version(torch_version, operation, version)
body: Yup, nice catch! 
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        requirement_version (`str`):
+            The version to compare the library version against
+    """
+    if operation not in STR_OPERATION_TO_FUNC.keys():
+        raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}")
+    operation = STR_OPERATION_TO_FUNC[operation]
+    if isinstance(library_or_version, str):
+        library_or_version = parse(importlib_metadata.version(library_or_version))
+    return operation(library_or_version, parse(requirement_version))
+
+
+def check_torch_version(operation: str, version: str):
+    """Compares the current PyTorch version to `version` with an `operation`
+
+    Args:
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        version (`str`):
+            A string version of PyTorch
+    """
+    return check_torch_version(torch_version, operation, version)
body: `is_torch_version` sounds better, will go with that. 
diff_hunk: @@ -173,7 +172,7 @@ def __init__(
             os.environ["USE_FSDP"] = "true"  # use FSDP if plugin is provided
 
         if os.environ.get("USE_FSDP", "false") == "true":
-            if version.parse(torch.__version__) < version.parse("1.12.0.dev20220418+cu113"):
+            if check_torch_version("<", "1.12.0.dev20220418+cu113"):
body: ```suggestion
        library_or_version (`str` or `packaging.version.Version`):
```
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
body: ```suggestion
            A library name or a version to check.
```
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
body: ```suggestion
    """
    Compares a library version to some requirement using a given operation.
```
I prefer to have complete English sentence describe the function :-)
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
body: ```suggestion
            A string representation of an operator, such as `">"` or `"<="`.
```
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
body: ```suggestion
    """
    Compares the current PyTorch version to a given reference with an operation.
```
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        requirement_version (`str`):
+            The version to compare the library version against
+    """
+    if operation not in STR_OPERATION_TO_FUNC.keys():
+        raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}")
+    operation = STR_OPERATION_TO_FUNC[operation]
+    if isinstance(library_or_version, str):
+        library_or_version = parse(importlib_metadata.version(library_or_version))
+    return operation(library_or_version, parse(requirement_version))
+
+
+def is_torch_version(operation: str, version: str):
+    """Compares the current PyTorch version to `version` with an `operation`
body: ```suggestion
            A string representation of an operator, such as `">"` or `"<="`
```
diff_hunk: @@ -23,3 +26,34 @@
     import importlib.metadata as importlib_metadata
 
 torch_version = parse(importlib_metadata.version("torch"))
+
+
+def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str):
+    """Compares `library_or_version` to `requirement_version` with an `operation`
+
+    Args:
+        library_or_version (`str`, `packaging.version.Version]`):
+            A library name or Version to check
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="
+        requirement_version (`str`):
+            The version to compare the library version against
+    """
+    if operation not in STR_OPERATION_TO_FUNC.keys():
+        raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}")
+    operation = STR_OPERATION_TO_FUNC[operation]
+    if isinstance(library_or_version, str):
+        library_or_version = parse(importlib_metadata.version(library_or_version))
+    return operation(library_or_version, parse(requirement_version))
+
+
+def is_torch_version(operation: str, version: str):
+    """Compares the current PyTorch version to `version` with an `operation`
+
+    Args:
+        operation (`str`):
+            A string representation of an operator, such as ">" or "<="

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._