diff --git a/setup.py b/setup.py
index cad067ad5..a59d1ecaf 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
 extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3", "hf-doc-builder >= 0.3.0"]
 extras["docs"] = []
 extras["test_prod"] = ["pytest", "pytest-xdist", "pytest-subtests", "parameterized"]
-extras["test_dev"] = ["datasets", "evaluate", "transformers", "scipy", "sklearn", "deepspeed"]
+extras["test_dev"] = ["datasets", "evaluate", "transformers", "scipy", "sklearn", "deepspeed", "tqdm"]
 extras["testing"] = extras["test_prod"] + extras["test_dev"]
 
 extras["test_trackers"] = ["wandb", "comet-ml", "tensorboard"]
diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index f6b83d2f9..af8ddd736 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -100,3 +100,4 @@
     write_basic_config,
 )
 from .random import set_seed, synchronize_rng_state, synchronize_rng_states
+from .tqdm import tqdm
diff --git a/src/accelerate/utils/imports.py b/src/accelerate/utils/imports.py
index b84a90c6e..4b796e62c 100644
--- a/src/accelerate/utils/imports.py
+++ b/src/accelerate/utils/imports.py
@@ -113,3 +113,7 @@ def is_boto3_available():
 
 def is_sagemaker_available():
     return importlib.util.find_spec("sagemaker") is not None
+
+
+def is_tqdm_available():
+    return importlib.util.find_spec("tqdm") is not None
diff --git a/src/accelerate/utils/tqdm.py b/src/accelerate/utils/tqdm.py
new file mode 100644
index 000000000..be489bd90
--- /dev/null
+++ b/src/accelerate/utils/tqdm.py
@@ -0,0 +1,37 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .imports import is_tqdm_available
+
+
+if is_tqdm_available():
+    import tqdm.auto as _tqdm
+
+from ..state import AcceleratorState
+
+
+def tqdm(main_process_only: bool = True, *args, **kwargs):
+    """
+    Wrapper around `tqdm.tqdm` that optionally displays only on the main process.
+
+    Args:
+        main_process_only (`bool`, *optional*):
+            Whether to display the progress bar only on the main process
+    """
+    if not is_tqdm_available():
+        raise ImportError("Accelerate's `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`.")
+    disable = False
+    if main_process_only:
+        disable = AcceleratorState().local_process_index == 0
+    return _tqdm(*args, **kwargs, disable=disable)


Review comments: body: Let's make tqdm a soft dependency.
diff_hunk: @@ -50,7 +50,7 @@
         ]
     },
     python_requires=">=3.7.0",
-    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0"],
+    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0", "tqdm"],

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._